i915_irq.c 130.7 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47
/**
 * DOC: interrupt handling
 *
 * These functions provide the basic support for enabling and disabling the
 * interrupt handling support. There's a lot more functionality in i915_irq.c
 * and related files, but that will be described in separate chapters.
 */

48 49 50 51
static const u32 hpd_ilk[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
};

52 53 54 55
static const u32 hpd_ivb[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
};

56 57 58 59
static const u32 hpd_bdw[HPD_NUM_PINS] = {
	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
};

60
static const u32 hpd_ibx[HPD_NUM_PINS] = {
61 62 63 64 65 66 67
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

68
static const u32 hpd_cpt[HPD_NUM_PINS] = {
69
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71 72 73 74 75
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

X
Xiong Zhang 已提交
76
static const u32 hpd_spt[HPD_NUM_PINS] = {
77
	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
X
Xiong Zhang 已提交
78 79 80 81 82 83
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
};

84
static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85 86 87 88 89 90 91 92
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

93
static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94 95 96 97 98 99 100 101
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

102
static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103 104 105 106 107 108 109 110
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

111 112
/* BXT hpd list */
static const u32 hpd_bxt[HPD_NUM_PINS] = {
113
	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114 115 116 117
	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
};

118
/* IIR can theoretically queue up two events. Be paranoid. */
119
#define GEN8_IRQ_RESET_NDX(type, which) do { \
120 121 122 123 124 125 126 127 128
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

129
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
130
	I915_WRITE(type##IMR, 0xffffffff); \
131
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
132
	I915_WRITE(type##IER, 0); \
133 134 135 136
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
137 138
} while (0)

139 140 141
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
142 143
static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
				    i915_reg_t reg)
144 145 146 147 148 149 150
{
	u32 val = I915_READ(reg);

	if (val == 0)
		return;

	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
151
	     i915_mmio_reg_offset(reg), val);
152 153 154 155 156
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
}
157

P
Paulo Zanoni 已提交
158
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
159
	gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
160
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
161 162
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
P
Paulo Zanoni 已提交
163 164 165
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
166
	gen5_assert_iir_is_zero(dev_priv, type##IIR); \
P
Paulo Zanoni 已提交
167
	I915_WRITE(type##IER, (ier_val)); \
168 169
	I915_WRITE(type##IMR, (imr_val)); \
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
170 171
} while (0)

172 173
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);

174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211
/* For display hotplug interrupt */
static inline void
i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
				     uint32_t mask,
				     uint32_t bits)
{
	uint32_t val;

	assert_spin_locked(&dev_priv->irq_lock);
	WARN_ON(bits & ~mask);

	val = I915_READ(PORT_HOTPLUG_EN);
	val &= ~mask;
	val |= bits;
	I915_WRITE(PORT_HOTPLUG_EN, val);
}

/**
 * i915_hotplug_interrupt_update - update hotplug interrupt enable
 * @dev_priv: driver private
 * @mask: bits to update
 * @bits: bits to enable
 * NOTE: the HPD enable bits are modified both inside and outside
 * of an interrupt context. To avoid that read-modify-write cycles
 * interfer, these bits are protected by a spinlock. Since this
 * function is usually not called from a context where the lock is
 * held already, this function acquires the lock itself. A non-locking
 * version is also available.
 */
void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
				   uint32_t mask,
				   uint32_t bits)
{
	spin_lock_irq(&dev_priv->irq_lock);
	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
	spin_unlock_irq(&dev_priv->irq_lock);
}

212 213 214 215 216 217
/**
 * ilk_update_display_irq - update DEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
218 219 220
void ilk_update_display_irq(struct drm_i915_private *dev_priv,
			    uint32_t interrupt_mask,
			    uint32_t enabled_irq_mask)
221
{
222 223
	uint32_t new_val;

224 225
	assert_spin_locked(&dev_priv->irq_lock);

226 227
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

228
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
229 230
		return;

231 232 233 234 235 236
	new_val = dev_priv->irq_mask;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->irq_mask) {
		dev_priv->irq_mask = new_val;
237
		I915_WRITE(DEIMR, dev_priv->irq_mask);
238
		POSTING_READ(DEIMR);
239 240 241
	}
}

P
Paulo Zanoni 已提交
242 243 244 245 246 247 248 249 250 251 252 253
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
	assert_spin_locked(&dev_priv->irq_lock);

254 255
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

256
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
257 258
		return;

P
Paulo Zanoni 已提交
259 260 261 262 263 264
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
	POSTING_READ(GTIMR);
}

265
void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
266 267 268 269
{
	ilk_update_gt_irq(dev_priv, mask, mask);
}

270
void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
271 272 273 274
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

275
static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
276 277 278 279
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
}

280
static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
281 282 283 284
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
}

285
static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
286 287 288 289
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
}

P
Paulo Zanoni 已提交
290
/**
291 292 293 294 295
 * snb_update_pm_irq - update GEN6_PMIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
P
Paulo Zanoni 已提交
296 297 298 299
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
300
	uint32_t new_val;
P
Paulo Zanoni 已提交
301

302 303
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

P
Paulo Zanoni 已提交
304 305
	assert_spin_locked(&dev_priv->irq_lock);

306
	new_val = dev_priv->pm_irq_mask;
307 308 309
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

310 311
	if (new_val != dev_priv->pm_irq_mask) {
		dev_priv->pm_irq_mask = new_val;
312 313
		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
		POSTING_READ(gen6_pm_imr(dev_priv));
314
	}
P
Paulo Zanoni 已提交
315 316
}

317
void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
318
{
319 320 321
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

P
Paulo Zanoni 已提交
322 323 324
	snb_update_pm_irq(dev_priv, mask, mask);
}

325 326
static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
				  uint32_t mask)
P
Paulo Zanoni 已提交
327 328 329 330
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

331 332 333 334 335 336 337 338
void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	__gen6_disable_pm_irq(dev_priv, mask);
}

339
void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
I
Imre Deak 已提交
340
{
341
	i915_reg_t reg = gen6_pm_iir(dev_priv);
I
Imre Deak 已提交
342 343 344 345 346

	spin_lock_irq(&dev_priv->irq_lock);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	POSTING_READ(reg);
347
	dev_priv->rps.pm_iir = 0;
I
Imre Deak 已提交
348 349 350
	spin_unlock_irq(&dev_priv->irq_lock);
}

351
void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
352 353
{
	spin_lock_irq(&dev_priv->irq_lock);
354

355
	WARN_ON(dev_priv->rps.pm_iir);
I
Imre Deak 已提交
356
	WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
I
Imre Deak 已提交
357
	dev_priv->rps.interrupts_enabled = true;
358 359
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
				dev_priv->pm_rps_events);
360
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
361

362 363 364
	spin_unlock_irq(&dev_priv->irq_lock);
}

365 366
u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
{
367
	return (mask & ~dev_priv->rps.pm_intr_keep);
368 369
}

370
void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
371
{
I
Imre Deak 已提交
372 373 374 375 376 377
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->rps.interrupts_enabled = false;
	spin_unlock_irq(&dev_priv->irq_lock);

	cancel_work_sync(&dev_priv->rps.work);

378 379
	spin_lock_irq(&dev_priv->irq_lock);

380
	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
381 382

	__gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
383 384
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
				~dev_priv->pm_rps_events);
385 386 387

	spin_unlock_irq(&dev_priv->irq_lock);

388
	synchronize_irq(dev_priv->dev->irq);
389 390
}

391
/**
392 393 394 395 396
 * bdw_update_port_irq - update DE port interrupt
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
				uint32_t interrupt_mask,
				uint32_t enabled_irq_mask)
{
	uint32_t new_val;
	uint32_t old_val;

	assert_spin_locked(&dev_priv->irq_lock);

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	old_val = I915_READ(GEN8_DE_PORT_IMR);

	new_val = old_val;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != old_val) {
		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		POSTING_READ(GEN8_DE_PORT_IMR);
	}
}

423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
/**
 * bdw_update_pipe_irq - update DE pipe interrupt
 * @dev_priv: driver private
 * @pipe: pipe whose interrupt to update
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
			 enum pipe pipe,
			 uint32_t interrupt_mask,
			 uint32_t enabled_irq_mask)
{
	uint32_t new_val;

	assert_spin_locked(&dev_priv->irq_lock);

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	new_val = dev_priv->de_irq_mask[pipe];
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->de_irq_mask[pipe]) {
		dev_priv->de_irq_mask[pipe] = new_val;
		I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
	}
}

455 456 457 458 459 460
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
461 462 463
void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
				  uint32_t interrupt_mask,
				  uint32_t enabled_irq_mask)
464 465 466 467 468
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

469 470
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

471 472
	assert_spin_locked(&dev_priv->irq_lock);

473
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
474 475
		return;

476 477 478
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
479

D
Daniel Vetter 已提交
480
static void
481 482
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
483
{
484
	i915_reg_t reg = PIPESTAT(pipe);
485
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
486

487
	assert_spin_locked(&dev_priv->irq_lock);
488
	WARN_ON(!intel_irqs_enabled(dev_priv));
489

490 491 492 493
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
494 495 496
		return;

	if ((pipestat & enable_mask) == enable_mask)
497 498
		return;

499 500
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

501
	/* Enable the interrupt, clear any pending status */
502
	pipestat |= enable_mask | status_mask;
503 504
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
505 506
}

D
Daniel Vetter 已提交
507
static void
508 509
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
510
{
511
	i915_reg_t reg = PIPESTAT(pipe);
512
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
513

514
	assert_spin_locked(&dev_priv->irq_lock);
515
	WARN_ON(!intel_irqs_enabled(dev_priv));
516

517 518 519 520
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
521 522
		return;

523 524 525
	if ((pipestat & enable_mask) == 0)
		return;

526 527
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

528
	pipestat &= ~enable_mask;
529 530
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
531 532
}

533 534 535 536 537
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
538 539
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
540 541 542
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
543 544 545 546 547 548
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
549 550 551 552 553 554 555 556 557 558 559 560

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

561 562 563 564 565 566
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

567
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
568 569 570 571
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
572 573 574 575 576 577 578 579 580
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

581
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
582 583 584 585
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
586 587 588
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

589
/**
590
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
591
 * @dev_priv: i915 device private
592
 */
593
static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
594
{
595
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev_priv))
596 597
		return;

598
	spin_lock_irq(&dev_priv->irq_lock);
599

600
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
601
	if (INTEL_GEN(dev_priv) >= 4)
602
		i915_enable_pipestat(dev_priv, PIPE_A,
603
				     PIPE_LEGACY_BLC_EVENT_STATUS);
604

605
	spin_unlock_irq(&dev_priv->irq_lock);
606 607
}

608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via PIPECONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on PIPECONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

658
static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
659 660 661 662 663
{
	/* Gen2 doesn't have a hardware frame counter */
	return 0;
}

664 665 666
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
667
static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
668
{
669
	struct drm_i915_private *dev_priv = dev->dev_private;
670
	i915_reg_t high_frame, low_frame;
671
	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
672 673
	struct intel_crtc *intel_crtc =
		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
674
	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
675

676 677 678 679 680
	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vbl_start = mode->crtc_vblank_start;
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
681

682 683 684 685 686 687
	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

688 689
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
690

691 692 693 694 695 696
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
697
		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
698
		low   = I915_READ(low_frame);
699
		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
700 701
	} while (high1 != high2);

702
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
703
	pixel = low & PIPE_PIXEL_MASK;
704
	low >>= PIPE_FRAME_LOW_SHIFT;
705 706 707 708 709 710

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
711
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
712 713
}

714
static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
715
{
716
	struct drm_i915_private *dev_priv = dev->dev_private;
717

718
	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
719 720
}

721
/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
722 723 724 725
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
726
	const struct drm_display_mode *mode = &crtc->base.hwmode;
727
	enum pipe pipe = crtc->pipe;
728
	int position, vtotal;
729

730
	vtotal = mode->crtc_vtotal;
731 732 733
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

734
	if (IS_GEN2(dev_priv))
735
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
736
	else
737
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
738

739 740 741 742 743 744 745 746 747 748 749 750
	/*
	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
	 * read it just before the start of vblank.  So try it again
	 * so we don't accidentally end up spanning a vblank frame
	 * increment, causing the pipe_update_end() code to squak at us.
	 *
	 * The nature of this problem means we can't simply check the ISR
	 * bit and return the vblank start value; nor can we use the scanline
	 * debug register in the transcoder as it appears to have the same
	 * problem.  We may need to extend this to include other platforms,
	 * but so far testing only shows the problem on HSW.
	 */
751
	if (HAS_DDI(dev_priv) && !position) {
752 753 754 755 756 757 758 759 760 761 762 763 764
		int i, temp;

		for (i = 0; i < 100; i++) {
			udelay(1);
			temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
				DSL_LINEMASK_GEN3;
			if (temp != position) {
				position = temp;
				break;
			}
		}
	}

765
	/*
766 767
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
768
	 */
769
	return (position + crtc->scanline_offset) % vtotal;
770 771
}

772
static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
773
				    unsigned int flags, int *vpos, int *hpos,
774 775
				    ktime_t *stime, ktime_t *etime,
				    const struct drm_display_mode *mode)
776
{
777 778 779
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
780
	int position;
781
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
782 783
	bool in_vbl = true;
	int ret = 0;
784
	unsigned long irqflags;
785

786
	if (WARN_ON(!mode->crtc_clock)) {
787
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
788
				 "pipe %c\n", pipe_name(pipe));
789 790 791
		return 0;
	}

792
	htotal = mode->crtc_htotal;
793
	hsync_start = mode->crtc_hsync_start;
794 795 796
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
797

798 799 800 801 802 803
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

804 805
	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

806 807 808 809 810 811
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
812

813 814 815 816 817 818
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

819
	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
820 821 822
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
823
		position = __intel_get_crtc_scanline(intel_crtc);
824 825 826 827 828
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
829
		position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
830

831 832 833 834
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
835

836 837 838 839 840 841 842 843 844 845 846 847
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

848 849 850 851 852 853 854 855 856 857
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
858 859
	}

860 861 862 863 864 865 866 867
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

868 869 870 871 872 873 874 875 876 877 878 879
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
880

881
	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
882 883 884 885 886 887
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
888 889 890

	/* In vblank? */
	if (in_vbl)
891
		ret |= DRM_SCANOUTPOS_IN_VBLANK;
892 893 894 895

	return ret;
}

896 897 898 899 900 901 902 903 904 905 906 907 908
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

909
static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
910 911 912 913
			      int *max_error,
			      struct timeval *vblank_time,
			      unsigned flags)
{
914
	struct drm_crtc *crtc;
915

916 917
	if (pipe >= INTEL_INFO(dev)->num_pipes) {
		DRM_ERROR("Invalid crtc %u\n", pipe);
918 919 920 921
		return -EINVAL;
	}

	/* Get drm_crtc to timestamp: */
922 923
	crtc = intel_get_crtc_for_pipe(dev, pipe);
	if (crtc == NULL) {
924
		DRM_ERROR("Invalid crtc %u\n", pipe);
925 926 927
		return -EINVAL;
	}

928
	if (!crtc->hwmode.crtc_clock) {
929
		DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
930 931
		return -EBUSY;
	}
932 933

	/* Helper routine in DRM core does all the work: */
934 935
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
						     vblank_time, flags,
936
						     &crtc->hwmode);
937 938
}

939
static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
940
{
941
	u32 busy_up, busy_down, max_avg, min_avg;
942 943
	u8 new_delay;

944
	spin_lock(&mchdev_lock);
945

946 947
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

948
	new_delay = dev_priv->ips.cur_delay;
949

950
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
951 952
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
953 954 955 956
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
957
	if (busy_up > max_avg) {
958 959 960 961
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
962
	} else if (busy_down < min_avg) {
963 964 965 966
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
967 968
	}

969
	if (ironlake_set_drps(dev_priv, new_delay))
970
		dev_priv->ips.cur_delay = new_delay;
971

972
	spin_unlock(&mchdev_lock);
973

974 975 976
	return;
}

977
static void notify_ring(struct intel_engine_cs *engine)
978
{
979
	smp_store_mb(engine->irq_posted, true);
980 981 982 983
	if (intel_engine_wakeup(engine)) {
		trace_i915_gem_request_notify(engine);
		engine->user_interrupts++;
	}
984 985
}

986 987
static void vlv_c0_read(struct drm_i915_private *dev_priv,
			struct intel_rps_ei *ei)
988
{
989 990 991 992
	ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
}
993

994 995 996 997 998 999
static bool vlv_c0_above(struct drm_i915_private *dev_priv,
			 const struct intel_rps_ei *old,
			 const struct intel_rps_ei *now,
			 int threshold)
{
	u64 time, c0;
1000
	unsigned int mul = 100;
1001

1002 1003
	if (old->cz_clock == 0)
		return false;
1004

1005 1006 1007
	if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
		mul <<= 8;

1008
	time = now->cz_clock - old->cz_clock;
1009
	time *= threshold * dev_priv->czclk_freq;
1010

1011 1012 1013
	/* Workload can be split between render + media, e.g. SwapBuffers
	 * being blitted in X after being rendered in mesa. To account for
	 * this we need to combine both engines into our activity counter.
1014
	 */
1015 1016
	c0 = now->render_c0 - old->render_c0;
	c0 += now->media_c0 - old->media_c0;
1017
	c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC;
1018

1019
	return c0 >= time;
1020 1021
}

1022
void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1023
{
1024 1025 1026
	vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
	dev_priv->rps.up_ei = dev_priv->rps.down_ei;
}
1027

1028 1029 1030 1031
static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
{
	struct intel_rps_ei now;
	u32 events = 0;
1032

1033
	if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
1034
		return 0;
1035

1036 1037 1038
	vlv_c0_read(dev_priv, &now);
	if (now.cz_clock == 0)
		return 0;
1039

1040 1041 1042
	if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
		if (!vlv_c0_above(dev_priv,
				  &dev_priv->rps.down_ei, &now,
1043
				  dev_priv->rps.down_threshold))
1044 1045 1046
			events |= GEN6_PM_RP_DOWN_THRESHOLD;
		dev_priv->rps.down_ei = now;
	}
1047

1048 1049 1050
	if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
		if (vlv_c0_above(dev_priv,
				 &dev_priv->rps.up_ei, &now,
1051
				 dev_priv->rps.up_threshold))
1052 1053
			events |= GEN6_PM_RP_UP_THRESHOLD;
		dev_priv->rps.up_ei = now;
1054 1055
	}

1056
	return events;
1057 1058
}

1059 1060
static bool any_waiters(struct drm_i915_private *dev_priv)
{
1061
	struct intel_engine_cs *engine;
1062

1063
	for_each_engine(engine, dev_priv)
1064
		if (intel_engine_has_waiter(engine))
1065 1066 1067 1068 1069
			return true;

	return false;
}

1070
static void gen6_pm_rps_work(struct work_struct *work)
1071
{
1072 1073
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
1074 1075
	bool client_boost;
	int new_delay, adj, min, max;
P
Paulo Zanoni 已提交
1076
	u32 pm_iir;
1077

1078
	spin_lock_irq(&dev_priv->irq_lock);
I
Imre Deak 已提交
1079 1080 1081 1082 1083
	/* Speed up work cancelation during disabling rps interrupts. */
	if (!dev_priv->rps.interrupts_enabled) {
		spin_unlock_irq(&dev_priv->irq_lock);
		return;
	}
1084 1085 1086 1087 1088 1089 1090 1091

	/*
	 * The RPS work is synced during runtime suspend, we don't require a
	 * wakeref. TODO: instead of disabling the asserts make sure that we
	 * always hold an RPM reference while the work is running.
	 */
	DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);

1092 1093
	pm_iir = dev_priv->rps.pm_iir;
	dev_priv->rps.pm_iir = 0;
1094 1095
	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
1096 1097
	client_boost = dev_priv->rps.client_boost;
	dev_priv->rps.client_boost = false;
1098
	spin_unlock_irq(&dev_priv->irq_lock);
1099

1100
	/* Make sure we didn't queue anything we're not going to process. */
1101
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1102

1103
	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1104
		goto out;
1105

1106
	mutex_lock(&dev_priv->rps.hw_lock);
1107

1108 1109
	pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);

1110
	adj = dev_priv->rps.last_adj;
1111
	new_delay = dev_priv->rps.cur_freq;
1112 1113 1114 1115 1116 1117 1118
	min = dev_priv->rps.min_freq_softlimit;
	max = dev_priv->rps.max_freq_softlimit;

	if (client_boost) {
		new_delay = dev_priv->rps.max_freq_softlimit;
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1119 1120
		if (adj > 0)
			adj *= 2;
1121 1122
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1123 1124 1125 1126
		/*
		 * For better performance, jump directly
		 * to RPe if we're below it.
		 */
1127
		if (new_delay < dev_priv->rps.efficient_freq - adj) {
1128
			new_delay = dev_priv->rps.efficient_freq;
1129 1130
			adj = 0;
		}
1131 1132
	} else if (any_waiters(dev_priv)) {
		adj = 0;
1133
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1134 1135
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1136
		else
1137
			new_delay = dev_priv->rps.min_freq_softlimit;
1138 1139 1140 1141
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
1142 1143
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1144
	} else { /* unknown event */
1145
		adj = 0;
1146
	}
1147

1148 1149
	dev_priv->rps.last_adj = adj;

1150 1151 1152
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1153
	new_delay += adj;
1154
	new_delay = clamp_t(int, new_delay, min, max);
1155

1156
	intel_set_rps(dev_priv, new_delay);
1157

1158
	mutex_unlock(&dev_priv->rps.hw_lock);
1159 1160
out:
	ENABLE_RPM_WAKEREF_ASSERTS(dev_priv);
1161 1162
}

1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1175 1176
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, l3_parity.error_work);
1177
	u32 error_status, row, bank, subbank;
1178
	char *parity_event[6];
1179
	uint32_t misccpctl;
1180
	uint8_t slice = 0;
1181 1182 1183 1184 1185 1186 1187

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
	mutex_lock(&dev_priv->dev->struct_mutex);

1188 1189 1190 1191
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1192 1193 1194 1195
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1196
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1197
		i915_reg_t reg;
1198

1199
		slice--;
1200
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv)))
1201
			break;
1202

1203
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1204

1205
		reg = GEN7_L3CDERRST1(slice);
1206

1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1222
		kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1223
				   KOBJ_CHANGE, parity_event);
1224

1225 1226
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1227

1228 1229 1230 1231 1232
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1233

1234
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1235

1236 1237
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
1238
	spin_lock_irq(&dev_priv->irq_lock);
1239
	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1240
	spin_unlock_irq(&dev_priv->irq_lock);
1241 1242

	mutex_unlock(&dev_priv->dev->struct_mutex);
1243 1244
}

1245 1246
static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv,
					       u32 iir)
1247
{
1248
	if (!HAS_L3_DPF(dev_priv))
1249 1250
		return;

1251
	spin_lock(&dev_priv->irq_lock);
1252
	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1253
	spin_unlock(&dev_priv->irq_lock);
1254

1255
	iir &= GT_PARITY_ERROR(dev_priv);
1256 1257 1258 1259 1260 1261
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1262
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1263 1264
}

1265
static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
1266 1267
			       u32 gt_iir)
{
1268
	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1269
		notify_ring(&dev_priv->engine[RCS]);
1270
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
1271
		notify_ring(&dev_priv->engine[VCS]);
1272 1273
}

1274
static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
1275 1276
			       u32 gt_iir)
{
1277
	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1278
		notify_ring(&dev_priv->engine[RCS]);
1279
	if (gt_iir & GT_BSD_USER_INTERRUPT)
1280
		notify_ring(&dev_priv->engine[VCS]);
1281
	if (gt_iir & GT_BLT_USER_INTERRUPT)
1282
		notify_ring(&dev_priv->engine[BCS]);
1283

1284 1285
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
1286 1287
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1288

1289 1290
	if (gt_iir & GT_PARITY_ERROR(dev_priv))
		ivybridge_parity_error_irq_handler(dev_priv, gt_iir);
1291 1292
}

1293
static __always_inline void
1294
gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
1295 1296
{
	if (iir & (GT_RENDER_USER_INTERRUPT << test_shift))
1297
		notify_ring(engine);
1298
	if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift))
1299
		tasklet_schedule(&engine->irq_tasklet);
1300 1301
}

1302 1303 1304
static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv,
				   u32 master_ctl,
				   u32 gt_iir[4])
1305 1306 1307 1308
{
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1309 1310 1311
		gt_iir[0] = I915_READ_FW(GEN8_GT_IIR(0));
		if (gt_iir[0]) {
			I915_WRITE_FW(GEN8_GT_IIR(0), gt_iir[0]);
1312 1313 1314 1315 1316
			ret = IRQ_HANDLED;
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1317
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1318 1319 1320
		gt_iir[1] = I915_READ_FW(GEN8_GT_IIR(1));
		if (gt_iir[1]) {
			I915_WRITE_FW(GEN8_GT_IIR(1), gt_iir[1]);
1321
			ret = IRQ_HANDLED;
1322
		} else
1323
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
1324 1325
	}

1326
	if (master_ctl & GEN8_GT_VECS_IRQ) {
1327 1328 1329
		gt_iir[3] = I915_READ_FW(GEN8_GT_IIR(3));
		if (gt_iir[3]) {
			I915_WRITE_FW(GEN8_GT_IIR(3), gt_iir[3]);
1330 1331 1332 1333 1334
			ret = IRQ_HANDLED;
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

1335
	if (master_ctl & GEN8_GT_PM_IRQ) {
1336 1337
		gt_iir[2] = I915_READ_FW(GEN8_GT_IIR(2));
		if (gt_iir[2] & dev_priv->pm_rps_events) {
1338
			I915_WRITE_FW(GEN8_GT_IIR(2),
1339
				      gt_iir[2] & dev_priv->pm_rps_events);
1340
			ret = IRQ_HANDLED;
1341 1342 1343 1344
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1345 1346 1347
	return ret;
}

1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
static void gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
				u32 gt_iir[4])
{
	if (gt_iir[0]) {
		gen8_cs_irq_handler(&dev_priv->engine[RCS],
				    gt_iir[0], GEN8_RCS_IRQ_SHIFT);
		gen8_cs_irq_handler(&dev_priv->engine[BCS],
				    gt_iir[0], GEN8_BCS_IRQ_SHIFT);
	}

	if (gt_iir[1]) {
		gen8_cs_irq_handler(&dev_priv->engine[VCS],
				    gt_iir[1], GEN8_VCS1_IRQ_SHIFT);
		gen8_cs_irq_handler(&dev_priv->engine[VCS2],
				    gt_iir[1], GEN8_VCS2_IRQ_SHIFT);
	}

	if (gt_iir[3])
		gen8_cs_irq_handler(&dev_priv->engine[VECS],
				    gt_iir[3], GEN8_VECS_IRQ_SHIFT);

	if (gt_iir[2] & dev_priv->pm_rps_events)
		gen6_rps_irq_handler(dev_priv, gt_iir[2]);
}

1373 1374 1375 1376
static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
1377
		return val & PORTA_HOTPLUG_LONG_DETECT;
1378 1379 1380 1381 1382 1383 1384 1385 1386
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1387 1388 1389 1390 1391 1392 1393 1394 1395 1396
static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_E:
		return val & PORTE_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412
static bool spt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & PORTA_HOTPLUG_LONG_DETECT;
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	case PORT_D:
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1413 1414 1415 1416 1417 1418 1419 1420 1421 1422
static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1423
static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1424 1425 1426
{
	switch (port) {
	case PORT_B:
1427
		return val & PORTB_HOTPLUG_LONG_DETECT;
1428
	case PORT_C:
1429
		return val & PORTC_HOTPLUG_LONG_DETECT;
1430
	case PORT_D:
1431 1432 1433
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
1434 1435 1436
	}
}

1437
static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1438 1439 1440
{
	switch (port) {
	case PORT_B:
1441
		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1442
	case PORT_C:
1443
		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1444
	case PORT_D:
1445 1446 1447
		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
	default:
		return false;
1448 1449 1450
	}
}

1451 1452 1453 1454 1455 1456 1457
/*
 * Get a bit mask of pins that have triggered, and which ones may be long.
 * This can be called multiple times with the same masks to accumulate
 * hotplug detection results from several registers.
 *
 * Note that the caller is expected to zero out the masks initially.
 */
1458
static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1459
			     u32 hotplug_trigger, u32 dig_hotplug_reg,
1460 1461
			     const u32 hpd[HPD_NUM_PINS],
			     bool long_pulse_detect(enum port port, u32 val))
1462
{
1463
	enum port port;
1464 1465 1466
	int i;

	for_each_hpd_pin(i) {
1467 1468
		if ((hpd[i] & hotplug_trigger) == 0)
			continue;
1469

1470 1471
		*pin_mask |= BIT(i);

1472 1473 1474
		if (!intel_hpd_pin_to_port(i, &port))
			continue;

1475
		if (long_pulse_detect(port, dig_hotplug_reg))
1476
			*long_mask |= BIT(i);
1477 1478 1479 1480 1481 1482 1483
	}

	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
			 hotplug_trigger, dig_hotplug_reg, *pin_mask);

}

1484
static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1485
{
1486
	wake_up_all(&dev_priv->gmbus_wait_queue);
1487 1488
}

1489
static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1490
{
1491
	wake_up_all(&dev_priv->gmbus_wait_queue);
1492 1493
}

1494
#if defined(CONFIG_DEBUG_FS)
1495 1496
static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
					 enum pipe pipe,
1497 1498 1499
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1500 1501 1502
{
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
1503
	int head, tail;
1504

1505 1506
	spin_lock(&pipe_crc->lock);

1507
	if (!pipe_crc->entries) {
1508
		spin_unlock(&pipe_crc->lock);
1509
		DRM_DEBUG_KMS("spurious interrupt\n");
1510 1511 1512
		return;
	}

1513 1514
	head = pipe_crc->head;
	tail = pipe_crc->tail;
1515 1516

	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1517
		spin_unlock(&pipe_crc->lock);
1518 1519 1520 1521 1522
		DRM_ERROR("CRC buffer overflowing\n");
		return;
	}

	entry = &pipe_crc->entries[head];
1523

1524 1525
	entry->frame = dev_priv->dev->driver->get_vblank_counter(dev_priv->dev,
								 pipe);
1526 1527 1528 1529 1530
	entry->crc[0] = crc0;
	entry->crc[1] = crc1;
	entry->crc[2] = crc2;
	entry->crc[3] = crc3;
	entry->crc[4] = crc4;
1531 1532

	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1533 1534 1535
	pipe_crc->head = head;

	spin_unlock(&pipe_crc->lock);
1536 1537

	wake_up_interruptible(&pipe_crc->wq);
1538
}
1539 1540
#else
static inline void
1541 1542
display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
			     enum pipe pipe,
1543 1544 1545 1546 1547
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1548

1549 1550
static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				     enum pipe pipe)
D
Daniel Vetter 已提交
1551
{
1552
	display_pipe_crc_irq_handler(dev_priv, pipe,
1553 1554
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1555 1556
}

1557 1558
static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				     enum pipe pipe)
1559
{
1560
	display_pipe_crc_irq_handler(dev_priv, pipe,
1561 1562 1563 1564 1565
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1566
}
1567

1568 1569
static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				      enum pipe pipe)
1570
{
1571 1572
	uint32_t res1, res2;

1573
	if (INTEL_GEN(dev_priv) >= 3)
1574 1575 1576 1577
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

1578
	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1579 1580 1581
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1582

1583
	display_pipe_crc_irq_handler(dev_priv, pipe,
1584 1585 1586 1587
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1588
}
1589

1590 1591 1592 1593
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1594
{
1595
	if (pm_iir & dev_priv->pm_rps_events) {
1596
		spin_lock(&dev_priv->irq_lock);
1597
		gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
I
Imre Deak 已提交
1598 1599 1600 1601
		if (dev_priv->rps.interrupts_enabled) {
			dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
			queue_work(dev_priv->wq, &dev_priv->rps.work);
		}
1602
		spin_unlock(&dev_priv->irq_lock);
1603 1604
	}

1605 1606 1607
	if (INTEL_INFO(dev_priv)->gen >= 8)
		return;

1608
	if (HAS_VEBOX(dev_priv)) {
1609
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1610
			notify_ring(&dev_priv->engine[VECS]);
B
Ben Widawsky 已提交
1611

1612 1613
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
B
Ben Widawsky 已提交
1614
	}
1615 1616
}

1617
static bool intel_pipe_handle_vblank(struct drm_i915_private *dev_priv,
1618
				     enum pipe pipe)
1619
{
1620 1621 1622 1623
	bool ret;

	ret = drm_handle_vblank(dev_priv->dev, pipe);
	if (ret)
1624
		intel_finish_page_flip_mmio(dev_priv, pipe);
1625 1626

	return ret;
1627 1628
}

1629 1630
static void valleyview_pipestat_irq_ack(struct drm_i915_private *dev_priv,
					u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1631 1632 1633
{
	int pipe;

1634
	spin_lock(&dev_priv->irq_lock);
1635 1636 1637 1638 1639 1640

	if (!dev_priv->display_irqs_enabled) {
		spin_unlock(&dev_priv->irq_lock);
		return;
	}

1641
	for_each_pipe(dev_priv, pipe) {
1642
		i915_reg_t reg;
1643
		u32 mask, iir_bit = 0;
1644

1645 1646 1647 1648 1649 1650 1651
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
1652 1653 1654

		/* fifo underruns are filterered in the underrun handler. */
		mask = PIPE_FIFO_UNDERRUN_STATUS;
1655 1656 1657 1658 1659 1660 1661 1662

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1663 1664 1665
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1666 1667 1668 1669 1670
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1671 1672 1673
			continue;

		reg = PIPESTAT(pipe);
1674 1675
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1676 1677 1678 1679

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1680 1681
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1682 1683
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1684
	spin_unlock(&dev_priv->irq_lock);
1685 1686
}

1687
static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1688 1689 1690
					    u32 pipe_stats[I915_MAX_PIPES])
{
	enum pipe pipe;
1691

1692
	for_each_pipe(dev_priv, pipe) {
1693 1694 1695
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
1696

1697
		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV)
1698
			intel_finish_page_flip_cs(dev_priv, pipe);
1699 1700

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1701
			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1702

1703 1704
		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1705 1706 1707
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1708
		gmbus_irq_handler(dev_priv);
1709 1710
}

1711
static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
1712 1713 1714
{
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);

1715 1716
	if (hotplug_status)
		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1717

1718 1719 1720
	return hotplug_status;
}

1721
static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1722 1723 1724
				 u32 hotplug_status)
{
	u32 pin_mask = 0, long_mask = 0;
1725

1726 1727
	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
	    IS_CHERRYVIEW(dev_priv)) {
1728
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1729

1730 1731 1732 1733 1734
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
					   hotplug_trigger, hpd_status_g4x,
					   i9xx_port_hotplug_long_detect);

1735
			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1736
		}
1737 1738

		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1739
			dp_aux_irq_handler(dev_priv);
1740 1741
	} else {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1742

1743 1744
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1745
					   hotplug_trigger, hpd_status_i915,
1746
					   i9xx_port_hotplug_long_detect);
1747
			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1748
		}
1749
	}
1750 1751
}

1752
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1753
{
1754
	struct drm_device *dev = arg;
1755
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
1756 1757
	irqreturn_t ret = IRQ_NONE;

1758 1759 1760
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1761 1762 1763
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1764
	do {
1765
		u32 iir, gt_iir, pm_iir;
1766
		u32 pipe_stats[I915_MAX_PIPES] = {};
1767
		u32 hotplug_status = 0;
1768
		u32 ier = 0;
1769

J
Jesse Barnes 已提交
1770 1771
		gt_iir = I915_READ(GTIIR);
		pm_iir = I915_READ(GEN6_PMIIR);
1772
		iir = I915_READ(VLV_IIR);
J
Jesse Barnes 已提交
1773 1774

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1775
			break;
J
Jesse Barnes 已提交
1776 1777 1778

		ret = IRQ_HANDLED;

1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
		 *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
		 * bits this time around.
		 */
1792
		I915_WRITE(VLV_MASTER_IER, 0);
1793 1794
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
1795 1796 1797 1798 1799 1800

		if (gt_iir)
			I915_WRITE(GTIIR, gt_iir);
		if (pm_iir)
			I915_WRITE(GEN6_PMIIR, pm_iir);

1801
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1802
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1803

1804 1805
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
1806
		valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1807 1808 1809 1810 1811 1812 1813

		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);
1814

1815
		I915_WRITE(VLV_IER, ier);
1816 1817
		I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
		POSTING_READ(VLV_MASTER_IER);
1818

1819
		if (gt_iir)
1820
			snb_gt_irq_handler(dev_priv, gt_iir);
1821 1822 1823
		if (pm_iir)
			gen6_rps_irq_handler(dev_priv, pm_iir);

1824
		if (hotplug_status)
1825
			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1826

1827
		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1828
	} while (0);
J
Jesse Barnes 已提交
1829

1830 1831
	enable_rpm_wakeref_asserts(dev_priv);

J
Jesse Barnes 已提交
1832 1833 1834
	return ret;
}

1835 1836
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1837
	struct drm_device *dev = arg;
1838 1839 1840
	struct drm_i915_private *dev_priv = dev->dev_private;
	irqreturn_t ret = IRQ_NONE;

1841 1842 1843
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1844 1845 1846
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1847
	do {
1848
		u32 master_ctl, iir;
1849
		u32 gt_iir[4] = {};
1850
		u32 pipe_stats[I915_MAX_PIPES] = {};
1851
		u32 hotplug_status = 0;
1852 1853
		u32 ier = 0;

1854 1855
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
1856

1857 1858
		if (master_ctl == 0 && iir == 0)
			break;
1859

1860 1861
		ret = IRQ_HANDLED;

1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
		 *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
		 * bits this time around.
		 */
1875
		I915_WRITE(GEN8_MASTER_IRQ, 0);
1876 1877
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
1878

1879
		gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
1880

1881
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1882
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1883

1884 1885
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
1886
		valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1887

1888 1889 1890 1891 1892 1893 1894
		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);

1895
		I915_WRITE(VLV_IER, ier);
1896
		I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
1897
		POSTING_READ(GEN8_MASTER_IRQ);
1898

1899 1900
		gen8_gt_irq_handler(dev_priv, gt_iir);

1901
		if (hotplug_status)
1902
			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1903

1904
		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1905
	} while (0);
1906

1907 1908
	enable_rpm_wakeref_asserts(dev_priv);

1909 1910 1911
	return ret;
}

1912 1913
static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
1914 1915 1916 1917
				const u32 hpd[HPD_NUM_PINS])
{
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

1918 1919 1920 1921 1922 1923
	/*
	 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
	 * unless we touch the hotplug register, even if hotplug_trigger is
	 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
	 * errors.
	 */
1924
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1925 1926 1927 1928 1929 1930 1931 1932
	if (!hotplug_trigger) {
		u32 mask = PORTA_HOTPLUG_STATUS_MASK |
			PORTD_HOTPLUG_STATUS_MASK |
			PORTC_HOTPLUG_STATUS_MASK |
			PORTB_HOTPLUG_STATUS_MASK;
		dig_hotplug_reg &= ~mask;
	}

1933
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1934 1935
	if (!hotplug_trigger)
		return;
1936 1937 1938 1939 1940

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   pch_port_hotplug_long_detect);

1941
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1942 1943
}

1944
static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
1945
{
1946
	int pipe;
1947
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1948

1949
	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx);
1950

1951 1952 1953
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
1954
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1955 1956
				 port_name(port));
	}
1957

1958
	if (pch_iir & SDE_AUX_MASK)
1959
		dp_aux_irq_handler(dev_priv);
1960

1961
	if (pch_iir & SDE_GMBUS)
1962
		gmbus_irq_handler(dev_priv);
1963 1964 1965 1966 1967 1968 1969 1970 1971 1972

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1973
	if (pch_iir & SDE_FDI_MASK)
1974
		for_each_pipe(dev_priv, pipe)
1975 1976 1977
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
1978 1979 1980 1981 1982 1983 1984 1985

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1986
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1987 1988

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1989
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1990 1991
}

1992
static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
1993 1994
{
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
1995
	enum pipe pipe;
1996

1997 1998 1999
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

2000
	for_each_pipe(dev_priv, pipe) {
2001 2002
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2003

D
Daniel Vetter 已提交
2004
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
2005 2006
			if (IS_IVYBRIDGE(dev_priv))
				ivb_pipe_crc_irq_handler(dev_priv, pipe);
D
Daniel Vetter 已提交
2007
			else
2008
				hsw_pipe_crc_irq_handler(dev_priv, pipe);
D
Daniel Vetter 已提交
2009 2010
		}
	}
2011

2012 2013 2014
	I915_WRITE(GEN7_ERR_INT, err_int);
}

2015
static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
2016 2017 2018
{
	u32 serr_int = I915_READ(SERR_INT);

2019 2020 2021
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2022
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
2023
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
2024 2025

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
2026
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
2027 2028

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
2029
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
2030 2031

	I915_WRITE(SERR_INT, serr_int);
2032 2033
}

2034
static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2035 2036
{
	int pipe;
2037
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
2038

2039
	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt);
2040

2041 2042 2043 2044 2045 2046
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
2047 2048

	if (pch_iir & SDE_AUX_MASK_CPT)
2049
		dp_aux_irq_handler(dev_priv);
2050 2051

	if (pch_iir & SDE_GMBUS_CPT)
2052
		gmbus_irq_handler(dev_priv);
2053 2054 2055 2056 2057 2058 2059 2060

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
2061
		for_each_pipe(dev_priv, pipe)
2062 2063 2064
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2065 2066

	if (pch_iir & SDE_ERROR_CPT)
2067
		cpt_serr_int_handler(dev_priv);
2068 2069
}

2070
static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084
{
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		~SDE_PORTE_HOTPLUG_SPT;
	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
	u32 pin_mask = 0, long_mask = 0;

	if (hotplug_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
				   dig_hotplug_reg, hpd_spt,
2085
				   spt_port_hotplug_long_detect);
2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099
	}

	if (hotplug2_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
				   dig_hotplug_reg, hpd_spt,
				   spt_port_hotplug2_long_detect);
	}

	if (pin_mask)
2100
		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2101 2102

	if (pch_iir & SDE_GMBUS_CPT)
2103
		gmbus_irq_handler(dev_priv);
2104 2105
}

2106 2107
static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118
				const u32 hpd[HPD_NUM_PINS])
{
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   ilk_port_hotplug_long_detect);

2119
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2120 2121
}

2122 2123
static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
				    u32 de_iir)
2124
{
2125
	enum pipe pipe;
2126 2127
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;

2128
	if (hotplug_trigger)
2129
		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk);
2130 2131

	if (de_iir & DE_AUX_CHANNEL_A)
2132
		dp_aux_irq_handler(dev_priv);
2133 2134

	if (de_iir & DE_GSE)
2135
		intel_opregion_asle_intr(dev_priv);
2136 2137 2138 2139

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2140
	for_each_pipe(dev_priv, pipe) {
2141 2142 2143
		if (de_iir & DE_PIPE_VBLANK(pipe) &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2144

2145
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2146
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2147

2148
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
2149
			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2150

2151
		/* plane/pipes map 1:1 on ilk+ */
2152
		if (de_iir & DE_PLANE_FLIP_DONE(pipe))
2153
			intel_finish_page_flip_cs(dev_priv, pipe);
2154 2155 2156 2157 2158 2159
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

2160 2161
		if (HAS_PCH_CPT(dev_priv))
			cpt_irq_handler(dev_priv, pch_iir);
2162
		else
2163
			ibx_irq_handler(dev_priv, pch_iir);
2164 2165 2166 2167 2168

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

2169 2170
	if (IS_GEN5(dev_priv) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev_priv);
2171 2172
}

2173 2174
static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
				    u32 de_iir)
2175
{
2176
	enum pipe pipe;
2177 2178
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;

2179
	if (hotplug_trigger)
2180
		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb);
2181 2182

	if (de_iir & DE_ERR_INT_IVB)
2183
		ivb_err_int_handler(dev_priv);
2184 2185

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
2186
		dp_aux_irq_handler(dev_priv);
2187 2188

	if (de_iir & DE_GSE_IVB)
2189
		intel_opregion_asle_intr(dev_priv);
2190

2191
	for_each_pipe(dev_priv, pipe) {
2192 2193 2194
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2195 2196

		/* plane/pipes map 1:1 on ilk+ */
2197
		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe))
2198
			intel_finish_page_flip_cs(dev_priv, pipe);
2199 2200 2201
	}

	/* check event from PCH */
2202
	if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
2203 2204
		u32 pch_iir = I915_READ(SDEIIR);

2205
		cpt_irq_handler(dev_priv, pch_iir);
2206 2207 2208 2209 2210 2211

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2212 2213 2214 2215 2216 2217 2218 2219
/*
 * To handle irqs with the minimum potential races with fresh interrupts, we:
 * 1 - Disable Master Interrupt Control.
 * 2 - Find the source(s) of the interrupt.
 * 3 - Clear the Interrupt Identity bits (IIR).
 * 4 - Process the interrupt(s) that had bits set in the IIRs.
 * 5 - Re-enable Master Interrupt Control.
 */
2220
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2221
{
2222
	struct drm_device *dev = arg;
2223
	struct drm_i915_private *dev_priv = dev->dev_private;
2224
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2225
	irqreturn_t ret = IRQ_NONE;
2226

2227 2228 2229
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2230 2231 2232
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

2233 2234 2235
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2236
	POSTING_READ(DEIER);
2237

2238 2239 2240 2241 2242
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2243
	if (!HAS_PCH_NOP(dev_priv)) {
2244 2245 2246 2247
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2248

2249 2250
	/* Find, clear, then process each source of interrupt */

2251
	gt_iir = I915_READ(GTIIR);
2252
	if (gt_iir) {
2253 2254
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2255
		if (INTEL_GEN(dev_priv) >= 6)
2256
			snb_gt_irq_handler(dev_priv, gt_iir);
2257
		else
2258
			ilk_gt_irq_handler(dev_priv, gt_iir);
2259 2260
	}

2261 2262
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2263 2264
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2265 2266
		if (INTEL_GEN(dev_priv) >= 7)
			ivb_display_irq_handler(dev_priv, de_iir);
2267
		else
2268
			ilk_display_irq_handler(dev_priv, de_iir);
2269 2270
	}

2271
	if (INTEL_GEN(dev_priv) >= 6) {
2272 2273 2274 2275
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
2276
			gen6_rps_irq_handler(dev_priv, pm_iir);
2277
		}
2278
	}
2279 2280 2281

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2282
	if (!HAS_PCH_NOP(dev_priv)) {
2283 2284 2285
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2286

2287 2288 2289
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	enable_rpm_wakeref_asserts(dev_priv);

2290 2291 2292
	return ret;
}

2293 2294
static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2295
				const u32 hpd[HPD_NUM_PINS])
2296
{
2297
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2298

2299 2300
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2301

2302
	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2303
			   dig_hotplug_reg, hpd,
2304
			   bxt_port_hotplug_long_detect);
2305

2306
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2307 2308
}

2309 2310
static irqreturn_t
gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2311 2312
{
	irqreturn_t ret = IRQ_NONE;
2313
	u32 iir;
2314
	enum pipe pipe;
J
Jesse Barnes 已提交
2315

2316
	if (master_ctl & GEN8_DE_MISC_IRQ) {
2317 2318 2319
		iir = I915_READ(GEN8_DE_MISC_IIR);
		if (iir) {
			I915_WRITE(GEN8_DE_MISC_IIR, iir);
2320
			ret = IRQ_HANDLED;
2321
			if (iir & GEN8_DE_MISC_GSE)
2322
				intel_opregion_asle_intr(dev_priv);
2323 2324
			else
				DRM_ERROR("Unexpected DE Misc interrupt\n");
2325
		}
2326 2327
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2328 2329
	}

2330
	if (master_ctl & GEN8_DE_PORT_IRQ) {
2331 2332 2333
		iir = I915_READ(GEN8_DE_PORT_IIR);
		if (iir) {
			u32 tmp_mask;
2334
			bool found = false;
2335

2336
			I915_WRITE(GEN8_DE_PORT_IIR, iir);
2337
			ret = IRQ_HANDLED;
J
Jesse Barnes 已提交
2338

2339 2340 2341 2342 2343 2344 2345
			tmp_mask = GEN8_AUX_CHANNEL_A;
			if (INTEL_INFO(dev_priv)->gen >= 9)
				tmp_mask |= GEN9_AUX_CHANNEL_B |
					    GEN9_AUX_CHANNEL_C |
					    GEN9_AUX_CHANNEL_D;

			if (iir & tmp_mask) {
2346
				dp_aux_irq_handler(dev_priv);
2347 2348 2349
				found = true;
			}

2350 2351 2352
			if (IS_BROXTON(dev_priv)) {
				tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
				if (tmp_mask) {
2353 2354
					bxt_hpd_irq_handler(dev_priv, tmp_mask,
							    hpd_bxt);
2355 2356 2357 2358 2359
					found = true;
				}
			} else if (IS_BROADWELL(dev_priv)) {
				tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
				if (tmp_mask) {
2360 2361
					ilk_hpd_irq_handler(dev_priv,
							    tmp_mask, hpd_bdw);
2362 2363
					found = true;
				}
2364 2365
			}

2366 2367
			if (IS_BROXTON(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
				gmbus_irq_handler(dev_priv);
S
Shashank Sharma 已提交
2368 2369 2370
				found = true;
			}

2371
			if (!found)
2372
				DRM_ERROR("Unexpected DE Port interrupt\n");
2373
		}
2374 2375
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2376 2377
	}

2378
	for_each_pipe(dev_priv, pipe) {
2379
		u32 flip_done, fault_errors;
2380

2381 2382
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2383

2384 2385 2386 2387 2388
		iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (!iir) {
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
			continue;
		}
2389

2390 2391
		ret = IRQ_HANDLED;
		I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2392

2393 2394 2395
		if (iir & GEN8_PIPE_VBLANK &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2396

2397 2398 2399 2400 2401
		flip_done = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			flip_done &= GEN9_PIPE_PLANE1_FLIP_DONE;
		else
			flip_done &= GEN8_PIPE_PRIMARY_FLIP_DONE;
2402

2403
		if (flip_done)
2404
			intel_finish_page_flip_cs(dev_priv, pipe);
2405

2406
		if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
2407
			hsw_pipe_crc_irq_handler(dev_priv, pipe);
2408

2409 2410
		if (iir & GEN8_PIPE_FIFO_UNDERRUN)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2411

2412 2413 2414 2415 2416
		fault_errors = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
		else
			fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2417

2418 2419 2420 2421
		if (fault_errors)
			DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
				  pipe_name(pipe),
				  fault_errors);
2422 2423
	}

2424
	if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2425
	    master_ctl & GEN8_DE_PCH_IRQ) {
2426 2427 2428 2429 2430
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
2431 2432 2433
		iir = I915_READ(SDEIIR);
		if (iir) {
			I915_WRITE(SDEIIR, iir);
2434
			ret = IRQ_HANDLED;
2435 2436

			if (HAS_PCH_SPT(dev_priv))
2437
				spt_irq_handler(dev_priv, iir);
2438
			else
2439
				cpt_irq_handler(dev_priv, iir);
2440 2441 2442 2443 2444 2445 2446
		} else {
			/*
			 * Like on previous PCH there seems to be something
			 * fishy going on with forwarding PCH interrupts.
			 */
			DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
		}
2447 2448
	}

2449 2450 2451 2452 2453 2454 2455 2456
	return ret;
}

static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl;
2457
	u32 gt_iir[4] = {};
2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473
	irqreturn_t ret;

	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);

	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	/* Find, clear, then process each source of interrupt */
2474 2475
	ret = gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
	gen8_gt_irq_handler(dev_priv, gt_iir);
2476 2477
	ret |= gen8_de_irq_handler(dev_priv, master_ctl);

2478 2479
	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ_FW(GEN8_MASTER_IRQ);
2480

2481 2482
	enable_rpm_wakeref_asserts(dev_priv);

2483 2484 2485
	return ret;
}

2486
static void i915_error_wake_up(struct drm_i915_private *dev_priv)
2487 2488 2489 2490 2491 2492 2493 2494 2495
{
	/*
	 * Notify all waiters for GPU completion events that reset state has
	 * been changed, and that they need to restart their wait after
	 * checking for potential errors (and bail out to drop locks if there is
	 * a gpu reset pending so that i915_error_work_func can acquire them).
	 */

	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2496
	wake_up_all(&dev_priv->gpu_error.wait_queue);
2497 2498 2499 2500 2501

	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
	wake_up_all(&dev_priv->pending_flip_queue);
}

2502
/**
2503
 * i915_reset_and_wakeup - do process context error handling work
2504
 * @dev_priv: i915 device private
2505 2506 2507 2508
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
2509
static void i915_reset_and_wakeup(struct drm_i915_private *dev_priv)
2510
{
2511
	struct kobject *kobj = &dev_priv->dev->primary->kdev->kobj;
2512 2513 2514
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2515
	int ret;
2516

2517
	kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);
2518

2519 2520 2521 2522 2523 2524 2525 2526 2527 2528
	/*
	 * Note that there's only one work item which does gpu resets, so we
	 * need not worry about concurrent gpu resets potentially incrementing
	 * error->reset_counter twice. We only need to take care of another
	 * racing irq/hangcheck declaring the gpu dead for a second time. A
	 * quick check for that is good enough: schedule_work ensures the
	 * correct ordering between hang detection and this work item, and since
	 * the reset in-progress bit is only ever set by code outside of this
	 * work we don't need to worry about any other races.
	 */
2529
	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
2530
		DRM_DEBUG_DRIVER("resetting chip\n");
2531
		kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);
2532

2533 2534 2535 2536 2537 2538 2539 2540
		/*
		 * In most cases it's guaranteed that we get here with an RPM
		 * reference held, for example because there is a pending GPU
		 * request that won't finish until the reset is done. This
		 * isn't the case at least when we get here by doing a
		 * simulated reset via debugs, so get an RPM reference.
		 */
		intel_runtime_pm_get(dev_priv);
2541

2542
		intel_prepare_reset(dev_priv);
2543

2544 2545 2546 2547 2548 2549
		/*
		 * All state reset _must_ be completed before we update the
		 * reset counter, for otherwise waiters might miss the reset
		 * pending state and not properly drop locks, resulting in
		 * deadlocks with the reset work.
		 */
2550
		ret = i915_reset(dev_priv);
2551

2552
		intel_finish_reset(dev_priv);
2553

2554 2555
		intel_runtime_pm_put(dev_priv);

2556
		if (ret == 0)
2557
			kobject_uevent_env(kobj,
2558
					   KOBJ_CHANGE, reset_done_event);
2559

2560 2561 2562 2563
		/*
		 * Note: The wake_up also serves as a memory barrier so that
		 * waiters see the update value of the reset counter atomic_t.
		 */
2564
		wake_up_all(&dev_priv->gpu_error.reset_queue);
2565
	}
2566 2567
}

2568
static void i915_report_and_clear_eir(struct drm_i915_private *dev_priv)
2569
{
2570
	uint32_t instdone[I915_NUM_INSTDONE_REG];
2571
	u32 eir = I915_READ(EIR);
2572
	int pipe, i;
2573

2574 2575
	if (!eir)
		return;
2576

2577
	pr_err("render error detected, EIR: 0x%08x\n", eir);
2578

2579
	i915_get_extra_instdone(dev_priv, instdone);
2580

2581
	if (IS_G4X(dev_priv)) {
2582 2583 2584
		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
			u32 ipeir = I915_READ(IPEIR_I965);

2585 2586
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2587 2588
			for (i = 0; i < ARRAY_SIZE(instdone); i++)
				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2589 2590
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2591
			I915_WRITE(IPEIR_I965, ipeir);
2592
			POSTING_READ(IPEIR_I965);
2593 2594 2595
		}
		if (eir & GM45_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2596 2597
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2598
			I915_WRITE(PGTBL_ER, pgtbl_err);
2599
			POSTING_READ(PGTBL_ER);
2600 2601 2602
		}
	}

2603
	if (!IS_GEN2(dev_priv)) {
2604 2605
		if (eir & I915_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2606 2607
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2608
			I915_WRITE(PGTBL_ER, pgtbl_err);
2609
			POSTING_READ(PGTBL_ER);
2610 2611 2612 2613
		}
	}

	if (eir & I915_ERROR_MEMORY_REFRESH) {
2614
		pr_err("memory refresh error:\n");
2615
		for_each_pipe(dev_priv, pipe)
2616
			pr_err("pipe %c stat: 0x%08x\n",
2617
			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2618 2619 2620
		/* pipestat has already been acked */
	}
	if (eir & I915_ERROR_INSTRUCTION) {
2621 2622
		pr_err("instruction error\n");
		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2623 2624
		for (i = 0; i < ARRAY_SIZE(instdone); i++)
			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2625
		if (INTEL_GEN(dev_priv) < 4) {
2626 2627
			u32 ipeir = I915_READ(IPEIR);

2628 2629 2630
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2631
			I915_WRITE(IPEIR, ipeir);
2632
			POSTING_READ(IPEIR);
2633 2634 2635
		} else {
			u32 ipeir = I915_READ(IPEIR_I965);

2636 2637 2638 2639
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2640
			I915_WRITE(IPEIR_I965, ipeir);
2641
			POSTING_READ(IPEIR_I965);
2642 2643 2644 2645
		}
	}

	I915_WRITE(EIR, eir);
2646
	POSTING_READ(EIR);
2647 2648 2649 2650 2651 2652 2653 2654 2655 2656
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2657 2658 2659
}

/**
2660
 * i915_handle_error - handle a gpu error
2661
 * @dev_priv: i915 device private
2662
 * @engine_mask: mask representing engines that are hung
2663
 * Do some basic checking of register state at error time and
2664 2665 2666 2667
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
2668
 * @fmt: Error message format string
2669
 */
2670 2671
void i915_handle_error(struct drm_i915_private *dev_priv,
		       u32 engine_mask,
2672
		       const char *fmt, ...)
2673
{
2674 2675
	va_list args;
	char error_msg[80];
2676

2677 2678 2679 2680
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

2681 2682
	i915_capture_error_state(dev_priv, engine_mask, error_msg);
	i915_report_and_clear_eir(dev_priv);
2683

2684
	if (engine_mask) {
2685
		atomic_or(I915_RESET_IN_PROGRESS_FLAG,
2686
				&dev_priv->gpu_error.reset_counter);
2687

2688
		/*
2689 2690 2691
		 * Wakeup waiting processes so that the reset function
		 * i915_reset_and_wakeup doesn't deadlock trying to grab
		 * various locks. By bumping the reset counter first, the woken
2692 2693 2694 2695 2696 2697 2698 2699
		 * processes will see a reset in progress and back off,
		 * releasing their locks and then wait for the reset completion.
		 * We must do this for _all_ gpu waiters that might hold locks
		 * that the reset work needs to acquire.
		 *
		 * Note: The wake_up serves as the required memory barrier to
		 * ensure that the waiters see the updated value of the reset
		 * counter atomic_t.
2700
		 */
2701
		i915_error_wake_up(dev_priv);
2702 2703
	}

2704
	i915_reset_and_wakeup(dev_priv);
2705 2706
}

2707 2708 2709
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2710
static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
2711
{
2712
	struct drm_i915_private *dev_priv = dev->dev_private;
2713
	unsigned long irqflags;
2714

2715
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2716
	if (INTEL_INFO(dev)->gen >= 4)
2717
		i915_enable_pipestat(dev_priv, pipe,
2718
				     PIPE_START_VBLANK_INTERRUPT_STATUS);
2719
	else
2720
		i915_enable_pipestat(dev_priv, pipe,
2721
				     PIPE_VBLANK_INTERRUPT_STATUS);
2722
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2723

2724 2725 2726
	return 0;
}

2727
static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
2728
{
2729
	struct drm_i915_private *dev_priv = dev->dev_private;
2730
	unsigned long irqflags;
2731
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2732
						     DE_PIPE_VBLANK(pipe);
2733 2734

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2735
	ilk_enable_display_irq(dev_priv, bit);
2736 2737 2738 2739 2740
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2741
static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2742
{
2743
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2744 2745 2746
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2747
	i915_enable_pipestat(dev_priv, pipe,
2748
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2749 2750 2751 2752 2753
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2754
static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
2755 2756 2757 2758 2759
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2760
	bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2761
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2762

2763 2764 2765
	return 0;
}

2766 2767 2768
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2769
static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
2770
{
2771
	struct drm_i915_private *dev_priv = dev->dev_private;
2772
	unsigned long irqflags;
2773

2774
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2775
	i915_disable_pipestat(dev_priv, pipe,
2776 2777
			      PIPE_VBLANK_INTERRUPT_STATUS |
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2778 2779 2780
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2781
static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
2782
{
2783
	struct drm_i915_private *dev_priv = dev->dev_private;
2784
	unsigned long irqflags;
2785
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2786
						     DE_PIPE_VBLANK(pipe);
2787 2788

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2789
	ilk_disable_display_irq(dev_priv, bit);
2790 2791 2792
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2793
static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2794
{
2795
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2796 2797 2798
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2799
	i915_disable_pipestat(dev_priv, pipe,
2800
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2801 2802 2803
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2804
static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
2805 2806 2807 2808 2809
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2810
	bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2811 2812 2813
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2814
static bool
2815
ring_idle(struct intel_engine_cs *engine, u32 seqno)
2816
{
2817 2818
	return i915_seqno_passed(seqno,
				 READ_ONCE(engine->last_submitted_seqno));
B
Ben Gamari 已提交
2819 2820
}

2821
static bool
2822
ipehr_is_semaphore_wait(struct drm_i915_private *dev_priv, u32 ipehr)
2823
{
2824
	if (INTEL_GEN(dev_priv) >= 8) {
2825
		return (ipehr >> 23) == 0x1c;
2826 2827 2828 2829 2830 2831 2832
	} else {
		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
				 MI_SEMAPHORE_REGISTER);
	}
}

2833
static struct intel_engine_cs *
2834 2835
semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr,
				 u64 offset)
2836
{
2837
	struct drm_i915_private *dev_priv = engine->i915;
2838
	struct intel_engine_cs *signaller;
2839

2840
	if (INTEL_GEN(dev_priv) >= 8) {
2841
		for_each_engine(signaller, dev_priv) {
2842
			if (engine == signaller)
2843 2844
				continue;

2845
			if (offset == signaller->semaphore.signal_ggtt[engine->id])
2846 2847
				return signaller;
		}
2848 2849 2850
	} else {
		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;

2851
		for_each_engine(signaller, dev_priv) {
2852
			if(engine == signaller)
2853 2854
				continue;

2855
			if (sync_bits == signaller->semaphore.mbox.wait[engine->id])
2856 2857 2858 2859
				return signaller;
		}
	}

2860
	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2861
		  engine->id, ipehr, offset);
2862 2863 2864 2865

	return NULL;
}

2866
static struct intel_engine_cs *
2867
semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
2868
{
2869
	struct drm_i915_private *dev_priv = engine->i915;
2870
	u32 cmd, ipehr, head;
2871 2872
	u64 offset = 0;
	int i, backwards;
2873

2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890
	/*
	 * This function does not support execlist mode - any attempt to
	 * proceed further into this function will result in a kernel panic
	 * when dereferencing ring->buffer, which is not set up in execlist
	 * mode.
	 *
	 * The correct way of doing it would be to derive the currently
	 * executing ring buffer from the current context, which is derived
	 * from the currently running request. Unfortunately, to get the
	 * current request we would have to grab the struct_mutex before doing
	 * anything else, which would be ill-advised since some other thread
	 * might have grabbed it already and managed to hang itself, causing
	 * the hang checker to deadlock.
	 *
	 * Therefore, this function does not support execlist mode in its
	 * current form. Just return NULL and move on.
	 */
2891
	if (engine->buffer == NULL)
2892 2893
		return NULL;

2894
	ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
2895
	if (!ipehr_is_semaphore_wait(engine->i915, ipehr))
2896
		return NULL;
2897

2898 2899 2900
	/*
	 * HEAD is likely pointing to the dword after the actual command,
	 * so scan backwards until we find the MBOX. But limit it to just 3
2901 2902
	 * or 4 dwords depending on the semaphore wait command size.
	 * Note that we don't care about ACTHD here since that might
2903 2904
	 * point at at batch, and semaphores are always emitted into the
	 * ringbuffer itself.
2905
	 */
2906
	head = I915_READ_HEAD(engine) & HEAD_ADDR;
2907
	backwards = (INTEL_GEN(dev_priv) >= 8) ? 5 : 4;
2908

2909
	for (i = backwards; i; --i) {
2910 2911 2912 2913 2914
		/*
		 * Be paranoid and presume the hw has gone off into the wild -
		 * our ring is smaller than what the hardware (and hence
		 * HEAD_ADDR) allows. Also handles wrap-around.
		 */
2915
		head &= engine->buffer->size - 1;
2916 2917

		/* This here seems to blow up */
2918
		cmd = ioread32(engine->buffer->virtual_start + head);
2919 2920 2921
		if (cmd == ipehr)
			break;

2922 2923
		head -= 4;
	}
2924

2925 2926
	if (!i)
		return NULL;
2927

2928
	*seqno = ioread32(engine->buffer->virtual_start + head + 4) + 1;
2929
	if (INTEL_GEN(dev_priv) >= 8) {
2930
		offset = ioread32(engine->buffer->virtual_start + head + 12);
2931
		offset <<= 32;
2932
		offset = ioread32(engine->buffer->virtual_start + head + 8);
2933
	}
2934
	return semaphore_wait_to_signaller_ring(engine, ipehr, offset);
2935 2936
}

2937
static int semaphore_passed(struct intel_engine_cs *engine)
2938
{
2939
	struct drm_i915_private *dev_priv = engine->i915;
2940
	struct intel_engine_cs *signaller;
2941
	u32 seqno;
2942

2943
	engine->hangcheck.deadlock++;
2944

2945
	signaller = semaphore_waits_for(engine, &seqno);
2946 2947 2948 2949
	if (signaller == NULL)
		return -1;

	/* Prevent pathological recursion due to driver bugs */
2950
	if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES)
2951 2952
		return -1;

2953
	if (i915_seqno_passed(intel_engine_get_seqno(signaller), seqno))
2954 2955
		return 1;

2956 2957 2958
	/* cursory check for an unkickable deadlock */
	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
	    semaphore_passed(signaller) < 0)
2959 2960 2961
		return -1;

	return 0;
2962 2963 2964 2965
}

static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
{
2966
	struct intel_engine_cs *engine;
2967

2968
	for_each_engine(engine, dev_priv)
2969
		engine->hangcheck.deadlock = 0;
2970 2971
}

2972
static bool subunits_stuck(struct intel_engine_cs *engine)
2973
{
2974 2975 2976 2977
	u32 instdone[I915_NUM_INSTDONE_REG];
	bool stuck;
	int i;

2978
	if (engine->id != RCS)
2979 2980
		return true;

2981
	i915_get_extra_instdone(engine->i915, instdone);
2982

2983 2984 2985 2986 2987 2988 2989
	/* There might be unstable subunit states even when
	 * actual head is not moving. Filter out the unstable ones by
	 * accumulating the undone -> done transitions and only
	 * consider those as progress.
	 */
	stuck = true;
	for (i = 0; i < I915_NUM_INSTDONE_REG; i++) {
2990
		const u32 tmp = instdone[i] | engine->hangcheck.instdone[i];
2991

2992
		if (tmp != engine->hangcheck.instdone[i])
2993 2994
			stuck = false;

2995
		engine->hangcheck.instdone[i] |= tmp;
2996 2997 2998 2999 3000 3001
	}

	return stuck;
}

static enum intel_ring_hangcheck_action
3002
head_stuck(struct intel_engine_cs *engine, u64 acthd)
3003
{
3004
	if (acthd != engine->hangcheck.acthd) {
3005 3006

		/* Clear subunit states on head movement */
3007 3008
		memset(engine->hangcheck.instdone, 0,
		       sizeof(engine->hangcheck.instdone));
3009

3010
		return HANGCHECK_ACTIVE;
3011
	}
3012

3013
	if (!subunits_stuck(engine))
3014 3015 3016 3017 3018 3019
		return HANGCHECK_ACTIVE;

	return HANGCHECK_HUNG;
}

static enum intel_ring_hangcheck_action
3020
ring_stuck(struct intel_engine_cs *engine, u64 acthd)
3021
{
3022
	struct drm_i915_private *dev_priv = engine->i915;
3023 3024 3025
	enum intel_ring_hangcheck_action ha;
	u32 tmp;

3026
	ha = head_stuck(engine, acthd);
3027 3028 3029
	if (ha != HANGCHECK_HUNG)
		return ha;

3030
	if (IS_GEN2(dev_priv))
3031
		return HANGCHECK_HUNG;
3032 3033 3034 3035 3036 3037

	/* Is the chip hanging on a WAIT_FOR_EVENT?
	 * If so we can simply poke the RB_WAIT bit
	 * and break the hang. This should work on
	 * all but the second generation chipsets.
	 */
3038
	tmp = I915_READ_CTL(engine);
3039
	if (tmp & RING_WAIT) {
3040
		i915_handle_error(dev_priv, 0,
3041
				  "Kicking stuck wait on %s",
3042 3043
				  engine->name);
		I915_WRITE_CTL(engine, tmp);
3044
		return HANGCHECK_KICK;
3045 3046
	}

3047
	if (INTEL_GEN(dev_priv) >= 6 && tmp & RING_WAIT_SEMAPHORE) {
3048
		switch (semaphore_passed(engine)) {
3049
		default:
3050
			return HANGCHECK_HUNG;
3051
		case 1:
3052
			i915_handle_error(dev_priv, 0,
3053
					  "Kicking stuck semaphore on %s",
3054 3055
					  engine->name);
			I915_WRITE_CTL(engine, tmp);
3056
			return HANGCHECK_KICK;
3057
		case 0:
3058
			return HANGCHECK_WAIT;
3059
		}
3060
	}
3061

3062
	return HANGCHECK_HUNG;
3063 3064
}

3065 3066
static unsigned kick_waiters(struct intel_engine_cs *engine)
{
3067
	struct drm_i915_private *i915 = engine->i915;
3068 3069 3070 3071
	unsigned user_interrupts = READ_ONCE(engine->user_interrupts);

	if (engine->hangcheck.user_interrupts == user_interrupts &&
	    !test_and_set_bit(engine->id, &i915->gpu_error.missed_irq_rings)) {
3072
		if (!test_bit(engine->id, &i915->gpu_error.test_irq_rings))
3073 3074 3075 3076 3077
			DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
				  engine->name);
		else
			DRM_INFO("Fake missed irq on %s\n",
				 engine->name);
3078 3079

		intel_engine_enable_fake_irq(engine);
3080 3081 3082 3083
	}

	return user_interrupts;
}
3084
/*
B
Ben Gamari 已提交
3085
 * This is called when the chip hasn't reported back with completed
3086 3087 3088 3089 3090
 * batchbuffers in a long time. We keep track per ring seqno progress and
 * if there are no progress, hangcheck score for that ring is increased.
 * Further, acthd is inspected to see if the ring is stuck. On stuck case
 * we kick the ring. If we see no progress on three subsequent calls
 * we assume chip is wedged and try to fix it by resetting the chip.
B
Ben Gamari 已提交
3091
 */
3092
static void i915_hangcheck_elapsed(struct work_struct *work)
B
Ben Gamari 已提交
3093
{
3094 3095 3096
	struct drm_i915_private *dev_priv =
		container_of(work, typeof(*dev_priv),
			     gpu_error.hangcheck_work.work);
3097
	struct intel_engine_cs *engine;
3098
	enum intel_engine_id id;
3099
	int busy_count = 0, rings_hung = 0;
3100
	bool stuck[I915_NUM_ENGINES] = { 0 };
3101 3102 3103
#define BUSY 1
#define KICK 5
#define HUNG 20
3104
#define ACTIVE_DECAY 15
3105

3106
	if (!i915.enable_hangcheck)
3107 3108
		return;

3109 3110 3111 3112 3113 3114 3115
	/*
	 * The hangcheck work is synced during runtime suspend, we don't
	 * require a wakeref. TODO: instead of disabling the asserts make
	 * sure that we hold a reference when this work is running.
	 */
	DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);

3116 3117 3118 3119 3120 3121
	/* As enabling the GPU requires fairly extensive mmio access,
	 * periodically arm the mmio checker to see if we are triggering
	 * any invalid access.
	 */
	intel_uncore_arm_unclaimed_mmio_detection(dev_priv);

3122
	for_each_engine_id(engine, dev_priv, id) {
3123
		bool busy = intel_engine_has_waiter(engine);
3124 3125
		u64 acthd;
		u32 seqno;
3126
		unsigned user_interrupts;
3127

3128 3129
		semaphore_clear_deadlocks(dev_priv);

3130 3131 3132 3133 3134 3135 3136 3137 3138 3139
		/* We don't strictly need an irq-barrier here, as we are not
		 * serving an interrupt request, be paranoid in case the
		 * barrier has side-effects (such as preventing a broken
		 * cacheline snoop) and so be sure that we can see the seqno
		 * advance. If the seqno should stick, due to a stale
		 * cacheline, we would erroneously declare the GPU hung.
		 */
		if (engine->irq_seqno_barrier)
			engine->irq_seqno_barrier(engine);

3140
		acthd = intel_ring_get_active_head(engine);
3141
		seqno = intel_engine_get_seqno(engine);
3142

3143 3144 3145
		/* Reset stuck interrupts between batch advances */
		user_interrupts = 0;

3146 3147 3148
		if (engine->hangcheck.seqno == seqno) {
			if (ring_idle(engine, seqno)) {
				engine->hangcheck.action = HANGCHECK_IDLE;
3149
				if (busy) {
3150
					/* Safeguard against driver failure */
3151
					user_interrupts = kick_waiters(engine);
3152
					engine->hangcheck.score += BUSY;
3153
				}
3154
			} else {
3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169
				/* We always increment the hangcheck score
				 * if the ring is busy and still processing
				 * the same request, so that no single request
				 * can run indefinitely (such as a chain of
				 * batches). The only time we do not increment
				 * the hangcheck score on this ring, if this
				 * ring is in a legitimate wait for another
				 * ring. In that case the waiting ring is a
				 * victim and we want to be sure we catch the
				 * right culprit. Then every time we do kick
				 * the ring, add a small increment to the
				 * score so that we can catch a batch that is
				 * being repeatedly kicked and so responsible
				 * for stalling the machine.
				 */
3170 3171
				engine->hangcheck.action = ring_stuck(engine,
								      acthd);
3172

3173
				switch (engine->hangcheck.action) {
3174
				case HANGCHECK_IDLE:
3175
				case HANGCHECK_WAIT:
3176
					break;
3177
				case HANGCHECK_ACTIVE:
3178
					engine->hangcheck.score += BUSY;
3179
					break;
3180
				case HANGCHECK_KICK:
3181
					engine->hangcheck.score += KICK;
3182
					break;
3183
				case HANGCHECK_HUNG:
3184
					engine->hangcheck.score += HUNG;
3185
					stuck[id] = true;
3186 3187
					break;
				}
3188
			}
3189
		} else {
3190
			engine->hangcheck.action = HANGCHECK_ACTIVE;
3191

3192 3193 3194
			/* Gradually reduce the count so that we catch DoS
			 * attempts across multiple batches.
			 */
3195 3196 3197 3198
			if (engine->hangcheck.score > 0)
				engine->hangcheck.score -= ACTIVE_DECAY;
			if (engine->hangcheck.score < 0)
				engine->hangcheck.score = 0;
3199

3200
			/* Clear head and subunit states on seqno movement */
3201
			acthd = 0;
3202

3203 3204
			memset(engine->hangcheck.instdone, 0,
			       sizeof(engine->hangcheck.instdone));
3205 3206
		}

3207 3208
		engine->hangcheck.seqno = seqno;
		engine->hangcheck.acthd = acthd;
3209
		engine->hangcheck.user_interrupts = user_interrupts;
3210
		busy_count += busy;
3211
	}
3212

3213
	for_each_engine_id(engine, dev_priv, id) {
3214
		if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
3215
			DRM_INFO("%s on %s\n",
3216
				 stuck[id] ? "stuck" : "no progress",
3217
				 engine->name);
3218
			rings_hung |= intel_engine_flag(engine);
3219 3220 3221
		}
	}

3222
	if (rings_hung) {
3223
		i915_handle_error(dev_priv, rings_hung, "Engine(s) hung");
3224 3225
		goto out;
	}
B
Ben Gamari 已提交
3226

3227
	/* Reset timer in case GPU hangs without another request being added */
3228
	if (busy_count)
3229
		i915_queue_hangcheck(dev_priv);
3230 3231 3232

out:
	ENABLE_RPM_WAKEREF_ASSERTS(dev_priv);
3233 3234
}

3235
static void ibx_irq_reset(struct drm_device *dev)
P
Paulo Zanoni 已提交
3236 3237 3238 3239 3240 3241
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

3242
	GEN5_IRQ_RESET(SDE);
3243 3244 3245

	if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
3246
}
3247

P
Paulo Zanoni 已提交
3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
3264 3265 3266 3267
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

3268
static void gen5_gt_irq_reset(struct drm_device *dev)
3269 3270 3271
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3272
	GEN5_IRQ_RESET(GT);
P
Paulo Zanoni 已提交
3273
	if (INTEL_INFO(dev)->gen >= 6)
3274
		GEN5_IRQ_RESET(GEN6_PM);
3275 3276
}

3277 3278 3279 3280
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
	enum pipe pipe;

3281 3282 3283 3284 3285
	if (IS_CHERRYVIEW(dev_priv))
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
	else
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);

3286
	i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
3287 3288
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

3289 3290 3291 3292 3293 3294
	for_each_pipe(dev_priv, pipe) {
		I915_WRITE(PIPESTAT(pipe),
			   PIPE_FIFO_UNDERRUN_STATUS |
			   PIPESTAT_INT_STATUS_MASK);
		dev_priv->pipestat_irq_mask[pipe] = 0;
	}
3295 3296

	GEN5_IRQ_RESET(VLV_);
3297
	dev_priv->irq_mask = ~0;
3298 3299
}

3300 3301 3302
static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
3303
	u32 enable_mask;
3304 3305 3306 3307 3308 3309 3310 3311 3312
	enum pipe pipe;

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		i915_enable_pipestat(dev_priv, pipe, pipestat_mask);

3313 3314 3315
	enable_mask = I915_DISPLAY_PORT_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3316
	if (IS_CHERRYVIEW(dev_priv))
3317
		enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3318 3319 3320

	WARN_ON(dev_priv->irq_mask != ~0);

3321 3322 3323
	dev_priv->irq_mask = ~enable_mask;

	GEN5_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask);
3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342
}

/* drm_dma.h hooks
*/
static void ironlake_irq_reset(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	I915_WRITE(HWSTAM, 0xffffffff);

	GEN5_IRQ_RESET(DE);
	if (IS_GEN7(dev))
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);

	gen5_gt_irq_reset(dev);

	ibx_irq_reset(dev);
}

J
Jesse Barnes 已提交
3343 3344
static void valleyview_irq_preinstall(struct drm_device *dev)
{
3345
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3346

3347 3348 3349
	I915_WRITE(VLV_MASTER_IER, 0);
	POSTING_READ(VLV_MASTER_IER);

3350
	gen5_gt_irq_reset(dev);
J
Jesse Barnes 已提交
3351

3352
	spin_lock_irq(&dev_priv->irq_lock);
3353 3354
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3355
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3356 3357
}

3358 3359 3360 3361 3362 3363 3364 3365
static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
{
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
}

P
Paulo Zanoni 已提交
3366
static void gen8_irq_reset(struct drm_device *dev)
3367 3368 3369 3370 3371 3372 3373
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3374
	gen8_gt_irq_reset(dev_priv);
3375

3376
	for_each_pipe(dev_priv, pipe)
3377 3378
		if (intel_display_power_is_enabled(dev_priv,
						   POWER_DOMAIN_PIPE(pipe)))
3379
			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3380

3381 3382 3383
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3384

3385 3386
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_reset(dev);
3387
}
3388

3389 3390
void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
3391
{
3392
	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3393
	enum pipe pipe;
3394

3395
	spin_lock_irq(&dev_priv->irq_lock);
3396 3397 3398 3399
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
				  dev_priv->de_irq_mask[pipe],
				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
3400
	spin_unlock_irq(&dev_priv->irq_lock);
3401 3402
}

3403 3404 3405
void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
{
3406 3407
	enum pipe pipe;

3408
	spin_lock_irq(&dev_priv->irq_lock);
3409 3410
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3411 3412 3413 3414 3415 3416
	spin_unlock_irq(&dev_priv->irq_lock);

	/* make sure we're done processing display irqs */
	synchronize_irq(dev_priv->dev->irq);
}

3417 3418 3419 3420 3421 3422 3423
static void cherryview_irq_preinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3424
	gen8_gt_irq_reset(dev_priv);
3425 3426 3427

	GEN5_IRQ_RESET(GEN8_PCU_);

3428
	spin_lock_irq(&dev_priv->irq_lock);
3429 3430
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3431
	spin_unlock_irq(&dev_priv->irq_lock);
3432 3433
}

3434
static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
3435 3436 3437 3438 3439
				  const u32 hpd[HPD_NUM_PINS])
{
	struct intel_encoder *encoder;
	u32 enabled_irqs = 0;

3440
	for_each_intel_encoder(dev_priv->dev, encoder)
3441 3442 3443 3444 3445 3446
		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
			enabled_irqs |= hpd[encoder->hpd_pin];

	return enabled_irqs;
}

3447
static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
3448
{
3449
	u32 hotplug_irqs, hotplug, enabled_irqs;
3450

3451
	if (HAS_PCH_IBX(dev_priv)) {
3452
		hotplug_irqs = SDE_HOTPLUG_MASK;
3453
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx);
3454
	} else {
3455
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3456
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt);
3457
	}
3458

3459
	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3460 3461 3462

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
3463 3464
	 * duration to 2ms (which is the minimum in the Display Port spec).
	 * The pulse duration bits are reserved on LPT+.
3465
	 */
3466 3467 3468 3469 3470
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3471 3472 3473 3474
	/*
	 * When CPU and PCH are on the same package, port A
	 * HPD must be enabled in both north and south.
	 */
3475
	if (HAS_PCH_LPT_LP(dev_priv))
3476
		hotplug |= PORTA_HOTPLUG_ENABLE;
3477
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3478
}
X
Xiong Zhang 已提交
3479

3480
static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3481 3482 3483 3484
{
	u32 hotplug_irqs, hotplug, enabled_irqs;

	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
3485
	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt);
3486 3487 3488 3489 3490 3491

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	/* Enable digital hotplug on the PCH */
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE |
3492
		PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE;
3493 3494 3495 3496 3497
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);

	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
	hotplug |= PORTE_HOTPLUG_ENABLE;
	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3498 3499
}

3500
static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3501 3502 3503
{
	u32 hotplug_irqs, hotplug, enabled_irqs;

3504
	if (INTEL_GEN(dev_priv) >= 8) {
3505
		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3506
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw);
3507 3508

		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3509
	} else if (INTEL_GEN(dev_priv) >= 7) {
3510
		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3511
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb);
3512 3513

		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3514 3515
	} else {
		hotplug_irqs = DE_DP_A_HOTPLUG;
3516
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk);
3517

3518 3519
		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
	}
3520 3521 3522 3523

	/*
	 * Enable digital hotplug on the CPU, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
3524
	 * The pulse duration bits are reserved on HSW+.
3525 3526 3527 3528 3529 3530
	 */
	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);

3531
	ibx_hpd_irq_setup(dev_priv);
3532 3533
}

3534
static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
3535
{
3536
	u32 hotplug_irqs, hotplug, enabled_irqs;
3537

3538
	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt);
3539
	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3540

3541
	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3542

3543 3544 3545
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE |
		PORTA_HOTPLUG_ENABLE;
3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565

	DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n",
		      hotplug, enabled_irqs);
	hotplug &= ~BXT_DDI_HPD_INVERT_MASK;

	/*
	 * For BXT invert bit has to be set based on AOB design
	 * for HPD detection logic, update it based on VBT fields.
	 */

	if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
		hotplug |= BXT_DDIA_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
		hotplug |= BXT_DDIB_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
		hotplug |= BXT_DDIC_HPD_INVERT;

3566
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3567 3568
}

P
Paulo Zanoni 已提交
3569 3570
static void ibx_irq_postinstall(struct drm_device *dev)
{
3571
	struct drm_i915_private *dev_priv = dev->dev_private;
3572
	u32 mask;
3573

D
Daniel Vetter 已提交
3574 3575 3576
	if (HAS_PCH_NOP(dev))
		return;

3577
	if (HAS_PCH_IBX(dev))
3578
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3579
	else
3580
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3581

3582
	gen5_assert_iir_is_zero(dev_priv, SDEIIR);
P
Paulo Zanoni 已提交
3583 3584 3585
	I915_WRITE(SDEIMR, ~mask);
}

3586 3587 3588 3589 3590 3591 3592 3593
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3594
	if (HAS_L3_DPF(dev)) {
3595
		/* L3 parity interrupt is always unmasked. */
3596 3597
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
		gt_irqs |= GT_PARITY_ERROR(dev);
3598 3599 3600 3601
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
	if (IS_GEN5(dev)) {
3602
		gt_irqs |= ILK_BSD_USER_INTERRUPT;
3603 3604 3605 3606
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3607
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3608 3609

	if (INTEL_INFO(dev)->gen >= 6) {
3610 3611 3612 3613
		/*
		 * RPS interrupts will get enabled/disabled on demand when RPS
		 * itself is enabled/disabled.
		 */
3614 3615 3616
		if (HAS_VEBOX(dev))
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;

3617
		dev_priv->pm_irq_mask = 0xffffffff;
P
Paulo Zanoni 已提交
3618
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3619 3620 3621
	}
}

3622
static int ironlake_irq_postinstall(struct drm_device *dev)
3623
{
3624
	struct drm_i915_private *dev_priv = dev->dev_private;
3625 3626 3627 3628 3629 3630
	u32 display_mask, extra_mask;

	if (INTEL_INFO(dev)->gen >= 7) {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3631
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3632
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3633 3634
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
			      DE_DP_A_HOTPLUG_IVB);
3635 3636 3637
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3638 3639 3640
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3641 3642 3643
		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
			      DE_DP_A_HOTPLUG);
3644
	}
3645

3646
	dev_priv->irq_mask = ~display_mask;
3647

3648 3649
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3650 3651
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3652
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3653

3654
	gen5_gt_irq_postinstall(dev);
3655

P
Paulo Zanoni 已提交
3656
	ibx_irq_postinstall(dev);
3657

3658
	if (IS_IRONLAKE_M(dev)) {
3659 3660 3661
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3662 3663
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
3664
		spin_lock_irq(&dev_priv->irq_lock);
3665
		ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
3666
		spin_unlock_irq(&dev_priv->irq_lock);
3667 3668
	}

3669 3670 3671
	return 0;
}

3672 3673 3674 3675 3676 3677 3678 3679 3680
void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

3681 3682
	if (intel_irqs_enabled(dev_priv)) {
		vlv_display_irq_reset(dev_priv);
3683
		vlv_display_irq_postinstall(dev_priv);
3684
	}
3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

3696
	if (intel_irqs_enabled(dev_priv))
3697
		vlv_display_irq_reset(dev_priv);
3698 3699
}

3700 3701 3702 3703 3704

static int valleyview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3705
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3706

3707
	spin_lock_irq(&dev_priv->irq_lock);
3708 3709
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3710 3711
	spin_unlock_irq(&dev_priv->irq_lock);

J
Jesse Barnes 已提交
3712
	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3713
	POSTING_READ(VLV_MASTER_IER);
3714 3715 3716 3717

	return 0;
}

3718 3719 3720 3721 3722
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3723 3724 3725
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3726
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3727 3728 3729
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3730
		0,
3731 3732
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3733 3734
		};

3735 3736 3737
	if (HAS_L3_DPF(dev_priv))
		gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;

3738
	dev_priv->pm_irq_mask = 0xffffffff;
3739 3740
	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3741 3742 3743 3744 3745
	/*
	 * RPS interrupts will get enabled/disabled on demand when RPS itself
	 * is enabled/disabled.
	 */
	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3746
	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3747 3748 3749 3750
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
3751 3752
	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
	uint32_t de_pipe_enables;
3753 3754
	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
	u32 de_port_enables;
3755
	u32 de_misc_masked = GEN8_DE_MISC_GSE;
3756
	enum pipe pipe;
3757

3758
	if (INTEL_INFO(dev_priv)->gen >= 9) {
3759 3760
		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3761 3762
		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
				  GEN9_AUX_CHANNEL_D;
S
Shashank Sharma 已提交
3763
		if (IS_BROXTON(dev_priv))
3764 3765
			de_port_masked |= BXT_DE_PORT_GMBUS;
	} else {
3766 3767
		de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
				  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3768
	}
3769 3770 3771 3772

	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
					   GEN8_PIPE_FIFO_UNDERRUN;

3773
	de_port_enables = de_port_masked;
3774 3775 3776
	if (IS_BROXTON(dev_priv))
		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
	else if (IS_BROADWELL(dev_priv))
3777 3778
		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;

3779 3780 3781
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3782

3783
	for_each_pipe(dev_priv, pipe)
3784
		if (intel_display_power_is_enabled(dev_priv,
3785 3786 3787 3788
				POWER_DOMAIN_PIPE(pipe)))
			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
					  dev_priv->de_irq_mask[pipe],
					  de_pipe_enables);
3789

3790
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3791
	GEN5_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
3792 3793 3794 3795 3796 3797
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3798 3799
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_pre_postinstall(dev);
P
Paulo Zanoni 已提交
3800

3801 3802 3803
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

3804 3805
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_postinstall(dev);
3806

3807
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3808 3809 3810 3811 3812
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3813 3814 3815 3816 3817 3818
static int cherryview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	gen8_gt_irq_postinstall(dev_priv);

3819
	spin_lock_irq(&dev_priv->irq_lock);
3820 3821
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3822 3823
	spin_unlock_irq(&dev_priv->irq_lock);

3824
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3825 3826 3827 3828 3829
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3830 3831 3832 3833 3834 3835 3836
static void gen8_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3837
	gen8_irq_reset(dev);
3838 3839
}

J
Jesse Barnes 已提交
3840 3841
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3842
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3843 3844 3845 3846

	if (!dev_priv)
		return;

3847
	I915_WRITE(VLV_MASTER_IER, 0);
3848
	POSTING_READ(VLV_MASTER_IER);
3849

3850 3851
	gen5_gt_irq_reset(dev);

J
Jesse Barnes 已提交
3852
	I915_WRITE(HWSTAM, 0xffffffff);
3853

3854
	spin_lock_irq(&dev_priv->irq_lock);
3855 3856
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3857
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3858 3859
}

3860 3861 3862 3863 3864 3865 3866 3867 3868 3869
static void cherryview_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3870
	gen8_gt_irq_reset(dev_priv);
3871

3872
	GEN5_IRQ_RESET(GEN8_PCU_);
3873

3874
	spin_lock_irq(&dev_priv->irq_lock);
3875 3876
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3877
	spin_unlock_irq(&dev_priv->irq_lock);
3878 3879
}

3880
static void ironlake_irq_uninstall(struct drm_device *dev)
3881
{
3882
	struct drm_i915_private *dev_priv = dev->dev_private;
3883 3884 3885 3886

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3887
	ironlake_irq_reset(dev);
3888 3889
}

3890
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3891
{
3892
	struct drm_i915_private *dev_priv = dev->dev_private;
3893
	int pipe;
3894

3895
	for_each_pipe(dev_priv, pipe)
3896
		I915_WRITE(PIPESTAT(pipe), 0);
3897 3898 3899
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3900 3901 3902 3903
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3904
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3905 3906 3907 3908 3909 3910 3911 3912 3913

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3914
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
C
Chris Wilson 已提交
3915 3916 3917 3918 3919 3920 3921 3922
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3923 3924
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3925
	spin_lock_irq(&dev_priv->irq_lock);
3926 3927
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3928
	spin_unlock_irq(&dev_priv->irq_lock);
3929

C
Chris Wilson 已提交
3930 3931 3932
	return 0;
}

3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963
/*
 * Returns true when a page flip has completed.
 */
static bool i8xx_handle_vblank(struct drm_i915_private *dev_priv,
			       int plane, int pipe, u32 iir)
{
	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

	if (!intel_pipe_handle_vblank(dev_priv, pipe))
		return false;

	if ((iir & flip_pending) == 0)
		goto check_page_flip;

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ16(ISR) & flip_pending)
		goto check_page_flip;

	intel_finish_page_flip_cs(dev_priv, pipe);
	return true;

check_page_flip:
	intel_check_page_flip(dev_priv, pipe);
	return false;
}

3964
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
3965
{
3966
	struct drm_device *dev = arg;
3967
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3968 3969 3970 3971 3972 3973
	u16 iir, new_iir;
	u32 pipe_stats[2];
	int pipe;
	u16 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3974
	irqreturn_t ret;
C
Chris Wilson 已提交
3975

3976 3977 3978
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

3979 3980 3981 3982
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	ret = IRQ_NONE;
C
Chris Wilson 已提交
3983 3984
	iir = I915_READ16(IIR);
	if (iir == 0)
3985
		goto out;
C
Chris Wilson 已提交
3986 3987 3988 3989 3990 3991 3992

	while (iir & ~flip_mask) {
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3993
		spin_lock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3994
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3995
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
C
Chris Wilson 已提交
3996

3997
		for_each_pipe(dev_priv, pipe) {
3998
			i915_reg_t reg = PIPESTAT(pipe);
C
Chris Wilson 已提交
3999 4000 4001 4002 4003
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
4004
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
4005 4006
				I915_WRITE(reg, pipe_stats[pipe]);
		}
4007
		spin_unlock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
4008 4009 4010 4011 4012

		I915_WRITE16(IIR, iir & ~flip_mask);
		new_iir = I915_READ16(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4013
			notify_ring(&dev_priv->engine[RCS]);
C
Chris Wilson 已提交
4014

4015
		for_each_pipe(dev_priv, pipe) {
4016 4017 4018 4019 4020 4021 4022
			int plane = pipe;
			if (HAS_FBC(dev_priv))
				plane = !plane;

			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
			    i8xx_handle_vblank(dev_priv, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
C
Chris Wilson 已提交
4023

4024
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4025
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
4026

4027 4028 4029
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
4030
		}
C
Chris Wilson 已提交
4031 4032 4033

		iir = new_iir;
	}
4034 4035 4036 4037
	ret = IRQ_HANDLED;

out:
	enable_rpm_wakeref_asserts(dev_priv);
C
Chris Wilson 已提交
4038

4039
	return ret;
C
Chris Wilson 已提交
4040 4041 4042 4043
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
4044
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
4045 4046
	int pipe;

4047
	for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
4048 4049 4050 4051 4052 4053 4054 4055 4056
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

4057 4058
static void i915_irq_preinstall(struct drm_device * dev)
{
4059
	struct drm_i915_private *dev_priv = dev->dev_private;
4060 4061 4062
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
4063
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4064 4065 4066
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4067
	I915_WRITE16(HWSTAM, 0xeffe);
4068
	for_each_pipe(dev_priv, pipe)
4069 4070 4071 4072 4073 4074 4075 4076
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
4077
	struct drm_i915_private *dev_priv = dev->dev_private;
4078
	u32 enable_mask;
4079

4080 4081 4082 4083 4084 4085 4086 4087
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4088
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4089 4090 4091 4092 4093 4094 4095

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_USER_INTERRUPT;

4096
	if (I915_HAS_HOTPLUG(dev)) {
4097
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4098 4099
		POSTING_READ(PORT_HOTPLUG_EN);

4100 4101 4102 4103 4104 4105 4106 4107 4108 4109
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4110
	i915_enable_asle_pipestat(dev_priv);
4111

4112 4113
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4114
	spin_lock_irq(&dev_priv->irq_lock);
4115 4116
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4117
	spin_unlock_irq(&dev_priv->irq_lock);
4118

4119 4120 4121
	return 0;
}

4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152
/*
 * Returns true when a page flip has completed.
 */
static bool i915_handle_vblank(struct drm_i915_private *dev_priv,
			       int plane, int pipe, u32 iir)
{
	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

	if (!intel_pipe_handle_vblank(dev_priv, pipe))
		return false;

	if ((iir & flip_pending) == 0)
		goto check_page_flip;

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ(ISR) & flip_pending)
		goto check_page_flip;

	intel_finish_page_flip_cs(dev_priv, pipe);
	return true;

check_page_flip:
	intel_check_page_flip(dev_priv, pipe);
	return false;
}

4153
static irqreturn_t i915_irq_handler(int irq, void *arg)
4154
{
4155
	struct drm_device *dev = arg;
4156
	struct drm_i915_private *dev_priv = dev->dev_private;
4157
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
4158 4159 4160 4161
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
	int pipe, ret = IRQ_NONE;
4162

4163 4164 4165
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4166 4167 4168
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

4169
	iir = I915_READ(IIR);
4170 4171
	do {
		bool irq_received = (iir & ~flip_mask) != 0;
4172
		bool blc_event = false;
4173 4174 4175 4176 4177 4178

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4179
		spin_lock(&dev_priv->irq_lock);
4180
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4181
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4182

4183
		for_each_pipe(dev_priv, pipe) {
4184
			i915_reg_t reg = PIPESTAT(pipe);
4185 4186
			pipe_stats[pipe] = I915_READ(reg);

4187
			/* Clear the PIPE*STAT regs before the IIR */
4188 4189
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4190
				irq_received = true;
4191 4192
			}
		}
4193
		spin_unlock(&dev_priv->irq_lock);
4194 4195 4196 4197 4198

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
4199
		if (I915_HAS_HOTPLUG(dev_priv) &&
4200 4201 4202
		    iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
4203
				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4204
		}
4205

4206
		I915_WRITE(IIR, iir & ~flip_mask);
4207 4208 4209
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4210
			notify_ring(&dev_priv->engine[RCS]);
4211

4212
		for_each_pipe(dev_priv, pipe) {
4213 4214 4215 4216 4217 4218 4219
			int plane = pipe;
			if (HAS_FBC(dev_priv))
				plane = !plane;

			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
			    i915_handle_vblank(dev_priv, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
4220 4221 4222

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4223 4224

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4225
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
4226

4227 4228 4229
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
4230 4231 4232
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
4233
			intel_opregion_asle_intr(dev_priv);
4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
4250
		ret = IRQ_HANDLED;
4251
		iir = new_iir;
4252
	} while (iir & ~flip_mask);
4253

4254 4255
	enable_rpm_wakeref_asserts(dev_priv);

4256 4257 4258 4259 4260
	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
4261
	struct drm_i915_private *dev_priv = dev->dev_private;
4262 4263 4264
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
4265
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4266 4267 4268
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4269
	I915_WRITE16(HWSTAM, 0xffff);
4270
	for_each_pipe(dev_priv, pipe) {
4271
		/* Clear enable bits; then clear status bits */
4272
		I915_WRITE(PIPESTAT(pipe), 0);
4273 4274
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
4275 4276 4277 4278 4279 4280 4281 4282
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
4283
	struct drm_i915_private *dev_priv = dev->dev_private;
4284 4285
	int pipe;

4286
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4287
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4288 4289

	I915_WRITE(HWSTAM, 0xeffe);
4290
	for_each_pipe(dev_priv, pipe)
4291 4292 4293 4294 4295 4296 4297 4298
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
4299
	struct drm_i915_private *dev_priv = dev->dev_private;
4300
	u32 enable_mask;
4301 4302 4303
	u32 error_mask;

	/* Unmask the interrupts that we always want on. */
4304
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4305
			       I915_DISPLAY_PORT_INTERRUPT |
4306 4307 4308 4309 4310 4311 4312
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
4313 4314
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4315 4316
	enable_mask |= I915_USER_INTERRUPT;

4317
	if (IS_G4X(dev_priv))
4318
		enable_mask |= I915_BSD_USER_INTERRUPT;
4319

4320 4321
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4322
	spin_lock_irq(&dev_priv->irq_lock);
4323 4324 4325
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4326
	spin_unlock_irq(&dev_priv->irq_lock);
4327 4328 4329 4330 4331

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
4332
	if (IS_G4X(dev_priv)) {
4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4347
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4348 4349
	POSTING_READ(PORT_HOTPLUG_EN);

4350
	i915_enable_asle_pipestat(dev_priv);
4351 4352 4353 4354

	return 0;
}

4355
static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
4356 4357 4358
{
	u32 hotplug_en;

4359 4360
	assert_spin_locked(&dev_priv->irq_lock);

4361 4362
	/* Note HDMI and DP share hotplug bits */
	/* enable bits are the same for all generations */
4363
	hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
4364 4365 4366 4367
	/* Programming the CRT detection parameters tends
	   to generate a spurious hotplug event about three
	   seconds later.  So just do it once.
	*/
4368
	if (IS_G4X(dev_priv))
4369 4370 4371 4372
		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;

	/* Ignore TV since it's buggy */
4373
	i915_hotplug_interrupt_update_locked(dev_priv,
4374 4375 4376 4377
					     HOTPLUG_INT_EN_MASK |
					     CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
					     CRT_HOTPLUG_ACTIVATION_PERIOD_64,
					     hotplug_en);
4378 4379
}

4380
static irqreturn_t i965_irq_handler(int irq, void *arg)
4381
{
4382
	struct drm_device *dev = arg;
4383
	struct drm_i915_private *dev_priv = dev->dev_private;
4384 4385 4386
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	int ret = IRQ_NONE, pipe;
4387 4388 4389
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4390

4391 4392 4393
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4394 4395 4396
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

4397 4398 4399
	iir = I915_READ(IIR);

	for (;;) {
4400
		bool irq_received = (iir & ~flip_mask) != 0;
4401 4402
		bool blc_event = false;

4403 4404 4405 4406 4407
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4408
		spin_lock(&dev_priv->irq_lock);
4409
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4410
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4411

4412
		for_each_pipe(dev_priv, pipe) {
4413
			i915_reg_t reg = PIPESTAT(pipe);
4414 4415 4416 4417 4418 4419 4420
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4421
				irq_received = true;
4422 4423
			}
		}
4424
		spin_unlock(&dev_priv->irq_lock);
4425 4426 4427 4428 4429 4430 4431

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4432 4433 4434
		if (iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
4435
				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4436
		}
4437

4438
		I915_WRITE(IIR, iir & ~flip_mask);
4439 4440 4441
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4442
			notify_ring(&dev_priv->engine[RCS]);
4443
		if (iir & I915_BSD_USER_INTERRUPT)
4444
			notify_ring(&dev_priv->engine[VCS]);
4445

4446
		for_each_pipe(dev_priv, pipe) {
4447 4448 4449
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
			    i915_handle_vblank(dev_priv, pipe, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4450 4451 4452

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4453 4454

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4455
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
4456

4457 4458
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4459
		}
4460 4461

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
4462
			intel_opregion_asle_intr(dev_priv);
4463

4464
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4465
			gmbus_irq_handler(dev_priv);
4466

4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

4485 4486
	enable_rpm_wakeref_asserts(dev_priv);

4487 4488 4489 4490 4491
	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4492
	struct drm_i915_private *dev_priv = dev->dev_private;
4493 4494 4495 4496 4497
	int pipe;

	if (!dev_priv)
		return;

4498
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4499
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4500 4501

	I915_WRITE(HWSTAM, 0xffffffff);
4502
	for_each_pipe(dev_priv, pipe)
4503 4504 4505 4506
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

4507
	for_each_pipe(dev_priv, pipe)
4508 4509 4510 4511 4512
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4513 4514 4515 4516 4517 4518 4519
/**
 * intel_irq_init - initializes irq support
 * @dev_priv: i915 device instance
 *
 * This function initializes all the irq support including work items, timers
 * and all the vtables. It does not setup the interrupt itself though.
 */
4520
void intel_irq_init(struct drm_i915_private *dev_priv)
4521
{
4522
	struct drm_device *dev = dev_priv->dev;
4523

4524 4525
	intel_hpd_init_work(dev_priv);

4526
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4527
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4528

4529
	/* Let's track the enabled rps events */
4530
	if (IS_VALLEYVIEW(dev_priv))
4531
		/* WaGsvRC0ResidencyMethod:vlv */
4532
		dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4533 4534
	else
		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4535

4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549
	dev_priv->rps.pm_intr_keep = 0;

	/*
	 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
	 * if GEN6_PM_UP_EI_EXPIRED is masked.
	 *
	 * TODO: verify if this can be reproduced on VLV,CHV.
	 */
	if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
		dev_priv->rps.pm_intr_keep |= GEN6_PM_RP_UP_EI_EXPIRED;

	if (INTEL_INFO(dev_priv)->gen >= 8)
		dev_priv->rps.pm_intr_keep |= GEN8_PMINTR_REDIRECT_TO_NON_DISP;

4550 4551
	INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
			  i915_hangcheck_elapsed);
4552

4553
	if (IS_GEN2(dev_priv)) {
4554 4555
		dev->max_vblank_count = 0;
		dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4556
	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4557
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4558
		dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4559 4560 4561
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4562 4563
	}

4564 4565 4566 4567 4568
	/*
	 * Opt out of the vblank disable timer on everything except gen2.
	 * Gen2 doesn't have a hardware frame counter and so depends on
	 * vblank interrupts to produce sane vblank seuquence numbers.
	 */
4569
	if (!IS_GEN2(dev_priv))
4570 4571
		dev->vblank_disable_immediate = true;

4572 4573
	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4574

4575
	if (IS_CHERRYVIEW(dev_priv)) {
4576 4577 4578 4579 4580 4581 4582
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4583
	} else if (IS_VALLEYVIEW(dev_priv)) {
J
Jesse Barnes 已提交
4584 4585 4586 4587 4588 4589
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
4590
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4591
	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4592
		dev->driver->irq_handler = gen8_irq_handler;
4593
		dev->driver->irq_preinstall = gen8_irq_reset;
4594 4595 4596 4597
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
4598
		if (IS_BROXTON(dev))
4599
			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4600 4601 4602
		else if (HAS_PCH_SPT(dev))
			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		else
4603
			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4604 4605
	} else if (HAS_PCH_SPLIT(dev)) {
		dev->driver->irq_handler = ironlake_irq_handler;
4606
		dev->driver->irq_preinstall = ironlake_irq_reset;
4607 4608 4609 4610
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4611
		dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4612
	} else {
4613
		if (IS_GEN2(dev_priv)) {
C
Chris Wilson 已提交
4614 4615 4616 4617
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4618
		} else if (IS_GEN3(dev_priv)) {
4619 4620 4621 4622
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
C
Chris Wilson 已提交
4623
		} else {
4624 4625 4626 4627
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
C
Chris Wilson 已提交
4628
		}
4629 4630
		if (I915_HAS_HOTPLUG(dev_priv))
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4631 4632 4633 4634
		dev->driver->enable_vblank = i915_enable_vblank;
		dev->driver->disable_vblank = i915_disable_vblank;
	}
}
4635

4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646
/**
 * intel_irq_install - enables the hardware interrupt
 * @dev_priv: i915 device instance
 *
 * This function enables the hardware interrupt handling, but leaves the hotplug
 * handling still disabled. It is called after intel_irq_init().
 *
 * In the driver load and resume code we need working interrupts in a few places
 * but don't want to deal with the hassle of concurrent probe and hotplug
 * workers. Hence the split into this two-stage approach.
 */
4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658
int intel_irq_install(struct drm_i915_private *dev_priv)
{
	/*
	 * We enable some interrupt sources in our postinstall hooks, so mark
	 * interrupts as enabled _before_ actually enabling them to avoid
	 * special cases in our ordering checks.
	 */
	dev_priv->pm.irqs_enabled = true;

	return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
}

4659 4660 4661 4662 4663 4664 4665
/**
 * intel_irq_uninstall - finilizes all irq handling
 * @dev_priv: i915 device instance
 *
 * This stops interrupt and hotplug handling and unregisters and frees all
 * resources acquired in the init functions.
 */
4666 4667 4668 4669 4670 4671 4672
void intel_irq_uninstall(struct drm_i915_private *dev_priv)
{
	drm_irq_uninstall(dev_priv->dev);
	intel_hpd_cancel_work(dev_priv);
	dev_priv->pm.irqs_enabled = false;
}

4673 4674 4675 4676 4677 4678 4679
/**
 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
 * @dev_priv: i915 device instance
 *
 * This function is used to disable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4680
void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4681
{
4682
	dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4683
	dev_priv->pm.irqs_enabled = false;
4684
	synchronize_irq(dev_priv->dev->irq);
4685 4686
}

4687 4688 4689 4690 4691 4692 4693
/**
 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
 * @dev_priv: i915 device instance
 *
 * This function is used to enable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4694
void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4695
{
4696
	dev_priv->pm.irqs_enabled = true;
4697 4698
	dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
	dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4699
}