i915_irq.c 131.9 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47
/**
 * DOC: interrupt handling
 *
 * These functions provide the basic support for enabling and disabling the
 * interrupt handling support. There's a lot more functionality in i915_irq.c
 * and related files, but that will be described in separate chapters.
 */

48 49 50 51
static const u32 hpd_ilk[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
};

52 53 54 55
static const u32 hpd_ivb[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
};

56 57 58 59
static const u32 hpd_bdw[HPD_NUM_PINS] = {
	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
};

60
static const u32 hpd_ibx[HPD_NUM_PINS] = {
61 62 63 64 65 66 67
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

68
static const u32 hpd_cpt[HPD_NUM_PINS] = {
69
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71 72 73 74 75
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

X
Xiong Zhang 已提交
76
static const u32 hpd_spt[HPD_NUM_PINS] = {
77
	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
X
Xiong Zhang 已提交
78 79 80 81 82 83
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
};

84
static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85 86 87 88 89 90 91 92
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

93
static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94 95 96 97 98 99 100 101
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

102
static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103 104 105 106 107 108 109 110
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

111 112
/* BXT hpd list */
static const u32 hpd_bxt[HPD_NUM_PINS] = {
113
	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114 115 116 117
	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
};

118
/* IIR can theoretically queue up two events. Be paranoid. */
119
#define GEN8_IRQ_RESET_NDX(type, which) do { \
120 121 122 123 124 125 126 127 128
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

129
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
130
	I915_WRITE(type##IMR, 0xffffffff); \
131
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
132
	I915_WRITE(type##IER, 0); \
133 134 135 136
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
137 138
} while (0)

139 140 141
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
142 143
static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
				    i915_reg_t reg)
144 145 146 147 148 149 150
{
	u32 val = I915_READ(reg);

	if (val == 0)
		return;

	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
151
	     i915_mmio_reg_offset(reg), val);
152 153 154 155 156
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
}
157

P
Paulo Zanoni 已提交
158
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
159
	gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
160
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
161 162
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
P
Paulo Zanoni 已提交
163 164 165
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
166
	gen5_assert_iir_is_zero(dev_priv, type##IIR); \
P
Paulo Zanoni 已提交
167
	I915_WRITE(type##IER, (ier_val)); \
168 169
	I915_WRITE(type##IMR, (imr_val)); \
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
170 171
} while (0)

172 173
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);

174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211
/* For display hotplug interrupt */
static inline void
i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
				     uint32_t mask,
				     uint32_t bits)
{
	uint32_t val;

	assert_spin_locked(&dev_priv->irq_lock);
	WARN_ON(bits & ~mask);

	val = I915_READ(PORT_HOTPLUG_EN);
	val &= ~mask;
	val |= bits;
	I915_WRITE(PORT_HOTPLUG_EN, val);
}

/**
 * i915_hotplug_interrupt_update - update hotplug interrupt enable
 * @dev_priv: driver private
 * @mask: bits to update
 * @bits: bits to enable
 * NOTE: the HPD enable bits are modified both inside and outside
 * of an interrupt context. To avoid that read-modify-write cycles
 * interfer, these bits are protected by a spinlock. Since this
 * function is usually not called from a context where the lock is
 * held already, this function acquires the lock itself. A non-locking
 * version is also available.
 */
void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
				   uint32_t mask,
				   uint32_t bits)
{
	spin_lock_irq(&dev_priv->irq_lock);
	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
	spin_unlock_irq(&dev_priv->irq_lock);
}

212 213 214 215 216 217
/**
 * ilk_update_display_irq - update DEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
218 219 220
void ilk_update_display_irq(struct drm_i915_private *dev_priv,
			    uint32_t interrupt_mask,
			    uint32_t enabled_irq_mask)
221
{
222 223
	uint32_t new_val;

224 225
	assert_spin_locked(&dev_priv->irq_lock);

226 227
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

228
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
229 230
		return;

231 232 233 234 235 236
	new_val = dev_priv->irq_mask;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->irq_mask) {
		dev_priv->irq_mask = new_val;
237
		I915_WRITE(DEIMR, dev_priv->irq_mask);
238
		POSTING_READ(DEIMR);
239 240 241
	}
}

P
Paulo Zanoni 已提交
242 243 244 245 246 247 248 249 250 251 252 253
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
	assert_spin_locked(&dev_priv->irq_lock);

254 255
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

256
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
257 258
		return;

P
Paulo Zanoni 已提交
259 260 261 262 263 264
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
	POSTING_READ(GTIMR);
}

265
void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
266 267 268 269
{
	ilk_update_gt_irq(dev_priv, mask, mask);
}

270
void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
271 272 273 274
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

275
static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
276 277 278 279
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
}

280
static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
281 282 283 284
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
}

285
static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
286 287 288 289
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
}

P
Paulo Zanoni 已提交
290
/**
291 292 293 294 295
 * snb_update_pm_irq - update GEN6_PMIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
P
Paulo Zanoni 已提交
296 297 298 299
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
300
	uint32_t new_val;
P
Paulo Zanoni 已提交
301

302 303
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

P
Paulo Zanoni 已提交
304 305
	assert_spin_locked(&dev_priv->irq_lock);

306
	new_val = dev_priv->pm_irq_mask;
307 308 309
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

310 311
	if (new_val != dev_priv->pm_irq_mask) {
		dev_priv->pm_irq_mask = new_val;
312 313
		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
		POSTING_READ(gen6_pm_imr(dev_priv));
314
	}
P
Paulo Zanoni 已提交
315 316
}

317
void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
318
{
319 320 321
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

P
Paulo Zanoni 已提交
322 323 324
	snb_update_pm_irq(dev_priv, mask, mask);
}

325 326
static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
				  uint32_t mask)
P
Paulo Zanoni 已提交
327 328 329 330
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

331 332 333 334 335 336 337 338
void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	__gen6_disable_pm_irq(dev_priv, mask);
}

I
Imre Deak 已提交
339 340 341
void gen6_reset_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
342
	i915_reg_t reg = gen6_pm_iir(dev_priv);
I
Imre Deak 已提交
343 344 345 346 347

	spin_lock_irq(&dev_priv->irq_lock);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	POSTING_READ(reg);
348
	dev_priv->rps.pm_iir = 0;
I
Imre Deak 已提交
349 350 351
	spin_unlock_irq(&dev_priv->irq_lock);
}

352 353 354 355 356
void gen6_enable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	spin_lock_irq(&dev_priv->irq_lock);
357

358
	WARN_ON(dev_priv->rps.pm_iir);
I
Imre Deak 已提交
359
	WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
I
Imre Deak 已提交
360
	dev_priv->rps.interrupts_enabled = true;
361 362
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
				dev_priv->pm_rps_events);
363
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
364

365 366 367
	spin_unlock_irq(&dev_priv->irq_lock);
}

368 369 370
u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
{
	/*
371
	 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
372
	 * if GEN6_PM_UP_EI_EXPIRED is masked.
373 374
	 *
	 * TODO: verify if this can be reproduced on VLV,CHV.
375 376 377 378 379 380 381 382 383 384
	 */
	if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
		mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;

	if (INTEL_INFO(dev_priv)->gen >= 8)
		mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;

	return mask;
}

385 386 387 388
void gen6_disable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

I
Imre Deak 已提交
389 390 391 392 393 394
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->rps.interrupts_enabled = false;
	spin_unlock_irq(&dev_priv->irq_lock);

	cancel_work_sync(&dev_priv->rps.work);

395 396
	spin_lock_irq(&dev_priv->irq_lock);

397
	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
398 399

	__gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
400 401
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
				~dev_priv->pm_rps_events);
402 403 404 405

	spin_unlock_irq(&dev_priv->irq_lock);

	synchronize_irq(dev->irq);
406 407
}

408
/**
409 410 411 412 413
 * bdw_update_port_irq - update DE port interrupt
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439
static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
				uint32_t interrupt_mask,
				uint32_t enabled_irq_mask)
{
	uint32_t new_val;
	uint32_t old_val;

	assert_spin_locked(&dev_priv->irq_lock);

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	old_val = I915_READ(GEN8_DE_PORT_IMR);

	new_val = old_val;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != old_val) {
		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		POSTING_READ(GEN8_DE_PORT_IMR);
	}
}

440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
/**
 * bdw_update_pipe_irq - update DE pipe interrupt
 * @dev_priv: driver private
 * @pipe: pipe whose interrupt to update
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
			 enum pipe pipe,
			 uint32_t interrupt_mask,
			 uint32_t enabled_irq_mask)
{
	uint32_t new_val;

	assert_spin_locked(&dev_priv->irq_lock);

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	new_val = dev_priv->de_irq_mask[pipe];
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->de_irq_mask[pipe]) {
		dev_priv->de_irq_mask[pipe] = new_val;
		I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
	}
}

472 473 474 475 476 477
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
478 479 480
void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
				  uint32_t interrupt_mask,
				  uint32_t enabled_irq_mask)
481 482 483 484 485
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

486 487
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

488 489
	assert_spin_locked(&dev_priv->irq_lock);

490
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
491 492
		return;

493 494 495
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
496

D
Daniel Vetter 已提交
497
static void
498 499
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
500
{
501
	i915_reg_t reg = PIPESTAT(pipe);
502
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
503

504
	assert_spin_locked(&dev_priv->irq_lock);
505
	WARN_ON(!intel_irqs_enabled(dev_priv));
506

507 508 509 510
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
511 512 513
		return;

	if ((pipestat & enable_mask) == enable_mask)
514 515
		return;

516 517
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

518
	/* Enable the interrupt, clear any pending status */
519
	pipestat |= enable_mask | status_mask;
520 521
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
522 523
}

D
Daniel Vetter 已提交
524
static void
525 526
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
527
{
528
	i915_reg_t reg = PIPESTAT(pipe);
529
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
530

531
	assert_spin_locked(&dev_priv->irq_lock);
532
	WARN_ON(!intel_irqs_enabled(dev_priv));
533

534 535 536 537
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
538 539
		return;

540 541 542
	if ((pipestat & enable_mask) == 0)
		return;

543 544
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

545
	pipestat &= ~enable_mask;
546 547
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
548 549
}

550 551 552 553 554
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
555 556
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
557 558 559
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
560 561 562 563 564 565
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
566 567 568 569 570 571 572 573 574 575 576 577

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

578 579 580 581 582 583
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

584
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
585 586 587 588
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
589 590 591 592 593 594 595 596 597
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

598
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
599 600 601 602
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
603 604 605
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

606
/**
607
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
608
 * @dev: drm device
609
 */
610
static void i915_enable_asle_pipestat(struct drm_device *dev)
611
{
612
	struct drm_i915_private *dev_priv = dev->dev_private;
613

614 615 616
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
		return;

617
	spin_lock_irq(&dev_priv->irq_lock);
618

619
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
620
	if (INTEL_INFO(dev)->gen >= 4)
621
		i915_enable_pipestat(dev_priv, PIPE_A,
622
				     PIPE_LEGACY_BLC_EVENT_STATUS);
623

624
	spin_unlock_irq(&dev_priv->irq_lock);
625 626
}

627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676
/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via PIPECONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on PIPECONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

677
static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
678 679 680 681 682
{
	/* Gen2 doesn't have a hardware frame counter */
	return 0;
}

683 684 685
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
686
static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
687
{
688
	struct drm_i915_private *dev_priv = dev->dev_private;
689
	i915_reg_t high_frame, low_frame;
690
	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
691 692
	struct intel_crtc *intel_crtc =
		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
693
	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
694

695 696 697 698 699
	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vbl_start = mode->crtc_vblank_start;
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
700

701 702 703 704 705 706
	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

707 708
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
709

710 711 712 713 714 715
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
716
		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
717
		low   = I915_READ(low_frame);
718
		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
719 720
	} while (high1 != high2);

721
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
722
	pixel = low & PIPE_PIXEL_MASK;
723
	low >>= PIPE_FRAME_LOW_SHIFT;
724 725 726 727 728 729

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
730
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
731 732
}

733
static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
734
{
735
	struct drm_i915_private *dev_priv = dev->dev_private;
736

737
	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
738 739
}

740
/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
741 742 743 744
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
745
	const struct drm_display_mode *mode = &crtc->base.hwmode;
746
	enum pipe pipe = crtc->pipe;
747
	int position, vtotal;
748

749
	vtotal = mode->crtc_vtotal;
750 751 752 753
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

	if (IS_GEN2(dev))
754
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
755
	else
756
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
757

758 759 760 761 762 763 764 765 766 767 768 769
	/*
	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
	 * read it just before the start of vblank.  So try it again
	 * so we don't accidentally end up spanning a vblank frame
	 * increment, causing the pipe_update_end() code to squak at us.
	 *
	 * The nature of this problem means we can't simply check the ISR
	 * bit and return the vblank start value; nor can we use the scanline
	 * debug register in the transcoder as it appears to have the same
	 * problem.  We may need to extend this to include other platforms,
	 * but so far testing only shows the problem on HSW.
	 */
770
	if (HAS_DDI(dev) && !position) {
771 772 773 774 775 776 777 778 779 780 781 782 783
		int i, temp;

		for (i = 0; i < 100; i++) {
			udelay(1);
			temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
				DSL_LINEMASK_GEN3;
			if (temp != position) {
				position = temp;
				break;
			}
		}
	}

784
	/*
785 786
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
787
	 */
788
	return (position + crtc->scanline_offset) % vtotal;
789 790
}

791
static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
792
				    unsigned int flags, int *vpos, int *hpos,
793 794
				    ktime_t *stime, ktime_t *etime,
				    const struct drm_display_mode *mode)
795
{
796 797 798
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
799
	int position;
800
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
801 802
	bool in_vbl = true;
	int ret = 0;
803
	unsigned long irqflags;
804

805
	if (WARN_ON(!mode->crtc_clock)) {
806
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
807
				 "pipe %c\n", pipe_name(pipe));
808 809 810
		return 0;
	}

811
	htotal = mode->crtc_htotal;
812
	hsync_start = mode->crtc_hsync_start;
813 814 815
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
816

817 818 819 820 821 822
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

823 824
	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

825 826 827 828 829 830
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
831

832 833 834 835 836 837
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

838
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
839 840 841
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
842
		position = __intel_get_crtc_scanline(intel_crtc);
843 844 845 846 847
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
848
		position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
849

850 851 852 853
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
854

855 856 857 858 859 860 861 862 863 864 865 866
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

867 868 869 870 871 872 873 874 875 876
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
877 878
	}

879 880 881 882 883 884 885 886
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

887 888 889 890 891 892 893 894 895 896 897 898
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
899

900
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
901 902 903 904 905 906
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
907 908 909

	/* In vblank? */
	if (in_vbl)
910
		ret |= DRM_SCANOUTPOS_IN_VBLANK;
911 912 913 914

	return ret;
}

915 916 917 918 919 920 921 922 923 924 925 926 927
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

928
static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
929 930 931 932
			      int *max_error,
			      struct timeval *vblank_time,
			      unsigned flags)
{
933
	struct drm_crtc *crtc;
934

935 936
	if (pipe >= INTEL_INFO(dev)->num_pipes) {
		DRM_ERROR("Invalid crtc %u\n", pipe);
937 938 939 940
		return -EINVAL;
	}

	/* Get drm_crtc to timestamp: */
941 942
	crtc = intel_get_crtc_for_pipe(dev, pipe);
	if (crtc == NULL) {
943
		DRM_ERROR("Invalid crtc %u\n", pipe);
944 945 946
		return -EINVAL;
	}

947
	if (!crtc->hwmode.crtc_clock) {
948
		DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
949 950
		return -EBUSY;
	}
951 952

	/* Helper routine in DRM core does all the work: */
953 954
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
						     vblank_time, flags,
955
						     &crtc->hwmode);
956 957
}

958
static void ironlake_rps_change_irq_handler(struct drm_device *dev)
959
{
960
	struct drm_i915_private *dev_priv = dev->dev_private;
961
	u32 busy_up, busy_down, max_avg, min_avg;
962 963
	u8 new_delay;

964
	spin_lock(&mchdev_lock);
965

966 967
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

968
	new_delay = dev_priv->ips.cur_delay;
969

970
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
971 972
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
973 974 975 976
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
977
	if (busy_up > max_avg) {
978 979 980 981
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
982
	} else if (busy_down < min_avg) {
983 984 985 986
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
987 988
	}

989
	if (ironlake_set_drps(dev, new_delay))
990
		dev_priv->ips.cur_delay = new_delay;
991

992
	spin_unlock(&mchdev_lock);
993

994 995 996
	return;
}

997
static void notify_ring(struct intel_engine_cs *engine)
998
{
999
	if (!intel_engine_initialized(engine))
1000 1001
		return;

1002
	trace_i915_gem_request_notify(engine);
1003
	engine->user_interrupts++;
1004

1005
	wake_up_all(&engine->irq_queue);
1006 1007
}

1008 1009
static void vlv_c0_read(struct drm_i915_private *dev_priv,
			struct intel_rps_ei *ei)
1010
{
1011 1012 1013 1014
	ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
}
1015

1016 1017 1018 1019 1020 1021
static bool vlv_c0_above(struct drm_i915_private *dev_priv,
			 const struct intel_rps_ei *old,
			 const struct intel_rps_ei *now,
			 int threshold)
{
	u64 time, c0;
1022
	unsigned int mul = 100;
1023

1024 1025
	if (old->cz_clock == 0)
		return false;
1026

1027 1028 1029
	if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
		mul <<= 8;

1030
	time = now->cz_clock - old->cz_clock;
1031
	time *= threshold * dev_priv->czclk_freq;
1032

1033 1034 1035
	/* Workload can be split between render + media, e.g. SwapBuffers
	 * being blitted in X after being rendered in mesa. To account for
	 * this we need to combine both engines into our activity counter.
1036
	 */
1037 1038
	c0 = now->render_c0 - old->render_c0;
	c0 += now->media_c0 - old->media_c0;
1039
	c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC;
1040

1041
	return c0 >= time;
1042 1043
}

1044
void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1045
{
1046 1047 1048
	vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
	dev_priv->rps.up_ei = dev_priv->rps.down_ei;
}
1049

1050 1051 1052 1053
static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
{
	struct intel_rps_ei now;
	u32 events = 0;
1054

1055
	if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
1056
		return 0;
1057

1058 1059 1060
	vlv_c0_read(dev_priv, &now);
	if (now.cz_clock == 0)
		return 0;
1061

1062 1063 1064
	if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
		if (!vlv_c0_above(dev_priv,
				  &dev_priv->rps.down_ei, &now,
1065
				  dev_priv->rps.down_threshold))
1066 1067 1068
			events |= GEN6_PM_RP_DOWN_THRESHOLD;
		dev_priv->rps.down_ei = now;
	}
1069

1070 1071 1072
	if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
		if (vlv_c0_above(dev_priv,
				 &dev_priv->rps.up_ei, &now,
1073
				 dev_priv->rps.up_threshold))
1074 1075
			events |= GEN6_PM_RP_UP_THRESHOLD;
		dev_priv->rps.up_ei = now;
1076 1077
	}

1078
	return events;
1079 1080
}

1081 1082
static bool any_waiters(struct drm_i915_private *dev_priv)
{
1083
	struct intel_engine_cs *engine;
1084

1085
	for_each_engine(engine, dev_priv)
1086
		if (engine->irq_refcount)
1087 1088 1089 1090 1091
			return true;

	return false;
}

1092
static void gen6_pm_rps_work(struct work_struct *work)
1093
{
1094 1095
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
1096 1097
	bool client_boost;
	int new_delay, adj, min, max;
P
Paulo Zanoni 已提交
1098
	u32 pm_iir;
1099

1100
	spin_lock_irq(&dev_priv->irq_lock);
I
Imre Deak 已提交
1101 1102 1103 1104 1105
	/* Speed up work cancelation during disabling rps interrupts. */
	if (!dev_priv->rps.interrupts_enabled) {
		spin_unlock_irq(&dev_priv->irq_lock);
		return;
	}
1106 1107 1108 1109 1110 1111 1112 1113

	/*
	 * The RPS work is synced during runtime suspend, we don't require a
	 * wakeref. TODO: instead of disabling the asserts make sure that we
	 * always hold an RPM reference while the work is running.
	 */
	DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);

1114 1115
	pm_iir = dev_priv->rps.pm_iir;
	dev_priv->rps.pm_iir = 0;
1116 1117
	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
1118 1119
	client_boost = dev_priv->rps.client_boost;
	dev_priv->rps.client_boost = false;
1120
	spin_unlock_irq(&dev_priv->irq_lock);
1121

1122
	/* Make sure we didn't queue anything we're not going to process. */
1123
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1124

1125
	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1126
		goto out;
1127

1128
	mutex_lock(&dev_priv->rps.hw_lock);
1129

1130 1131
	pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);

1132
	adj = dev_priv->rps.last_adj;
1133
	new_delay = dev_priv->rps.cur_freq;
1134 1135 1136 1137 1138 1139 1140
	min = dev_priv->rps.min_freq_softlimit;
	max = dev_priv->rps.max_freq_softlimit;

	if (client_boost) {
		new_delay = dev_priv->rps.max_freq_softlimit;
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1141 1142
		if (adj > 0)
			adj *= 2;
1143 1144
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1145 1146 1147 1148
		/*
		 * For better performance, jump directly
		 * to RPe if we're below it.
		 */
1149
		if (new_delay < dev_priv->rps.efficient_freq - adj) {
1150
			new_delay = dev_priv->rps.efficient_freq;
1151 1152
			adj = 0;
		}
1153 1154
	} else if (any_waiters(dev_priv)) {
		adj = 0;
1155
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1156 1157
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1158
		else
1159
			new_delay = dev_priv->rps.min_freq_softlimit;
1160 1161 1162 1163
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
1164 1165
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1166
	} else { /* unknown event */
1167
		adj = 0;
1168
	}
1169

1170 1171
	dev_priv->rps.last_adj = adj;

1172 1173 1174
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1175
	new_delay += adj;
1176
	new_delay = clamp_t(int, new_delay, min, max);
1177

1178
	intel_set_rps(dev_priv->dev, new_delay);
1179

1180
	mutex_unlock(&dev_priv->rps.hw_lock);
1181 1182
out:
	ENABLE_RPM_WAKEREF_ASSERTS(dev_priv);
1183 1184
}

1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1197 1198
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, l3_parity.error_work);
1199
	u32 error_status, row, bank, subbank;
1200
	char *parity_event[6];
1201
	uint32_t misccpctl;
1202
	uint8_t slice = 0;
1203 1204 1205 1206 1207 1208 1209

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
	mutex_lock(&dev_priv->dev->struct_mutex);

1210 1211 1212 1213
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1214 1215 1216 1217
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1218
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1219
		i915_reg_t reg;
1220

1221
		slice--;
1222
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv)))
1223
			break;
1224

1225
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1226

1227
		reg = GEN7_L3CDERRST1(slice);
1228

1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1244
		kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1245
				   KOBJ_CHANGE, parity_event);
1246

1247 1248
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1249

1250 1251 1252 1253 1254
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1255

1256
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1257

1258 1259
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
1260
	spin_lock_irq(&dev_priv->irq_lock);
1261
	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1262
	spin_unlock_irq(&dev_priv->irq_lock);
1263 1264

	mutex_unlock(&dev_priv->dev->struct_mutex);
1265 1266
}

1267
static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1268
{
1269
	struct drm_i915_private *dev_priv = dev->dev_private;
1270

1271
	if (!HAS_L3_DPF(dev))
1272 1273
		return;

1274
	spin_lock(&dev_priv->irq_lock);
1275
	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1276
	spin_unlock(&dev_priv->irq_lock);
1277

1278 1279 1280 1281 1282 1283 1284
	iir &= GT_PARITY_ERROR(dev);
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1285
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1286 1287
}

1288 1289 1290 1291 1292 1293
static void ilk_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1294
		notify_ring(&dev_priv->engine[RCS]);
1295
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
1296
		notify_ring(&dev_priv->engine[VCS]);
1297 1298
}

1299 1300 1301 1302 1303
static void snb_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{

1304 1305
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1306
		notify_ring(&dev_priv->engine[RCS]);
1307
	if (gt_iir & GT_BSD_USER_INTERRUPT)
1308
		notify_ring(&dev_priv->engine[VCS]);
1309
	if (gt_iir & GT_BLT_USER_INTERRUPT)
1310
		notify_ring(&dev_priv->engine[BCS]);
1311

1312 1313
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
1314 1315
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1316

1317 1318
	if (gt_iir & GT_PARITY_ERROR(dev))
		ivybridge_parity_error_irq_handler(dev, gt_iir);
1319 1320
}

1321
static __always_inline void
1322
gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
1323 1324
{
	if (iir & (GT_RENDER_USER_INTERRUPT << test_shift))
1325
		notify_ring(engine);
1326
	if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift))
1327
		tasklet_schedule(&engine->irq_tasklet);
1328 1329
}

C
Chris Wilson 已提交
1330
static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1331 1332 1333 1334 1335
				       u32 master_ctl)
{
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1336 1337 1338
		u32 iir = I915_READ_FW(GEN8_GT_IIR(0));
		if (iir) {
			I915_WRITE_FW(GEN8_GT_IIR(0), iir);
1339
			ret = IRQ_HANDLED;
1340

1341 1342
			gen8_cs_irq_handler(&dev_priv->engine[RCS],
					    iir, GEN8_RCS_IRQ_SHIFT);
C
Chris Wilson 已提交
1343

1344 1345
			gen8_cs_irq_handler(&dev_priv->engine[BCS],
					    iir, GEN8_BCS_IRQ_SHIFT);
1346 1347 1348 1349
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1350
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1351 1352 1353
		u32 iir = I915_READ_FW(GEN8_GT_IIR(1));
		if (iir) {
			I915_WRITE_FW(GEN8_GT_IIR(1), iir);
1354
			ret = IRQ_HANDLED;
1355

1356 1357
			gen8_cs_irq_handler(&dev_priv->engine[VCS],
					    iir, GEN8_VCS1_IRQ_SHIFT);
1358

1359 1360
			gen8_cs_irq_handler(&dev_priv->engine[VCS2],
					    iir, GEN8_VCS2_IRQ_SHIFT);
1361
		} else
1362
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
1363 1364
	}

1365
	if (master_ctl & GEN8_GT_VECS_IRQ) {
1366 1367 1368
		u32 iir = I915_READ_FW(GEN8_GT_IIR(3));
		if (iir) {
			I915_WRITE_FW(GEN8_GT_IIR(3), iir);
1369
			ret = IRQ_HANDLED;
1370

1371 1372
			gen8_cs_irq_handler(&dev_priv->engine[VECS],
					    iir, GEN8_VECS_IRQ_SHIFT);
1373 1374 1375 1376
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

1377
	if (master_ctl & GEN8_GT_PM_IRQ) {
1378 1379
		u32 iir = I915_READ_FW(GEN8_GT_IIR(2));
		if (iir & dev_priv->pm_rps_events) {
1380
			I915_WRITE_FW(GEN8_GT_IIR(2),
1381
				      iir & dev_priv->pm_rps_events);
1382
			ret = IRQ_HANDLED;
1383
			gen6_rps_irq_handler(dev_priv, iir);
1384 1385 1386 1387
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1388 1389 1390
	return ret;
}

1391 1392 1393 1394
static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
1395
		return val & PORTA_HOTPLUG_LONG_DETECT;
1396 1397 1398 1399 1400 1401 1402 1403 1404
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1405 1406 1407 1408 1409 1410 1411 1412 1413 1414
static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_E:
		return val & PORTE_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430
static bool spt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & PORTA_HOTPLUG_LONG_DETECT;
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	case PORT_D:
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1431 1432 1433 1434 1435 1436 1437 1438 1439 1440
static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1441
static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1442 1443 1444
{
	switch (port) {
	case PORT_B:
1445
		return val & PORTB_HOTPLUG_LONG_DETECT;
1446
	case PORT_C:
1447
		return val & PORTC_HOTPLUG_LONG_DETECT;
1448
	case PORT_D:
1449 1450 1451
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
1452 1453 1454
	}
}

1455
static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1456 1457 1458
{
	switch (port) {
	case PORT_B:
1459
		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1460
	case PORT_C:
1461
		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1462
	case PORT_D:
1463 1464 1465
		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
	default:
		return false;
1466 1467 1468
	}
}

1469 1470 1471 1472 1473 1474 1475
/*
 * Get a bit mask of pins that have triggered, and which ones may be long.
 * This can be called multiple times with the same masks to accumulate
 * hotplug detection results from several registers.
 *
 * Note that the caller is expected to zero out the masks initially.
 */
1476
static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1477
			     u32 hotplug_trigger, u32 dig_hotplug_reg,
1478 1479
			     const u32 hpd[HPD_NUM_PINS],
			     bool long_pulse_detect(enum port port, u32 val))
1480
{
1481
	enum port port;
1482 1483 1484
	int i;

	for_each_hpd_pin(i) {
1485 1486
		if ((hpd[i] & hotplug_trigger) == 0)
			continue;
1487

1488 1489
		*pin_mask |= BIT(i);

1490 1491 1492
		if (!intel_hpd_pin_to_port(i, &port))
			continue;

1493
		if (long_pulse_detect(port, dig_hotplug_reg))
1494
			*long_mask |= BIT(i);
1495 1496 1497 1498 1499 1500 1501
	}

	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
			 hotplug_trigger, dig_hotplug_reg, *pin_mask);

}

1502 1503
static void gmbus_irq_handler(struct drm_device *dev)
{
1504
	struct drm_i915_private *dev_priv = dev->dev_private;
1505 1506

	wake_up_all(&dev_priv->gmbus_wait_queue);
1507 1508
}

1509 1510
static void dp_aux_irq_handler(struct drm_device *dev)
{
1511
	struct drm_i915_private *dev_priv = dev->dev_private;
1512 1513

	wake_up_all(&dev_priv->gmbus_wait_queue);
1514 1515
}

1516
#if defined(CONFIG_DEBUG_FS)
1517 1518 1519 1520
static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1521 1522 1523 1524
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
1525
	int head, tail;
1526

1527 1528
	spin_lock(&pipe_crc->lock);

1529
	if (!pipe_crc->entries) {
1530
		spin_unlock(&pipe_crc->lock);
1531
		DRM_DEBUG_KMS("spurious interrupt\n");
1532 1533 1534
		return;
	}

1535 1536
	head = pipe_crc->head;
	tail = pipe_crc->tail;
1537 1538

	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1539
		spin_unlock(&pipe_crc->lock);
1540 1541 1542 1543 1544
		DRM_ERROR("CRC buffer overflowing\n");
		return;
	}

	entry = &pipe_crc->entries[head];
1545

1546
	entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1547 1548 1549 1550 1551
	entry->crc[0] = crc0;
	entry->crc[1] = crc1;
	entry->crc[2] = crc2;
	entry->crc[3] = crc3;
	entry->crc[4] = crc4;
1552 1553

	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1554 1555 1556
	pipe_crc->head = head;

	spin_unlock(&pipe_crc->lock);
1557 1558

	wake_up_interruptible(&pipe_crc->wq);
1559
}
1560 1561 1562 1563 1564 1565 1566 1567
#else
static inline void
display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1568

1569
static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
D
Daniel Vetter 已提交
1570 1571 1572
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1573 1574 1575
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1576 1577
}

1578
static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1579 1580 1581
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1582 1583 1584 1585 1586 1587
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1588
}
1589

1590
static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1591 1592
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
	uint32_t res1, res2;

	if (INTEL_INFO(dev)->gen >= 3)
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

	if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1604

1605 1606 1607 1608 1609
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1610
}
1611

1612 1613 1614 1615
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1616
{
1617
	if (pm_iir & dev_priv->pm_rps_events) {
1618
		spin_lock(&dev_priv->irq_lock);
1619
		gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
I
Imre Deak 已提交
1620 1621 1622 1623
		if (dev_priv->rps.interrupts_enabled) {
			dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
			queue_work(dev_priv->wq, &dev_priv->rps.work);
		}
1624
		spin_unlock(&dev_priv->irq_lock);
1625 1626
	}

1627 1628 1629
	if (INTEL_INFO(dev_priv)->gen >= 8)
		return;

1630
	if (HAS_VEBOX(dev_priv)) {
1631
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1632
			notify_ring(&dev_priv->engine[VECS]);
B
Ben Widawsky 已提交
1633

1634 1635
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
B
Ben Widawsky 已提交
1636
	}
1637 1638
}

1639 1640 1641 1642 1643 1644 1645 1646
static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
{
	if (!drm_handle_vblank(dev, pipe))
		return false;

	return true;
}

1647 1648
static void valleyview_pipestat_irq_ack(struct drm_device *dev, u32 iir,
					u32 pipe_stats[I915_MAX_PIPES])
1649 1650 1651 1652
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int pipe;

1653
	spin_lock(&dev_priv->irq_lock);
1654 1655 1656 1657 1658 1659

	if (!dev_priv->display_irqs_enabled) {
		spin_unlock(&dev_priv->irq_lock);
		return;
	}

1660
	for_each_pipe(dev_priv, pipe) {
1661
		i915_reg_t reg;
1662
		u32 mask, iir_bit = 0;
1663

1664 1665 1666 1667 1668 1669 1670
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
1671 1672 1673

		/* fifo underruns are filterered in the underrun handler. */
		mask = PIPE_FIFO_UNDERRUN_STATUS;
1674 1675 1676 1677 1678 1679 1680 1681

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1682 1683 1684
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1685 1686 1687 1688 1689
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1690 1691 1692
			continue;

		reg = PIPESTAT(pipe);
1693 1694
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1695 1696 1697 1698

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1699 1700
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1701 1702
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1703
	spin_unlock(&dev_priv->irq_lock);
1704 1705 1706 1707 1708 1709 1710
}

static void valleyview_pipestat_irq_handler(struct drm_device *dev,
					    u32 pipe_stats[I915_MAX_PIPES])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	enum pipe pipe;
1711

1712
	for_each_pipe(dev_priv, pipe) {
1713 1714 1715
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
1716

1717
		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1718 1719 1720 1721 1722 1723 1724
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip(dev, pipe);
		}

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
			i9xx_pipe_crc_irq_handler(dev, pipe);

1725 1726
		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1727 1728 1729 1730 1731 1732
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		gmbus_irq_handler(dev);
}

1733
static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
1734 1735 1736
{
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);

1737 1738
	if (hotplug_status)
		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1739

1740 1741 1742 1743 1744 1745 1746
	return hotplug_status;
}

static void i9xx_hpd_irq_handler(struct drm_device *dev,
				 u32 hotplug_status)
{
	u32 pin_mask = 0, long_mask = 0;
1747

1748
	if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
1749
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1750

1751 1752 1753 1754 1755 1756 1757
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
					   hotplug_trigger, hpd_status_g4x,
					   i9xx_port_hotplug_long_detect);

			intel_hpd_irq_handler(dev, pin_mask, long_mask);
		}
1758 1759 1760

		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
			dp_aux_irq_handler(dev);
1761 1762
	} else {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1763

1764 1765
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1766
					   hotplug_trigger, hpd_status_i915,
1767 1768 1769
					   i9xx_port_hotplug_long_detect);
			intel_hpd_irq_handler(dev, pin_mask, long_mask);
		}
1770
	}
1771 1772
}

1773
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1774
{
1775
	struct drm_device *dev = arg;
1776
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
1777 1778
	irqreturn_t ret = IRQ_NONE;

1779 1780 1781
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1782 1783 1784
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1785
	do {
1786
		u32 iir, gt_iir, pm_iir;
1787
		u32 pipe_stats[I915_MAX_PIPES] = {};
1788
		u32 hotplug_status = 0;
1789
		u32 ier = 0;
1790

J
Jesse Barnes 已提交
1791 1792
		gt_iir = I915_READ(GTIIR);
		pm_iir = I915_READ(GEN6_PMIIR);
1793
		iir = I915_READ(VLV_IIR);
J
Jesse Barnes 已提交
1794 1795

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1796
			break;
J
Jesse Barnes 已提交
1797 1798 1799

		ret = IRQ_HANDLED;

1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
		 *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
		 * bits this time around.
		 */
1813
		I915_WRITE(VLV_MASTER_IER, 0);
1814 1815
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
1816 1817 1818 1819 1820 1821

		if (gt_iir)
			I915_WRITE(GTIIR, gt_iir);
		if (pm_iir)
			I915_WRITE(GEN6_PMIIR, pm_iir);

1822 1823
		if (gt_iir)
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
1824
		if (pm_iir)
1825
			gen6_rps_irq_handler(dev_priv, pm_iir);
1826 1827

		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1828
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1829

1830 1831
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
1832
		valleyview_pipestat_irq_ack(dev, iir, pipe_stats);
1833 1834 1835 1836 1837 1838 1839

		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);
1840

1841
		I915_WRITE(VLV_IER, ier);
1842 1843
		I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
		POSTING_READ(VLV_MASTER_IER);
1844 1845 1846

		if (hotplug_status)
			i9xx_hpd_irq_handler(dev, hotplug_status);
1847 1848

		valleyview_pipestat_irq_handler(dev, pipe_stats);
1849
	} while (0);
J
Jesse Barnes 已提交
1850

1851 1852
	enable_rpm_wakeref_asserts(dev_priv);

J
Jesse Barnes 已提交
1853 1854 1855
	return ret;
}

1856 1857
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1858
	struct drm_device *dev = arg;
1859 1860 1861
	struct drm_i915_private *dev_priv = dev->dev_private;
	irqreturn_t ret = IRQ_NONE;

1862 1863 1864
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1865 1866 1867
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1868
	do {
1869
		u32 master_ctl, iir;
1870
		u32 pipe_stats[I915_MAX_PIPES] = {};
1871
		u32 hotplug_status = 0;
1872 1873
		u32 ier = 0;

1874 1875
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
1876

1877 1878
		if (master_ctl == 0 && iir == 0)
			break;
1879

1880 1881
		ret = IRQ_HANDLED;

1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
		 *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
		 * bits this time around.
		 */
1895
		I915_WRITE(GEN8_MASTER_IRQ, 0);
1896 1897
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
1898

C
Chris Wilson 已提交
1899
		gen8_gt_irq_handler(dev_priv, master_ctl);
1900

1901
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1902
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1903

1904 1905
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
1906
		valleyview_pipestat_irq_ack(dev, iir, pipe_stats);
1907

1908 1909 1910 1911 1912 1913 1914
		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);

1915
		I915_WRITE(VLV_IER, ier);
1916
		I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
1917
		POSTING_READ(GEN8_MASTER_IRQ);
1918 1919 1920

		if (hotplug_status)
			i9xx_hpd_irq_handler(dev, hotplug_status);
1921 1922

		valleyview_pipestat_irq_handler(dev, pipe_stats);
1923
	} while (0);
1924

1925 1926
	enable_rpm_wakeref_asserts(dev_priv);

1927 1928 1929
	return ret;
}

1930 1931 1932 1933 1934 1935
static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

1936 1937 1938 1939 1940 1941
	/*
	 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
	 * unless we touch the hotplug register, even if hotplug_trigger is
	 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
	 * errors.
	 */
1942
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1943 1944 1945 1946 1947 1948 1949 1950
	if (!hotplug_trigger) {
		u32 mask = PORTA_HOTPLUG_STATUS_MASK |
			PORTD_HOTPLUG_STATUS_MASK |
			PORTC_HOTPLUG_STATUS_MASK |
			PORTB_HOTPLUG_STATUS_MASK;
		dig_hotplug_reg &= ~mask;
	}

1951
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1952 1953
	if (!hotplug_trigger)
		return;
1954 1955 1956 1957 1958 1959 1960 1961

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   pch_port_hotplug_long_detect);

	intel_hpd_irq_handler(dev, pin_mask, long_mask);
}

1962
static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1963
{
1964
	struct drm_i915_private *dev_priv = dev->dev_private;
1965
	int pipe;
1966
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1967

1968
	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1969

1970 1971 1972
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
1973
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1974 1975
				 port_name(port));
	}
1976

1977 1978 1979
	if (pch_iir & SDE_AUX_MASK)
		dp_aux_irq_handler(dev);

1980
	if (pch_iir & SDE_GMBUS)
1981
		gmbus_irq_handler(dev);
1982 1983 1984 1985 1986 1987 1988 1989 1990 1991

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1992
	if (pch_iir & SDE_FDI_MASK)
1993
		for_each_pipe(dev_priv, pipe)
1994 1995 1996
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
1997 1998 1999 2000 2001 2002 2003 2004

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
2005
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
2006 2007

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
2008
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
2009 2010 2011 2012 2013 2014
}

static void ivb_err_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
2015
	enum pipe pipe;
2016

2017 2018 2019
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

2020
	for_each_pipe(dev_priv, pipe) {
2021 2022
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2023

D
Daniel Vetter 已提交
2024 2025
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
			if (IS_IVYBRIDGE(dev))
2026
				ivb_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
2027
			else
2028
				hsw_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
2029 2030
		}
	}
2031

2032 2033 2034 2035 2036 2037 2038 2039
	I915_WRITE(GEN7_ERR_INT, err_int);
}

static void cpt_serr_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 serr_int = I915_READ(SERR_INT);

2040 2041 2042
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2043
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
2044
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
2045 2046

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
2047
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
2048 2049

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
2050
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
2051 2052

	I915_WRITE(SERR_INT, serr_int);
2053 2054
}

2055 2056
static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
{
2057
	struct drm_i915_private *dev_priv = dev->dev_private;
2058
	int pipe;
2059
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
2060

2061
	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
2062

2063 2064 2065 2066 2067 2068
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
2069 2070

	if (pch_iir & SDE_AUX_MASK_CPT)
2071
		dp_aux_irq_handler(dev);
2072 2073

	if (pch_iir & SDE_GMBUS_CPT)
2074
		gmbus_irq_handler(dev);
2075 2076 2077 2078 2079 2080 2081 2082

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
2083
		for_each_pipe(dev_priv, pipe)
2084 2085 2086
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2087 2088 2089

	if (pch_iir & SDE_ERROR_CPT)
		cpt_serr_int_handler(dev);
2090 2091
}

2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107
static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		~SDE_PORTE_HOTPLUG_SPT;
	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
	u32 pin_mask = 0, long_mask = 0;

	if (hotplug_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
				   dig_hotplug_reg, hpd_spt,
2108
				   spt_port_hotplug_long_detect);
2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
	}

	if (hotplug2_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
				   dig_hotplug_reg, hpd_spt,
				   spt_port_hotplug2_long_detect);
	}

	if (pin_mask)
		intel_hpd_irq_handler(dev, pin_mask, long_mask);

	if (pch_iir & SDE_GMBUS_CPT)
		gmbus_irq_handler(dev);
}

2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   ilk_port_hotplug_long_detect);

	intel_hpd_irq_handler(dev, pin_mask, long_mask);
}

2145 2146 2147
static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2148
	enum pipe pipe;
2149 2150
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;

2151 2152
	if (hotplug_trigger)
		ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
2153 2154 2155 2156 2157 2158 2159 2160 2161 2162

	if (de_iir & DE_AUX_CHANNEL_A)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE)
		intel_opregion_asle_intr(dev);

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2163
	for_each_pipe(dev_priv, pipe) {
2164 2165 2166
		if (de_iir & DE_PIPE_VBLANK(pipe) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2167

2168
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2169
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2170

2171 2172
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
			i9xx_pipe_crc_irq_handler(dev, pipe);
2173

2174 2175 2176 2177 2178
		/* plane/pipes map 1:1 on ilk+ */
		if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
		}
2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

		if (HAS_PCH_CPT(dev))
			cpt_irq_handler(dev, pch_iir);
		else
			ibx_irq_handler(dev, pch_iir);

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

	if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev);
}

2198 2199 2200
static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2201
	enum pipe pipe;
2202 2203
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;

2204 2205
	if (hotplug_trigger)
		ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
2206 2207 2208 2209 2210 2211 2212 2213 2214 2215

	if (de_iir & DE_ERR_INT_IVB)
		ivb_err_int_handler(dev);

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE_IVB)
		intel_opregion_asle_intr(dev);

2216
	for_each_pipe(dev_priv, pipe) {
2217 2218 2219
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2220 2221

		/* plane/pipes map 1:1 on ilk+ */
2222 2223 2224
		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238
		}
	}

	/* check event from PCH */
	if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
		u32 pch_iir = I915_READ(SDEIIR);

		cpt_irq_handler(dev, pch_iir);

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2239 2240 2241 2242 2243 2244 2245 2246
/*
 * To handle irqs with the minimum potential races with fresh interrupts, we:
 * 1 - Disable Master Interrupt Control.
 * 2 - Find the source(s) of the interrupt.
 * 3 - Clear the Interrupt Identity bits (IIR).
 * 4 - Process the interrupt(s) that had bits set in the IIRs.
 * 5 - Re-enable Master Interrupt Control.
 */
2247
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2248
{
2249
	struct drm_device *dev = arg;
2250
	struct drm_i915_private *dev_priv = dev->dev_private;
2251
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2252
	irqreturn_t ret = IRQ_NONE;
2253

2254 2255 2256
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2257 2258 2259
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

2260 2261 2262
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2263
	POSTING_READ(DEIER);
2264

2265 2266 2267 2268 2269
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2270 2271 2272 2273 2274
	if (!HAS_PCH_NOP(dev)) {
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2275

2276 2277
	/* Find, clear, then process each source of interrupt */

2278
	gt_iir = I915_READ(GTIIR);
2279
	if (gt_iir) {
2280 2281
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2282
		if (INTEL_INFO(dev)->gen >= 6)
2283
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
2284 2285
		else
			ilk_gt_irq_handler(dev, dev_priv, gt_iir);
2286 2287
	}

2288 2289
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2290 2291
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2292 2293 2294 2295
		if (INTEL_INFO(dev)->gen >= 7)
			ivb_display_irq_handler(dev, de_iir);
		else
			ilk_display_irq_handler(dev, de_iir);
2296 2297
	}

2298 2299 2300 2301 2302
	if (INTEL_INFO(dev)->gen >= 6) {
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
2303
			gen6_rps_irq_handler(dev_priv, pm_iir);
2304
		}
2305
	}
2306 2307 2308

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2309 2310 2311 2312
	if (!HAS_PCH_NOP(dev)) {
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2313

2314 2315 2316
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	enable_rpm_wakeref_asserts(dev_priv);

2317 2318 2319
	return ret;
}

2320 2321
static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
2322
{
2323 2324
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2325

2326 2327
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2328

2329
	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2330
			   dig_hotplug_reg, hpd,
2331
			   bxt_port_hotplug_long_detect);
2332

2333
	intel_hpd_irq_handler(dev, pin_mask, long_mask);
2334 2335
}

2336 2337
static irqreturn_t
gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2338
{
2339
	struct drm_device *dev = dev_priv->dev;
2340
	irqreturn_t ret = IRQ_NONE;
2341
	u32 iir;
2342
	enum pipe pipe;
J
Jesse Barnes 已提交
2343

2344
	if (master_ctl & GEN8_DE_MISC_IRQ) {
2345 2346 2347
		iir = I915_READ(GEN8_DE_MISC_IIR);
		if (iir) {
			I915_WRITE(GEN8_DE_MISC_IIR, iir);
2348
			ret = IRQ_HANDLED;
2349
			if (iir & GEN8_DE_MISC_GSE)
2350 2351 2352
				intel_opregion_asle_intr(dev);
			else
				DRM_ERROR("Unexpected DE Misc interrupt\n");
2353
		}
2354 2355
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2356 2357
	}

2358
	if (master_ctl & GEN8_DE_PORT_IRQ) {
2359 2360 2361
		iir = I915_READ(GEN8_DE_PORT_IIR);
		if (iir) {
			u32 tmp_mask;
2362
			bool found = false;
2363

2364
			I915_WRITE(GEN8_DE_PORT_IIR, iir);
2365
			ret = IRQ_HANDLED;
J
Jesse Barnes 已提交
2366

2367 2368 2369 2370 2371 2372 2373
			tmp_mask = GEN8_AUX_CHANNEL_A;
			if (INTEL_INFO(dev_priv)->gen >= 9)
				tmp_mask |= GEN9_AUX_CHANNEL_B |
					    GEN9_AUX_CHANNEL_C |
					    GEN9_AUX_CHANNEL_D;

			if (iir & tmp_mask) {
2374
				dp_aux_irq_handler(dev);
2375 2376 2377
				found = true;
			}

2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389
			if (IS_BROXTON(dev_priv)) {
				tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
				if (tmp_mask) {
					bxt_hpd_irq_handler(dev, tmp_mask, hpd_bxt);
					found = true;
				}
			} else if (IS_BROADWELL(dev_priv)) {
				tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
				if (tmp_mask) {
					ilk_hpd_irq_handler(dev, tmp_mask, hpd_bdw);
					found = true;
				}
2390 2391
			}

2392
			if (IS_BROXTON(dev) && (iir & BXT_DE_PORT_GMBUS)) {
S
Shashank Sharma 已提交
2393 2394 2395 2396
				gmbus_irq_handler(dev);
				found = true;
			}

2397
			if (!found)
2398
				DRM_ERROR("Unexpected DE Port interrupt\n");
2399
		}
2400 2401
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2402 2403
	}

2404
	for_each_pipe(dev_priv, pipe) {
2405
		u32 flip_done, fault_errors;
2406

2407 2408
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2409

2410 2411 2412 2413 2414
		iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (!iir) {
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
			continue;
		}
2415

2416 2417
		ret = IRQ_HANDLED;
		I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2418

2419 2420 2421
		if (iir & GEN8_PIPE_VBLANK &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2422

2423 2424 2425 2426 2427
		flip_done = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			flip_done &= GEN9_PIPE_PLANE1_FLIP_DONE;
		else
			flip_done &= GEN8_PIPE_PRIMARY_FLIP_DONE;
2428

2429 2430 2431 2432
		if (flip_done) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
		}
2433

2434 2435
		if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
			hsw_pipe_crc_irq_handler(dev, pipe);
2436

2437 2438
		if (iir & GEN8_PIPE_FIFO_UNDERRUN)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2439

2440 2441 2442 2443 2444
		fault_errors = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
		else
			fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2445

2446 2447 2448 2449
		if (fault_errors)
			DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
				  pipe_name(pipe),
				  fault_errors);
2450 2451
	}

2452 2453
	if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
	    master_ctl & GEN8_DE_PCH_IRQ) {
2454 2455 2456 2457 2458
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
2459 2460 2461
		iir = I915_READ(SDEIIR);
		if (iir) {
			I915_WRITE(SDEIIR, iir);
2462
			ret = IRQ_HANDLED;
2463 2464

			if (HAS_PCH_SPT(dev_priv))
2465
				spt_irq_handler(dev, iir);
2466
			else
2467
				cpt_irq_handler(dev, iir);
2468 2469 2470 2471 2472 2473 2474
		} else {
			/*
			 * Like on previous PCH there seems to be something
			 * fishy going on with forwarding PCH interrupts.
			 */
			DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
		}
2475 2476
	}

2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503
	return ret;
}

static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl;
	irqreturn_t ret;

	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);

	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	/* Find, clear, then process each source of interrupt */
	ret = gen8_gt_irq_handler(dev_priv, master_ctl);
	ret |= gen8_de_irq_handler(dev_priv, master_ctl);

2504 2505
	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ_FW(GEN8_MASTER_IRQ);
2506

2507 2508
	enable_rpm_wakeref_asserts(dev_priv);

2509 2510 2511
	return ret;
}

2512 2513 2514
static void i915_error_wake_up(struct drm_i915_private *dev_priv,
			       bool reset_completed)
{
2515
	struct intel_engine_cs *engine;
2516 2517 2518 2519 2520 2521 2522 2523 2524

	/*
	 * Notify all waiters for GPU completion events that reset state has
	 * been changed, and that they need to restart their wait after
	 * checking for potential errors (and bail out to drop locks if there is
	 * a gpu reset pending so that i915_error_work_func can acquire them).
	 */

	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2525
	for_each_engine(engine, dev_priv)
2526
		wake_up_all(&engine->irq_queue);
2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538

	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
	wake_up_all(&dev_priv->pending_flip_queue);

	/*
	 * Signal tasks blocked in i915_gem_wait_for_error that the pending
	 * reset state is cleared.
	 */
	if (reset_completed)
		wake_up_all(&dev_priv->gpu_error.reset_queue);
}

2539
/**
2540
 * i915_reset_and_wakeup - do process context error handling work
2541
 * @dev: drm device
2542 2543 2544 2545
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
2546
static void i915_reset_and_wakeup(struct drm_device *dev)
2547
{
2548
	struct drm_i915_private *dev_priv = to_i915(dev);
2549 2550 2551
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2552
	int ret;
2553

2554
	kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2555

2556 2557 2558 2559 2560 2561 2562 2563 2564 2565
	/*
	 * Note that there's only one work item which does gpu resets, so we
	 * need not worry about concurrent gpu resets potentially incrementing
	 * error->reset_counter twice. We only need to take care of another
	 * racing irq/hangcheck declaring the gpu dead for a second time. A
	 * quick check for that is good enough: schedule_work ensures the
	 * correct ordering between hang detection and this work item, and since
	 * the reset in-progress bit is only ever set by code outside of this
	 * work we don't need to worry about any other races.
	 */
2566
	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
2567
		DRM_DEBUG_DRIVER("resetting chip\n");
2568
		kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2569
				   reset_event);
2570

2571 2572 2573 2574 2575 2576 2577 2578
		/*
		 * In most cases it's guaranteed that we get here with an RPM
		 * reference held, for example because there is a pending GPU
		 * request that won't finish until the reset is done. This
		 * isn't the case at least when we get here by doing a
		 * simulated reset via debugs, so get an RPM reference.
		 */
		intel_runtime_pm_get(dev_priv);
2579 2580 2581

		intel_prepare_reset(dev);

2582 2583 2584 2585 2586 2587
		/*
		 * All state reset _must_ be completed before we update the
		 * reset counter, for otherwise waiters might miss the reset
		 * pending state and not properly drop locks, resulting in
		 * deadlocks with the reset work.
		 */
2588 2589
		ret = i915_reset(dev);

2590
		intel_finish_reset(dev);
2591

2592 2593
		intel_runtime_pm_put(dev_priv);

2594
		if (ret == 0)
2595
			kobject_uevent_env(&dev->primary->kdev->kobj,
2596
					   KOBJ_CHANGE, reset_done_event);
2597

2598 2599 2600 2601 2602
		/*
		 * Note: The wake_up also serves as a memory barrier so that
		 * waiters see the update value of the reset counter atomic_t.
		 */
		i915_error_wake_up(dev_priv, true);
2603
	}
2604 2605
}

2606
static void i915_report_and_clear_eir(struct drm_device *dev)
2607 2608
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2609
	uint32_t instdone[I915_NUM_INSTDONE_REG];
2610
	u32 eir = I915_READ(EIR);
2611
	int pipe, i;
2612

2613 2614
	if (!eir)
		return;
2615

2616
	pr_err("render error detected, EIR: 0x%08x\n", eir);
2617

2618 2619
	i915_get_extra_instdone(dev, instdone);

2620 2621 2622 2623
	if (IS_G4X(dev)) {
		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
			u32 ipeir = I915_READ(IPEIR_I965);

2624 2625
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2626 2627
			for (i = 0; i < ARRAY_SIZE(instdone); i++)
				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2628 2629
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2630
			I915_WRITE(IPEIR_I965, ipeir);
2631
			POSTING_READ(IPEIR_I965);
2632 2633 2634
		}
		if (eir & GM45_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2635 2636
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2637
			I915_WRITE(PGTBL_ER, pgtbl_err);
2638
			POSTING_READ(PGTBL_ER);
2639 2640 2641
		}
	}

2642
	if (!IS_GEN2(dev)) {
2643 2644
		if (eir & I915_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2645 2646
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2647
			I915_WRITE(PGTBL_ER, pgtbl_err);
2648
			POSTING_READ(PGTBL_ER);
2649 2650 2651 2652
		}
	}

	if (eir & I915_ERROR_MEMORY_REFRESH) {
2653
		pr_err("memory refresh error:\n");
2654
		for_each_pipe(dev_priv, pipe)
2655
			pr_err("pipe %c stat: 0x%08x\n",
2656
			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2657 2658 2659
		/* pipestat has already been acked */
	}
	if (eir & I915_ERROR_INSTRUCTION) {
2660 2661
		pr_err("instruction error\n");
		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2662 2663
		for (i = 0; i < ARRAY_SIZE(instdone); i++)
			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2664
		if (INTEL_INFO(dev)->gen < 4) {
2665 2666
			u32 ipeir = I915_READ(IPEIR);

2667 2668 2669
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2670
			I915_WRITE(IPEIR, ipeir);
2671
			POSTING_READ(IPEIR);
2672 2673 2674
		} else {
			u32 ipeir = I915_READ(IPEIR_I965);

2675 2676 2677 2678
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2679
			I915_WRITE(IPEIR_I965, ipeir);
2680
			POSTING_READ(IPEIR_I965);
2681 2682 2683 2684
		}
	}

	I915_WRITE(EIR, eir);
2685
	POSTING_READ(EIR);
2686 2687 2688 2689 2690 2691 2692 2693 2694 2695
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2696 2697 2698
}

/**
2699
 * i915_handle_error - handle a gpu error
2700
 * @dev: drm device
2701
 * @engine_mask: mask representing engines that are hung
2702
 * Do some basic checking of register state at error time and
2703 2704 2705 2706 2707
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
2708
void i915_handle_error(struct drm_device *dev, u32 engine_mask,
2709
		       const char *fmt, ...)
2710 2711
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2712 2713
	va_list args;
	char error_msg[80];
2714

2715 2716 2717 2718
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

2719
	i915_capture_error_state(dev, engine_mask, error_msg);
2720
	i915_report_and_clear_eir(dev);
2721

2722
	if (engine_mask) {
2723
		atomic_or(I915_RESET_IN_PROGRESS_FLAG,
2724
				&dev_priv->gpu_error.reset_counter);
2725

2726
		/*
2727 2728 2729
		 * Wakeup waiting processes so that the reset function
		 * i915_reset_and_wakeup doesn't deadlock trying to grab
		 * various locks. By bumping the reset counter first, the woken
2730 2731 2732 2733 2734 2735 2736 2737
		 * processes will see a reset in progress and back off,
		 * releasing their locks and then wait for the reset completion.
		 * We must do this for _all_ gpu waiters that might hold locks
		 * that the reset work needs to acquire.
		 *
		 * Note: The wake_up serves as the required memory barrier to
		 * ensure that the waiters see the updated value of the reset
		 * counter atomic_t.
2738
		 */
2739
		i915_error_wake_up(dev_priv, false);
2740 2741
	}

2742
	i915_reset_and_wakeup(dev);
2743 2744
}

2745 2746 2747
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2748
static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
2749
{
2750
	struct drm_i915_private *dev_priv = dev->dev_private;
2751
	unsigned long irqflags;
2752

2753
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2754
	if (INTEL_INFO(dev)->gen >= 4)
2755
		i915_enable_pipestat(dev_priv, pipe,
2756
				     PIPE_START_VBLANK_INTERRUPT_STATUS);
2757
	else
2758
		i915_enable_pipestat(dev_priv, pipe,
2759
				     PIPE_VBLANK_INTERRUPT_STATUS);
2760
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2761

2762 2763 2764
	return 0;
}

2765
static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
2766
{
2767
	struct drm_i915_private *dev_priv = dev->dev_private;
2768
	unsigned long irqflags;
2769
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2770
						     DE_PIPE_VBLANK(pipe);
2771 2772

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2773
	ilk_enable_display_irq(dev_priv, bit);
2774 2775 2776 2777 2778
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2779
static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2780
{
2781
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2782 2783 2784
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2785
	i915_enable_pipestat(dev_priv, pipe,
2786
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2787 2788 2789 2790 2791
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2792
static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
2793 2794 2795 2796 2797
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2798
	bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2799
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2800

2801 2802 2803
	return 0;
}

2804 2805 2806
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2807
static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
2808
{
2809
	struct drm_i915_private *dev_priv = dev->dev_private;
2810
	unsigned long irqflags;
2811

2812
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2813
	i915_disable_pipestat(dev_priv, pipe,
2814 2815
			      PIPE_VBLANK_INTERRUPT_STATUS |
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2816 2817 2818
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2819
static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
2820
{
2821
	struct drm_i915_private *dev_priv = dev->dev_private;
2822
	unsigned long irqflags;
2823
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2824
						     DE_PIPE_VBLANK(pipe);
2825 2826

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2827
	ilk_disable_display_irq(dev_priv, bit);
2828 2829 2830
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2831
static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2832
{
2833
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2834 2835 2836
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2837
	i915_disable_pipestat(dev_priv, pipe,
2838
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2839 2840 2841
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2842
static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
2843 2844 2845 2846 2847
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2848
	bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2849 2850 2851
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2852
static bool
2853
ring_idle(struct intel_engine_cs *engine, u32 seqno)
2854
{
2855 2856
	return i915_seqno_passed(seqno,
				 READ_ONCE(engine->last_submitted_seqno));
B
Ben Gamari 已提交
2857 2858
}

2859 2860 2861 2862
static bool
ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
{
	if (INTEL_INFO(dev)->gen >= 8) {
2863
		return (ipehr >> 23) == 0x1c;
2864 2865 2866 2867 2868 2869 2870
	} else {
		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
				 MI_SEMAPHORE_REGISTER);
	}
}

2871
static struct intel_engine_cs *
2872 2873
semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr,
				 u64 offset)
2874
{
2875
	struct drm_i915_private *dev_priv = engine->dev->dev_private;
2876
	struct intel_engine_cs *signaller;
2877

2878
	if (INTEL_INFO(dev_priv)->gen >= 8) {
2879
		for_each_engine(signaller, dev_priv) {
2880
			if (engine == signaller)
2881 2882
				continue;

2883
			if (offset == signaller->semaphore.signal_ggtt[engine->id])
2884 2885
				return signaller;
		}
2886 2887 2888
	} else {
		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;

2889
		for_each_engine(signaller, dev_priv) {
2890
			if(engine == signaller)
2891 2892
				continue;

2893
			if (sync_bits == signaller->semaphore.mbox.wait[engine->id])
2894 2895 2896 2897
				return signaller;
		}
	}

2898
	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2899
		  engine->id, ipehr, offset);
2900 2901 2902 2903

	return NULL;
}

2904
static struct intel_engine_cs *
2905
semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
2906
{
2907
	struct drm_i915_private *dev_priv = engine->dev->dev_private;
2908
	u32 cmd, ipehr, head;
2909 2910
	u64 offset = 0;
	int i, backwards;
2911

2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928
	/*
	 * This function does not support execlist mode - any attempt to
	 * proceed further into this function will result in a kernel panic
	 * when dereferencing ring->buffer, which is not set up in execlist
	 * mode.
	 *
	 * The correct way of doing it would be to derive the currently
	 * executing ring buffer from the current context, which is derived
	 * from the currently running request. Unfortunately, to get the
	 * current request we would have to grab the struct_mutex before doing
	 * anything else, which would be ill-advised since some other thread
	 * might have grabbed it already and managed to hang itself, causing
	 * the hang checker to deadlock.
	 *
	 * Therefore, this function does not support execlist mode in its
	 * current form. Just return NULL and move on.
	 */
2929
	if (engine->buffer == NULL)
2930 2931
		return NULL;

2932 2933
	ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
	if (!ipehr_is_semaphore_wait(engine->dev, ipehr))
2934
		return NULL;
2935

2936 2937 2938
	/*
	 * HEAD is likely pointing to the dword after the actual command,
	 * so scan backwards until we find the MBOX. But limit it to just 3
2939 2940
	 * or 4 dwords depending on the semaphore wait command size.
	 * Note that we don't care about ACTHD here since that might
2941 2942
	 * point at at batch, and semaphores are always emitted into the
	 * ringbuffer itself.
2943
	 */
2944 2945
	head = I915_READ_HEAD(engine) & HEAD_ADDR;
	backwards = (INTEL_INFO(engine->dev)->gen >= 8) ? 5 : 4;
2946

2947
	for (i = backwards; i; --i) {
2948 2949 2950 2951 2952
		/*
		 * Be paranoid and presume the hw has gone off into the wild -
		 * our ring is smaller than what the hardware (and hence
		 * HEAD_ADDR) allows. Also handles wrap-around.
		 */
2953
		head &= engine->buffer->size - 1;
2954 2955

		/* This here seems to blow up */
2956
		cmd = ioread32(engine->buffer->virtual_start + head);
2957 2958 2959
		if (cmd == ipehr)
			break;

2960 2961
		head -= 4;
	}
2962

2963 2964
	if (!i)
		return NULL;
2965

2966 2967 2968
	*seqno = ioread32(engine->buffer->virtual_start + head + 4) + 1;
	if (INTEL_INFO(engine->dev)->gen >= 8) {
		offset = ioread32(engine->buffer->virtual_start + head + 12);
2969
		offset <<= 32;
2970
		offset = ioread32(engine->buffer->virtual_start + head + 8);
2971
	}
2972
	return semaphore_wait_to_signaller_ring(engine, ipehr, offset);
2973 2974
}

2975
static int semaphore_passed(struct intel_engine_cs *engine)
2976
{
2977
	struct drm_i915_private *dev_priv = engine->dev->dev_private;
2978
	struct intel_engine_cs *signaller;
2979
	u32 seqno;
2980

2981
	engine->hangcheck.deadlock++;
2982

2983
	signaller = semaphore_waits_for(engine, &seqno);
2984 2985 2986 2987
	if (signaller == NULL)
		return -1;

	/* Prevent pathological recursion due to driver bugs */
2988
	if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES)
2989 2990
		return -1;

2991
	if (i915_seqno_passed(signaller->get_seqno(signaller), seqno))
2992 2993
		return 1;

2994 2995 2996
	/* cursory check for an unkickable deadlock */
	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
	    semaphore_passed(signaller) < 0)
2997 2998 2999
		return -1;

	return 0;
3000 3001 3002 3003
}

static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
{
3004
	struct intel_engine_cs *engine;
3005

3006
	for_each_engine(engine, dev_priv)
3007
		engine->hangcheck.deadlock = 0;
3008 3009
}

3010
static bool subunits_stuck(struct intel_engine_cs *engine)
3011
{
3012 3013 3014 3015
	u32 instdone[I915_NUM_INSTDONE_REG];
	bool stuck;
	int i;

3016
	if (engine->id != RCS)
3017 3018
		return true;

3019
	i915_get_extra_instdone(engine->dev, instdone);
3020

3021 3022 3023 3024 3025 3026 3027
	/* There might be unstable subunit states even when
	 * actual head is not moving. Filter out the unstable ones by
	 * accumulating the undone -> done transitions and only
	 * consider those as progress.
	 */
	stuck = true;
	for (i = 0; i < I915_NUM_INSTDONE_REG; i++) {
3028
		const u32 tmp = instdone[i] | engine->hangcheck.instdone[i];
3029

3030
		if (tmp != engine->hangcheck.instdone[i])
3031 3032
			stuck = false;

3033
		engine->hangcheck.instdone[i] |= tmp;
3034 3035 3036 3037 3038 3039
	}

	return stuck;
}

static enum intel_ring_hangcheck_action
3040
head_stuck(struct intel_engine_cs *engine, u64 acthd)
3041
{
3042
	if (acthd != engine->hangcheck.acthd) {
3043 3044

		/* Clear subunit states on head movement */
3045 3046
		memset(engine->hangcheck.instdone, 0,
		       sizeof(engine->hangcheck.instdone));
3047

3048
		return HANGCHECK_ACTIVE;
3049
	}
3050

3051
	if (!subunits_stuck(engine))
3052 3053 3054 3055 3056 3057
		return HANGCHECK_ACTIVE;

	return HANGCHECK_HUNG;
}

static enum intel_ring_hangcheck_action
3058
ring_stuck(struct intel_engine_cs *engine, u64 acthd)
3059
{
3060
	struct drm_device *dev = engine->dev;
3061 3062 3063 3064
	struct drm_i915_private *dev_priv = dev->dev_private;
	enum intel_ring_hangcheck_action ha;
	u32 tmp;

3065
	ha = head_stuck(engine, acthd);
3066 3067 3068
	if (ha != HANGCHECK_HUNG)
		return ha;

3069
	if (IS_GEN2(dev))
3070
		return HANGCHECK_HUNG;
3071 3072 3073 3074 3075 3076

	/* Is the chip hanging on a WAIT_FOR_EVENT?
	 * If so we can simply poke the RB_WAIT bit
	 * and break the hang. This should work on
	 * all but the second generation chipsets.
	 */
3077
	tmp = I915_READ_CTL(engine);
3078
	if (tmp & RING_WAIT) {
3079
		i915_handle_error(dev, 0,
3080
				  "Kicking stuck wait on %s",
3081 3082
				  engine->name);
		I915_WRITE_CTL(engine, tmp);
3083
		return HANGCHECK_KICK;
3084 3085 3086
	}

	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
3087
		switch (semaphore_passed(engine)) {
3088
		default:
3089
			return HANGCHECK_HUNG;
3090
		case 1:
3091
			i915_handle_error(dev, 0,
3092
					  "Kicking stuck semaphore on %s",
3093 3094
					  engine->name);
			I915_WRITE_CTL(engine, tmp);
3095
			return HANGCHECK_KICK;
3096
		case 0:
3097
			return HANGCHECK_WAIT;
3098
		}
3099
	}
3100

3101
	return HANGCHECK_HUNG;
3102 3103
}

3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121
static unsigned kick_waiters(struct intel_engine_cs *engine)
{
	struct drm_i915_private *i915 = to_i915(engine->dev);
	unsigned user_interrupts = READ_ONCE(engine->user_interrupts);

	if (engine->hangcheck.user_interrupts == user_interrupts &&
	    !test_and_set_bit(engine->id, &i915->gpu_error.missed_irq_rings)) {
		if (!(i915->gpu_error.test_irq_rings & intel_engine_flag(engine)))
			DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
				  engine->name);
		else
			DRM_INFO("Fake missed irq on %s\n",
				 engine->name);
		wake_up_all(&engine->irq_queue);
	}

	return user_interrupts;
}
3122
/*
B
Ben Gamari 已提交
3123
 * This is called when the chip hasn't reported back with completed
3124 3125 3126 3127 3128
 * batchbuffers in a long time. We keep track per ring seqno progress and
 * if there are no progress, hangcheck score for that ring is increased.
 * Further, acthd is inspected to see if the ring is stuck. On stuck case
 * we kick the ring. If we see no progress on three subsequent calls
 * we assume chip is wedged and try to fix it by resetting the chip.
B
Ben Gamari 已提交
3129
 */
3130
static void i915_hangcheck_elapsed(struct work_struct *work)
B
Ben Gamari 已提交
3131
{
3132 3133 3134 3135
	struct drm_i915_private *dev_priv =
		container_of(work, typeof(*dev_priv),
			     gpu_error.hangcheck_work.work);
	struct drm_device *dev = dev_priv->dev;
3136
	struct intel_engine_cs *engine;
3137
	enum intel_engine_id id;
3138
	int busy_count = 0, rings_hung = 0;
3139
	bool stuck[I915_NUM_ENGINES] = { 0 };
3140 3141 3142
#define BUSY 1
#define KICK 5
#define HUNG 20
3143
#define ACTIVE_DECAY 15
3144

3145
	if (!i915.enable_hangcheck)
3146 3147
		return;

3148 3149 3150 3151 3152 3153 3154
	/*
	 * The hangcheck work is synced during runtime suspend, we don't
	 * require a wakeref. TODO: instead of disabling the asserts make
	 * sure that we hold a reference when this work is running.
	 */
	DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);

3155 3156 3157 3158 3159 3160
	/* As enabling the GPU requires fairly extensive mmio access,
	 * periodically arm the mmio checker to see if we are triggering
	 * any invalid access.
	 */
	intel_uncore_arm_unclaimed_mmio_detection(dev_priv);

3161
	for_each_engine_id(engine, dev_priv, id) {
3162 3163
		u64 acthd;
		u32 seqno;
3164
		unsigned user_interrupts;
3165
		bool busy = true;
3166

3167 3168
		semaphore_clear_deadlocks(dev_priv);

3169 3170 3171 3172 3173 3174 3175 3176 3177 3178
		/* We don't strictly need an irq-barrier here, as we are not
		 * serving an interrupt request, be paranoid in case the
		 * barrier has side-effects (such as preventing a broken
		 * cacheline snoop) and so be sure that we can see the seqno
		 * advance. If the seqno should stick, due to a stale
		 * cacheline, we would erroneously declare the GPU hung.
		 */
		if (engine->irq_seqno_barrier)
			engine->irq_seqno_barrier(engine);

3179
		acthd = intel_ring_get_active_head(engine);
3180
		seqno = engine->get_seqno(engine);
3181

3182 3183 3184
		/* Reset stuck interrupts between batch advances */
		user_interrupts = 0;

3185 3186 3187 3188
		if (engine->hangcheck.seqno == seqno) {
			if (ring_idle(engine, seqno)) {
				engine->hangcheck.action = HANGCHECK_IDLE;
				if (waitqueue_active(&engine->irq_queue)) {
3189
					/* Safeguard against driver failure */
3190
					user_interrupts = kick_waiters(engine);
3191
					engine->hangcheck.score += BUSY;
3192 3193
				} else
					busy = false;
3194
			} else {
3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209
				/* We always increment the hangcheck score
				 * if the ring is busy and still processing
				 * the same request, so that no single request
				 * can run indefinitely (such as a chain of
				 * batches). The only time we do not increment
				 * the hangcheck score on this ring, if this
				 * ring is in a legitimate wait for another
				 * ring. In that case the waiting ring is a
				 * victim and we want to be sure we catch the
				 * right culprit. Then every time we do kick
				 * the ring, add a small increment to the
				 * score so that we can catch a batch that is
				 * being repeatedly kicked and so responsible
				 * for stalling the machine.
				 */
3210 3211
				engine->hangcheck.action = ring_stuck(engine,
								      acthd);
3212

3213
				switch (engine->hangcheck.action) {
3214
				case HANGCHECK_IDLE:
3215
				case HANGCHECK_WAIT:
3216
					break;
3217
				case HANGCHECK_ACTIVE:
3218
					engine->hangcheck.score += BUSY;
3219
					break;
3220
				case HANGCHECK_KICK:
3221
					engine->hangcheck.score += KICK;
3222
					break;
3223
				case HANGCHECK_HUNG:
3224
					engine->hangcheck.score += HUNG;
3225
					stuck[id] = true;
3226 3227
					break;
				}
3228
			}
3229
		} else {
3230
			engine->hangcheck.action = HANGCHECK_ACTIVE;
3231

3232 3233 3234
			/* Gradually reduce the count so that we catch DoS
			 * attempts across multiple batches.
			 */
3235 3236 3237 3238
			if (engine->hangcheck.score > 0)
				engine->hangcheck.score -= ACTIVE_DECAY;
			if (engine->hangcheck.score < 0)
				engine->hangcheck.score = 0;
3239

3240
			/* Clear head and subunit states on seqno movement */
3241
			acthd = 0;
3242

3243 3244
			memset(engine->hangcheck.instdone, 0,
			       sizeof(engine->hangcheck.instdone));
3245 3246
		}

3247 3248
		engine->hangcheck.seqno = seqno;
		engine->hangcheck.acthd = acthd;
3249
		engine->hangcheck.user_interrupts = user_interrupts;
3250
		busy_count += busy;
3251
	}
3252

3253
	for_each_engine_id(engine, dev_priv, id) {
3254
		if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
3255
			DRM_INFO("%s on %s\n",
3256
				 stuck[id] ? "stuck" : "no progress",
3257
				 engine->name);
3258
			rings_hung |= intel_engine_flag(engine);
3259 3260 3261
		}
	}

3262
	if (rings_hung) {
3263
		i915_handle_error(dev, rings_hung, "Engine(s) hung");
3264 3265
		goto out;
	}
B
Ben Gamari 已提交
3266

3267 3268 3269
	if (busy_count)
		/* Reset timer case chip hangs without another request
		 * being added */
3270
		i915_queue_hangcheck(dev);
3271 3272 3273

out:
	ENABLE_RPM_WAKEREF_ASSERTS(dev_priv);
3274 3275 3276 3277
}

void i915_queue_hangcheck(struct drm_device *dev)
{
3278
	struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
3279

3280
	if (!i915.enable_hangcheck)
3281 3282
		return;

3283 3284 3285 3286 3287 3288 3289
	/* Don't continually defer the hangcheck so that it is always run at
	 * least once after work has been scheduled on any ring. Otherwise,
	 * we will ignore a hung ring if a second ring is kept busy.
	 */

	queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
			   round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
B
Ben Gamari 已提交
3290 3291
}

3292
static void ibx_irq_reset(struct drm_device *dev)
P
Paulo Zanoni 已提交
3293 3294 3295 3296 3297 3298
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

3299
	GEN5_IRQ_RESET(SDE);
3300 3301 3302

	if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
3303
}
3304

P
Paulo Zanoni 已提交
3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
3321 3322 3323 3324
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

3325
static void gen5_gt_irq_reset(struct drm_device *dev)
3326 3327 3328
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3329
	GEN5_IRQ_RESET(GT);
P
Paulo Zanoni 已提交
3330
	if (INTEL_INFO(dev)->gen >= 6)
3331
		GEN5_IRQ_RESET(GEN6_PM);
3332 3333
}

3334 3335 3336 3337
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
	enum pipe pipe;

3338 3339 3340 3341 3342
	if (IS_CHERRYVIEW(dev_priv))
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
	else
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);

3343
	i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
3344 3345
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

3346 3347 3348 3349 3350 3351
	for_each_pipe(dev_priv, pipe) {
		I915_WRITE(PIPESTAT(pipe),
			   PIPE_FIFO_UNDERRUN_STATUS |
			   PIPESTAT_INT_STATUS_MASK);
		dev_priv->pipestat_irq_mask[pipe] = 0;
	}
3352 3353

	GEN5_IRQ_RESET(VLV_);
3354
	dev_priv->irq_mask = ~0;
3355 3356
}

3357 3358 3359
static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
3360
	u32 enable_mask;
3361 3362 3363 3364 3365 3366 3367 3368 3369
	enum pipe pipe;

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		i915_enable_pipestat(dev_priv, pipe, pipestat_mask);

3370 3371 3372
	enable_mask = I915_DISPLAY_PORT_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3373
	if (IS_CHERRYVIEW(dev_priv))
3374
		enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3375 3376 3377

	WARN_ON(dev_priv->irq_mask != ~0);

3378 3379 3380
	dev_priv->irq_mask = ~enable_mask;

	GEN5_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask);
3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399
}

/* drm_dma.h hooks
*/
static void ironlake_irq_reset(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	I915_WRITE(HWSTAM, 0xffffffff);

	GEN5_IRQ_RESET(DE);
	if (IS_GEN7(dev))
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);

	gen5_gt_irq_reset(dev);

	ibx_irq_reset(dev);
}

J
Jesse Barnes 已提交
3400 3401
static void valleyview_irq_preinstall(struct drm_device *dev)
{
3402
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3403

3404 3405 3406
	I915_WRITE(VLV_MASTER_IER, 0);
	POSTING_READ(VLV_MASTER_IER);

3407
	gen5_gt_irq_reset(dev);
J
Jesse Barnes 已提交
3408

3409
	spin_lock_irq(&dev_priv->irq_lock);
3410 3411
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3412
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3413 3414
}

3415 3416 3417 3418 3419 3420 3421 3422
static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
{
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
}

P
Paulo Zanoni 已提交
3423
static void gen8_irq_reset(struct drm_device *dev)
3424 3425 3426 3427 3428 3429 3430
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3431
	gen8_gt_irq_reset(dev_priv);
3432

3433
	for_each_pipe(dev_priv, pipe)
3434 3435
		if (intel_display_power_is_enabled(dev_priv,
						   POWER_DOMAIN_PIPE(pipe)))
3436
			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3437

3438 3439 3440
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3441

3442 3443
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_reset(dev);
3444
}
3445

3446 3447
void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
3448
{
3449
	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3450
	enum pipe pipe;
3451

3452
	spin_lock_irq(&dev_priv->irq_lock);
3453 3454 3455 3456
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
				  dev_priv->de_irq_mask[pipe],
				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
3457
	spin_unlock_irq(&dev_priv->irq_lock);
3458 3459
}

3460 3461 3462
void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
{
3463 3464
	enum pipe pipe;

3465
	spin_lock_irq(&dev_priv->irq_lock);
3466 3467
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3468 3469 3470 3471 3472 3473
	spin_unlock_irq(&dev_priv->irq_lock);

	/* make sure we're done processing display irqs */
	synchronize_irq(dev_priv->dev->irq);
}

3474 3475 3476 3477 3478 3479 3480
static void cherryview_irq_preinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3481
	gen8_gt_irq_reset(dev_priv);
3482 3483 3484

	GEN5_IRQ_RESET(GEN8_PCU_);

3485
	spin_lock_irq(&dev_priv->irq_lock);
3486 3487
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3488
	spin_unlock_irq(&dev_priv->irq_lock);
3489 3490
}

3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504
static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
				  const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct intel_encoder *encoder;
	u32 enabled_irqs = 0;

	for_each_intel_encoder(dev, encoder)
		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
			enabled_irqs |= hpd[encoder->hpd_pin];

	return enabled_irqs;
}

3505
static void ibx_hpd_irq_setup(struct drm_device *dev)
3506
{
3507
	struct drm_i915_private *dev_priv = dev->dev_private;
3508
	u32 hotplug_irqs, hotplug, enabled_irqs;
3509 3510

	if (HAS_PCH_IBX(dev)) {
3511
		hotplug_irqs = SDE_HOTPLUG_MASK;
3512
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
3513
	} else {
3514
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3515
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
3516
	}
3517

3518
	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3519 3520 3521

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
3522 3523
	 * duration to 2ms (which is the minimum in the Display Port spec).
	 * The pulse duration bits are reserved on LPT+.
3524
	 */
3525 3526 3527 3528 3529
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3530 3531 3532 3533 3534 3535
	/*
	 * When CPU and PCH are on the same package, port A
	 * HPD must be enabled in both north and south.
	 */
	if (HAS_PCH_LPT_LP(dev))
		hotplug |= PORTA_HOTPLUG_ENABLE;
3536
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3537
}
X
Xiong Zhang 已提交
3538

3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551
static void spt_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_irqs, hotplug, enabled_irqs;

	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	/* Enable digital hotplug on the PCH */
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE |
3552
		PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE;
3553 3554 3555 3556 3557
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);

	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
	hotplug |= PORTE_HOTPLUG_ENABLE;
	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3558 3559
}

3560 3561 3562 3563 3564
static void ilk_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_irqs, hotplug, enabled_irqs;

3565 3566 3567 3568 3569 3570
	if (INTEL_INFO(dev)->gen >= 8) {
		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);

		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
	} else if (INTEL_INFO(dev)->gen >= 7) {
3571 3572
		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
3573 3574

		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3575 3576 3577
	} else {
		hotplug_irqs = DE_DP_A_HOTPLUG;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
3578

3579 3580
		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
	}
3581 3582 3583 3584

	/*
	 * Enable digital hotplug on the CPU, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
3585
	 * The pulse duration bits are reserved on HSW+.
3586 3587 3588 3589 3590 3591 3592 3593 3594
	 */
	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);

	ibx_hpd_irq_setup(dev);
}

3595 3596 3597
static void bxt_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
3598
	u32 hotplug_irqs, hotplug, enabled_irqs;
3599

3600 3601
	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3602

3603
	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3604

3605 3606 3607
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE |
		PORTA_HOTPLUG_ENABLE;
3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627

	DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n",
		      hotplug, enabled_irqs);
	hotplug &= ~BXT_DDI_HPD_INVERT_MASK;

	/*
	 * For BXT invert bit has to be set based on AOB design
	 * for HPD detection logic, update it based on VBT fields.
	 */

	if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
		hotplug |= BXT_DDIA_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
		hotplug |= BXT_DDIB_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
		hotplug |= BXT_DDIC_HPD_INVERT;

3628
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3629 3630
}

P
Paulo Zanoni 已提交
3631 3632
static void ibx_irq_postinstall(struct drm_device *dev)
{
3633
	struct drm_i915_private *dev_priv = dev->dev_private;
3634
	u32 mask;
3635

D
Daniel Vetter 已提交
3636 3637 3638
	if (HAS_PCH_NOP(dev))
		return;

3639
	if (HAS_PCH_IBX(dev))
3640
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3641
	else
3642
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3643

3644
	gen5_assert_iir_is_zero(dev_priv, SDEIIR);
P
Paulo Zanoni 已提交
3645 3646 3647
	I915_WRITE(SDEIMR, ~mask);
}

3648 3649 3650 3651 3652 3653 3654 3655
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3656
	if (HAS_L3_DPF(dev)) {
3657
		/* L3 parity interrupt is always unmasked. */
3658 3659
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
		gt_irqs |= GT_PARITY_ERROR(dev);
3660 3661 3662 3663 3664 3665 3666 3667 3668 3669
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
	if (IS_GEN5(dev)) {
		gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
			   ILK_BSD_USER_INTERRUPT;
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3670
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3671 3672

	if (INTEL_INFO(dev)->gen >= 6) {
3673 3674 3675 3676
		/*
		 * RPS interrupts will get enabled/disabled on demand when RPS
		 * itself is enabled/disabled.
		 */
3677 3678 3679
		if (HAS_VEBOX(dev))
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;

3680
		dev_priv->pm_irq_mask = 0xffffffff;
P
Paulo Zanoni 已提交
3681
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3682 3683 3684
	}
}

3685
static int ironlake_irq_postinstall(struct drm_device *dev)
3686
{
3687
	struct drm_i915_private *dev_priv = dev->dev_private;
3688 3689 3690 3691 3692 3693
	u32 display_mask, extra_mask;

	if (INTEL_INFO(dev)->gen >= 7) {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3694
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3695
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3696 3697
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
			      DE_DP_A_HOTPLUG_IVB);
3698 3699 3700
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3701 3702 3703
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3704 3705 3706
		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
			      DE_DP_A_HOTPLUG);
3707
	}
3708

3709
	dev_priv->irq_mask = ~display_mask;
3710

3711 3712
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3713 3714
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3715
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3716

3717
	gen5_gt_irq_postinstall(dev);
3718

P
Paulo Zanoni 已提交
3719
	ibx_irq_postinstall(dev);
3720

3721
	if (IS_IRONLAKE_M(dev)) {
3722 3723 3724
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3725 3726
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
3727
		spin_lock_irq(&dev_priv->irq_lock);
3728
		ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
3729
		spin_unlock_irq(&dev_priv->irq_lock);
3730 3731
	}

3732 3733 3734
	return 0;
}

3735 3736 3737 3738 3739 3740 3741 3742 3743
void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

3744 3745
	if (intel_irqs_enabled(dev_priv)) {
		vlv_display_irq_reset(dev_priv);
3746
		vlv_display_irq_postinstall(dev_priv);
3747
	}
3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

3759
	if (intel_irqs_enabled(dev_priv))
3760
		vlv_display_irq_reset(dev_priv);
3761 3762
}

3763 3764 3765 3766 3767

static int valleyview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3768
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3769

3770
	spin_lock_irq(&dev_priv->irq_lock);
3771 3772
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3773 3774
	spin_unlock_irq(&dev_priv->irq_lock);

J
Jesse Barnes 已提交
3775
	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3776
	POSTING_READ(VLV_MASTER_IER);
3777 3778 3779 3780

	return 0;
}

3781 3782 3783 3784 3785
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3786
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3787
			GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3788 3789
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3790
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3791 3792 3793
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3794
		0,
3795 3796
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3797 3798
		};

3799
	dev_priv->pm_irq_mask = 0xffffffff;
3800 3801
	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3802 3803 3804 3805 3806
	/*
	 * RPS interrupts will get enabled/disabled on demand when RPS itself
	 * is enabled/disabled.
	 */
	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3807
	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3808 3809 3810 3811
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
3812 3813
	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
	uint32_t de_pipe_enables;
3814 3815 3816
	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
	u32 de_port_enables;
	enum pipe pipe;
3817

3818
	if (INTEL_INFO(dev_priv)->gen >= 9) {
3819 3820
		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3821 3822
		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
				  GEN9_AUX_CHANNEL_D;
S
Shashank Sharma 已提交
3823
		if (IS_BROXTON(dev_priv))
3824 3825
			de_port_masked |= BXT_DE_PORT_GMBUS;
	} else {
3826 3827
		de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
				  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3828
	}
3829 3830 3831 3832

	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
					   GEN8_PIPE_FIFO_UNDERRUN;

3833
	de_port_enables = de_port_masked;
3834 3835 3836
	if (IS_BROXTON(dev_priv))
		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
	else if (IS_BROADWELL(dev_priv))
3837 3838
		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;

3839 3840 3841
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3842

3843
	for_each_pipe(dev_priv, pipe)
3844
		if (intel_display_power_is_enabled(dev_priv,
3845 3846 3847 3848
				POWER_DOMAIN_PIPE(pipe)))
			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
					  dev_priv->de_irq_mask[pipe],
					  de_pipe_enables);
3849

3850
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3851 3852 3853 3854 3855 3856
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3857 3858
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_pre_postinstall(dev);
P
Paulo Zanoni 已提交
3859

3860 3861 3862
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

3863 3864
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_postinstall(dev);
3865

3866
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3867 3868 3869 3870 3871
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3872 3873 3874 3875 3876 3877
static int cherryview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	gen8_gt_irq_postinstall(dev_priv);

3878
	spin_lock_irq(&dev_priv->irq_lock);
3879 3880
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3881 3882
	spin_unlock_irq(&dev_priv->irq_lock);

3883
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3884 3885 3886 3887 3888
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3889 3890 3891 3892 3893 3894 3895
static void gen8_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3896
	gen8_irq_reset(dev);
3897 3898
}

J
Jesse Barnes 已提交
3899 3900
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3901
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3902 3903 3904 3905

	if (!dev_priv)
		return;

3906
	I915_WRITE(VLV_MASTER_IER, 0);
3907
	POSTING_READ(VLV_MASTER_IER);
3908

3909 3910
	gen5_gt_irq_reset(dev);

J
Jesse Barnes 已提交
3911
	I915_WRITE(HWSTAM, 0xffffffff);
3912

3913
	spin_lock_irq(&dev_priv->irq_lock);
3914 3915
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3916
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3917 3918
}

3919 3920 3921 3922 3923 3924 3925 3926 3927 3928
static void cherryview_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3929
	gen8_gt_irq_reset(dev_priv);
3930

3931
	GEN5_IRQ_RESET(GEN8_PCU_);
3932

3933
	spin_lock_irq(&dev_priv->irq_lock);
3934 3935
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3936
	spin_unlock_irq(&dev_priv->irq_lock);
3937 3938
}

3939
static void ironlake_irq_uninstall(struct drm_device *dev)
3940
{
3941
	struct drm_i915_private *dev_priv = dev->dev_private;
3942 3943 3944 3945

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3946
	ironlake_irq_reset(dev);
3947 3948
}

3949
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3950
{
3951
	struct drm_i915_private *dev_priv = dev->dev_private;
3952
	int pipe;
3953

3954
	for_each_pipe(dev_priv, pipe)
3955
		I915_WRITE(PIPESTAT(pipe), 0);
3956 3957 3958
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3959 3960 3961 3962
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3963
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3964 3965 3966 3967 3968 3969 3970 3971 3972

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3973
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
C
Chris Wilson 已提交
3974 3975 3976 3977 3978 3979 3980 3981
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3982 3983
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3984
	spin_lock_irq(&dev_priv->irq_lock);
3985 3986
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3987
	spin_unlock_irq(&dev_priv->irq_lock);
3988

C
Chris Wilson 已提交
3989 3990 3991
	return 0;
}

3992 3993 3994 3995
/*
 * Returns true when a page flip has completed.
 */
static bool i8xx_handle_vblank(struct drm_device *dev,
3996
			       int plane, int pipe, u32 iir)
3997
{
3998
	struct drm_i915_private *dev_priv = dev->dev_private;
3999
	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
4000

4001
	if (!intel_pipe_handle_vblank(dev, pipe))
4002 4003 4004
		return false;

	if ((iir & flip_pending) == 0)
4005
		goto check_page_flip;
4006 4007 4008 4009 4010 4011 4012 4013

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ16(ISR) & flip_pending)
4014
		goto check_page_flip;
4015

4016
	intel_prepare_page_flip(dev, plane);
4017 4018
	intel_finish_page_flip(dev, pipe);
	return true;
4019 4020 4021 4022

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
4023 4024
}

4025
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
4026
{
4027
	struct drm_device *dev = arg;
4028
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
4029 4030 4031 4032 4033 4034
	u16 iir, new_iir;
	u32 pipe_stats[2];
	int pipe;
	u16 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4035
	irqreturn_t ret;
C
Chris Wilson 已提交
4036

4037 4038 4039
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4040 4041 4042 4043
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	ret = IRQ_NONE;
C
Chris Wilson 已提交
4044 4045
	iir = I915_READ16(IIR);
	if (iir == 0)
4046
		goto out;
C
Chris Wilson 已提交
4047 4048 4049 4050 4051 4052 4053

	while (iir & ~flip_mask) {
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4054
		spin_lock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
4055
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4056
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
C
Chris Wilson 已提交
4057

4058
		for_each_pipe(dev_priv, pipe) {
4059
			i915_reg_t reg = PIPESTAT(pipe);
C
Chris Wilson 已提交
4060 4061 4062 4063 4064
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
4065
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
4066 4067
				I915_WRITE(reg, pipe_stats[pipe]);
		}
4068
		spin_unlock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
4069 4070 4071 4072 4073

		I915_WRITE16(IIR, iir & ~flip_mask);
		new_iir = I915_READ16(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4074
			notify_ring(&dev_priv->engine[RCS]);
C
Chris Wilson 已提交
4075

4076
		for_each_pipe(dev_priv, pipe) {
4077
			int plane = pipe;
4078
			if (HAS_FBC(dev))
4079 4080
				plane = !plane;

4081
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
4082 4083
			    i8xx_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
C
Chris Wilson 已提交
4084

4085
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4086
				i9xx_pipe_crc_irq_handler(dev, pipe);
4087

4088 4089 4090
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
4091
		}
C
Chris Wilson 已提交
4092 4093 4094

		iir = new_iir;
	}
4095 4096 4097 4098
	ret = IRQ_HANDLED;

out:
	enable_rpm_wakeref_asserts(dev_priv);
C
Chris Wilson 已提交
4099

4100
	return ret;
C
Chris Wilson 已提交
4101 4102 4103 4104
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
4105
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
4106 4107
	int pipe;

4108
	for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
4109 4110 4111 4112 4113 4114 4115 4116 4117
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

4118 4119
static void i915_irq_preinstall(struct drm_device * dev)
{
4120
	struct drm_i915_private *dev_priv = dev->dev_private;
4121 4122 4123
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
4124
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4125 4126 4127
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4128
	I915_WRITE16(HWSTAM, 0xeffe);
4129
	for_each_pipe(dev_priv, pipe)
4130 4131 4132 4133 4134 4135 4136 4137
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
4138
	struct drm_i915_private *dev_priv = dev->dev_private;
4139
	u32 enable_mask;
4140

4141 4142 4143 4144 4145 4146 4147 4148
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4149
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4150 4151 4152 4153 4154 4155 4156

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_USER_INTERRUPT;

4157
	if (I915_HAS_HOTPLUG(dev)) {
4158
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4159 4160
		POSTING_READ(PORT_HOTPLUG_EN);

4161 4162 4163 4164 4165 4166 4167 4168 4169 4170
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4171
	i915_enable_asle_pipestat(dev);
4172

4173 4174
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4175
	spin_lock_irq(&dev_priv->irq_lock);
4176 4177
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4178
	spin_unlock_irq(&dev_priv->irq_lock);
4179

4180 4181 4182
	return 0;
}

4183 4184 4185 4186 4187 4188
/*
 * Returns true when a page flip has completed.
 */
static bool i915_handle_vblank(struct drm_device *dev,
			       int plane, int pipe, u32 iir)
{
4189
	struct drm_i915_private *dev_priv = dev->dev_private;
4190 4191
	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

4192
	if (!intel_pipe_handle_vblank(dev, pipe))
4193 4194 4195
		return false;

	if ((iir & flip_pending) == 0)
4196
		goto check_page_flip;
4197 4198 4199 4200 4201 4202 4203 4204

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ(ISR) & flip_pending)
4205
		goto check_page_flip;
4206

4207
	intel_prepare_page_flip(dev, plane);
4208 4209
	intel_finish_page_flip(dev, pipe);
	return true;
4210 4211 4212 4213

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
4214 4215
}

4216
static irqreturn_t i915_irq_handler(int irq, void *arg)
4217
{
4218
	struct drm_device *dev = arg;
4219
	struct drm_i915_private *dev_priv = dev->dev_private;
4220
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
4221 4222 4223 4224
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
	int pipe, ret = IRQ_NONE;
4225

4226 4227 4228
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4229 4230 4231
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

4232
	iir = I915_READ(IIR);
4233 4234
	do {
		bool irq_received = (iir & ~flip_mask) != 0;
4235
		bool blc_event = false;
4236 4237 4238 4239 4240 4241

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4242
		spin_lock(&dev_priv->irq_lock);
4243
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4244
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4245

4246
		for_each_pipe(dev_priv, pipe) {
4247
			i915_reg_t reg = PIPESTAT(pipe);
4248 4249
			pipe_stats[pipe] = I915_READ(reg);

4250
			/* Clear the PIPE*STAT regs before the IIR */
4251 4252
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4253
				irq_received = true;
4254 4255
			}
		}
4256
		spin_unlock(&dev_priv->irq_lock);
4257 4258 4259 4260 4261

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
4262
		if (I915_HAS_HOTPLUG(dev) &&
4263 4264 4265 4266 4267
		    iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
				i9xx_hpd_irq_handler(dev, hotplug_status);
		}
4268

4269
		I915_WRITE(IIR, iir & ~flip_mask);
4270 4271 4272
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4273
			notify_ring(&dev_priv->engine[RCS]);
4274

4275
		for_each_pipe(dev_priv, pipe) {
4276
			int plane = pipe;
4277
			if (HAS_FBC(dev))
4278
				plane = !plane;
4279

4280
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
4281 4282
			    i915_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
4283 4284 4285

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4286 4287

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4288
				i9xx_pipe_crc_irq_handler(dev, pipe);
4289

4290 4291 4292
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
4313
		ret = IRQ_HANDLED;
4314
		iir = new_iir;
4315
	} while (iir & ~flip_mask);
4316

4317 4318
	enable_rpm_wakeref_asserts(dev_priv);

4319 4320 4321 4322 4323
	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
4324
	struct drm_i915_private *dev_priv = dev->dev_private;
4325 4326 4327
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
4328
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4329 4330 4331
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4332
	I915_WRITE16(HWSTAM, 0xffff);
4333
	for_each_pipe(dev_priv, pipe) {
4334
		/* Clear enable bits; then clear status bits */
4335
		I915_WRITE(PIPESTAT(pipe), 0);
4336 4337
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
4338 4339 4340 4341 4342 4343 4344 4345
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
4346
	struct drm_i915_private *dev_priv = dev->dev_private;
4347 4348
	int pipe;

4349
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4350
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4351 4352

	I915_WRITE(HWSTAM, 0xeffe);
4353
	for_each_pipe(dev_priv, pipe)
4354 4355 4356 4357 4358 4359 4360 4361
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
4362
	struct drm_i915_private *dev_priv = dev->dev_private;
4363
	u32 enable_mask;
4364 4365 4366
	u32 error_mask;

	/* Unmask the interrupts that we always want on. */
4367
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4368
			       I915_DISPLAY_PORT_INTERRUPT |
4369 4370 4371 4372 4373 4374 4375
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
4376 4377
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4378 4379 4380 4381
	enable_mask |= I915_USER_INTERRUPT;

	if (IS_G4X(dev))
		enable_mask |= I915_BSD_USER_INTERRUPT;
4382

4383 4384
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4385
	spin_lock_irq(&dev_priv->irq_lock);
4386 4387 4388
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4389
	spin_unlock_irq(&dev_priv->irq_lock);
4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
	if (IS_G4X(dev)) {
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4410
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4411 4412
	POSTING_READ(PORT_HOTPLUG_EN);

4413
	i915_enable_asle_pipestat(dev);
4414 4415 4416 4417

	return 0;
}

4418
static void i915_hpd_irq_setup(struct drm_device *dev)
4419
{
4420
	struct drm_i915_private *dev_priv = dev->dev_private;
4421 4422
	u32 hotplug_en;

4423 4424
	assert_spin_locked(&dev_priv->irq_lock);

4425 4426
	/* Note HDMI and DP share hotplug bits */
	/* enable bits are the same for all generations */
4427
	hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
4428 4429 4430 4431 4432 4433 4434 4435 4436
	/* Programming the CRT detection parameters tends
	   to generate a spurious hotplug event about three
	   seconds later.  So just do it once.
	*/
	if (IS_G4X(dev))
		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;

	/* Ignore TV since it's buggy */
4437
	i915_hotplug_interrupt_update_locked(dev_priv,
4438 4439 4440 4441
					     HOTPLUG_INT_EN_MASK |
					     CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
					     CRT_HOTPLUG_ACTIVATION_PERIOD_64,
					     hotplug_en);
4442 4443
}

4444
static irqreturn_t i965_irq_handler(int irq, void *arg)
4445
{
4446
	struct drm_device *dev = arg;
4447
	struct drm_i915_private *dev_priv = dev->dev_private;
4448 4449 4450
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	int ret = IRQ_NONE, pipe;
4451 4452 4453
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4454

4455 4456 4457
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4458 4459 4460
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

4461 4462 4463
	iir = I915_READ(IIR);

	for (;;) {
4464
		bool irq_received = (iir & ~flip_mask) != 0;
4465 4466
		bool blc_event = false;

4467 4468 4469 4470 4471
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4472
		spin_lock(&dev_priv->irq_lock);
4473
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4474
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4475

4476
		for_each_pipe(dev_priv, pipe) {
4477
			i915_reg_t reg = PIPESTAT(pipe);
4478 4479 4480 4481 4482 4483 4484
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4485
				irq_received = true;
4486 4487
			}
		}
4488
		spin_unlock(&dev_priv->irq_lock);
4489 4490 4491 4492 4493 4494 4495

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4496 4497 4498 4499 4500
		if (iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
				i9xx_hpd_irq_handler(dev, hotplug_status);
		}
4501

4502
		I915_WRITE(IIR, iir & ~flip_mask);
4503 4504 4505
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4506
			notify_ring(&dev_priv->engine[RCS]);
4507
		if (iir & I915_BSD_USER_INTERRUPT)
4508
			notify_ring(&dev_priv->engine[VCS]);
4509

4510
		for_each_pipe(dev_priv, pipe) {
4511
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4512 4513
			    i915_handle_vblank(dev, pipe, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4514 4515 4516

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4517 4518

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4519
				i9xx_pipe_crc_irq_handler(dev, pipe);
4520

4521 4522
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4523
		}
4524 4525 4526 4527

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

4528 4529 4530
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
			gmbus_irq_handler(dev);

4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

4549 4550
	enable_rpm_wakeref_asserts(dev_priv);

4551 4552 4553 4554 4555
	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4556
	struct drm_i915_private *dev_priv = dev->dev_private;
4557 4558 4559 4560 4561
	int pipe;

	if (!dev_priv)
		return;

4562
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4563
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4564 4565

	I915_WRITE(HWSTAM, 0xffffffff);
4566
	for_each_pipe(dev_priv, pipe)
4567 4568 4569 4570
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

4571
	for_each_pipe(dev_priv, pipe)
4572 4573 4574 4575 4576
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4577 4578 4579 4580 4581 4582 4583
/**
 * intel_irq_init - initializes irq support
 * @dev_priv: i915 device instance
 *
 * This function initializes all the irq support including work items, timers
 * and all the vtables. It does not setup the interrupt itself though.
 */
4584
void intel_irq_init(struct drm_i915_private *dev_priv)
4585
{
4586
	struct drm_device *dev = dev_priv->dev;
4587

4588 4589
	intel_hpd_init_work(dev_priv);

4590
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4591
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4592

4593
	/* Let's track the enabled rps events */
4594
	if (IS_VALLEYVIEW(dev_priv))
4595
		/* WaGsvRC0ResidencyMethod:vlv */
4596
		dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4597 4598
	else
		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4599

4600 4601
	INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
			  i915_hangcheck_elapsed);
4602

4603
	if (IS_GEN2(dev_priv)) {
4604 4605
		dev->max_vblank_count = 0;
		dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4606
	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4607
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4608
		dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4609 4610 4611
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4612 4613
	}

4614 4615 4616 4617 4618
	/*
	 * Opt out of the vblank disable timer on everything except gen2.
	 * Gen2 doesn't have a hardware frame counter and so depends on
	 * vblank interrupts to produce sane vblank seuquence numbers.
	 */
4619
	if (!IS_GEN2(dev_priv))
4620 4621
		dev->vblank_disable_immediate = true;

4622 4623
	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4624

4625
	if (IS_CHERRYVIEW(dev_priv)) {
4626 4627 4628 4629 4630 4631 4632
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4633
	} else if (IS_VALLEYVIEW(dev_priv)) {
J
Jesse Barnes 已提交
4634 4635 4636 4637 4638 4639
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
4640
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4641
	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4642
		dev->driver->irq_handler = gen8_irq_handler;
4643
		dev->driver->irq_preinstall = gen8_irq_reset;
4644 4645 4646 4647
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
4648
		if (IS_BROXTON(dev))
4649
			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4650 4651 4652
		else if (HAS_PCH_SPT(dev))
			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		else
4653
			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4654 4655
	} else if (HAS_PCH_SPLIT(dev)) {
		dev->driver->irq_handler = ironlake_irq_handler;
4656
		dev->driver->irq_preinstall = ironlake_irq_reset;
4657 4658 4659 4660
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4661
		dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4662
	} else {
4663
		if (INTEL_INFO(dev_priv)->gen == 2) {
C
Chris Wilson 已提交
4664 4665 4666 4667
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4668
		} else if (INTEL_INFO(dev_priv)->gen == 3) {
4669 4670 4671 4672
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
C
Chris Wilson 已提交
4673
		} else {
4674 4675 4676 4677
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
C
Chris Wilson 已提交
4678
		}
4679 4680
		if (I915_HAS_HOTPLUG(dev_priv))
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4681 4682 4683 4684
		dev->driver->enable_vblank = i915_enable_vblank;
		dev->driver->disable_vblank = i915_disable_vblank;
	}
}
4685

4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696
/**
 * intel_irq_install - enables the hardware interrupt
 * @dev_priv: i915 device instance
 *
 * This function enables the hardware interrupt handling, but leaves the hotplug
 * handling still disabled. It is called after intel_irq_init().
 *
 * In the driver load and resume code we need working interrupts in a few places
 * but don't want to deal with the hassle of concurrent probe and hotplug
 * workers. Hence the split into this two-stage approach.
 */
4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708
int intel_irq_install(struct drm_i915_private *dev_priv)
{
	/*
	 * We enable some interrupt sources in our postinstall hooks, so mark
	 * interrupts as enabled _before_ actually enabling them to avoid
	 * special cases in our ordering checks.
	 */
	dev_priv->pm.irqs_enabled = true;

	return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
}

4709 4710 4711 4712 4713 4714 4715
/**
 * intel_irq_uninstall - finilizes all irq handling
 * @dev_priv: i915 device instance
 *
 * This stops interrupt and hotplug handling and unregisters and frees all
 * resources acquired in the init functions.
 */
4716 4717 4718 4719 4720 4721 4722
void intel_irq_uninstall(struct drm_i915_private *dev_priv)
{
	drm_irq_uninstall(dev_priv->dev);
	intel_hpd_cancel_work(dev_priv);
	dev_priv->pm.irqs_enabled = false;
}

4723 4724 4725 4726 4727 4728 4729
/**
 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
 * @dev_priv: i915 device instance
 *
 * This function is used to disable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4730
void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4731
{
4732
	dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4733
	dev_priv->pm.irqs_enabled = false;
4734
	synchronize_irq(dev_priv->dev->irq);
4735 4736
}

4737 4738 4739 4740 4741 4742 4743
/**
 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
 * @dev_priv: i915 device instance
 *
 * This function is used to enable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4744
void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4745
{
4746
	dev_priv->pm.irqs_enabled = true;
4747 4748
	dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
	dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4749
}