i915_irq.c 132.0 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47
/**
 * DOC: interrupt handling
 *
 * These functions provide the basic support for enabling and disabling the
 * interrupt handling support. There's a lot more functionality in i915_irq.c
 * and related files, but that will be described in separate chapters.
 */

48 49 50 51
static const u32 hpd_ilk[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
};

52 53 54 55
static const u32 hpd_ivb[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
};

56 57 58 59
static const u32 hpd_bdw[HPD_NUM_PINS] = {
	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
};

60
static const u32 hpd_ibx[HPD_NUM_PINS] = {
61 62 63 64 65 66 67
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

68
static const u32 hpd_cpt[HPD_NUM_PINS] = {
69
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71 72 73 74 75
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

X
Xiong Zhang 已提交
76
static const u32 hpd_spt[HPD_NUM_PINS] = {
77
	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
X
Xiong Zhang 已提交
78 79 80 81 82 83
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
};

84
static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85 86 87 88 89 90 91 92
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

93
static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94 95 96 97 98 99 100 101
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

102
static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103 104 105 106 107 108 109 110
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

111 112
/* BXT hpd list */
static const u32 hpd_bxt[HPD_NUM_PINS] = {
113
	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114 115 116 117
	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
};

118
/* IIR can theoretically queue up two events. Be paranoid. */
119
#define GEN8_IRQ_RESET_NDX(type, which) do { \
120 121 122 123 124 125 126 127 128
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

129
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
130
	I915_WRITE(type##IMR, 0xffffffff); \
131
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
132
	I915_WRITE(type##IER, 0); \
133 134 135 136
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
137 138
} while (0)

139 140 141
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
142 143
static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
				    i915_reg_t reg)
144 145 146 147 148 149 150
{
	u32 val = I915_READ(reg);

	if (val == 0)
		return;

	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
151
	     i915_mmio_reg_offset(reg), val);
152 153 154 155 156
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
}
157

P
Paulo Zanoni 已提交
158
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
159
	gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
160
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
161 162
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
P
Paulo Zanoni 已提交
163 164 165
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
166
	gen5_assert_iir_is_zero(dev_priv, type##IIR); \
P
Paulo Zanoni 已提交
167
	I915_WRITE(type##IER, (ier_val)); \
168 169
	I915_WRITE(type##IMR, (imr_val)); \
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
170 171
} while (0)

172 173
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);

174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211
/* For display hotplug interrupt */
static inline void
i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
				     uint32_t mask,
				     uint32_t bits)
{
	uint32_t val;

	assert_spin_locked(&dev_priv->irq_lock);
	WARN_ON(bits & ~mask);

	val = I915_READ(PORT_HOTPLUG_EN);
	val &= ~mask;
	val |= bits;
	I915_WRITE(PORT_HOTPLUG_EN, val);
}

/**
 * i915_hotplug_interrupt_update - update hotplug interrupt enable
 * @dev_priv: driver private
 * @mask: bits to update
 * @bits: bits to enable
 * NOTE: the HPD enable bits are modified both inside and outside
 * of an interrupt context. To avoid that read-modify-write cycles
 * interfer, these bits are protected by a spinlock. Since this
 * function is usually not called from a context where the lock is
 * held already, this function acquires the lock itself. A non-locking
 * version is also available.
 */
void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
				   uint32_t mask,
				   uint32_t bits)
{
	spin_lock_irq(&dev_priv->irq_lock);
	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
	spin_unlock_irq(&dev_priv->irq_lock);
}

212 213 214 215 216 217
/**
 * ilk_update_display_irq - update DEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
218 219 220
void ilk_update_display_irq(struct drm_i915_private *dev_priv,
			    uint32_t interrupt_mask,
			    uint32_t enabled_irq_mask)
221
{
222 223
	uint32_t new_val;

224 225
	assert_spin_locked(&dev_priv->irq_lock);

226 227
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

228
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
229 230
		return;

231 232 233 234 235 236
	new_val = dev_priv->irq_mask;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->irq_mask) {
		dev_priv->irq_mask = new_val;
237
		I915_WRITE(DEIMR, dev_priv->irq_mask);
238
		POSTING_READ(DEIMR);
239 240 241
	}
}

P
Paulo Zanoni 已提交
242 243 244 245 246 247 248 249 250 251 252 253
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
	assert_spin_locked(&dev_priv->irq_lock);

254 255
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

256
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
257 258
		return;

P
Paulo Zanoni 已提交
259 260 261 262 263 264
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
	POSTING_READ(GTIMR);
}

265
void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
266 267 268 269
{
	ilk_update_gt_irq(dev_priv, mask, mask);
}

270
void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
271 272 273 274
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

275
static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
276 277 278 279
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
}

280
static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
281 282 283 284
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
}

285
static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
286 287 288 289
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
}

P
Paulo Zanoni 已提交
290
/**
291 292 293 294 295
 * snb_update_pm_irq - update GEN6_PMIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
P
Paulo Zanoni 已提交
296 297 298 299
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
300
	uint32_t new_val;
P
Paulo Zanoni 已提交
301

302 303
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

P
Paulo Zanoni 已提交
304 305
	assert_spin_locked(&dev_priv->irq_lock);

306
	new_val = dev_priv->pm_irq_mask;
307 308 309
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

310 311
	if (new_val != dev_priv->pm_irq_mask) {
		dev_priv->pm_irq_mask = new_val;
312 313
		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
		POSTING_READ(gen6_pm_imr(dev_priv));
314
	}
P
Paulo Zanoni 已提交
315 316
}

317
void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
318
{
319 320 321
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

P
Paulo Zanoni 已提交
322 323 324
	snb_update_pm_irq(dev_priv, mask, mask);
}

325 326
static void __gen6_disable_pm_irq(struct drm_i915_private *dev_priv,
				  uint32_t mask)
P
Paulo Zanoni 已提交
327 328 329 330
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

331 332 333 334 335 336 337 338
void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	__gen6_disable_pm_irq(dev_priv, mask);
}

I
Imre Deak 已提交
339 340 341
void gen6_reset_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
342
	i915_reg_t reg = gen6_pm_iir(dev_priv);
I
Imre Deak 已提交
343 344 345 346 347

	spin_lock_irq(&dev_priv->irq_lock);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	I915_WRITE(reg, dev_priv->pm_rps_events);
	POSTING_READ(reg);
348
	dev_priv->rps.pm_iir = 0;
I
Imre Deak 已提交
349 350 351
	spin_unlock_irq(&dev_priv->irq_lock);
}

352 353 354 355 356
void gen6_enable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	spin_lock_irq(&dev_priv->irq_lock);
357

358
	WARN_ON(dev_priv->rps.pm_iir);
I
Imre Deak 已提交
359
	WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
I
Imre Deak 已提交
360
	dev_priv->rps.interrupts_enabled = true;
361 362
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
				dev_priv->pm_rps_events);
363
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
364

365 366 367
	spin_unlock_irq(&dev_priv->irq_lock);
}

368 369 370
u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
{
	/*
371
	 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
372
	 * if GEN6_PM_UP_EI_EXPIRED is masked.
373 374
	 *
	 * TODO: verify if this can be reproduced on VLV,CHV.
375 376 377 378 379 380 381 382 383 384
	 */
	if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
		mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;

	if (INTEL_INFO(dev_priv)->gen >= 8)
		mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;

	return mask;
}

385 386 387 388
void gen6_disable_rps_interrupts(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

I
Imre Deak 已提交
389 390 391 392 393 394
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->rps.interrupts_enabled = false;
	spin_unlock_irq(&dev_priv->irq_lock);

	cancel_work_sync(&dev_priv->rps.work);

395 396
	spin_lock_irq(&dev_priv->irq_lock);

397
	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
398 399

	__gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
400 401
	I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
				~dev_priv->pm_rps_events);
402 403 404 405

	spin_unlock_irq(&dev_priv->irq_lock);

	synchronize_irq(dev->irq);
406 407
}

408
/**
409 410 411 412 413
 * bdw_update_port_irq - update DE port interrupt
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439
static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
				uint32_t interrupt_mask,
				uint32_t enabled_irq_mask)
{
	uint32_t new_val;
	uint32_t old_val;

	assert_spin_locked(&dev_priv->irq_lock);

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	old_val = I915_READ(GEN8_DE_PORT_IMR);

	new_val = old_val;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != old_val) {
		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		POSTING_READ(GEN8_DE_PORT_IMR);
	}
}

440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
/**
 * bdw_update_pipe_irq - update DE pipe interrupt
 * @dev_priv: driver private
 * @pipe: pipe whose interrupt to update
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
			 enum pipe pipe,
			 uint32_t interrupt_mask,
			 uint32_t enabled_irq_mask)
{
	uint32_t new_val;

	assert_spin_locked(&dev_priv->irq_lock);

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	new_val = dev_priv->de_irq_mask[pipe];
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->de_irq_mask[pipe]) {
		dev_priv->de_irq_mask[pipe] = new_val;
		I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
	}
}

472 473 474 475 476 477
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
478 479 480
void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
				  uint32_t interrupt_mask,
				  uint32_t enabled_irq_mask)
481 482 483 484 485
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

486 487
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

488 489
	assert_spin_locked(&dev_priv->irq_lock);

490
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
491 492
		return;

493 494 495
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
496

D
Daniel Vetter 已提交
497
static void
498 499
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
500
{
501
	i915_reg_t reg = PIPESTAT(pipe);
502
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
503

504
	assert_spin_locked(&dev_priv->irq_lock);
505
	WARN_ON(!intel_irqs_enabled(dev_priv));
506

507 508 509 510
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
511 512 513
		return;

	if ((pipestat & enable_mask) == enable_mask)
514 515
		return;

516 517
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

518
	/* Enable the interrupt, clear any pending status */
519
	pipestat |= enable_mask | status_mask;
520 521
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
522 523
}

D
Daniel Vetter 已提交
524
static void
525 526
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
527
{
528
	i915_reg_t reg = PIPESTAT(pipe);
529
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
530

531
	assert_spin_locked(&dev_priv->irq_lock);
532
	WARN_ON(!intel_irqs_enabled(dev_priv));
533

534 535 536 537
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
538 539
		return;

540 541 542
	if ((pipestat & enable_mask) == 0)
		return;

543 544
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

545
	pipestat &= ~enable_mask;
546 547
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
548 549
}

550 551 552 553 554
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
555 556
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
557 558 559
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
560 561 562 563 564 565
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
566 567 568 569 570 571 572 573 574 575 576 577

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

578 579 580 581 582 583
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

584
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
585 586 587 588
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
589 590 591 592 593 594 595 596 597
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

598
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
599 600 601 602
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
603 604 605
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

606
/**
607
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
608
 * @dev: drm device
609
 */
610
static void i915_enable_asle_pipestat(struct drm_device *dev)
611
{
612
	struct drm_i915_private *dev_priv = dev->dev_private;
613

614 615 616
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
		return;

617
	spin_lock_irq(&dev_priv->irq_lock);
618

619
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
620
	if (INTEL_INFO(dev)->gen >= 4)
621
		i915_enable_pipestat(dev_priv, PIPE_A,
622
				     PIPE_LEGACY_BLC_EVENT_STATUS);
623

624
	spin_unlock_irq(&dev_priv->irq_lock);
625 626
}

627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676
/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via PIPECONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on PIPECONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

677
static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
678 679 680 681 682
{
	/* Gen2 doesn't have a hardware frame counter */
	return 0;
}

683 684 685
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
686
static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
687
{
688
	struct drm_i915_private *dev_priv = dev->dev_private;
689
	i915_reg_t high_frame, low_frame;
690
	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
691 692
	struct intel_crtc *intel_crtc =
		to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
693
	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
694

695 696 697 698 699
	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vbl_start = mode->crtc_vblank_start;
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
700

701 702 703 704 705 706
	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

707 708
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
709

710 711 712 713 714 715
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
716
		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
717
		low   = I915_READ(low_frame);
718
		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
719 720
	} while (high1 != high2);

721
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
722
	pixel = low & PIPE_PIXEL_MASK;
723
	low >>= PIPE_FRAME_LOW_SHIFT;
724 725 726 727 728 729

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
730
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
731 732
}

733
static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
734
{
735
	struct drm_i915_private *dev_priv = dev->dev_private;
736

737
	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
738 739
}

740
/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
741 742 743 744
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
745
	const struct drm_display_mode *mode = &crtc->base.hwmode;
746
	enum pipe pipe = crtc->pipe;
747
	int position, vtotal;
748

749
	vtotal = mode->crtc_vtotal;
750 751 752 753
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

	if (IS_GEN2(dev))
754
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
755
	else
756
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
757

758 759 760 761 762 763 764 765 766 767 768 769
	/*
	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
	 * read it just before the start of vblank.  So try it again
	 * so we don't accidentally end up spanning a vblank frame
	 * increment, causing the pipe_update_end() code to squak at us.
	 *
	 * The nature of this problem means we can't simply check the ISR
	 * bit and return the vblank start value; nor can we use the scanline
	 * debug register in the transcoder as it appears to have the same
	 * problem.  We may need to extend this to include other platforms,
	 * but so far testing only shows the problem on HSW.
	 */
770
	if (HAS_DDI(dev) && !position) {
771 772 773 774 775 776 777 778 779 780 781 782 783
		int i, temp;

		for (i = 0; i < 100; i++) {
			udelay(1);
			temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
				DSL_LINEMASK_GEN3;
			if (temp != position) {
				position = temp;
				break;
			}
		}
	}

784
	/*
785 786
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
787
	 */
788
	return (position + crtc->scanline_offset) % vtotal;
789 790
}

791
static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
792
				    unsigned int flags, int *vpos, int *hpos,
793 794
				    ktime_t *stime, ktime_t *etime,
				    const struct drm_display_mode *mode)
795
{
796 797 798
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
799
	int position;
800
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
801 802
	bool in_vbl = true;
	int ret = 0;
803
	unsigned long irqflags;
804

805
	if (WARN_ON(!mode->crtc_clock)) {
806
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
807
				 "pipe %c\n", pipe_name(pipe));
808 809 810
		return 0;
	}

811
	htotal = mode->crtc_htotal;
812
	hsync_start = mode->crtc_hsync_start;
813 814 815
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
816

817 818 819 820 821 822
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

823 824
	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

825 826 827 828 829 830
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
831

832 833 834 835 836 837
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

838
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
839 840 841
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
842
		position = __intel_get_crtc_scanline(intel_crtc);
843 844 845 846 847
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
848
		position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
849

850 851 852 853
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
854

855 856 857 858 859 860 861 862 863 864 865 866
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

867 868 869 870 871 872 873 874 875 876
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
877 878
	}

879 880 881 882 883 884 885 886
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

887 888 889 890 891 892 893 894 895 896 897 898
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
899

900
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
901 902 903 904 905 906
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
907 908 909

	/* In vblank? */
	if (in_vbl)
910
		ret |= DRM_SCANOUTPOS_IN_VBLANK;
911 912 913 914

	return ret;
}

915 916 917 918 919 920 921 922 923 924 925 926 927
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

928
static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
929 930 931 932
			      int *max_error,
			      struct timeval *vblank_time,
			      unsigned flags)
{
933
	struct drm_crtc *crtc;
934

935 936
	if (pipe >= INTEL_INFO(dev)->num_pipes) {
		DRM_ERROR("Invalid crtc %u\n", pipe);
937 938 939 940
		return -EINVAL;
	}

	/* Get drm_crtc to timestamp: */
941 942
	crtc = intel_get_crtc_for_pipe(dev, pipe);
	if (crtc == NULL) {
943
		DRM_ERROR("Invalid crtc %u\n", pipe);
944 945 946
		return -EINVAL;
	}

947
	if (!crtc->hwmode.crtc_clock) {
948
		DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
949 950
		return -EBUSY;
	}
951 952

	/* Helper routine in DRM core does all the work: */
953 954
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
						     vblank_time, flags,
955
						     &crtc->hwmode);
956 957
}

958
static void ironlake_rps_change_irq_handler(struct drm_device *dev)
959
{
960
	struct drm_i915_private *dev_priv = dev->dev_private;
961
	u32 busy_up, busy_down, max_avg, min_avg;
962 963
	u8 new_delay;

964
	spin_lock(&mchdev_lock);
965

966 967
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

968
	new_delay = dev_priv->ips.cur_delay;
969

970
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
971 972
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
973 974 975 976
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
977
	if (busy_up > max_avg) {
978 979 980 981
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
982
	} else if (busy_down < min_avg) {
983 984 985 986
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
987 988
	}

989
	if (ironlake_set_drps(dev, new_delay))
990
		dev_priv->ips.cur_delay = new_delay;
991

992
	spin_unlock(&mchdev_lock);
993

994 995 996
	return;
}

997
static void notify_ring(struct intel_engine_cs *engine)
998
{
999
	if (!intel_engine_initialized(engine))
1000 1001
		return;

1002
	trace_i915_gem_request_notify(engine);
1003
	engine->user_interrupts++;
1004

1005
	wake_up_all(&engine->irq_queue);
1006 1007
}

1008 1009
static void vlv_c0_read(struct drm_i915_private *dev_priv,
			struct intel_rps_ei *ei)
1010
{
1011 1012 1013 1014
	ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
}
1015

1016 1017 1018 1019 1020 1021
static bool vlv_c0_above(struct drm_i915_private *dev_priv,
			 const struct intel_rps_ei *old,
			 const struct intel_rps_ei *now,
			 int threshold)
{
	u64 time, c0;
1022
	unsigned int mul = 100;
1023

1024 1025
	if (old->cz_clock == 0)
		return false;
1026

1027 1028 1029
	if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
		mul <<= 8;

1030
	time = now->cz_clock - old->cz_clock;
1031
	time *= threshold * dev_priv->czclk_freq;
1032

1033 1034 1035
	/* Workload can be split between render + media, e.g. SwapBuffers
	 * being blitted in X after being rendered in mesa. To account for
	 * this we need to combine both engines into our activity counter.
1036
	 */
1037 1038
	c0 = now->render_c0 - old->render_c0;
	c0 += now->media_c0 - old->media_c0;
1039
	c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC;
1040

1041
	return c0 >= time;
1042 1043
}

1044
void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1045
{
1046 1047 1048
	vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
	dev_priv->rps.up_ei = dev_priv->rps.down_ei;
}
1049

1050 1051 1052 1053
static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
{
	struct intel_rps_ei now;
	u32 events = 0;
1054

1055
	if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
1056
		return 0;
1057

1058 1059 1060
	vlv_c0_read(dev_priv, &now);
	if (now.cz_clock == 0)
		return 0;
1061

1062 1063 1064
	if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
		if (!vlv_c0_above(dev_priv,
				  &dev_priv->rps.down_ei, &now,
1065
				  dev_priv->rps.down_threshold))
1066 1067 1068
			events |= GEN6_PM_RP_DOWN_THRESHOLD;
		dev_priv->rps.down_ei = now;
	}
1069

1070 1071 1072
	if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
		if (vlv_c0_above(dev_priv,
				 &dev_priv->rps.up_ei, &now,
1073
				 dev_priv->rps.up_threshold))
1074 1075
			events |= GEN6_PM_RP_UP_THRESHOLD;
		dev_priv->rps.up_ei = now;
1076 1077
	}

1078
	return events;
1079 1080
}

1081 1082
static bool any_waiters(struct drm_i915_private *dev_priv)
{
1083
	struct intel_engine_cs *engine;
1084

1085
	for_each_engine(engine, dev_priv)
1086
		if (engine->irq_refcount)
1087 1088 1089 1090 1091
			return true;

	return false;
}

1092
static void gen6_pm_rps_work(struct work_struct *work)
1093
{
1094 1095
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
1096 1097
	bool client_boost;
	int new_delay, adj, min, max;
P
Paulo Zanoni 已提交
1098
	u32 pm_iir;
1099

1100
	spin_lock_irq(&dev_priv->irq_lock);
I
Imre Deak 已提交
1101 1102 1103 1104 1105
	/* Speed up work cancelation during disabling rps interrupts. */
	if (!dev_priv->rps.interrupts_enabled) {
		spin_unlock_irq(&dev_priv->irq_lock);
		return;
	}
1106 1107 1108 1109 1110 1111 1112 1113

	/*
	 * The RPS work is synced during runtime suspend, we don't require a
	 * wakeref. TODO: instead of disabling the asserts make sure that we
	 * always hold an RPM reference while the work is running.
	 */
	DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);

1114 1115
	pm_iir = dev_priv->rps.pm_iir;
	dev_priv->rps.pm_iir = 0;
1116 1117
	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
1118 1119
	client_boost = dev_priv->rps.client_boost;
	dev_priv->rps.client_boost = false;
1120
	spin_unlock_irq(&dev_priv->irq_lock);
1121

1122
	/* Make sure we didn't queue anything we're not going to process. */
1123
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1124

1125
	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1126
		goto out;
1127

1128
	mutex_lock(&dev_priv->rps.hw_lock);
1129

1130 1131
	pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);

1132
	adj = dev_priv->rps.last_adj;
1133
	new_delay = dev_priv->rps.cur_freq;
1134 1135 1136 1137 1138 1139 1140
	min = dev_priv->rps.min_freq_softlimit;
	max = dev_priv->rps.max_freq_softlimit;

	if (client_boost) {
		new_delay = dev_priv->rps.max_freq_softlimit;
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1141 1142
		if (adj > 0)
			adj *= 2;
1143 1144
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1145 1146 1147 1148
		/*
		 * For better performance, jump directly
		 * to RPe if we're below it.
		 */
1149
		if (new_delay < dev_priv->rps.efficient_freq - adj) {
1150
			new_delay = dev_priv->rps.efficient_freq;
1151 1152
			adj = 0;
		}
1153 1154
	} else if (any_waiters(dev_priv)) {
		adj = 0;
1155
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1156 1157
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1158
		else
1159
			new_delay = dev_priv->rps.min_freq_softlimit;
1160 1161 1162 1163
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
1164 1165
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1166
	} else { /* unknown event */
1167
		adj = 0;
1168
	}
1169

1170 1171
	dev_priv->rps.last_adj = adj;

1172 1173 1174
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1175
	new_delay += adj;
1176
	new_delay = clamp_t(int, new_delay, min, max);
1177

1178
	intel_set_rps(dev_priv->dev, new_delay);
1179

1180
	mutex_unlock(&dev_priv->rps.hw_lock);
1181 1182
out:
	ENABLE_RPM_WAKEREF_ASSERTS(dev_priv);
1183 1184
}

1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1197 1198
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, l3_parity.error_work);
1199
	u32 error_status, row, bank, subbank;
1200
	char *parity_event[6];
1201
	uint32_t misccpctl;
1202
	uint8_t slice = 0;
1203 1204 1205 1206 1207 1208 1209

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
	mutex_lock(&dev_priv->dev->struct_mutex);

1210 1211 1212 1213
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1214 1215 1216 1217
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1218
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1219
		i915_reg_t reg;
1220

1221
		slice--;
1222
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv)))
1223
			break;
1224

1225
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1226

1227
		reg = GEN7_L3CDERRST1(slice);
1228

1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1244
		kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1245
				   KOBJ_CHANGE, parity_event);
1246

1247 1248
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1249

1250 1251 1252 1253 1254
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1255

1256
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1257

1258 1259
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
1260
	spin_lock_irq(&dev_priv->irq_lock);
1261
	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1262
	spin_unlock_irq(&dev_priv->irq_lock);
1263 1264

	mutex_unlock(&dev_priv->dev->struct_mutex);
1265 1266
}

1267
static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1268
{
1269
	struct drm_i915_private *dev_priv = dev->dev_private;
1270

1271
	if (!HAS_L3_DPF(dev))
1272 1273
		return;

1274
	spin_lock(&dev_priv->irq_lock);
1275
	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1276
	spin_unlock(&dev_priv->irq_lock);
1277

1278 1279 1280 1281 1282 1283 1284
	iir &= GT_PARITY_ERROR(dev);
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1285
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1286 1287
}

1288 1289 1290 1291 1292 1293
static void ilk_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1294
		notify_ring(&dev_priv->engine[RCS]);
1295
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
1296
		notify_ring(&dev_priv->engine[VCS]);
1297 1298
}

1299 1300 1301 1302 1303
static void snb_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{

1304 1305
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1306
		notify_ring(&dev_priv->engine[RCS]);
1307
	if (gt_iir & GT_BSD_USER_INTERRUPT)
1308
		notify_ring(&dev_priv->engine[VCS]);
1309
	if (gt_iir & GT_BLT_USER_INTERRUPT)
1310
		notify_ring(&dev_priv->engine[BCS]);
1311

1312 1313
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
1314 1315
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1316

1317 1318
	if (gt_iir & GT_PARITY_ERROR(dev))
		ivybridge_parity_error_irq_handler(dev, gt_iir);
1319 1320
}

1321
static __always_inline void
1322
gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
1323 1324
{
	if (iir & (GT_RENDER_USER_INTERRUPT << test_shift))
1325
		notify_ring(engine);
1326
	if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift))
1327
		tasklet_schedule(&engine->irq_tasklet);
1328 1329
}

C
Chris Wilson 已提交
1330
static irqreturn_t gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
1331 1332 1333 1334 1335
				       u32 master_ctl)
{
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1336 1337 1338
		u32 iir = I915_READ_FW(GEN8_GT_IIR(0));
		if (iir) {
			I915_WRITE_FW(GEN8_GT_IIR(0), iir);
1339
			ret = IRQ_HANDLED;
1340

1341 1342
			gen8_cs_irq_handler(&dev_priv->engine[RCS],
					    iir, GEN8_RCS_IRQ_SHIFT);
C
Chris Wilson 已提交
1343

1344 1345
			gen8_cs_irq_handler(&dev_priv->engine[BCS],
					    iir, GEN8_BCS_IRQ_SHIFT);
1346 1347 1348 1349
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1350
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1351 1352 1353
		u32 iir = I915_READ_FW(GEN8_GT_IIR(1));
		if (iir) {
			I915_WRITE_FW(GEN8_GT_IIR(1), iir);
1354
			ret = IRQ_HANDLED;
1355

1356 1357
			gen8_cs_irq_handler(&dev_priv->engine[VCS],
					    iir, GEN8_VCS1_IRQ_SHIFT);
1358

1359 1360
			gen8_cs_irq_handler(&dev_priv->engine[VCS2],
					    iir, GEN8_VCS2_IRQ_SHIFT);
1361
		} else
1362
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
1363 1364
	}

1365
	if (master_ctl & GEN8_GT_VECS_IRQ) {
1366 1367 1368
		u32 iir = I915_READ_FW(GEN8_GT_IIR(3));
		if (iir) {
			I915_WRITE_FW(GEN8_GT_IIR(3), iir);
1369
			ret = IRQ_HANDLED;
1370

1371 1372
			gen8_cs_irq_handler(&dev_priv->engine[VECS],
					    iir, GEN8_VECS_IRQ_SHIFT);
1373 1374 1375 1376
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

1377
	if (master_ctl & GEN8_GT_PM_IRQ) {
1378 1379
		u32 iir = I915_READ_FW(GEN8_GT_IIR(2));
		if (iir & dev_priv->pm_rps_events) {
1380
			I915_WRITE_FW(GEN8_GT_IIR(2),
1381
				      iir & dev_priv->pm_rps_events);
1382
			ret = IRQ_HANDLED;
1383
			gen6_rps_irq_handler(dev_priv, iir);
1384 1385 1386 1387
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1388 1389 1390
	return ret;
}

1391 1392 1393 1394
static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
1395
		return val & PORTA_HOTPLUG_LONG_DETECT;
1396 1397 1398 1399 1400 1401 1402 1403 1404
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1405 1406 1407 1408 1409 1410 1411 1412 1413 1414
static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_E:
		return val & PORTE_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430
static bool spt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & PORTA_HOTPLUG_LONG_DETECT;
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	case PORT_D:
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1431 1432 1433 1434 1435 1436 1437 1438 1439 1440
static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1441
static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1442 1443 1444
{
	switch (port) {
	case PORT_B:
1445
		return val & PORTB_HOTPLUG_LONG_DETECT;
1446
	case PORT_C:
1447
		return val & PORTC_HOTPLUG_LONG_DETECT;
1448
	case PORT_D:
1449 1450 1451
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
1452 1453 1454
	}
}

1455
static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1456 1457 1458
{
	switch (port) {
	case PORT_B:
1459
		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1460
	case PORT_C:
1461
		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1462
	case PORT_D:
1463 1464 1465
		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
	default:
		return false;
1466 1467 1468
	}
}

1469 1470 1471 1472 1473 1474 1475
/*
 * Get a bit mask of pins that have triggered, and which ones may be long.
 * This can be called multiple times with the same masks to accumulate
 * hotplug detection results from several registers.
 *
 * Note that the caller is expected to zero out the masks initially.
 */
1476
static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1477
			     u32 hotplug_trigger, u32 dig_hotplug_reg,
1478 1479
			     const u32 hpd[HPD_NUM_PINS],
			     bool long_pulse_detect(enum port port, u32 val))
1480
{
1481
	enum port port;
1482 1483 1484
	int i;

	for_each_hpd_pin(i) {
1485 1486
		if ((hpd[i] & hotplug_trigger) == 0)
			continue;
1487

1488 1489
		*pin_mask |= BIT(i);

1490 1491 1492
		if (!intel_hpd_pin_to_port(i, &port))
			continue;

1493
		if (long_pulse_detect(port, dig_hotplug_reg))
1494
			*long_mask |= BIT(i);
1495 1496 1497 1498 1499 1500 1501
	}

	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
			 hotplug_trigger, dig_hotplug_reg, *pin_mask);

}

1502 1503
static void gmbus_irq_handler(struct drm_device *dev)
{
1504
	struct drm_i915_private *dev_priv = dev->dev_private;
1505 1506

	wake_up_all(&dev_priv->gmbus_wait_queue);
1507 1508
}

1509 1510
static void dp_aux_irq_handler(struct drm_device *dev)
{
1511
	struct drm_i915_private *dev_priv = dev->dev_private;
1512 1513

	wake_up_all(&dev_priv->gmbus_wait_queue);
1514 1515
}

1516
#if defined(CONFIG_DEBUG_FS)
1517 1518 1519 1520
static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1521 1522 1523 1524
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
1525
	int head, tail;
1526

1527 1528
	spin_lock(&pipe_crc->lock);

1529
	if (!pipe_crc->entries) {
1530
		spin_unlock(&pipe_crc->lock);
1531
		DRM_DEBUG_KMS("spurious interrupt\n");
1532 1533 1534
		return;
	}

1535 1536
	head = pipe_crc->head;
	tail = pipe_crc->tail;
1537 1538

	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1539
		spin_unlock(&pipe_crc->lock);
1540 1541 1542 1543 1544
		DRM_ERROR("CRC buffer overflowing\n");
		return;
	}

	entry = &pipe_crc->entries[head];
1545

1546
	entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1547 1548 1549 1550 1551
	entry->crc[0] = crc0;
	entry->crc[1] = crc1;
	entry->crc[2] = crc2;
	entry->crc[3] = crc3;
	entry->crc[4] = crc4;
1552 1553

	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1554 1555 1556
	pipe_crc->head = head;

	spin_unlock(&pipe_crc->lock);
1557 1558

	wake_up_interruptible(&pipe_crc->wq);
1559
}
1560 1561 1562 1563 1564 1565 1566 1567
#else
static inline void
display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1568

1569
static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
D
Daniel Vetter 已提交
1570 1571 1572
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1573 1574 1575
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1576 1577
}

1578
static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1579 1580 1581
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1582 1583 1584 1585 1586 1587
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1588
}
1589

1590
static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1591 1592
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
	uint32_t res1, res2;

	if (INTEL_INFO(dev)->gen >= 3)
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

	if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1604

1605 1606 1607 1608 1609
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1610
}
1611

1612 1613 1614 1615
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1616
{
1617
	if (pm_iir & dev_priv->pm_rps_events) {
1618
		spin_lock(&dev_priv->irq_lock);
1619
		gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
I
Imre Deak 已提交
1620 1621 1622 1623
		if (dev_priv->rps.interrupts_enabled) {
			dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
			queue_work(dev_priv->wq, &dev_priv->rps.work);
		}
1624
		spin_unlock(&dev_priv->irq_lock);
1625 1626
	}

1627 1628 1629
	if (INTEL_INFO(dev_priv)->gen >= 8)
		return;

1630
	if (HAS_VEBOX(dev_priv)) {
1631
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1632
			notify_ring(&dev_priv->engine[VECS]);
B
Ben Widawsky 已提交
1633

1634 1635
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
B
Ben Widawsky 已提交
1636
	}
1637 1638
}

1639 1640 1641 1642 1643 1644 1645 1646
static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
{
	if (!drm_handle_vblank(dev, pipe))
		return false;

	return true;
}

1647 1648 1649
static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1650
	u32 pipe_stats[I915_MAX_PIPES] = { };
1651 1652
	int pipe;

1653
	spin_lock(&dev_priv->irq_lock);
1654 1655 1656 1657 1658 1659

	if (!dev_priv->display_irqs_enabled) {
		spin_unlock(&dev_priv->irq_lock);
		return;
	}

1660
	for_each_pipe(dev_priv, pipe) {
1661
		i915_reg_t reg;
1662
		u32 mask, iir_bit = 0;
1663

1664 1665 1666 1667 1668 1669 1670
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
1671 1672 1673

		/* fifo underruns are filterered in the underrun handler. */
		mask = PIPE_FIFO_UNDERRUN_STATUS;
1674 1675 1676 1677 1678 1679 1680 1681

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1682 1683 1684
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1685 1686 1687 1688 1689
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1690 1691 1692
			continue;

		reg = PIPESTAT(pipe);
1693 1694
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1695 1696 1697 1698

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1699 1700
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1701 1702
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1703
	spin_unlock(&dev_priv->irq_lock);
1704

1705
	for_each_pipe(dev_priv, pipe) {
1706 1707 1708
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
1709

1710
		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1711 1712 1713 1714 1715 1716 1717
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip(dev, pipe);
		}

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
			i9xx_pipe_crc_irq_handler(dev, pipe);

1718 1719
		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1720 1721 1722 1723 1724 1725
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		gmbus_irq_handler(dev);
}

1726 1727 1728 1729
static void i9xx_hpd_irq_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1730
	u32 pin_mask = 0, long_mask = 0;
1731

1732 1733
	if (!hotplug_status)
		return;
1734

1735 1736 1737 1738 1739 1740
	I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
	/*
	 * Make sure hotplug status is cleared before we clear IIR, or else we
	 * may miss hotplug events.
	 */
	POSTING_READ(PORT_HOTPLUG_STAT);
1741

1742
	if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
1743
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1744

1745 1746 1747 1748 1749 1750 1751
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
					   hotplug_trigger, hpd_status_g4x,
					   i9xx_port_hotplug_long_detect);

			intel_hpd_irq_handler(dev, pin_mask, long_mask);
		}
1752 1753 1754

		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
			dp_aux_irq_handler(dev);
1755 1756
	} else {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1757

1758 1759
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1760
					   hotplug_trigger, hpd_status_i915,
1761 1762 1763
					   i9xx_port_hotplug_long_detect);
			intel_hpd_irq_handler(dev, pin_mask, long_mask);
		}
1764
	}
1765 1766
}

1767
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1768
{
1769
	struct drm_device *dev = arg;
1770
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
1771 1772 1773
	u32 iir, gt_iir, pm_iir;
	irqreturn_t ret = IRQ_NONE;

1774 1775 1776
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1777 1778 1779
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

J
Jesse Barnes 已提交
1780
	while (true) {
1781 1782
		/* Find, clear, then process each source of interrupt */

J
Jesse Barnes 已提交
1783
		gt_iir = I915_READ(GTIIR);
1784 1785 1786
		if (gt_iir)
			I915_WRITE(GTIIR, gt_iir);

J
Jesse Barnes 已提交
1787
		pm_iir = I915_READ(GEN6_PMIIR);
1788 1789 1790 1791 1792 1793 1794 1795 1796 1797
		if (pm_iir)
			I915_WRITE(GEN6_PMIIR, pm_iir);

		iir = I915_READ(VLV_IIR);
		if (iir) {
			/* Consume port before clearing IIR or we'll miss events */
			if (iir & I915_DISPLAY_PORT_INTERRUPT)
				i9xx_hpd_irq_handler(dev);
			I915_WRITE(VLV_IIR, iir);
		}
J
Jesse Barnes 已提交
1798 1799 1800 1801 1802 1803

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
			goto out;

		ret = IRQ_HANDLED;

1804 1805
		if (gt_iir)
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
1806
		if (pm_iir)
1807
			gen6_rps_irq_handler(dev_priv, pm_iir);
1808 1809 1810
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
		valleyview_pipestat_irq_handler(dev, iir);
J
Jesse Barnes 已提交
1811 1812 1813
	}

out:
1814 1815
	enable_rpm_wakeref_asserts(dev_priv);

J
Jesse Barnes 已提交
1816 1817 1818
	return ret;
}

1819 1820
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1821
	struct drm_device *dev = arg;
1822 1823 1824 1825
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl, iir;
	irqreturn_t ret = IRQ_NONE;

1826 1827 1828
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1829 1830 1831
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1832
	do {
1833 1834
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
1835

1836 1837
		if (master_ctl == 0 && iir == 0)
			break;
1838

1839 1840
		ret = IRQ_HANDLED;

1841
		I915_WRITE(GEN8_MASTER_IRQ, 0);
1842

1843
		/* Find, clear, then process each source of interrupt */
1844

1845 1846 1847 1848 1849 1850
		if (iir) {
			/* Consume port before clearing IIR or we'll miss events */
			if (iir & I915_DISPLAY_PORT_INTERRUPT)
				i9xx_hpd_irq_handler(dev);
			I915_WRITE(VLV_IIR, iir);
		}
1851

C
Chris Wilson 已提交
1852
		gen8_gt_irq_handler(dev_priv, master_ctl);
1853

1854 1855 1856
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
		valleyview_pipestat_irq_handler(dev, iir);
1857

1858 1859
		I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		POSTING_READ(GEN8_MASTER_IRQ);
1860
	} while (0);
1861

1862 1863
	enable_rpm_wakeref_asserts(dev_priv);

1864 1865 1866
	return ret;
}

1867 1868 1869 1870 1871 1872
static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

1873 1874 1875 1876 1877 1878
	/*
	 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
	 * unless we touch the hotplug register, even if hotplug_trigger is
	 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
	 * errors.
	 */
1879
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1880 1881 1882 1883 1884 1885 1886 1887
	if (!hotplug_trigger) {
		u32 mask = PORTA_HOTPLUG_STATUS_MASK |
			PORTD_HOTPLUG_STATUS_MASK |
			PORTC_HOTPLUG_STATUS_MASK |
			PORTB_HOTPLUG_STATUS_MASK;
		dig_hotplug_reg &= ~mask;
	}

1888
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1889 1890
	if (!hotplug_trigger)
		return;
1891 1892 1893 1894 1895 1896 1897 1898

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   pch_port_hotplug_long_detect);

	intel_hpd_irq_handler(dev, pin_mask, long_mask);
}

1899
static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1900
{
1901
	struct drm_i915_private *dev_priv = dev->dev_private;
1902
	int pipe;
1903
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1904

1905
	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
1906

1907 1908 1909
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
1910
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1911 1912
				 port_name(port));
	}
1913

1914 1915 1916
	if (pch_iir & SDE_AUX_MASK)
		dp_aux_irq_handler(dev);

1917
	if (pch_iir & SDE_GMBUS)
1918
		gmbus_irq_handler(dev);
1919 1920 1921 1922 1923 1924 1925 1926 1927 1928

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1929
	if (pch_iir & SDE_FDI_MASK)
1930
		for_each_pipe(dev_priv, pipe)
1931 1932 1933
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
1934 1935 1936 1937 1938 1939 1940 1941

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1942
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1943 1944

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
1945
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1946 1947 1948 1949 1950 1951
}

static void ivb_err_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
1952
	enum pipe pipe;
1953

1954 1955 1956
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

1957
	for_each_pipe(dev_priv, pipe) {
1958 1959
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1960

D
Daniel Vetter 已提交
1961 1962
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
			if (IS_IVYBRIDGE(dev))
1963
				ivb_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1964
			else
1965
				hsw_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1966 1967
		}
	}
1968

1969 1970 1971 1972 1973 1974 1975 1976
	I915_WRITE(GEN7_ERR_INT, err_int);
}

static void cpt_serr_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 serr_int = I915_READ(SERR_INT);

1977 1978 1979
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1980
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
1981
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
1982 1983

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
1984
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
1985 1986

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
1987
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
1988 1989

	I915_WRITE(SERR_INT, serr_int);
1990 1991
}

1992 1993
static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
{
1994
	struct drm_i915_private *dev_priv = dev->dev_private;
1995
	int pipe;
1996
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1997

1998
	ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
1999

2000 2001 2002 2003 2004 2005
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
2006 2007

	if (pch_iir & SDE_AUX_MASK_CPT)
2008
		dp_aux_irq_handler(dev);
2009 2010

	if (pch_iir & SDE_GMBUS_CPT)
2011
		gmbus_irq_handler(dev);
2012 2013 2014 2015 2016 2017 2018 2019

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
2020
		for_each_pipe(dev_priv, pipe)
2021 2022 2023
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2024 2025 2026

	if (pch_iir & SDE_ERROR_CPT)
		cpt_serr_int_handler(dev);
2027 2028
}

2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044
static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		~SDE_PORTE_HOTPLUG_SPT;
	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
	u32 pin_mask = 0, long_mask = 0;

	if (hotplug_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
				   dig_hotplug_reg, hpd_spt,
2045
				   spt_port_hotplug_long_detect);
2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065
	}

	if (hotplug2_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
				   dig_hotplug_reg, hpd_spt,
				   spt_port_hotplug2_long_detect);
	}

	if (pin_mask)
		intel_hpd_irq_handler(dev, pin_mask, long_mask);

	if (pch_iir & SDE_GMBUS_CPT)
		gmbus_irq_handler(dev);
}

2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081
static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   ilk_port_hotplug_long_detect);

	intel_hpd_irq_handler(dev, pin_mask, long_mask);
}

2082 2083 2084
static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2085
	enum pipe pipe;
2086 2087
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;

2088 2089
	if (hotplug_trigger)
		ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
2090 2091 2092 2093 2094 2095 2096 2097 2098 2099

	if (de_iir & DE_AUX_CHANNEL_A)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE)
		intel_opregion_asle_intr(dev);

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2100
	for_each_pipe(dev_priv, pipe) {
2101 2102 2103
		if (de_iir & DE_PIPE_VBLANK(pipe) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2104

2105
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2106
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2107

2108 2109
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
			i9xx_pipe_crc_irq_handler(dev, pipe);
2110

2111 2112 2113 2114 2115
		/* plane/pipes map 1:1 on ilk+ */
		if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
		}
2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

		if (HAS_PCH_CPT(dev))
			cpt_irq_handler(dev, pch_iir);
		else
			ibx_irq_handler(dev, pch_iir);

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

	if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev);
}

2135 2136 2137
static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2138
	enum pipe pipe;
2139 2140
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;

2141 2142
	if (hotplug_trigger)
		ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
2143 2144 2145 2146 2147 2148 2149 2150 2151 2152

	if (de_iir & DE_ERR_INT_IVB)
		ivb_err_int_handler(dev);

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE_IVB)
		intel_opregion_asle_intr(dev);

2153
	for_each_pipe(dev_priv, pipe) {
2154 2155 2156
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2157 2158

		/* plane/pipes map 1:1 on ilk+ */
2159 2160 2161
		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175
		}
	}

	/* check event from PCH */
	if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
		u32 pch_iir = I915_READ(SDEIIR);

		cpt_irq_handler(dev, pch_iir);

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2176 2177 2178 2179 2180 2181 2182 2183
/*
 * To handle irqs with the minimum potential races with fresh interrupts, we:
 * 1 - Disable Master Interrupt Control.
 * 2 - Find the source(s) of the interrupt.
 * 3 - Clear the Interrupt Identity bits (IIR).
 * 4 - Process the interrupt(s) that had bits set in the IIRs.
 * 5 - Re-enable Master Interrupt Control.
 */
2184
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2185
{
2186
	struct drm_device *dev = arg;
2187
	struct drm_i915_private *dev_priv = dev->dev_private;
2188
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2189
	irqreturn_t ret = IRQ_NONE;
2190

2191 2192 2193
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2194 2195 2196
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

2197 2198 2199
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2200
	POSTING_READ(DEIER);
2201

2202 2203 2204 2205 2206
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2207 2208 2209 2210 2211
	if (!HAS_PCH_NOP(dev)) {
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2212

2213 2214
	/* Find, clear, then process each source of interrupt */

2215
	gt_iir = I915_READ(GTIIR);
2216
	if (gt_iir) {
2217 2218
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2219
		if (INTEL_INFO(dev)->gen >= 6)
2220
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
2221 2222
		else
			ilk_gt_irq_handler(dev, dev_priv, gt_iir);
2223 2224
	}

2225 2226
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2227 2228
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2229 2230 2231 2232
		if (INTEL_INFO(dev)->gen >= 7)
			ivb_display_irq_handler(dev, de_iir);
		else
			ilk_display_irq_handler(dev, de_iir);
2233 2234
	}

2235 2236 2237 2238 2239
	if (INTEL_INFO(dev)->gen >= 6) {
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
2240
			gen6_rps_irq_handler(dev_priv, pm_iir);
2241
		}
2242
	}
2243 2244 2245

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2246 2247 2248 2249
	if (!HAS_PCH_NOP(dev)) {
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2250

2251 2252 2253
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	enable_rpm_wakeref_asserts(dev_priv);

2254 2255 2256
	return ret;
}

2257 2258
static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
				const u32 hpd[HPD_NUM_PINS])
2259
{
2260 2261
	struct drm_i915_private *dev_priv = to_i915(dev);
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2262

2263 2264
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2265

2266
	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2267
			   dig_hotplug_reg, hpd,
2268
			   bxt_port_hotplug_long_detect);
2269

2270
	intel_hpd_irq_handler(dev, pin_mask, long_mask);
2271 2272
}

2273 2274
static irqreturn_t
gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2275
{
2276
	struct drm_device *dev = dev_priv->dev;
2277
	irqreturn_t ret = IRQ_NONE;
2278
	u32 iir;
2279
	enum pipe pipe;
J
Jesse Barnes 已提交
2280

2281
	if (master_ctl & GEN8_DE_MISC_IRQ) {
2282 2283 2284
		iir = I915_READ(GEN8_DE_MISC_IIR);
		if (iir) {
			I915_WRITE(GEN8_DE_MISC_IIR, iir);
2285
			ret = IRQ_HANDLED;
2286
			if (iir & GEN8_DE_MISC_GSE)
2287 2288 2289
				intel_opregion_asle_intr(dev);
			else
				DRM_ERROR("Unexpected DE Misc interrupt\n");
2290
		}
2291 2292
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2293 2294
	}

2295
	if (master_ctl & GEN8_DE_PORT_IRQ) {
2296 2297 2298
		iir = I915_READ(GEN8_DE_PORT_IIR);
		if (iir) {
			u32 tmp_mask;
2299
			bool found = false;
2300

2301
			I915_WRITE(GEN8_DE_PORT_IIR, iir);
2302
			ret = IRQ_HANDLED;
J
Jesse Barnes 已提交
2303

2304 2305 2306 2307 2308 2309 2310
			tmp_mask = GEN8_AUX_CHANNEL_A;
			if (INTEL_INFO(dev_priv)->gen >= 9)
				tmp_mask |= GEN9_AUX_CHANNEL_B |
					    GEN9_AUX_CHANNEL_C |
					    GEN9_AUX_CHANNEL_D;

			if (iir & tmp_mask) {
2311
				dp_aux_irq_handler(dev);
2312 2313 2314
				found = true;
			}

2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326
			if (IS_BROXTON(dev_priv)) {
				tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
				if (tmp_mask) {
					bxt_hpd_irq_handler(dev, tmp_mask, hpd_bxt);
					found = true;
				}
			} else if (IS_BROADWELL(dev_priv)) {
				tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
				if (tmp_mask) {
					ilk_hpd_irq_handler(dev, tmp_mask, hpd_bdw);
					found = true;
				}
2327 2328
			}

2329
			if (IS_BROXTON(dev) && (iir & BXT_DE_PORT_GMBUS)) {
S
Shashank Sharma 已提交
2330 2331 2332 2333
				gmbus_irq_handler(dev);
				found = true;
			}

2334
			if (!found)
2335
				DRM_ERROR("Unexpected DE Port interrupt\n");
2336
		}
2337 2338
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2339 2340
	}

2341
	for_each_pipe(dev_priv, pipe) {
2342
		u32 flip_done, fault_errors;
2343

2344 2345
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2346

2347 2348 2349 2350 2351
		iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (!iir) {
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
			continue;
		}
2352

2353 2354
		ret = IRQ_HANDLED;
		I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2355

2356 2357 2358
		if (iir & GEN8_PIPE_VBLANK &&
		    intel_pipe_handle_vblank(dev, pipe))
			intel_check_page_flip(dev, pipe);
2359

2360 2361 2362 2363 2364
		flip_done = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			flip_done &= GEN9_PIPE_PLANE1_FLIP_DONE;
		else
			flip_done &= GEN8_PIPE_PRIMARY_FLIP_DONE;
2365

2366 2367 2368 2369
		if (flip_done) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
		}
2370

2371 2372
		if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
			hsw_pipe_crc_irq_handler(dev, pipe);
2373

2374 2375
		if (iir & GEN8_PIPE_FIFO_UNDERRUN)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2376

2377 2378 2379 2380 2381
		fault_errors = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
		else
			fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2382

2383 2384 2385 2386
		if (fault_errors)
			DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
				  pipe_name(pipe),
				  fault_errors);
2387 2388
	}

2389 2390
	if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
	    master_ctl & GEN8_DE_PCH_IRQ) {
2391 2392 2393 2394 2395
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
2396 2397 2398
		iir = I915_READ(SDEIIR);
		if (iir) {
			I915_WRITE(SDEIIR, iir);
2399
			ret = IRQ_HANDLED;
2400 2401

			if (HAS_PCH_SPT(dev_priv))
2402
				spt_irq_handler(dev, iir);
2403
			else
2404
				cpt_irq_handler(dev, iir);
2405 2406 2407 2408 2409 2410 2411
		} else {
			/*
			 * Like on previous PCH there seems to be something
			 * fishy going on with forwarding PCH interrupts.
			 */
			DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
		}
2412 2413
	}

2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440
	return ret;
}

static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl;
	irqreturn_t ret;

	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);

	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	/* Find, clear, then process each source of interrupt */
	ret = gen8_gt_irq_handler(dev_priv, master_ctl);
	ret |= gen8_de_irq_handler(dev_priv, master_ctl);

2441 2442
	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ_FW(GEN8_MASTER_IRQ);
2443

2444 2445
	enable_rpm_wakeref_asserts(dev_priv);

2446 2447 2448
	return ret;
}

2449 2450 2451
static void i915_error_wake_up(struct drm_i915_private *dev_priv,
			       bool reset_completed)
{
2452
	struct intel_engine_cs *engine;
2453 2454 2455 2456 2457 2458 2459 2460 2461

	/*
	 * Notify all waiters for GPU completion events that reset state has
	 * been changed, and that they need to restart their wait after
	 * checking for potential errors (and bail out to drop locks if there is
	 * a gpu reset pending so that i915_error_work_func can acquire them).
	 */

	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2462
	for_each_engine(engine, dev_priv)
2463
		wake_up_all(&engine->irq_queue);
2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475

	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
	wake_up_all(&dev_priv->pending_flip_queue);

	/*
	 * Signal tasks blocked in i915_gem_wait_for_error that the pending
	 * reset state is cleared.
	 */
	if (reset_completed)
		wake_up_all(&dev_priv->gpu_error.reset_queue);
}

2476
/**
2477
 * i915_reset_and_wakeup - do process context error handling work
2478
 * @dev: drm device
2479 2480 2481 2482
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
2483
static void i915_reset_and_wakeup(struct drm_device *dev)
2484
{
2485 2486
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct i915_gpu_error *error = &dev_priv->gpu_error;
2487 2488 2489
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2490
	int ret;
2491

2492
	kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2493

2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504
	/*
	 * Note that there's only one work item which does gpu resets, so we
	 * need not worry about concurrent gpu resets potentially incrementing
	 * error->reset_counter twice. We only need to take care of another
	 * racing irq/hangcheck declaring the gpu dead for a second time. A
	 * quick check for that is good enough: schedule_work ensures the
	 * correct ordering between hang detection and this work item, and since
	 * the reset in-progress bit is only ever set by code outside of this
	 * work we don't need to worry about any other races.
	 */
	if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2505
		DRM_DEBUG_DRIVER("resetting chip\n");
2506
		kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2507
				   reset_event);
2508

2509 2510 2511 2512 2513 2514 2515 2516
		/*
		 * In most cases it's guaranteed that we get here with an RPM
		 * reference held, for example because there is a pending GPU
		 * request that won't finish until the reset is done. This
		 * isn't the case at least when we get here by doing a
		 * simulated reset via debugs, so get an RPM reference.
		 */
		intel_runtime_pm_get(dev_priv);
2517 2518 2519

		intel_prepare_reset(dev);

2520 2521 2522 2523 2524 2525
		/*
		 * All state reset _must_ be completed before we update the
		 * reset counter, for otherwise waiters might miss the reset
		 * pending state and not properly drop locks, resulting in
		 * deadlocks with the reset work.
		 */
2526 2527
		ret = i915_reset(dev);

2528
		intel_finish_reset(dev);
2529

2530 2531
		intel_runtime_pm_put(dev_priv);

2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542
		if (ret == 0) {
			/*
			 * After all the gem state is reset, increment the reset
			 * counter and wake up everyone waiting for the reset to
			 * complete.
			 *
			 * Since unlock operations are a one-sided barrier only,
			 * we need to insert a barrier here to order any seqno
			 * updates before
			 * the counter increment.
			 */
2543
			smp_mb__before_atomic();
2544 2545
			atomic_inc(&dev_priv->gpu_error.reset_counter);

2546
			kobject_uevent_env(&dev->primary->kdev->kobj,
2547
					   KOBJ_CHANGE, reset_done_event);
2548
		} else {
2549
			atomic_or(I915_WEDGED, &error->reset_counter);
2550
		}
2551

2552 2553 2554 2555 2556
		/*
		 * Note: The wake_up also serves as a memory barrier so that
		 * waiters see the update value of the reset counter atomic_t.
		 */
		i915_error_wake_up(dev_priv, true);
2557
	}
2558 2559
}

2560
static void i915_report_and_clear_eir(struct drm_device *dev)
2561 2562
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2563
	uint32_t instdone[I915_NUM_INSTDONE_REG];
2564
	u32 eir = I915_READ(EIR);
2565
	int pipe, i;
2566

2567 2568
	if (!eir)
		return;
2569

2570
	pr_err("render error detected, EIR: 0x%08x\n", eir);
2571

2572 2573
	i915_get_extra_instdone(dev, instdone);

2574 2575 2576 2577
	if (IS_G4X(dev)) {
		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
			u32 ipeir = I915_READ(IPEIR_I965);

2578 2579
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2580 2581
			for (i = 0; i < ARRAY_SIZE(instdone); i++)
				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2582 2583
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2584
			I915_WRITE(IPEIR_I965, ipeir);
2585
			POSTING_READ(IPEIR_I965);
2586 2587 2588
		}
		if (eir & GM45_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2589 2590
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2591
			I915_WRITE(PGTBL_ER, pgtbl_err);
2592
			POSTING_READ(PGTBL_ER);
2593 2594 2595
		}
	}

2596
	if (!IS_GEN2(dev)) {
2597 2598
		if (eir & I915_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2599 2600
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2601
			I915_WRITE(PGTBL_ER, pgtbl_err);
2602
			POSTING_READ(PGTBL_ER);
2603 2604 2605 2606
		}
	}

	if (eir & I915_ERROR_MEMORY_REFRESH) {
2607
		pr_err("memory refresh error:\n");
2608
		for_each_pipe(dev_priv, pipe)
2609
			pr_err("pipe %c stat: 0x%08x\n",
2610
			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2611 2612 2613
		/* pipestat has already been acked */
	}
	if (eir & I915_ERROR_INSTRUCTION) {
2614 2615
		pr_err("instruction error\n");
		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2616 2617
		for (i = 0; i < ARRAY_SIZE(instdone); i++)
			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2618
		if (INTEL_INFO(dev)->gen < 4) {
2619 2620
			u32 ipeir = I915_READ(IPEIR);

2621 2622 2623
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2624
			I915_WRITE(IPEIR, ipeir);
2625
			POSTING_READ(IPEIR);
2626 2627 2628
		} else {
			u32 ipeir = I915_READ(IPEIR_I965);

2629 2630 2631 2632
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2633
			I915_WRITE(IPEIR_I965, ipeir);
2634
			POSTING_READ(IPEIR_I965);
2635 2636 2637 2638
		}
	}

	I915_WRITE(EIR, eir);
2639
	POSTING_READ(EIR);
2640 2641 2642 2643 2644 2645 2646 2647 2648 2649
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2650 2651 2652
}

/**
2653
 * i915_handle_error - handle a gpu error
2654
 * @dev: drm device
2655
 * @engine_mask: mask representing engines that are hung
2656
 * Do some basic checking of register state at error time and
2657 2658 2659 2660 2661
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
2662
void i915_handle_error(struct drm_device *dev, u32 engine_mask,
2663
		       const char *fmt, ...)
2664 2665
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2666 2667
	va_list args;
	char error_msg[80];
2668

2669 2670 2671 2672
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

2673
	i915_capture_error_state(dev, engine_mask, error_msg);
2674
	i915_report_and_clear_eir(dev);
2675

2676
	if (engine_mask) {
2677
		atomic_or(I915_RESET_IN_PROGRESS_FLAG,
2678
				&dev_priv->gpu_error.reset_counter);
2679

2680
		/*
2681 2682 2683
		 * Wakeup waiting processes so that the reset function
		 * i915_reset_and_wakeup doesn't deadlock trying to grab
		 * various locks. By bumping the reset counter first, the woken
2684 2685 2686 2687 2688 2689 2690 2691
		 * processes will see a reset in progress and back off,
		 * releasing their locks and then wait for the reset completion.
		 * We must do this for _all_ gpu waiters that might hold locks
		 * that the reset work needs to acquire.
		 *
		 * Note: The wake_up serves as the required memory barrier to
		 * ensure that the waiters see the updated value of the reset
		 * counter atomic_t.
2692
		 */
2693
		i915_error_wake_up(dev_priv, false);
2694 2695
	}

2696
	i915_reset_and_wakeup(dev);
2697 2698
}

2699 2700 2701
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2702
static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
2703
{
2704
	struct drm_i915_private *dev_priv = dev->dev_private;
2705
	unsigned long irqflags;
2706

2707
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2708
	if (INTEL_INFO(dev)->gen >= 4)
2709
		i915_enable_pipestat(dev_priv, pipe,
2710
				     PIPE_START_VBLANK_INTERRUPT_STATUS);
2711
	else
2712
		i915_enable_pipestat(dev_priv, pipe,
2713
				     PIPE_VBLANK_INTERRUPT_STATUS);
2714
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2715

2716 2717 2718
	return 0;
}

2719
static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
2720
{
2721
	struct drm_i915_private *dev_priv = dev->dev_private;
2722
	unsigned long irqflags;
2723
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2724
						     DE_PIPE_VBLANK(pipe);
2725 2726

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2727
	ilk_enable_display_irq(dev_priv, bit);
2728 2729 2730 2731 2732
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2733
static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2734
{
2735
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2736 2737 2738
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2739
	i915_enable_pipestat(dev_priv, pipe,
2740
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2741 2742 2743 2744 2745
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2746
static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
2747 2748 2749 2750 2751
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2752
	bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2753
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2754

2755 2756 2757
	return 0;
}

2758 2759 2760
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2761
static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
2762
{
2763
	struct drm_i915_private *dev_priv = dev->dev_private;
2764
	unsigned long irqflags;
2765

2766
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2767
	i915_disable_pipestat(dev_priv, pipe,
2768 2769
			      PIPE_VBLANK_INTERRUPT_STATUS |
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2770 2771 2772
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2773
static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
2774
{
2775
	struct drm_i915_private *dev_priv = dev->dev_private;
2776
	unsigned long irqflags;
2777
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2778
						     DE_PIPE_VBLANK(pipe);
2779 2780

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2781
	ilk_disable_display_irq(dev_priv, bit);
2782 2783 2784
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2785
static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2786
{
2787
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2788 2789 2790
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2791
	i915_disable_pipestat(dev_priv, pipe,
2792
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2793 2794 2795
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2796
static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
2797 2798 2799 2800 2801
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2802
	bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2803 2804 2805
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2806
static bool
2807
ring_idle(struct intel_engine_cs *engine, u32 seqno)
2808
{
2809 2810
	return i915_seqno_passed(seqno,
				 READ_ONCE(engine->last_submitted_seqno));
B
Ben Gamari 已提交
2811 2812
}

2813 2814 2815 2816
static bool
ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
{
	if (INTEL_INFO(dev)->gen >= 8) {
2817
		return (ipehr >> 23) == 0x1c;
2818 2819 2820 2821 2822 2823 2824
	} else {
		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
				 MI_SEMAPHORE_REGISTER);
	}
}

2825
static struct intel_engine_cs *
2826 2827
semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr,
				 u64 offset)
2828
{
2829
	struct drm_i915_private *dev_priv = engine->dev->dev_private;
2830
	struct intel_engine_cs *signaller;
2831

2832
	if (INTEL_INFO(dev_priv)->gen >= 8) {
2833
		for_each_engine(signaller, dev_priv) {
2834
			if (engine == signaller)
2835 2836
				continue;

2837
			if (offset == signaller->semaphore.signal_ggtt[engine->id])
2838 2839
				return signaller;
		}
2840 2841 2842
	} else {
		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;

2843
		for_each_engine(signaller, dev_priv) {
2844
			if(engine == signaller)
2845 2846
				continue;

2847
			if (sync_bits == signaller->semaphore.mbox.wait[engine->id])
2848 2849 2850 2851
				return signaller;
		}
	}

2852
	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2853
		  engine->id, ipehr, offset);
2854 2855 2856 2857

	return NULL;
}

2858
static struct intel_engine_cs *
2859
semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
2860
{
2861
	struct drm_i915_private *dev_priv = engine->dev->dev_private;
2862
	u32 cmd, ipehr, head;
2863 2864
	u64 offset = 0;
	int i, backwards;
2865

2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882
	/*
	 * This function does not support execlist mode - any attempt to
	 * proceed further into this function will result in a kernel panic
	 * when dereferencing ring->buffer, which is not set up in execlist
	 * mode.
	 *
	 * The correct way of doing it would be to derive the currently
	 * executing ring buffer from the current context, which is derived
	 * from the currently running request. Unfortunately, to get the
	 * current request we would have to grab the struct_mutex before doing
	 * anything else, which would be ill-advised since some other thread
	 * might have grabbed it already and managed to hang itself, causing
	 * the hang checker to deadlock.
	 *
	 * Therefore, this function does not support execlist mode in its
	 * current form. Just return NULL and move on.
	 */
2883
	if (engine->buffer == NULL)
2884 2885
		return NULL;

2886 2887
	ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
	if (!ipehr_is_semaphore_wait(engine->dev, ipehr))
2888
		return NULL;
2889

2890 2891 2892
	/*
	 * HEAD is likely pointing to the dword after the actual command,
	 * so scan backwards until we find the MBOX. But limit it to just 3
2893 2894
	 * or 4 dwords depending on the semaphore wait command size.
	 * Note that we don't care about ACTHD here since that might
2895 2896
	 * point at at batch, and semaphores are always emitted into the
	 * ringbuffer itself.
2897
	 */
2898 2899
	head = I915_READ_HEAD(engine) & HEAD_ADDR;
	backwards = (INTEL_INFO(engine->dev)->gen >= 8) ? 5 : 4;
2900

2901
	for (i = backwards; i; --i) {
2902 2903 2904 2905 2906
		/*
		 * Be paranoid and presume the hw has gone off into the wild -
		 * our ring is smaller than what the hardware (and hence
		 * HEAD_ADDR) allows. Also handles wrap-around.
		 */
2907
		head &= engine->buffer->size - 1;
2908 2909

		/* This here seems to blow up */
2910
		cmd = ioread32(engine->buffer->virtual_start + head);
2911 2912 2913
		if (cmd == ipehr)
			break;

2914 2915
		head -= 4;
	}
2916

2917 2918
	if (!i)
		return NULL;
2919

2920 2921 2922
	*seqno = ioread32(engine->buffer->virtual_start + head + 4) + 1;
	if (INTEL_INFO(engine->dev)->gen >= 8) {
		offset = ioread32(engine->buffer->virtual_start + head + 12);
2923
		offset <<= 32;
2924
		offset = ioread32(engine->buffer->virtual_start + head + 8);
2925
	}
2926
	return semaphore_wait_to_signaller_ring(engine, ipehr, offset);
2927 2928
}

2929
static int semaphore_passed(struct intel_engine_cs *engine)
2930
{
2931
	struct drm_i915_private *dev_priv = engine->dev->dev_private;
2932
	struct intel_engine_cs *signaller;
2933
	u32 seqno;
2934

2935
	engine->hangcheck.deadlock++;
2936

2937
	signaller = semaphore_waits_for(engine, &seqno);
2938 2939 2940 2941
	if (signaller == NULL)
		return -1;

	/* Prevent pathological recursion due to driver bugs */
2942
	if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES)
2943 2944
		return -1;

2945
	if (i915_seqno_passed(signaller->get_seqno(signaller), seqno))
2946 2947
		return 1;

2948 2949 2950
	/* cursory check for an unkickable deadlock */
	if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
	    semaphore_passed(signaller) < 0)
2951 2952 2953
		return -1;

	return 0;
2954 2955 2956 2957
}

static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
{
2958
	struct intel_engine_cs *engine;
2959

2960
	for_each_engine(engine, dev_priv)
2961
		engine->hangcheck.deadlock = 0;
2962 2963
}

2964
static bool subunits_stuck(struct intel_engine_cs *engine)
2965
{
2966 2967 2968 2969
	u32 instdone[I915_NUM_INSTDONE_REG];
	bool stuck;
	int i;

2970
	if (engine->id != RCS)
2971 2972
		return true;

2973
	i915_get_extra_instdone(engine->dev, instdone);
2974

2975 2976 2977 2978 2979 2980 2981
	/* There might be unstable subunit states even when
	 * actual head is not moving. Filter out the unstable ones by
	 * accumulating the undone -> done transitions and only
	 * consider those as progress.
	 */
	stuck = true;
	for (i = 0; i < I915_NUM_INSTDONE_REG; i++) {
2982
		const u32 tmp = instdone[i] | engine->hangcheck.instdone[i];
2983

2984
		if (tmp != engine->hangcheck.instdone[i])
2985 2986
			stuck = false;

2987
		engine->hangcheck.instdone[i] |= tmp;
2988 2989 2990 2991 2992 2993
	}

	return stuck;
}

static enum intel_ring_hangcheck_action
2994
head_stuck(struct intel_engine_cs *engine, u64 acthd)
2995
{
2996
	if (acthd != engine->hangcheck.acthd) {
2997 2998

		/* Clear subunit states on head movement */
2999 3000
		memset(engine->hangcheck.instdone, 0,
		       sizeof(engine->hangcheck.instdone));
3001

3002
		return HANGCHECK_ACTIVE;
3003
	}
3004

3005
	if (!subunits_stuck(engine))
3006 3007 3008 3009 3010 3011
		return HANGCHECK_ACTIVE;

	return HANGCHECK_HUNG;
}

static enum intel_ring_hangcheck_action
3012
ring_stuck(struct intel_engine_cs *engine, u64 acthd)
3013
{
3014
	struct drm_device *dev = engine->dev;
3015 3016 3017 3018
	struct drm_i915_private *dev_priv = dev->dev_private;
	enum intel_ring_hangcheck_action ha;
	u32 tmp;

3019
	ha = head_stuck(engine, acthd);
3020 3021 3022
	if (ha != HANGCHECK_HUNG)
		return ha;

3023
	if (IS_GEN2(dev))
3024
		return HANGCHECK_HUNG;
3025 3026 3027 3028 3029 3030

	/* Is the chip hanging on a WAIT_FOR_EVENT?
	 * If so we can simply poke the RB_WAIT bit
	 * and break the hang. This should work on
	 * all but the second generation chipsets.
	 */
3031
	tmp = I915_READ_CTL(engine);
3032
	if (tmp & RING_WAIT) {
3033
		i915_handle_error(dev, 0,
3034
				  "Kicking stuck wait on %s",
3035 3036
				  engine->name);
		I915_WRITE_CTL(engine, tmp);
3037
		return HANGCHECK_KICK;
3038 3039 3040
	}

	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
3041
		switch (semaphore_passed(engine)) {
3042
		default:
3043
			return HANGCHECK_HUNG;
3044
		case 1:
3045
			i915_handle_error(dev, 0,
3046
					  "Kicking stuck semaphore on %s",
3047 3048
					  engine->name);
			I915_WRITE_CTL(engine, tmp);
3049
			return HANGCHECK_KICK;
3050
		case 0:
3051
			return HANGCHECK_WAIT;
3052
		}
3053
	}
3054

3055
	return HANGCHECK_HUNG;
3056 3057
}

3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075
static unsigned kick_waiters(struct intel_engine_cs *engine)
{
	struct drm_i915_private *i915 = to_i915(engine->dev);
	unsigned user_interrupts = READ_ONCE(engine->user_interrupts);

	if (engine->hangcheck.user_interrupts == user_interrupts &&
	    !test_and_set_bit(engine->id, &i915->gpu_error.missed_irq_rings)) {
		if (!(i915->gpu_error.test_irq_rings & intel_engine_flag(engine)))
			DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
				  engine->name);
		else
			DRM_INFO("Fake missed irq on %s\n",
				 engine->name);
		wake_up_all(&engine->irq_queue);
	}

	return user_interrupts;
}
3076
/*
B
Ben Gamari 已提交
3077
 * This is called when the chip hasn't reported back with completed
3078 3079 3080 3081 3082
 * batchbuffers in a long time. We keep track per ring seqno progress and
 * if there are no progress, hangcheck score for that ring is increased.
 * Further, acthd is inspected to see if the ring is stuck. On stuck case
 * we kick the ring. If we see no progress on three subsequent calls
 * we assume chip is wedged and try to fix it by resetting the chip.
B
Ben Gamari 已提交
3083
 */
3084
static void i915_hangcheck_elapsed(struct work_struct *work)
B
Ben Gamari 已提交
3085
{
3086 3087 3088 3089
	struct drm_i915_private *dev_priv =
		container_of(work, typeof(*dev_priv),
			     gpu_error.hangcheck_work.work);
	struct drm_device *dev = dev_priv->dev;
3090
	struct intel_engine_cs *engine;
3091
	enum intel_engine_id id;
3092
	int busy_count = 0, rings_hung = 0;
3093
	bool stuck[I915_NUM_ENGINES] = { 0 };
3094 3095 3096
#define BUSY 1
#define KICK 5
#define HUNG 20
3097
#define ACTIVE_DECAY 15
3098

3099
	if (!i915.enable_hangcheck)
3100 3101
		return;

3102 3103 3104 3105 3106 3107 3108
	/*
	 * The hangcheck work is synced during runtime suspend, we don't
	 * require a wakeref. TODO: instead of disabling the asserts make
	 * sure that we hold a reference when this work is running.
	 */
	DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);

3109 3110 3111 3112 3113 3114
	/* As enabling the GPU requires fairly extensive mmio access,
	 * periodically arm the mmio checker to see if we are triggering
	 * any invalid access.
	 */
	intel_uncore_arm_unclaimed_mmio_detection(dev_priv);

3115
	for_each_engine_id(engine, dev_priv, id) {
3116 3117
		u64 acthd;
		u32 seqno;
3118
		unsigned user_interrupts;
3119
		bool busy = true;
3120

3121 3122
		semaphore_clear_deadlocks(dev_priv);

3123 3124 3125 3126 3127 3128 3129 3130 3131 3132
		/* We don't strictly need an irq-barrier here, as we are not
		 * serving an interrupt request, be paranoid in case the
		 * barrier has side-effects (such as preventing a broken
		 * cacheline snoop) and so be sure that we can see the seqno
		 * advance. If the seqno should stick, due to a stale
		 * cacheline, we would erroneously declare the GPU hung.
		 */
		if (engine->irq_seqno_barrier)
			engine->irq_seqno_barrier(engine);

3133
		acthd = intel_ring_get_active_head(engine);
3134
		seqno = engine->get_seqno(engine);
3135

3136 3137 3138
		/* Reset stuck interrupts between batch advances */
		user_interrupts = 0;

3139 3140 3141 3142
		if (engine->hangcheck.seqno == seqno) {
			if (ring_idle(engine, seqno)) {
				engine->hangcheck.action = HANGCHECK_IDLE;
				if (waitqueue_active(&engine->irq_queue)) {
3143
					/* Safeguard against driver failure */
3144
					user_interrupts = kick_waiters(engine);
3145
					engine->hangcheck.score += BUSY;
3146 3147
				} else
					busy = false;
3148
			} else {
3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163
				/* We always increment the hangcheck score
				 * if the ring is busy and still processing
				 * the same request, so that no single request
				 * can run indefinitely (such as a chain of
				 * batches). The only time we do not increment
				 * the hangcheck score on this ring, if this
				 * ring is in a legitimate wait for another
				 * ring. In that case the waiting ring is a
				 * victim and we want to be sure we catch the
				 * right culprit. Then every time we do kick
				 * the ring, add a small increment to the
				 * score so that we can catch a batch that is
				 * being repeatedly kicked and so responsible
				 * for stalling the machine.
				 */
3164 3165
				engine->hangcheck.action = ring_stuck(engine,
								      acthd);
3166

3167
				switch (engine->hangcheck.action) {
3168
				case HANGCHECK_IDLE:
3169
				case HANGCHECK_WAIT:
3170
					break;
3171
				case HANGCHECK_ACTIVE:
3172
					engine->hangcheck.score += BUSY;
3173
					break;
3174
				case HANGCHECK_KICK:
3175
					engine->hangcheck.score += KICK;
3176
					break;
3177
				case HANGCHECK_HUNG:
3178
					engine->hangcheck.score += HUNG;
3179
					stuck[id] = true;
3180 3181
					break;
				}
3182
			}
3183
		} else {
3184
			engine->hangcheck.action = HANGCHECK_ACTIVE;
3185

3186 3187 3188
			/* Gradually reduce the count so that we catch DoS
			 * attempts across multiple batches.
			 */
3189 3190 3191 3192
			if (engine->hangcheck.score > 0)
				engine->hangcheck.score -= ACTIVE_DECAY;
			if (engine->hangcheck.score < 0)
				engine->hangcheck.score = 0;
3193

3194
			/* Clear head and subunit states on seqno movement */
3195
			acthd = 0;
3196

3197 3198
			memset(engine->hangcheck.instdone, 0,
			       sizeof(engine->hangcheck.instdone));
3199 3200
		}

3201 3202
		engine->hangcheck.seqno = seqno;
		engine->hangcheck.acthd = acthd;
3203
		engine->hangcheck.user_interrupts = user_interrupts;
3204
		busy_count += busy;
3205
	}
3206

3207
	for_each_engine_id(engine, dev_priv, id) {
3208
		if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
3209
			DRM_INFO("%s on %s\n",
3210
				 stuck[id] ? "stuck" : "no progress",
3211
				 engine->name);
3212
			rings_hung |= intel_engine_flag(engine);
3213 3214 3215
		}
	}

3216
	if (rings_hung) {
3217
		i915_handle_error(dev, rings_hung, "Engine(s) hung");
3218 3219
		goto out;
	}
B
Ben Gamari 已提交
3220

3221 3222 3223
	if (busy_count)
		/* Reset timer case chip hangs without another request
		 * being added */
3224
		i915_queue_hangcheck(dev);
3225 3226 3227

out:
	ENABLE_RPM_WAKEREF_ASSERTS(dev_priv);
3228 3229 3230 3231
}

void i915_queue_hangcheck(struct drm_device *dev)
{
3232
	struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
3233

3234
	if (!i915.enable_hangcheck)
3235 3236
		return;

3237 3238 3239 3240 3241 3242 3243
	/* Don't continually defer the hangcheck so that it is always run at
	 * least once after work has been scheduled on any ring. Otherwise,
	 * we will ignore a hung ring if a second ring is kept busy.
	 */

	queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
			   round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
B
Ben Gamari 已提交
3244 3245
}

3246
static void ibx_irq_reset(struct drm_device *dev)
P
Paulo Zanoni 已提交
3247 3248 3249 3250 3251 3252
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

3253
	GEN5_IRQ_RESET(SDE);
3254 3255 3256

	if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
3257
}
3258

P
Paulo Zanoni 已提交
3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
3275 3276 3277 3278
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

3279
static void gen5_gt_irq_reset(struct drm_device *dev)
3280 3281 3282
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3283
	GEN5_IRQ_RESET(GT);
P
Paulo Zanoni 已提交
3284
	if (INTEL_INFO(dev)->gen >= 6)
3285
		GEN5_IRQ_RESET(GEN6_PM);
3286 3287
}

L
Linus Torvalds 已提交
3288 3289
/* drm_dma.h hooks
*/
P
Paulo Zanoni 已提交
3290
static void ironlake_irq_reset(struct drm_device *dev)
3291
{
3292
	struct drm_i915_private *dev_priv = dev->dev_private;
3293

3294
	I915_WRITE(HWSTAM, 0xffffffff);
3295

3296
	GEN5_IRQ_RESET(DE);
3297 3298
	if (IS_GEN7(dev))
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3299

3300
	gen5_gt_irq_reset(dev);
3301

3302
	ibx_irq_reset(dev);
3303
}
3304

3305 3306 3307 3308
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
	enum pipe pipe;

3309
	i915_hotplug_interrupt_update(dev_priv, 0xFFFFFFFF, 0);
3310 3311 3312 3313 3314 3315 3316 3317
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), 0xffff);

	GEN5_IRQ_RESET(VLV_);
}

J
Jesse Barnes 已提交
3318 3319
static void valleyview_irq_preinstall(struct drm_device *dev)
{
3320
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3321

3322
	gen5_gt_irq_reset(dev);
J
Jesse Barnes 已提交
3323

3324
	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
J
Jesse Barnes 已提交
3325

3326
	vlv_display_irq_reset(dev_priv);
J
Jesse Barnes 已提交
3327 3328
}

3329 3330 3331 3332 3333 3334 3335 3336
static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
{
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
}

P
Paulo Zanoni 已提交
3337
static void gen8_irq_reset(struct drm_device *dev)
3338 3339 3340 3341 3342 3343 3344
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3345
	gen8_gt_irq_reset(dev_priv);
3346

3347
	for_each_pipe(dev_priv, pipe)
3348 3349
		if (intel_display_power_is_enabled(dev_priv,
						   POWER_DOMAIN_PIPE(pipe)))
3350
			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3351

3352 3353 3354
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3355

3356 3357
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_reset(dev);
3358
}
3359

3360 3361
void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
3362
{
3363
	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3364
	enum pipe pipe;
3365

3366
	spin_lock_irq(&dev_priv->irq_lock);
3367 3368 3369 3370
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
				  dev_priv->de_irq_mask[pipe],
				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
3371
	spin_unlock_irq(&dev_priv->irq_lock);
3372 3373
}

3374 3375 3376
void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
{
3377 3378
	enum pipe pipe;

3379
	spin_lock_irq(&dev_priv->irq_lock);
3380 3381
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3382 3383 3384 3385 3386 3387
	spin_unlock_irq(&dev_priv->irq_lock);

	/* make sure we're done processing display irqs */
	synchronize_irq(dev_priv->dev->irq);
}

3388 3389 3390 3391 3392 3393 3394
static void cherryview_irq_preinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3395
	gen8_gt_irq_reset(dev_priv);
3396 3397 3398 3399 3400

	GEN5_IRQ_RESET(GEN8_PCU_);

	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);

3401
	vlv_display_irq_reset(dev_priv);
3402 3403
}

3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417
static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
				  const u32 hpd[HPD_NUM_PINS])
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct intel_encoder *encoder;
	u32 enabled_irqs = 0;

	for_each_intel_encoder(dev, encoder)
		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
			enabled_irqs |= hpd[encoder->hpd_pin];

	return enabled_irqs;
}

3418
static void ibx_hpd_irq_setup(struct drm_device *dev)
3419
{
3420
	struct drm_i915_private *dev_priv = dev->dev_private;
3421
	u32 hotplug_irqs, hotplug, enabled_irqs;
3422 3423

	if (HAS_PCH_IBX(dev)) {
3424
		hotplug_irqs = SDE_HOTPLUG_MASK;
3425
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
3426
	} else {
3427
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3428
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
3429
	}
3430

3431
	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3432 3433 3434

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
3435 3436
	 * duration to 2ms (which is the minimum in the Display Port spec).
	 * The pulse duration bits are reserved on LPT+.
3437
	 */
3438 3439 3440 3441 3442
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3443 3444 3445 3446 3447 3448
	/*
	 * When CPU and PCH are on the same package, port A
	 * HPD must be enabled in both north and south.
	 */
	if (HAS_PCH_LPT_LP(dev))
		hotplug |= PORTA_HOTPLUG_ENABLE;
3449
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3450
}
X
Xiong Zhang 已提交
3451

3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464
static void spt_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_irqs, hotplug, enabled_irqs;

	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	/* Enable digital hotplug on the PCH */
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTC_HOTPLUG_ENABLE |
3465
		PORTB_HOTPLUG_ENABLE | PORTA_HOTPLUG_ENABLE;
3466 3467 3468 3469 3470
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);

	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
	hotplug |= PORTE_HOTPLUG_ENABLE;
	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3471 3472
}

3473 3474 3475 3476 3477
static void ilk_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_irqs, hotplug, enabled_irqs;

3478 3479 3480 3481 3482 3483
	if (INTEL_INFO(dev)->gen >= 8) {
		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);

		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
	} else if (INTEL_INFO(dev)->gen >= 7) {
3484 3485
		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
3486 3487

		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3488 3489 3490
	} else {
		hotplug_irqs = DE_DP_A_HOTPLUG;
		enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
3491

3492 3493
		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
	}
3494 3495 3496 3497

	/*
	 * Enable digital hotplug on the CPU, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
3498
	 * The pulse duration bits are reserved on HSW+.
3499 3500 3501 3502 3503 3504 3505 3506 3507
	 */
	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);

	ibx_hpd_irq_setup(dev);
}

3508 3509 3510
static void bxt_hpd_irq_setup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
3511
	u32 hotplug_irqs, hotplug, enabled_irqs;
3512

3513 3514
	enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
3515

3516
	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3517

3518 3519 3520
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTB_HOTPLUG_ENABLE |
		PORTA_HOTPLUG_ENABLE;
3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540

	DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n",
		      hotplug, enabled_irqs);
	hotplug &= ~BXT_DDI_HPD_INVERT_MASK;

	/*
	 * For BXT invert bit has to be set based on AOB design
	 * for HPD detection logic, update it based on VBT fields.
	 */

	if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
		hotplug |= BXT_DDIA_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
		hotplug |= BXT_DDIB_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
		hotplug |= BXT_DDIC_HPD_INVERT;

3541
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3542 3543
}

P
Paulo Zanoni 已提交
3544 3545
static void ibx_irq_postinstall(struct drm_device *dev)
{
3546
	struct drm_i915_private *dev_priv = dev->dev_private;
3547
	u32 mask;
3548

D
Daniel Vetter 已提交
3549 3550 3551
	if (HAS_PCH_NOP(dev))
		return;

3552
	if (HAS_PCH_IBX(dev))
3553
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3554
	else
3555
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3556

3557
	gen5_assert_iir_is_zero(dev_priv, SDEIIR);
P
Paulo Zanoni 已提交
3558 3559 3560
	I915_WRITE(SDEIMR, ~mask);
}

3561 3562 3563 3564 3565 3566 3567 3568
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3569
	if (HAS_L3_DPF(dev)) {
3570
		/* L3 parity interrupt is always unmasked. */
3571 3572
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
		gt_irqs |= GT_PARITY_ERROR(dev);
3573 3574 3575 3576 3577 3578 3579 3580 3581 3582
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
	if (IS_GEN5(dev)) {
		gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
			   ILK_BSD_USER_INTERRUPT;
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3583
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3584 3585

	if (INTEL_INFO(dev)->gen >= 6) {
3586 3587 3588 3589
		/*
		 * RPS interrupts will get enabled/disabled on demand when RPS
		 * itself is enabled/disabled.
		 */
3590 3591 3592
		if (HAS_VEBOX(dev))
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;

3593
		dev_priv->pm_irq_mask = 0xffffffff;
P
Paulo Zanoni 已提交
3594
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3595 3596 3597
	}
}

3598
static int ironlake_irq_postinstall(struct drm_device *dev)
3599
{
3600
	struct drm_i915_private *dev_priv = dev->dev_private;
3601 3602 3603 3604 3605 3606
	u32 display_mask, extra_mask;

	if (INTEL_INFO(dev)->gen >= 7) {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3607
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3608
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3609 3610
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
			      DE_DP_A_HOTPLUG_IVB);
3611 3612 3613
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3614 3615 3616
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3617 3618 3619
		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
			      DE_DP_A_HOTPLUG);
3620
	}
3621

3622
	dev_priv->irq_mask = ~display_mask;
3623

3624 3625
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3626 3627
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3628
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3629

3630
	gen5_gt_irq_postinstall(dev);
3631

P
Paulo Zanoni 已提交
3632
	ibx_irq_postinstall(dev);
3633

3634
	if (IS_IRONLAKE_M(dev)) {
3635 3636 3637
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3638 3639
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
3640
		spin_lock_irq(&dev_priv->irq_lock);
3641
		ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
3642
		spin_unlock_irq(&dev_priv->irq_lock);
3643 3644
	}

3645 3646 3647
	return 0;
}

3648 3649 3650 3651
static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;
3652
	enum pipe pipe;
3653 3654 3655 3656

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;

3657 3658
	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3659 3660 3661 3662 3663
	POSTING_READ(PIPESTAT(PIPE_A));

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

3664 3665 3666
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		      i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
3667 3668 3669 3670

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3671 3672
	if (IS_CHERRYVIEW(dev_priv))
		iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3673 3674 3675 3676 3677
	dev_priv->irq_mask &= ~iir_mask;

	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3678 3679
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	POSTING_READ(VLV_IMR);
3680 3681 3682 3683 3684 3685
}

static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;
3686
	enum pipe pipe;
3687 3688 3689

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3690
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3691 3692
	if (IS_CHERRYVIEW(dev_priv))
		iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3693 3694 3695

	dev_priv->irq_mask |= iir_mask;
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3696
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3697 3698 3699 3700 3701 3702 3703
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	POSTING_READ(VLV_IIR);

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

3704 3705 3706
	i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
3707 3708 3709

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;
3710 3711 3712

	for_each_pipe(dev_priv, pipe)
		I915_WRITE(PIPESTAT(pipe), pipestat_mask);
3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724
	POSTING_READ(PIPESTAT(PIPE_A));
}

void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

3725
	if (intel_irqs_enabled(dev_priv))
3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737
		valleyview_display_irqs_install(dev_priv);
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

3738
	if (intel_irqs_enabled(dev_priv))
3739 3740 3741
		valleyview_display_irqs_uninstall(dev_priv);
}

3742
static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
J
Jesse Barnes 已提交
3743
{
3744
	dev_priv->irq_mask = ~0;
J
Jesse Barnes 已提交
3745

3746
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3747 3748
	POSTING_READ(PORT_HOTPLUG_EN);

J
Jesse Barnes 已提交
3749
	I915_WRITE(VLV_IIR, 0xffffffff);
3750 3751 3752 3753
	I915_WRITE(VLV_IIR, 0xffffffff);
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	POSTING_READ(VLV_IMR);
J
Jesse Barnes 已提交
3754

3755 3756
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3757
	spin_lock_irq(&dev_priv->irq_lock);
3758 3759
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_install(dev_priv);
3760
	spin_unlock_irq(&dev_priv->irq_lock);
3761 3762 3763 3764 3765 3766 3767
}

static int valleyview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	vlv_display_irq_postinstall(dev_priv);
J
Jesse Barnes 已提交
3768

3769
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3770 3771 3772 3773 3774 3775 3776 3777

	/* ack & enable invalid PTE error interrupts */
#if 0 /* FIXME: add support to irq handler for checking these bits */
	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
#endif

	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3778 3779 3780 3781

	return 0;
}

3782 3783 3784 3785 3786
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3787
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3788
			GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
3789 3790
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3791
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3792 3793 3794
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3795
		0,
3796 3797
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3798 3799
		};

3800
	dev_priv->pm_irq_mask = 0xffffffff;
3801 3802
	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3803 3804 3805 3806 3807
	/*
	 * RPS interrupts will get enabled/disabled on demand when RPS itself
	 * is enabled/disabled.
	 */
	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, 0);
3808
	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3809 3810 3811 3812
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
3813 3814
	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
	uint32_t de_pipe_enables;
3815 3816 3817
	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
	u32 de_port_enables;
	enum pipe pipe;
3818

3819
	if (INTEL_INFO(dev_priv)->gen >= 9) {
3820 3821
		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3822 3823
		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
				  GEN9_AUX_CHANNEL_D;
S
Shashank Sharma 已提交
3824
		if (IS_BROXTON(dev_priv))
3825 3826
			de_port_masked |= BXT_DE_PORT_GMBUS;
	} else {
3827 3828
		de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
				  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3829
	}
3830 3831 3832 3833

	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
					   GEN8_PIPE_FIFO_UNDERRUN;

3834
	de_port_enables = de_port_masked;
3835 3836 3837
	if (IS_BROXTON(dev_priv))
		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
	else if (IS_BROADWELL(dev_priv))
3838 3839
		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;

3840 3841 3842
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3843

3844
	for_each_pipe(dev_priv, pipe)
3845
		if (intel_display_power_is_enabled(dev_priv,
3846 3847 3848 3849
				POWER_DOMAIN_PIPE(pipe)))
			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
					  dev_priv->de_irq_mask[pipe],
					  de_pipe_enables);
3850

3851
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3852 3853 3854 3855 3856 3857
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3858 3859
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_pre_postinstall(dev);
P
Paulo Zanoni 已提交
3860

3861 3862 3863
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

3864 3865
	if (HAS_PCH_SPLIT(dev))
		ibx_irq_postinstall(dev);
3866 3867 3868 3869 3870 3871 3872

	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3873 3874 3875 3876
static int cherryview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3877
	vlv_display_irq_postinstall(dev_priv);
3878 3879 3880 3881 3882 3883 3884 3885 3886

	gen8_gt_irq_postinstall(dev_priv);

	I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3887 3888 3889 3890 3891 3892 3893
static void gen8_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3894
	gen8_irq_reset(dev);
3895 3896
}

3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907
static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
{
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
	spin_lock_irq(&dev_priv->irq_lock);
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_uninstall(dev_priv);
	spin_unlock_irq(&dev_priv->irq_lock);

	vlv_display_irq_reset(dev_priv);

3908
	dev_priv->irq_mask = ~0;
3909 3910
}

J
Jesse Barnes 已提交
3911 3912
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3913
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3914 3915 3916 3917

	if (!dev_priv)
		return;

3918 3919
	I915_WRITE(VLV_MASTER_IER, 0);

3920 3921
	gen5_gt_irq_reset(dev);

J
Jesse Barnes 已提交
3922
	I915_WRITE(HWSTAM, 0xffffffff);
3923

3924
	vlv_display_irq_uninstall(dev_priv);
J
Jesse Barnes 已提交
3925 3926
}

3927 3928 3929 3930 3931 3932 3933 3934 3935 3936
static void cherryview_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3937
	gen8_gt_irq_reset(dev_priv);
3938

3939
	GEN5_IRQ_RESET(GEN8_PCU_);
3940

3941
	vlv_display_irq_uninstall(dev_priv);
3942 3943
}

3944
static void ironlake_irq_uninstall(struct drm_device *dev)
3945
{
3946
	struct drm_i915_private *dev_priv = dev->dev_private;
3947 3948 3949 3950

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3951
	ironlake_irq_reset(dev);
3952 3953
}

3954
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3955
{
3956
	struct drm_i915_private *dev_priv = dev->dev_private;
3957
	int pipe;
3958

3959
	for_each_pipe(dev_priv, pipe)
3960
		I915_WRITE(PIPESTAT(pipe), 0);
3961 3962 3963
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3964 3965 3966 3967
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3968
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3969 3970 3971 3972 3973 3974 3975 3976 3977

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3978
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
C
Chris Wilson 已提交
3979 3980 3981 3982 3983 3984 3985 3986
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3987 3988
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3989
	spin_lock_irq(&dev_priv->irq_lock);
3990 3991
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3992
	spin_unlock_irq(&dev_priv->irq_lock);
3993

C
Chris Wilson 已提交
3994 3995 3996
	return 0;
}

3997 3998 3999 4000
/*
 * Returns true when a page flip has completed.
 */
static bool i8xx_handle_vblank(struct drm_device *dev,
4001
			       int plane, int pipe, u32 iir)
4002
{
4003
	struct drm_i915_private *dev_priv = dev->dev_private;
4004
	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
4005

4006
	if (!intel_pipe_handle_vblank(dev, pipe))
4007 4008 4009
		return false;

	if ((iir & flip_pending) == 0)
4010
		goto check_page_flip;
4011 4012 4013 4014 4015 4016 4017 4018

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ16(ISR) & flip_pending)
4019
		goto check_page_flip;
4020

4021
	intel_prepare_page_flip(dev, plane);
4022 4023
	intel_finish_page_flip(dev, pipe);
	return true;
4024 4025 4026 4027

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
4028 4029
}

4030
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
4031
{
4032
	struct drm_device *dev = arg;
4033
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
4034 4035 4036 4037 4038 4039
	u16 iir, new_iir;
	u32 pipe_stats[2];
	int pipe;
	u16 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4040
	irqreturn_t ret;
C
Chris Wilson 已提交
4041

4042 4043 4044
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4045 4046 4047 4048
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	ret = IRQ_NONE;
C
Chris Wilson 已提交
4049 4050
	iir = I915_READ16(IIR);
	if (iir == 0)
4051
		goto out;
C
Chris Wilson 已提交
4052 4053 4054 4055 4056 4057 4058

	while (iir & ~flip_mask) {
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4059
		spin_lock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
4060
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4061
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
C
Chris Wilson 已提交
4062

4063
		for_each_pipe(dev_priv, pipe) {
4064
			i915_reg_t reg = PIPESTAT(pipe);
C
Chris Wilson 已提交
4065 4066 4067 4068 4069
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
4070
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
4071 4072
				I915_WRITE(reg, pipe_stats[pipe]);
		}
4073
		spin_unlock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
4074 4075 4076 4077 4078

		I915_WRITE16(IIR, iir & ~flip_mask);
		new_iir = I915_READ16(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4079
			notify_ring(&dev_priv->engine[RCS]);
C
Chris Wilson 已提交
4080

4081
		for_each_pipe(dev_priv, pipe) {
4082
			int plane = pipe;
4083
			if (HAS_FBC(dev))
4084 4085
				plane = !plane;

4086
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
4087 4088
			    i8xx_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
C
Chris Wilson 已提交
4089

4090
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4091
				i9xx_pipe_crc_irq_handler(dev, pipe);
4092

4093 4094 4095
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
4096
		}
C
Chris Wilson 已提交
4097 4098 4099

		iir = new_iir;
	}
4100 4101 4102 4103
	ret = IRQ_HANDLED;

out:
	enable_rpm_wakeref_asserts(dev_priv);
C
Chris Wilson 已提交
4104

4105
	return ret;
C
Chris Wilson 已提交
4106 4107 4108 4109
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
4110
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
4111 4112
	int pipe;

4113
	for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
4114 4115 4116 4117 4118 4119 4120 4121 4122
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

4123 4124
static void i915_irq_preinstall(struct drm_device * dev)
{
4125
	struct drm_i915_private *dev_priv = dev->dev_private;
4126 4127 4128
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
4129
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4130 4131 4132
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4133
	I915_WRITE16(HWSTAM, 0xeffe);
4134
	for_each_pipe(dev_priv, pipe)
4135 4136 4137 4138 4139 4140 4141 4142
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
4143
	struct drm_i915_private *dev_priv = dev->dev_private;
4144
	u32 enable_mask;
4145

4146 4147 4148 4149 4150 4151 4152 4153
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4154
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4155 4156 4157 4158 4159 4160 4161

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_USER_INTERRUPT;

4162
	if (I915_HAS_HOTPLUG(dev)) {
4163
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4164 4165
		POSTING_READ(PORT_HOTPLUG_EN);

4166 4167 4168 4169 4170 4171 4172 4173 4174 4175
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4176
	i915_enable_asle_pipestat(dev);
4177

4178 4179
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4180
	spin_lock_irq(&dev_priv->irq_lock);
4181 4182
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4183
	spin_unlock_irq(&dev_priv->irq_lock);
4184

4185 4186 4187
	return 0;
}

4188 4189 4190 4191 4192 4193
/*
 * Returns true when a page flip has completed.
 */
static bool i915_handle_vblank(struct drm_device *dev,
			       int plane, int pipe, u32 iir)
{
4194
	struct drm_i915_private *dev_priv = dev->dev_private;
4195 4196
	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

4197
	if (!intel_pipe_handle_vblank(dev, pipe))
4198 4199 4200
		return false;

	if ((iir & flip_pending) == 0)
4201
		goto check_page_flip;
4202 4203 4204 4205 4206 4207 4208 4209

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ(ISR) & flip_pending)
4210
		goto check_page_flip;
4211

4212
	intel_prepare_page_flip(dev, plane);
4213 4214
	intel_finish_page_flip(dev, pipe);
	return true;
4215 4216 4217 4218

check_page_flip:
	intel_check_page_flip(dev, pipe);
	return false;
4219 4220
}

4221
static irqreturn_t i915_irq_handler(int irq, void *arg)
4222
{
4223
	struct drm_device *dev = arg;
4224
	struct drm_i915_private *dev_priv = dev->dev_private;
4225
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
4226 4227 4228 4229
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
	int pipe, ret = IRQ_NONE;
4230

4231 4232 4233
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4234 4235 4236
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

4237
	iir = I915_READ(IIR);
4238 4239
	do {
		bool irq_received = (iir & ~flip_mask) != 0;
4240
		bool blc_event = false;
4241 4242 4243 4244 4245 4246

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4247
		spin_lock(&dev_priv->irq_lock);
4248
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4249
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4250

4251
		for_each_pipe(dev_priv, pipe) {
4252
			i915_reg_t reg = PIPESTAT(pipe);
4253 4254
			pipe_stats[pipe] = I915_READ(reg);

4255
			/* Clear the PIPE*STAT regs before the IIR */
4256 4257
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4258
				irq_received = true;
4259 4260
			}
		}
4261
		spin_unlock(&dev_priv->irq_lock);
4262 4263 4264 4265 4266

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
4267 4268 4269
		if (I915_HAS_HOTPLUG(dev) &&
		    iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
4270

4271
		I915_WRITE(IIR, iir & ~flip_mask);
4272 4273 4274
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4275
			notify_ring(&dev_priv->engine[RCS]);
4276

4277
		for_each_pipe(dev_priv, pipe) {
4278
			int plane = pipe;
4279
			if (HAS_FBC(dev))
4280
				plane = !plane;
4281

4282
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
4283 4284
			    i915_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
4285 4286 4287

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4288 4289

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4290
				i9xx_pipe_crc_irq_handler(dev, pipe);
4291

4292 4293 4294
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
4315
		ret = IRQ_HANDLED;
4316
		iir = new_iir;
4317
	} while (iir & ~flip_mask);
4318

4319 4320
	enable_rpm_wakeref_asserts(dev_priv);

4321 4322 4323 4324 4325
	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
4326
	struct drm_i915_private *dev_priv = dev->dev_private;
4327 4328 4329
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
4330
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4331 4332 4333
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4334
	I915_WRITE16(HWSTAM, 0xffff);
4335
	for_each_pipe(dev_priv, pipe) {
4336
		/* Clear enable bits; then clear status bits */
4337
		I915_WRITE(PIPESTAT(pipe), 0);
4338 4339
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
4340 4341 4342 4343 4344 4345 4346 4347
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
4348
	struct drm_i915_private *dev_priv = dev->dev_private;
4349 4350
	int pipe;

4351
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4352
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4353 4354

	I915_WRITE(HWSTAM, 0xeffe);
4355
	for_each_pipe(dev_priv, pipe)
4356 4357 4358 4359 4360 4361 4362 4363
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
4364
	struct drm_i915_private *dev_priv = dev->dev_private;
4365
	u32 enable_mask;
4366 4367 4368
	u32 error_mask;

	/* Unmask the interrupts that we always want on. */
4369
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4370
			       I915_DISPLAY_PORT_INTERRUPT |
4371 4372 4373 4374 4375 4376 4377
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
4378 4379
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4380 4381 4382 4383
	enable_mask |= I915_USER_INTERRUPT;

	if (IS_G4X(dev))
		enable_mask |= I915_BSD_USER_INTERRUPT;
4384

4385 4386
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4387
	spin_lock_irq(&dev_priv->irq_lock);
4388 4389 4390
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4391
	spin_unlock_irq(&dev_priv->irq_lock);
4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
	if (IS_G4X(dev)) {
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4412
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4413 4414
	POSTING_READ(PORT_HOTPLUG_EN);

4415
	i915_enable_asle_pipestat(dev);
4416 4417 4418 4419

	return 0;
}

4420
static void i915_hpd_irq_setup(struct drm_device *dev)
4421
{
4422
	struct drm_i915_private *dev_priv = dev->dev_private;
4423 4424
	u32 hotplug_en;

4425 4426
	assert_spin_locked(&dev_priv->irq_lock);

4427 4428
	/* Note HDMI and DP share hotplug bits */
	/* enable bits are the same for all generations */
4429
	hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
4430 4431 4432 4433 4434 4435 4436 4437 4438
	/* Programming the CRT detection parameters tends
	   to generate a spurious hotplug event about three
	   seconds later.  So just do it once.
	*/
	if (IS_G4X(dev))
		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;

	/* Ignore TV since it's buggy */
4439
	i915_hotplug_interrupt_update_locked(dev_priv,
4440 4441 4442 4443
					     HOTPLUG_INT_EN_MASK |
					     CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
					     CRT_HOTPLUG_ACTIVATION_PERIOD_64,
					     hotplug_en);
4444 4445
}

4446
static irqreturn_t i965_irq_handler(int irq, void *arg)
4447
{
4448
	struct drm_device *dev = arg;
4449
	struct drm_i915_private *dev_priv = dev->dev_private;
4450 4451 4452
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	int ret = IRQ_NONE, pipe;
4453 4454 4455
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4456

4457 4458 4459
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4460 4461 4462
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

4463 4464 4465
	iir = I915_READ(IIR);

	for (;;) {
4466
		bool irq_received = (iir & ~flip_mask) != 0;
4467 4468
		bool blc_event = false;

4469 4470 4471 4472 4473
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4474
		spin_lock(&dev_priv->irq_lock);
4475
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4476
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4477

4478
		for_each_pipe(dev_priv, pipe) {
4479
			i915_reg_t reg = PIPESTAT(pipe);
4480 4481 4482 4483 4484 4485 4486
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4487
				irq_received = true;
4488 4489
			}
		}
4490
		spin_unlock(&dev_priv->irq_lock);
4491 4492 4493 4494 4495 4496 4497

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4498 4499
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
4500

4501
		I915_WRITE(IIR, iir & ~flip_mask);
4502 4503 4504
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4505
			notify_ring(&dev_priv->engine[RCS]);
4506
		if (iir & I915_BSD_USER_INTERRUPT)
4507
			notify_ring(&dev_priv->engine[VCS]);
4508

4509
		for_each_pipe(dev_priv, pipe) {
4510
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4511 4512
			    i915_handle_vblank(dev, pipe, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4513 4514 4515

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4516 4517

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4518
				i9xx_pipe_crc_irq_handler(dev, pipe);
4519

4520 4521
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4522
		}
4523 4524 4525 4526

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

4527 4528 4529
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
			gmbus_irq_handler(dev);

4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

4548 4549
	enable_rpm_wakeref_asserts(dev_priv);

4550 4551 4552 4553 4554
	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4555
	struct drm_i915_private *dev_priv = dev->dev_private;
4556 4557 4558 4559 4560
	int pipe;

	if (!dev_priv)
		return;

4561
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4562
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4563 4564

	I915_WRITE(HWSTAM, 0xffffffff);
4565
	for_each_pipe(dev_priv, pipe)
4566 4567 4568 4569
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

4570
	for_each_pipe(dev_priv, pipe)
4571 4572 4573 4574 4575
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4576 4577 4578 4579 4580 4581 4582
/**
 * intel_irq_init - initializes irq support
 * @dev_priv: i915 device instance
 *
 * This function initializes all the irq support including work items, timers
 * and all the vtables. It does not setup the interrupt itself though.
 */
4583
void intel_irq_init(struct drm_i915_private *dev_priv)
4584
{
4585
	struct drm_device *dev = dev_priv->dev;
4586

4587 4588
	intel_hpd_init_work(dev_priv);

4589
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4590
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4591

4592
	/* Let's track the enabled rps events */
4593
	if (IS_VALLEYVIEW(dev_priv))
4594
		/* WaGsvRC0ResidencyMethod:vlv */
4595
		dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4596 4597
	else
		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4598

4599 4600
	INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
			  i915_hangcheck_elapsed);
4601

4602
	if (IS_GEN2(dev_priv)) {
4603 4604
		dev->max_vblank_count = 0;
		dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
4605
	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4606
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4607
		dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4608 4609 4610
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4611 4612
	}

4613 4614 4615 4616 4617
	/*
	 * Opt out of the vblank disable timer on everything except gen2.
	 * Gen2 doesn't have a hardware frame counter and so depends on
	 * vblank interrupts to produce sane vblank seuquence numbers.
	 */
4618
	if (!IS_GEN2(dev_priv))
4619 4620
		dev->vblank_disable_immediate = true;

4621 4622
	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4623

4624
	if (IS_CHERRYVIEW(dev_priv)) {
4625 4626 4627 4628 4629 4630 4631
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4632
	} else if (IS_VALLEYVIEW(dev_priv)) {
J
Jesse Barnes 已提交
4633 4634 4635 4636 4637 4638
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
4639
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4640
	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4641
		dev->driver->irq_handler = gen8_irq_handler;
4642
		dev->driver->irq_preinstall = gen8_irq_reset;
4643 4644 4645 4646
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
4647
		if (IS_BROXTON(dev))
4648
			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4649 4650 4651
		else if (HAS_PCH_SPT(dev))
			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		else
4652
			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4653 4654
	} else if (HAS_PCH_SPLIT(dev)) {
		dev->driver->irq_handler = ironlake_irq_handler;
4655
		dev->driver->irq_preinstall = ironlake_irq_reset;
4656 4657 4658 4659
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4660
		dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4661
	} else {
4662
		if (INTEL_INFO(dev_priv)->gen == 2) {
C
Chris Wilson 已提交
4663 4664 4665 4666
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4667
		} else if (INTEL_INFO(dev_priv)->gen == 3) {
4668 4669 4670 4671
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
C
Chris Wilson 已提交
4672
		} else {
4673 4674 4675 4676
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
C
Chris Wilson 已提交
4677
		}
4678 4679
		if (I915_HAS_HOTPLUG(dev_priv))
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4680 4681 4682 4683
		dev->driver->enable_vblank = i915_enable_vblank;
		dev->driver->disable_vblank = i915_disable_vblank;
	}
}
4684

4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695
/**
 * intel_irq_install - enables the hardware interrupt
 * @dev_priv: i915 device instance
 *
 * This function enables the hardware interrupt handling, but leaves the hotplug
 * handling still disabled. It is called after intel_irq_init().
 *
 * In the driver load and resume code we need working interrupts in a few places
 * but don't want to deal with the hassle of concurrent probe and hotplug
 * workers. Hence the split into this two-stage approach.
 */
4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707
int intel_irq_install(struct drm_i915_private *dev_priv)
{
	/*
	 * We enable some interrupt sources in our postinstall hooks, so mark
	 * interrupts as enabled _before_ actually enabling them to avoid
	 * special cases in our ordering checks.
	 */
	dev_priv->pm.irqs_enabled = true;

	return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
}

4708 4709 4710 4711 4712 4713 4714
/**
 * intel_irq_uninstall - finilizes all irq handling
 * @dev_priv: i915 device instance
 *
 * This stops interrupt and hotplug handling and unregisters and frees all
 * resources acquired in the init functions.
 */
4715 4716 4717 4718 4719 4720 4721
void intel_irq_uninstall(struct drm_i915_private *dev_priv)
{
	drm_irq_uninstall(dev_priv->dev);
	intel_hpd_cancel_work(dev_priv);
	dev_priv->pm.irqs_enabled = false;
}

4722 4723 4724 4725 4726 4727 4728
/**
 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
 * @dev_priv: i915 device instance
 *
 * This function is used to disable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4729
void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4730
{
4731
	dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
4732
	dev_priv->pm.irqs_enabled = false;
4733
	synchronize_irq(dev_priv->dev->irq);
4734 4735
}

4736 4737 4738 4739 4740 4741 4742
/**
 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
 * @dev_priv: i915 device instance
 *
 * This function is used to enable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4743
void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4744
{
4745
	dev_priv->pm.irqs_enabled = true;
4746 4747
	dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
	dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
4748
}