i915_irq.c 125.3 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47
/**
 * DOC: interrupt handling
 *
 * These functions provide the basic support for enabling and disabling the
 * interrupt handling support. There's a lot more functionality in i915_irq.c
 * and related files, but that will be described in separate chapters.
 */

48 49 50 51
static const u32 hpd_ilk[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
};

52 53 54 55
static const u32 hpd_ivb[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
};

56 57 58 59
static const u32 hpd_bdw[HPD_NUM_PINS] = {
	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
};

60
static const u32 hpd_ibx[HPD_NUM_PINS] = {
61 62 63 64 65 66 67
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

68
static const u32 hpd_cpt[HPD_NUM_PINS] = {
69
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71 72 73 74 75
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

X
Xiong Zhang 已提交
76
static const u32 hpd_spt[HPD_NUM_PINS] = {
77
	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
X
Xiong Zhang 已提交
78 79 80 81 82 83
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
};

84
static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85 86 87 88 89 90 91 92
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

93
static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94 95 96 97 98 99 100 101
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

102
static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103 104 105 106 107 108 109 110
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

111 112
/* BXT hpd list */
static const u32 hpd_bxt[HPD_NUM_PINS] = {
113
	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114 115 116 117
	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
};

118
/* IIR can theoretically queue up two events. Be paranoid. */
119
#define GEN8_IRQ_RESET_NDX(type, which) do { \
120 121 122 123 124 125 126 127 128
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

129
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
130
	I915_WRITE(type##IMR, 0xffffffff); \
131
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
132
	I915_WRITE(type##IER, 0); \
133 134 135 136
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
137 138
} while (0)

139 140 141
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
142 143
static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
				    i915_reg_t reg)
144 145 146 147 148 149 150
{
	u32 val = I915_READ(reg);

	if (val == 0)
		return;

	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
151
	     i915_mmio_reg_offset(reg), val);
152 153 154 155 156
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
}
157

P
Paulo Zanoni 已提交
158
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
159
	gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
160
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
161 162
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
P
Paulo Zanoni 已提交
163 164 165
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
166
	gen5_assert_iir_is_zero(dev_priv, type##IIR); \
P
Paulo Zanoni 已提交
167
	I915_WRITE(type##IER, (ier_val)); \
168 169
	I915_WRITE(type##IMR, (imr_val)); \
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
170 171
} while (0)

172
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
173
static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
174

175 176 177 178 179 180 181 182
/* For display hotplug interrupt */
static inline void
i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
				     uint32_t mask,
				     uint32_t bits)
{
	uint32_t val;

183
	lockdep_assert_held(&dev_priv->irq_lock);
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
	WARN_ON(bits & ~mask);

	val = I915_READ(PORT_HOTPLUG_EN);
	val &= ~mask;
	val |= bits;
	I915_WRITE(PORT_HOTPLUG_EN, val);
}

/**
 * i915_hotplug_interrupt_update - update hotplug interrupt enable
 * @dev_priv: driver private
 * @mask: bits to update
 * @bits: bits to enable
 * NOTE: the HPD enable bits are modified both inside and outside
 * of an interrupt context. To avoid that read-modify-write cycles
 * interfer, these bits are protected by a spinlock. Since this
 * function is usually not called from a context where the lock is
 * held already, this function acquires the lock itself. A non-locking
 * version is also available.
 */
void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
				   uint32_t mask,
				   uint32_t bits)
{
	spin_lock_irq(&dev_priv->irq_lock);
	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
	spin_unlock_irq(&dev_priv->irq_lock);
}

213 214 215 216 217 218
/**
 * ilk_update_display_irq - update DEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
219 220 221
void ilk_update_display_irq(struct drm_i915_private *dev_priv,
			    uint32_t interrupt_mask,
			    uint32_t enabled_irq_mask)
222
{
223 224
	uint32_t new_val;

225
	lockdep_assert_held(&dev_priv->irq_lock);
226

227 228
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

229
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
230 231
		return;

232 233 234 235 236 237
	new_val = dev_priv->irq_mask;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->irq_mask) {
		dev_priv->irq_mask = new_val;
238
		I915_WRITE(DEIMR, dev_priv->irq_mask);
239
		POSTING_READ(DEIMR);
240 241 242
	}
}

P
Paulo Zanoni 已提交
243 244 245 246 247 248 249 250 251 252
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
253
	lockdep_assert_held(&dev_priv->irq_lock);
P
Paulo Zanoni 已提交
254

255 256
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

257
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
258 259
		return;

P
Paulo Zanoni 已提交
260 261 262 263 264
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
}

265
void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
266 267
{
	ilk_update_gt_irq(dev_priv, mask, mask);
268
	POSTING_READ_FW(GTIMR);
P
Paulo Zanoni 已提交
269 270
}

271
void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
272 273 274 275
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

276
static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
277 278 279 280
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
}

281
static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
282 283 284 285
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
}

286
static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
287 288 289 290
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
}

P
Paulo Zanoni 已提交
291
/**
292 293 294 295 296
 * snb_update_pm_irq - update GEN6_PMIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
P
Paulo Zanoni 已提交
297 298 299 300
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
301
	uint32_t new_val;
P
Paulo Zanoni 已提交
302

303 304
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

305
	lockdep_assert_held(&dev_priv->irq_lock);
P
Paulo Zanoni 已提交
306

307
	new_val = dev_priv->pm_imr;
308 309 310
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

311 312 313
	if (new_val != dev_priv->pm_imr) {
		dev_priv->pm_imr = new_val;
		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_imr);
314
		POSTING_READ(gen6_pm_imr(dev_priv));
315
	}
P
Paulo Zanoni 已提交
316 317
}

318
void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
P
Paulo Zanoni 已提交
319
{
320 321 322
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

P
Paulo Zanoni 已提交
323 324 325
	snb_update_pm_irq(dev_priv, mask, mask);
}

326
static void __gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
P
Paulo Zanoni 已提交
327 328 329 330
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

331
void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
332 333 334 335
{
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

336
	__gen6_mask_pm_irq(dev_priv, mask);
337 338
}

339
void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 reset_mask)
I
Imre Deak 已提交
340
{
341
	i915_reg_t reg = gen6_pm_iir(dev_priv);
I
Imre Deak 已提交
342

343
	lockdep_assert_held(&dev_priv->irq_lock);
344 345 346

	I915_WRITE(reg, reset_mask);
	I915_WRITE(reg, reset_mask);
I
Imre Deak 已提交
347
	POSTING_READ(reg);
348 349 350 351
}

void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mask)
{
352
	lockdep_assert_held(&dev_priv->irq_lock);
353 354 355 356 357 358 359 360 361

	dev_priv->pm_ier |= enable_mask;
	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
	gen6_unmask_pm_irq(dev_priv, enable_mask);
	/* unmask_pm_irq provides an implicit barrier (POSTING_READ) */
}

void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_mask)
{
362
	lockdep_assert_held(&dev_priv->irq_lock);
363 364 365 366 367 368 369 370 371 372 373

	dev_priv->pm_ier &= ~disable_mask;
	__gen6_mask_pm_irq(dev_priv, disable_mask);
	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
	/* though a barrier is missing here, but don't really need a one */
}

void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	gen6_reset_pm_iir(dev_priv, dev_priv->pm_rps_events);
374
	dev_priv->rps.pm_iir = 0;
I
Imre Deak 已提交
375 376 377
	spin_unlock_irq(&dev_priv->irq_lock);
}

378
void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
379
{
380 381 382
	if (READ_ONCE(dev_priv->rps.interrupts_enabled))
		return;

383
	spin_lock_irq(&dev_priv->irq_lock);
384 385
	WARN_ON_ONCE(dev_priv->rps.pm_iir);
	WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
I
Imre Deak 已提交
386
	dev_priv->rps.interrupts_enabled = true;
387
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
388

389 390 391
	spin_unlock_irq(&dev_priv->irq_lock);
}

392 393
u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
{
394
	return (mask & ~dev_priv->rps.pm_intr_keep);
395 396
}

397
void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
398
{
399 400 401
	if (!READ_ONCE(dev_priv->rps.interrupts_enabled))
		return;

I
Imre Deak 已提交
402 403
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->rps.interrupts_enabled = false;
404

405
	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u));
406

407
	gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
408 409

	spin_unlock_irq(&dev_priv->irq_lock);
410
	synchronize_irq(dev_priv->drm.irq);
411 412 413 414 415 416 417 418

	/* Now that we will not be generating any more work, flush any
	 * outsanding tasks. As we are called on the RPS idle path,
	 * we will reset the GPU to minimum frequencies, so the current
	 * state of the worker can be discarded.
	 */
	cancel_work_sync(&dev_priv->rps.work);
	gen6_reset_rps_interrupts(dev_priv);
419 420
}

421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452
void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	gen6_reset_pm_iir(dev_priv, dev_priv->pm_guc_events);
	spin_unlock_irq(&dev_priv->irq_lock);
}

void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	if (!dev_priv->guc.interrupts_enabled) {
		WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) &
				       dev_priv->pm_guc_events);
		dev_priv->guc.interrupts_enabled = true;
		gen6_enable_pm_irq(dev_priv, dev_priv->pm_guc_events);
	}
	spin_unlock_irq(&dev_priv->irq_lock);
}

void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->guc.interrupts_enabled = false;

	gen6_disable_pm_irq(dev_priv, dev_priv->pm_guc_events);

	spin_unlock_irq(&dev_priv->irq_lock);
	synchronize_irq(dev_priv->drm.irq);

	gen9_reset_guc_interrupts(dev_priv);
}

453
/**
454 455 456 457 458
 * bdw_update_port_irq - update DE port interrupt
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
459 460 461 462 463 464 465
static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
				uint32_t interrupt_mask,
				uint32_t enabled_irq_mask)
{
	uint32_t new_val;
	uint32_t old_val;

466
	lockdep_assert_held(&dev_priv->irq_lock);
467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	old_val = I915_READ(GEN8_DE_PORT_IMR);

	new_val = old_val;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != old_val) {
		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		POSTING_READ(GEN8_DE_PORT_IMR);
	}
}

485 486 487 488 489 490 491 492 493 494 495 496 497 498
/**
 * bdw_update_pipe_irq - update DE pipe interrupt
 * @dev_priv: driver private
 * @pipe: pipe whose interrupt to update
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
			 enum pipe pipe,
			 uint32_t interrupt_mask,
			 uint32_t enabled_irq_mask)
{
	uint32_t new_val;

499
	lockdep_assert_held(&dev_priv->irq_lock);
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	new_val = dev_priv->de_irq_mask[pipe];
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->de_irq_mask[pipe]) {
		dev_priv->de_irq_mask[pipe] = new_val;
		I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
	}
}

517 518 519 520 521 522
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
523 524 525
void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
				  uint32_t interrupt_mask,
				  uint32_t enabled_irq_mask)
526 527 528 529 530
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

531 532
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

533
	lockdep_assert_held(&dev_priv->irq_lock);
534

535
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
536 537
		return;

538 539 540
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
541

D
Daniel Vetter 已提交
542
static void
543 544
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
545
{
546
	i915_reg_t reg = PIPESTAT(pipe);
547
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
548

549
	lockdep_assert_held(&dev_priv->irq_lock);
550
	WARN_ON(!intel_irqs_enabled(dev_priv));
551

552 553 554 555
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
556 557 558
		return;

	if ((pipestat & enable_mask) == enable_mask)
559 560
		return;

561 562
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

563
	/* Enable the interrupt, clear any pending status */
564
	pipestat |= enable_mask | status_mask;
565 566
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
567 568
}

D
Daniel Vetter 已提交
569
static void
570 571
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
572
{
573
	i915_reg_t reg = PIPESTAT(pipe);
574
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
575

576
	lockdep_assert_held(&dev_priv->irq_lock);
577
	WARN_ON(!intel_irqs_enabled(dev_priv));
578

579 580 581 582
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
583 584
		return;

585 586 587
	if ((pipestat & enable_mask) == 0)
		return;

588 589
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

590
	pipestat &= ~enable_mask;
591 592
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
593 594
}

595 596 597 598 599
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
600 601
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
602 603 604
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
605 606 607 608 609 610
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
611 612 613 614 615 616 617 618 619 620 621 622

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

623 624 625 626 627 628
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

629
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
630
		enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
631 632 633
							   status_mask);
	else
		enable_mask = status_mask << 16;
634 635 636 637 638 639 640 641 642
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

643
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
644
		enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
645 646 647
							   status_mask);
	else
		enable_mask = status_mask << 16;
648 649 650
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

651
/**
652
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
653
 * @dev_priv: i915 device private
654
 */
655
static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
656
{
657
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev_priv))
658 659
		return;

660
	spin_lock_irq(&dev_priv->irq_lock);
661

662
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
663
	if (INTEL_GEN(dev_priv) >= 4)
664
		i915_enable_pipestat(dev_priv, PIPE_A,
665
				     PIPE_LEGACY_BLC_EVENT_STATUS);
666

667
	spin_unlock_irq(&dev_priv->irq_lock);
668 669
}

670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719
/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via PIPECONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on PIPECONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

720 721 722
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
723
static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
724
{
725
	struct drm_i915_private *dev_priv = to_i915(dev);
726
	i915_reg_t high_frame, low_frame;
727
	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
728 729
	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
								pipe);
730
	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
731

732 733 734 735 736
	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vbl_start = mode->crtc_vblank_start;
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
737

738 739 740 741 742 743
	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

744 745
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
746

747 748 749 750 751 752
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
753
		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
754
		low   = I915_READ(low_frame);
755
		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
756 757
	} while (high1 != high2);

758
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
759
	pixel = low & PIPE_PIXEL_MASK;
760
	low >>= PIPE_FRAME_LOW_SHIFT;
761 762 763 764 765 766

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
767
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
768 769
}

770
static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
771
{
772
	struct drm_i915_private *dev_priv = to_i915(dev);
773

774
	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
775 776
}

777
/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
778 779 780
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
781
	struct drm_i915_private *dev_priv = to_i915(dev);
782
	const struct drm_display_mode *mode = &crtc->base.hwmode;
783
	enum pipe pipe = crtc->pipe;
784
	int position, vtotal;
785

786 787 788
	if (!crtc->active)
		return -1;

789
	vtotal = mode->crtc_vtotal;
790 791 792
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

793
	if (IS_GEN2(dev_priv))
794
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
795
	else
796
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
797

798 799 800 801 802 803 804 805 806 807 808 809
	/*
	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
	 * read it just before the start of vblank.  So try it again
	 * so we don't accidentally end up spanning a vblank frame
	 * increment, causing the pipe_update_end() code to squak at us.
	 *
	 * The nature of this problem means we can't simply check the ISR
	 * bit and return the vblank start value; nor can we use the scanline
	 * debug register in the transcoder as it appears to have the same
	 * problem.  We may need to extend this to include other platforms,
	 * but so far testing only shows the problem on HSW.
	 */
810
	if (HAS_DDI(dev_priv) && !position) {
811 812 813 814 815 816 817 818 819 820 821 822 823
		int i, temp;

		for (i = 0; i < 100; i++) {
			udelay(1);
			temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
				DSL_LINEMASK_GEN3;
			if (temp != position) {
				position = temp;
				break;
			}
		}
	}

824
	/*
825 826
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
827
	 */
828
	return (position + crtc->scanline_offset) % vtotal;
829 830
}

831
static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
832
				    unsigned int flags, int *vpos, int *hpos,
833 834
				    ktime_t *stime, ktime_t *etime,
				    const struct drm_display_mode *mode)
835
{
836
	struct drm_i915_private *dev_priv = to_i915(dev);
837 838
	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
								pipe);
839
	int position;
840
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
841 842
	bool in_vbl = true;
	int ret = 0;
843
	unsigned long irqflags;
844

845
	if (WARN_ON(!mode->crtc_clock)) {
846
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
847
				 "pipe %c\n", pipe_name(pipe));
848 849 850
		return 0;
	}

851
	htotal = mode->crtc_htotal;
852
	hsync_start = mode->crtc_hsync_start;
853 854 855
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
856

857 858 859 860 861 862
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

863 864
	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

865 866 867 868 869 870
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
871

872 873 874 875 876 877
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

878
	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
879 880 881
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
882
		position = __intel_get_crtc_scanline(intel_crtc);
883 884 885 886 887
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
888
		position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
889

890 891 892 893
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
894

895 896 897 898 899 900 901 902 903 904 905 906
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

907 908 909 910 911 912 913 914 915 916
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
917 918
	}

919 920 921 922 923 924 925 926
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

927 928 929 930 931 932 933 934 935 936 937 938
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
939

940
	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
941 942 943 944 945 946
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
947 948 949

	/* In vblank? */
	if (in_vbl)
950
		ret |= DRM_SCANOUTPOS_IN_VBLANK;
951 952 953 954

	return ret;
}

955 956
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
957
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
958 959 960 961 962 963 964 965 966 967
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

968
static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
969 970 971 972
			      int *max_error,
			      struct timeval *vblank_time,
			      unsigned flags)
{
973
	struct drm_i915_private *dev_priv = to_i915(dev);
974
	struct intel_crtc *crtc;
975

976
	if (pipe >= INTEL_INFO(dev_priv)->num_pipes) {
977
		DRM_ERROR("Invalid crtc %u\n", pipe);
978 979 980 981
		return -EINVAL;
	}

	/* Get drm_crtc to timestamp: */
982
	crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
983
	if (crtc == NULL) {
984
		DRM_ERROR("Invalid crtc %u\n", pipe);
985 986 987
		return -EINVAL;
	}

988
	if (!crtc->base.hwmode.crtc_clock) {
989
		DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
990 991
		return -EBUSY;
	}
992 993

	/* Helper routine in DRM core does all the work: */
994 995
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
						     vblank_time, flags,
996
						     &crtc->base.hwmode);
997 998
}

999
static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
1000
{
1001
	u32 busy_up, busy_down, max_avg, min_avg;
1002 1003
	u8 new_delay;

1004
	spin_lock(&mchdev_lock);
1005

1006 1007
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

1008
	new_delay = dev_priv->ips.cur_delay;
1009

1010
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
1011 1012
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
1013 1014 1015 1016
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
1017
	if (busy_up > max_avg) {
1018 1019 1020 1021
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
1022
	} else if (busy_down < min_avg) {
1023 1024 1025 1026
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
1027 1028
	}

1029
	if (ironlake_set_drps(dev_priv, new_delay))
1030
		dev_priv->ips.cur_delay = new_delay;
1031

1032
	spin_unlock(&mchdev_lock);
1033

1034 1035 1036
	return;
}

1037
static void notify_ring(struct intel_engine_cs *engine)
1038
{
1039 1040
	struct drm_i915_gem_request *rq = NULL;
	struct intel_wait *wait;
1041

1042
	atomic_inc(&engine->irq_count);
1043
	set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
1044

1045 1046
	spin_lock(&engine->breadcrumbs.irq_lock);
	wait = engine->breadcrumbs.irq_wait;
1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
	if (wait) {
		/* We use a callback from the dma-fence to submit
		 * requests after waiting on our own requests. To
		 * ensure minimum delay in queuing the next request to
		 * hardware, signal the fence now rather than wait for
		 * the signaler to be woken up. We still wake up the
		 * waiter in order to handle the irq-seqno coherency
		 * issues (we may receive the interrupt before the
		 * seqno is written, see __i915_request_irq_complete())
		 * and to handle coalescing of multiple seqno updates
		 * and many waiters.
		 */
		if (i915_seqno_passed(intel_engine_get_seqno(engine),
				      wait->seqno))
1061
			rq = i915_gem_request_get(wait->request);
1062 1063

		wake_up_process(wait->tsk);
1064 1065
	} else {
		__intel_engine_disarm_breadcrumbs(engine);
1066
	}
1067
	spin_unlock(&engine->breadcrumbs.irq_lock);
1068

1069
	if (rq) {
1070
		dma_fence_signal(&rq->fence);
1071 1072
		i915_gem_request_put(rq);
	}
1073 1074

	trace_intel_engine_notify(engine, wait);
1075 1076
}

1077 1078
static void vlv_c0_read(struct drm_i915_private *dev_priv,
			struct intel_rps_ei *ei)
1079
{
1080 1081 1082 1083
	ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
}
1084

1085 1086 1087 1088 1089 1090
static bool vlv_c0_above(struct drm_i915_private *dev_priv,
			 const struct intel_rps_ei *old,
			 const struct intel_rps_ei *now,
			 int threshold)
{
	u64 time, c0;
1091
	unsigned int mul = 100;
1092

1093 1094
	if (old->cz_clock == 0)
		return false;
1095

1096 1097 1098
	if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
		mul <<= 8;

1099
	time = now->cz_clock - old->cz_clock;
1100
	time *= threshold * dev_priv->czclk_freq;
1101

1102 1103 1104
	/* Workload can be split between render + media, e.g. SwapBuffers
	 * being blitted in X after being rendered in mesa. To account for
	 * this we need to combine both engines into our activity counter.
1105
	 */
1106 1107
	c0 = now->render_c0 - old->render_c0;
	c0 += now->media_c0 - old->media_c0;
1108
	c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC;
1109

1110
	return c0 >= time;
1111 1112
}

1113
void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1114
{
1115 1116 1117
	vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
	dev_priv->rps.up_ei = dev_priv->rps.down_ei;
}
1118

1119 1120 1121 1122
static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
{
	struct intel_rps_ei now;
	u32 events = 0;
1123

1124
	if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
1125
		return 0;
1126

1127 1128 1129
	vlv_c0_read(dev_priv, &now);
	if (now.cz_clock == 0)
		return 0;
1130

1131 1132 1133
	if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
		if (!vlv_c0_above(dev_priv,
				  &dev_priv->rps.down_ei, &now,
1134
				  dev_priv->rps.down_threshold))
1135 1136 1137
			events |= GEN6_PM_RP_DOWN_THRESHOLD;
		dev_priv->rps.down_ei = now;
	}
1138

1139 1140 1141
	if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
		if (vlv_c0_above(dev_priv,
				 &dev_priv->rps.up_ei, &now,
1142
				 dev_priv->rps.up_threshold))
1143 1144
			events |= GEN6_PM_RP_UP_THRESHOLD;
		dev_priv->rps.up_ei = now;
1145 1146
	}

1147
	return events;
1148 1149
}

1150 1151
static bool any_waiters(struct drm_i915_private *dev_priv)
{
1152
	struct intel_engine_cs *engine;
1153
	enum intel_engine_id id;
1154

1155
	for_each_engine(engine, dev_priv, id)
1156
		if (intel_engine_has_waiter(engine))
1157 1158 1159 1160 1161
			return true;

	return false;
}

1162
static void gen6_pm_rps_work(struct work_struct *work)
1163
{
1164 1165
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
1166 1167
	bool client_boost;
	int new_delay, adj, min, max;
P
Paulo Zanoni 已提交
1168
	u32 pm_iir;
1169

1170
	spin_lock_irq(&dev_priv->irq_lock);
I
Imre Deak 已提交
1171 1172 1173 1174 1175
	/* Speed up work cancelation during disabling rps interrupts. */
	if (!dev_priv->rps.interrupts_enabled) {
		spin_unlock_irq(&dev_priv->irq_lock);
		return;
	}
1176

1177 1178
	pm_iir = dev_priv->rps.pm_iir;
	dev_priv->rps.pm_iir = 0;
1179
	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
1180
	gen6_unmask_pm_irq(dev_priv, dev_priv->pm_rps_events);
1181 1182
	client_boost = dev_priv->rps.client_boost;
	dev_priv->rps.client_boost = false;
1183
	spin_unlock_irq(&dev_priv->irq_lock);
1184

1185
	/* Make sure we didn't queue anything we're not going to process. */
1186
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1187

1188
	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1189
		return;
1190

1191
	mutex_lock(&dev_priv->rps.hw_lock);
1192

1193 1194
	pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);

1195
	adj = dev_priv->rps.last_adj;
1196
	new_delay = dev_priv->rps.cur_freq;
1197 1198
	min = dev_priv->rps.min_freq_softlimit;
	max = dev_priv->rps.max_freq_softlimit;
1199 1200 1201 1202
	if (client_boost || any_waiters(dev_priv))
		max = dev_priv->rps.max_freq;
	if (client_boost && new_delay < dev_priv->rps.boost_freq) {
		new_delay = dev_priv->rps.boost_freq;
1203 1204
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1205 1206
		if (adj > 0)
			adj *= 2;
1207 1208
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1209 1210 1211

		if (new_delay >= dev_priv->rps.max_freq_softlimit)
			adj = 0;
1212
	} else if (client_boost || any_waiters(dev_priv)) {
1213
		adj = 0;
1214
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1215 1216
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1217
		else if (dev_priv->rps.cur_freq > dev_priv->rps.min_freq_softlimit)
1218
			new_delay = dev_priv->rps.min_freq_softlimit;
1219 1220 1221 1222
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
1223 1224
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1225 1226 1227

		if (new_delay <= dev_priv->rps.min_freq_softlimit)
			adj = 0;
1228
	} else { /* unknown event */
1229
		adj = 0;
1230
	}
1231

1232 1233
	dev_priv->rps.last_adj = adj;

1234 1235 1236
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1237
	new_delay += adj;
1238
	new_delay = clamp_t(int, new_delay, min, max);
1239

1240 1241 1242 1243
	if (intel_set_rps(dev_priv, new_delay)) {
		DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
		dev_priv->rps.last_adj = 0;
	}
1244

1245
	mutex_unlock(&dev_priv->rps.hw_lock);
1246 1247
}

1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1260 1261
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, l3_parity.error_work);
1262
	u32 error_status, row, bank, subbank;
1263
	char *parity_event[6];
1264
	uint32_t misccpctl;
1265
	uint8_t slice = 0;
1266 1267 1268 1269 1270

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
1271
	mutex_lock(&dev_priv->drm.struct_mutex);
1272

1273 1274 1275 1276
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1277 1278 1279 1280
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1281
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1282
		i915_reg_t reg;
1283

1284
		slice--;
1285
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv)))
1286
			break;
1287

1288
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1289

1290
		reg = GEN7_L3CDERRST1(slice);
1291

1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1307
		kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
1308
				   KOBJ_CHANGE, parity_event);
1309

1310 1311
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1312

1313 1314 1315 1316 1317
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1318

1319
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1320

1321 1322
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
1323
	spin_lock_irq(&dev_priv->irq_lock);
1324
	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1325
	spin_unlock_irq(&dev_priv->irq_lock);
1326

1327
	mutex_unlock(&dev_priv->drm.struct_mutex);
1328 1329
}

1330 1331
static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv,
					       u32 iir)
1332
{
1333
	if (!HAS_L3_DPF(dev_priv))
1334 1335
		return;

1336
	spin_lock(&dev_priv->irq_lock);
1337
	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1338
	spin_unlock(&dev_priv->irq_lock);
1339

1340
	iir &= GT_PARITY_ERROR(dev_priv);
1341 1342 1343 1344 1345 1346
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1347
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1348 1349
}

1350
static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
1351 1352
			       u32 gt_iir)
{
1353
	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1354
		notify_ring(dev_priv->engine[RCS]);
1355
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
1356
		notify_ring(dev_priv->engine[VCS]);
1357 1358
}

1359
static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
1360 1361
			       u32 gt_iir)
{
1362
	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1363
		notify_ring(dev_priv->engine[RCS]);
1364
	if (gt_iir & GT_BSD_USER_INTERRUPT)
1365
		notify_ring(dev_priv->engine[VCS]);
1366
	if (gt_iir & GT_BLT_USER_INTERRUPT)
1367
		notify_ring(dev_priv->engine[BCS]);
1368

1369 1370
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
1371 1372
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1373

1374 1375
	if (gt_iir & GT_PARITY_ERROR(dev_priv))
		ivybridge_parity_error_irq_handler(dev_priv, gt_iir);
1376 1377
}

1378
static __always_inline void
1379
gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
1380 1381
{
	if (iir & (GT_RENDER_USER_INTERRUPT << test_shift))
1382
		notify_ring(engine);
1383 1384 1385 1386 1387

	if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) {
		set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
		tasklet_hi_schedule(&engine->irq_tasklet);
	}
1388 1389
}

1390 1391 1392
static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv,
				   u32 master_ctl,
				   u32 gt_iir[4])
1393 1394 1395 1396
{
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1397 1398 1399
		gt_iir[0] = I915_READ_FW(GEN8_GT_IIR(0));
		if (gt_iir[0]) {
			I915_WRITE_FW(GEN8_GT_IIR(0), gt_iir[0]);
1400 1401 1402 1403 1404
			ret = IRQ_HANDLED;
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1405
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1406 1407 1408
		gt_iir[1] = I915_READ_FW(GEN8_GT_IIR(1));
		if (gt_iir[1]) {
			I915_WRITE_FW(GEN8_GT_IIR(1), gt_iir[1]);
1409
			ret = IRQ_HANDLED;
1410
		} else
1411
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
1412 1413
	}

1414
	if (master_ctl & GEN8_GT_VECS_IRQ) {
1415 1416 1417
		gt_iir[3] = I915_READ_FW(GEN8_GT_IIR(3));
		if (gt_iir[3]) {
			I915_WRITE_FW(GEN8_GT_IIR(3), gt_iir[3]);
1418 1419 1420 1421 1422
			ret = IRQ_HANDLED;
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

1423
	if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
1424
		gt_iir[2] = I915_READ_FW(GEN8_GT_IIR(2));
1425 1426
		if (gt_iir[2] & (dev_priv->pm_rps_events |
				 dev_priv->pm_guc_events)) {
1427
			I915_WRITE_FW(GEN8_GT_IIR(2),
1428 1429
				      gt_iir[2] & (dev_priv->pm_rps_events |
						   dev_priv->pm_guc_events));
1430
			ret = IRQ_HANDLED;
1431 1432 1433 1434
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1435 1436 1437
	return ret;
}

1438 1439 1440 1441
static void gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
				u32 gt_iir[4])
{
	if (gt_iir[0]) {
1442
		gen8_cs_irq_handler(dev_priv->engine[RCS],
1443
				    gt_iir[0], GEN8_RCS_IRQ_SHIFT);
1444
		gen8_cs_irq_handler(dev_priv->engine[BCS],
1445 1446 1447 1448
				    gt_iir[0], GEN8_BCS_IRQ_SHIFT);
	}

	if (gt_iir[1]) {
1449
		gen8_cs_irq_handler(dev_priv->engine[VCS],
1450
				    gt_iir[1], GEN8_VCS1_IRQ_SHIFT);
1451
		gen8_cs_irq_handler(dev_priv->engine[VCS2],
1452 1453 1454 1455
				    gt_iir[1], GEN8_VCS2_IRQ_SHIFT);
	}

	if (gt_iir[3])
1456
		gen8_cs_irq_handler(dev_priv->engine[VECS],
1457 1458 1459 1460
				    gt_iir[3], GEN8_VECS_IRQ_SHIFT);

	if (gt_iir[2] & dev_priv->pm_rps_events)
		gen6_rps_irq_handler(dev_priv, gt_iir[2]);
1461 1462 1463

	if (gt_iir[2] & dev_priv->pm_guc_events)
		gen9_guc_irq_handler(dev_priv, gt_iir[2]);
1464 1465
}

1466 1467 1468 1469
static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
1470
		return val & PORTA_HOTPLUG_LONG_DETECT;
1471 1472 1473 1474 1475 1476 1477 1478 1479
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1480 1481 1482 1483 1484 1485 1486 1487 1488 1489
static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_E:
		return val & PORTE_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505
static bool spt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & PORTA_HOTPLUG_LONG_DETECT;
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	case PORT_D:
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1516
static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1517 1518 1519
{
	switch (port) {
	case PORT_B:
1520
		return val & PORTB_HOTPLUG_LONG_DETECT;
1521
	case PORT_C:
1522
		return val & PORTC_HOTPLUG_LONG_DETECT;
1523
	case PORT_D:
1524 1525 1526
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
1527 1528 1529
	}
}

1530
static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1531 1532 1533
{
	switch (port) {
	case PORT_B:
1534
		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1535
	case PORT_C:
1536
		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1537
	case PORT_D:
1538 1539 1540
		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
	default:
		return false;
1541 1542 1543
	}
}

1544 1545 1546 1547 1548 1549 1550
/*
 * Get a bit mask of pins that have triggered, and which ones may be long.
 * This can be called multiple times with the same masks to accumulate
 * hotplug detection results from several registers.
 *
 * Note that the caller is expected to zero out the masks initially.
 */
1551
static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1552
			     u32 hotplug_trigger, u32 dig_hotplug_reg,
1553 1554
			     const u32 hpd[HPD_NUM_PINS],
			     bool long_pulse_detect(enum port port, u32 val))
1555
{
1556
	enum port port;
1557 1558 1559
	int i;

	for_each_hpd_pin(i) {
1560 1561
		if ((hpd[i] & hotplug_trigger) == 0)
			continue;
1562

1563 1564
		*pin_mask |= BIT(i);

1565 1566 1567
		if (!intel_hpd_pin_to_port(i, &port))
			continue;

1568
		if (long_pulse_detect(port, dig_hotplug_reg))
1569
			*long_mask |= BIT(i);
1570 1571 1572 1573 1574 1575 1576
	}

	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
			 hotplug_trigger, dig_hotplug_reg, *pin_mask);

}

1577
static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1578
{
1579
	wake_up_all(&dev_priv->gmbus_wait_queue);
1580 1581
}

1582
static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1583
{
1584
	wake_up_all(&dev_priv->gmbus_wait_queue);
1585 1586
}

1587
#if defined(CONFIG_DEBUG_FS)
1588 1589
static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
					 enum pipe pipe,
1590 1591 1592
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1593 1594 1595
{
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
T
Tomeu Vizoso 已提交
1596 1597 1598
	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
	struct drm_driver *driver = dev_priv->drm.driver;
	uint32_t crcs[5];
1599
	int head, tail;
1600

1601
	spin_lock(&pipe_crc->lock);
T
Tomeu Vizoso 已提交
1602 1603 1604 1605 1606 1607
	if (pipe_crc->source) {
		if (!pipe_crc->entries) {
			spin_unlock(&pipe_crc->lock);
			DRM_DEBUG_KMS("spurious interrupt\n");
			return;
		}
1608

T
Tomeu Vizoso 已提交
1609 1610
		head = pipe_crc->head;
		tail = pipe_crc->tail;
1611

T
Tomeu Vizoso 已提交
1612 1613 1614 1615 1616
		if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
			spin_unlock(&pipe_crc->lock);
			DRM_ERROR("CRC buffer overflowing\n");
			return;
		}
1617

T
Tomeu Vizoso 已提交
1618
		entry = &pipe_crc->entries[head];
1619

T
Tomeu Vizoso 已提交
1620 1621 1622 1623 1624 1625
		entry->frame = driver->get_vblank_counter(&dev_priv->drm, pipe);
		entry->crc[0] = crc0;
		entry->crc[1] = crc1;
		entry->crc[2] = crc2;
		entry->crc[3] = crc3;
		entry->crc[4] = crc4;
1626

T
Tomeu Vizoso 已提交
1627 1628
		head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
		pipe_crc->head = head;
1629

T
Tomeu Vizoso 已提交
1630
		spin_unlock(&pipe_crc->lock);
1631

T
Tomeu Vizoso 已提交
1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653
		wake_up_interruptible(&pipe_crc->wq);
	} else {
		/*
		 * For some not yet identified reason, the first CRC is
		 * bonkers. So let's just wait for the next vblank and read
		 * out the buggy result.
		 *
		 * On CHV sometimes the second CRC is bonkers as well, so
		 * don't trust that one either.
		 */
		if (pipe_crc->skipped == 0 ||
		    (IS_CHERRYVIEW(dev_priv) && pipe_crc->skipped == 1)) {
			pipe_crc->skipped++;
			spin_unlock(&pipe_crc->lock);
			return;
		}
		spin_unlock(&pipe_crc->lock);
		crcs[0] = crc0;
		crcs[1] = crc1;
		crcs[2] = crc2;
		crcs[3] = crc3;
		crcs[4] = crc4;
1654 1655 1656
		drm_crtc_add_crc_entry(&crtc->base, true,
				       drm_accurate_vblank_count(&crtc->base),
				       crcs);
T
Tomeu Vizoso 已提交
1657
	}
1658
}
1659 1660
#else
static inline void
1661 1662
display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
			     enum pipe pipe,
1663 1664 1665 1666 1667
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1668

1669 1670
static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				     enum pipe pipe)
D
Daniel Vetter 已提交
1671
{
1672
	display_pipe_crc_irq_handler(dev_priv, pipe,
1673 1674
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1675 1676
}

1677 1678
static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				     enum pipe pipe)
1679
{
1680
	display_pipe_crc_irq_handler(dev_priv, pipe,
1681 1682 1683 1684 1685
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1686
}
1687

1688 1689
static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				      enum pipe pipe)
1690
{
1691 1692
	uint32_t res1, res2;

1693
	if (INTEL_GEN(dev_priv) >= 3)
1694 1695 1696 1697
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

1698
	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1699 1700 1701
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1702

1703
	display_pipe_crc_irq_handler(dev_priv, pipe,
1704 1705 1706 1707
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1708
}
1709

1710 1711 1712 1713
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1714
{
1715
	if (pm_iir & dev_priv->pm_rps_events) {
1716
		spin_lock(&dev_priv->irq_lock);
1717
		gen6_mask_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
I
Imre Deak 已提交
1718 1719
		if (dev_priv->rps.interrupts_enabled) {
			dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1720
			schedule_work(&dev_priv->rps.work);
I
Imre Deak 已提交
1721
		}
1722
		spin_unlock(&dev_priv->irq_lock);
1723 1724
	}

1725 1726 1727
	if (INTEL_INFO(dev_priv)->gen >= 8)
		return;

1728
	if (HAS_VEBOX(dev_priv)) {
1729
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1730
			notify_ring(dev_priv->engine[VECS]);
B
Ben Widawsky 已提交
1731

1732 1733
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
B
Ben Widawsky 已提交
1734
	}
1735 1736
}

1737 1738 1739
static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
{
	if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT) {
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
		/* Sample the log buffer flush related bits & clear them out now
		 * itself from the message identity register to minimize the
		 * probability of losing a flush interrupt, when there are back
		 * to back flush interrupts.
		 * There can be a new flush interrupt, for different log buffer
		 * type (like for ISR), whilst Host is handling one (for DPC).
		 * Since same bit is used in message register for ISR & DPC, it
		 * could happen that GuC sets the bit for 2nd interrupt but Host
		 * clears out the bit on handling the 1st interrupt.
		 */
		u32 msg, flush;

		msg = I915_READ(SOFT_SCRATCH(15));
1753 1754
		flush = msg & (INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED |
			       INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER);
1755 1756 1757 1758 1759 1760 1761
		if (flush) {
			/* Clear the message bits that are handled */
			I915_WRITE(SOFT_SCRATCH(15), msg & ~flush);

			/* Handle flush interrupt in bottom half */
			queue_work(dev_priv->guc.log.flush_wq,
				   &dev_priv->guc.log.flush_work);
1762 1763

			dev_priv->guc.log.flush_interrupt_count++;
1764 1765 1766 1767 1768
		} else {
			/* Not clearing of unhandled event bits won't result in
			 * re-triggering of the interrupt.
			 */
		}
1769 1770 1771
	}
}

1772
static bool intel_pipe_handle_vblank(struct drm_i915_private *dev_priv,
1773
				     enum pipe pipe)
1774
{
1775 1776
	bool ret;

1777
	ret = drm_handle_vblank(&dev_priv->drm, pipe);
1778
	if (ret)
1779
		intel_finish_page_flip_mmio(dev_priv, pipe);
1780 1781

	return ret;
1782 1783
}

1784 1785
static void valleyview_pipestat_irq_ack(struct drm_i915_private *dev_priv,
					u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1786 1787 1788
{
	int pipe;

1789
	spin_lock(&dev_priv->irq_lock);
1790 1791 1792 1793 1794 1795

	if (!dev_priv->display_irqs_enabled) {
		spin_unlock(&dev_priv->irq_lock);
		return;
	}

1796
	for_each_pipe(dev_priv, pipe) {
1797
		i915_reg_t reg;
1798
		u32 mask, iir_bit = 0;
1799

1800 1801 1802 1803 1804 1805 1806
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
1807 1808 1809

		/* fifo underruns are filterered in the underrun handler. */
		mask = PIPE_FIFO_UNDERRUN_STATUS;
1810 1811 1812 1813 1814 1815 1816 1817

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1818 1819 1820
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1821 1822 1823 1824 1825
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1826 1827 1828
			continue;

		reg = PIPESTAT(pipe);
1829 1830
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1831 1832 1833 1834

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1835 1836
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1837 1838
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1839
	spin_unlock(&dev_priv->irq_lock);
1840 1841
}

1842
static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1843 1844 1845
					    u32 pipe_stats[I915_MAX_PIPES])
{
	enum pipe pipe;
1846

1847
	for_each_pipe(dev_priv, pipe) {
1848 1849 1850
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
1851

1852
		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV)
1853
			intel_finish_page_flip_cs(dev_priv, pipe);
1854 1855

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1856
			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1857

1858 1859
		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1860 1861 1862
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1863
		gmbus_irq_handler(dev_priv);
1864 1865
}

1866
static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
1867 1868 1869
{
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);

1870 1871
	if (hotplug_status)
		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1872

1873 1874 1875
	return hotplug_status;
}

1876
static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1877 1878 1879
				 u32 hotplug_status)
{
	u32 pin_mask = 0, long_mask = 0;
1880

1881 1882
	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
	    IS_CHERRYVIEW(dev_priv)) {
1883
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1884

1885 1886 1887 1888 1889
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
					   hotplug_trigger, hpd_status_g4x,
					   i9xx_port_hotplug_long_detect);

1890
			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1891
		}
1892 1893

		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1894
			dp_aux_irq_handler(dev_priv);
1895 1896
	} else {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1897

1898 1899
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1900
					   hotplug_trigger, hpd_status_i915,
1901
					   i9xx_port_hotplug_long_detect);
1902
			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1903
		}
1904
	}
1905 1906
}

1907
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1908
{
1909
	struct drm_device *dev = arg;
1910
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
1911 1912
	irqreturn_t ret = IRQ_NONE;

1913 1914 1915
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1916 1917 1918
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1919
	do {
1920
		u32 iir, gt_iir, pm_iir;
1921
		u32 pipe_stats[I915_MAX_PIPES] = {};
1922
		u32 hotplug_status = 0;
1923
		u32 ier = 0;
1924

J
Jesse Barnes 已提交
1925 1926
		gt_iir = I915_READ(GTIIR);
		pm_iir = I915_READ(GEN6_PMIIR);
1927
		iir = I915_READ(VLV_IIR);
J
Jesse Barnes 已提交
1928 1929

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1930
			break;
J
Jesse Barnes 已提交
1931 1932 1933

		ret = IRQ_HANDLED;

1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
		 *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
		 * bits this time around.
		 */
1947
		I915_WRITE(VLV_MASTER_IER, 0);
1948 1949
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
1950 1951 1952 1953 1954 1955

		if (gt_iir)
			I915_WRITE(GTIIR, gt_iir);
		if (pm_iir)
			I915_WRITE(GEN6_PMIIR, pm_iir);

1956
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1957
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1958

1959 1960
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
1961
		valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1962

1963 1964 1965 1966
		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
			   I915_LPE_PIPE_B_INTERRUPT))
			intel_lpe_audio_irq_handler(dev_priv);

1967 1968 1969 1970 1971 1972
		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);
1973

1974
		I915_WRITE(VLV_IER, ier);
1975 1976
		I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
		POSTING_READ(VLV_MASTER_IER);
1977

1978
		if (gt_iir)
1979
			snb_gt_irq_handler(dev_priv, gt_iir);
1980 1981 1982
		if (pm_iir)
			gen6_rps_irq_handler(dev_priv, pm_iir);

1983
		if (hotplug_status)
1984
			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1985

1986
		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1987
	} while (0);
J
Jesse Barnes 已提交
1988

1989 1990
	enable_rpm_wakeref_asserts(dev_priv);

J
Jesse Barnes 已提交
1991 1992 1993
	return ret;
}

1994 1995
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1996
	struct drm_device *dev = arg;
1997
	struct drm_i915_private *dev_priv = to_i915(dev);
1998 1999
	irqreturn_t ret = IRQ_NONE;

2000 2001 2002
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2003 2004 2005
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

2006
	do {
2007
		u32 master_ctl, iir;
2008
		u32 gt_iir[4] = {};
2009
		u32 pipe_stats[I915_MAX_PIPES] = {};
2010
		u32 hotplug_status = 0;
2011 2012
		u32 ier = 0;

2013 2014
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
2015

2016 2017
		if (master_ctl == 0 && iir == 0)
			break;
2018

2019 2020
		ret = IRQ_HANDLED;

2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
		 *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
		 * bits this time around.
		 */
2034
		I915_WRITE(GEN8_MASTER_IRQ, 0);
2035 2036
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
2037

2038
		gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
2039

2040
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
2041
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
2042

2043 2044
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
2045
		valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
2046

2047 2048 2049 2050 2051
		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
			   I915_LPE_PIPE_B_INTERRUPT |
			   I915_LPE_PIPE_C_INTERRUPT))
			intel_lpe_audio_irq_handler(dev_priv);

2052 2053 2054 2055 2056 2057 2058
		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);

2059
		I915_WRITE(VLV_IER, ier);
2060
		I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2061
		POSTING_READ(GEN8_MASTER_IRQ);
2062

2063 2064
		gen8_gt_irq_handler(dev_priv, gt_iir);

2065
		if (hotplug_status)
2066
			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
2067

2068
		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
2069
	} while (0);
2070

2071 2072
	enable_rpm_wakeref_asserts(dev_priv);

2073 2074 2075
	return ret;
}

2076 2077
static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2078 2079 2080 2081
				const u32 hpd[HPD_NUM_PINS])
{
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

2082 2083 2084 2085 2086 2087
	/*
	 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
	 * unless we touch the hotplug register, even if hotplug_trigger is
	 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
	 * errors.
	 */
2088
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2089 2090 2091 2092 2093 2094 2095 2096
	if (!hotplug_trigger) {
		u32 mask = PORTA_HOTPLUG_STATUS_MASK |
			PORTD_HOTPLUG_STATUS_MASK |
			PORTC_HOTPLUG_STATUS_MASK |
			PORTB_HOTPLUG_STATUS_MASK;
		dig_hotplug_reg &= ~mask;
	}

2097
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2098 2099
	if (!hotplug_trigger)
		return;
2100 2101 2102 2103 2104

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   pch_port_hotplug_long_detect);

2105
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2106 2107
}

2108
static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2109
{
2110
	int pipe;
2111
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
2112

2113
	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx);
2114

2115 2116 2117
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
2118
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
2119 2120
				 port_name(port));
	}
2121

2122
	if (pch_iir & SDE_AUX_MASK)
2123
		dp_aux_irq_handler(dev_priv);
2124

2125
	if (pch_iir & SDE_GMBUS)
2126
		gmbus_irq_handler(dev_priv);
2127 2128 2129 2130 2131 2132 2133 2134 2135 2136

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2137
	if (pch_iir & SDE_FDI_MASK)
2138
		for_each_pipe(dev_priv, pipe)
2139 2140 2141
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2142 2143 2144 2145 2146 2147 2148 2149

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
2150
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
2151 2152

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
2153
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
2154 2155
}

2156
static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
2157 2158
{
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
2159
	enum pipe pipe;
2160

2161 2162 2163
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

2164
	for_each_pipe(dev_priv, pipe) {
2165 2166
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2167

D
Daniel Vetter 已提交
2168
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
2169 2170
			if (IS_IVYBRIDGE(dev_priv))
				ivb_pipe_crc_irq_handler(dev_priv, pipe);
D
Daniel Vetter 已提交
2171
			else
2172
				hsw_pipe_crc_irq_handler(dev_priv, pipe);
D
Daniel Vetter 已提交
2173 2174
		}
	}
2175

2176 2177 2178
	I915_WRITE(GEN7_ERR_INT, err_int);
}

2179
static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
2180 2181 2182
{
	u32 serr_int = I915_READ(SERR_INT);

2183 2184 2185
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2186
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
2187
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
2188 2189

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
2190
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
2191 2192

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
2193
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
2194 2195

	I915_WRITE(SERR_INT, serr_int);
2196 2197
}

2198
static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2199 2200
{
	int pipe;
2201
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
2202

2203
	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt);
2204

2205 2206 2207 2208 2209 2210
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
2211 2212

	if (pch_iir & SDE_AUX_MASK_CPT)
2213
		dp_aux_irq_handler(dev_priv);
2214 2215

	if (pch_iir & SDE_GMBUS_CPT)
2216
		gmbus_irq_handler(dev_priv);
2217 2218 2219 2220 2221 2222 2223 2224

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
2225
		for_each_pipe(dev_priv, pipe)
2226 2227 2228
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2229 2230

	if (pch_iir & SDE_ERROR_CPT)
2231
		cpt_serr_int_handler(dev_priv);
2232 2233
}

2234
static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248
{
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		~SDE_PORTE_HOTPLUG_SPT;
	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
	u32 pin_mask = 0, long_mask = 0;

	if (hotplug_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
				   dig_hotplug_reg, hpd_spt,
2249
				   spt_port_hotplug_long_detect);
2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263
	}

	if (hotplug2_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
				   dig_hotplug_reg, hpd_spt,
				   spt_port_hotplug2_long_detect);
	}

	if (pin_mask)
2264
		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2265 2266

	if (pch_iir & SDE_GMBUS_CPT)
2267
		gmbus_irq_handler(dev_priv);
2268 2269
}

2270 2271
static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282
				const u32 hpd[HPD_NUM_PINS])
{
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   ilk_port_hotplug_long_detect);

2283
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2284 2285
}

2286 2287
static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
				    u32 de_iir)
2288
{
2289
	enum pipe pipe;
2290 2291
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;

2292
	if (hotplug_trigger)
2293
		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk);
2294 2295

	if (de_iir & DE_AUX_CHANNEL_A)
2296
		dp_aux_irq_handler(dev_priv);
2297 2298

	if (de_iir & DE_GSE)
2299
		intel_opregion_asle_intr(dev_priv);
2300 2301 2302 2303

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2304
	for_each_pipe(dev_priv, pipe) {
2305 2306 2307
		if (de_iir & DE_PIPE_VBLANK(pipe) &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2308

2309
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2310
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2311

2312
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
2313
			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2314

2315
		/* plane/pipes map 1:1 on ilk+ */
2316
		if (de_iir & DE_PLANE_FLIP_DONE(pipe))
2317
			intel_finish_page_flip_cs(dev_priv, pipe);
2318 2319 2320 2321 2322 2323
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

2324 2325
		if (HAS_PCH_CPT(dev_priv))
			cpt_irq_handler(dev_priv, pch_iir);
2326
		else
2327
			ibx_irq_handler(dev_priv, pch_iir);
2328 2329 2330 2331 2332

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

2333 2334
	if (IS_GEN5(dev_priv) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev_priv);
2335 2336
}

2337 2338
static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
				    u32 de_iir)
2339
{
2340
	enum pipe pipe;
2341 2342
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;

2343
	if (hotplug_trigger)
2344
		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb);
2345 2346

	if (de_iir & DE_ERR_INT_IVB)
2347
		ivb_err_int_handler(dev_priv);
2348 2349

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
2350
		dp_aux_irq_handler(dev_priv);
2351 2352

	if (de_iir & DE_GSE_IVB)
2353
		intel_opregion_asle_intr(dev_priv);
2354

2355
	for_each_pipe(dev_priv, pipe) {
2356 2357 2358
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2359 2360

		/* plane/pipes map 1:1 on ilk+ */
2361
		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe))
2362
			intel_finish_page_flip_cs(dev_priv, pipe);
2363 2364 2365
	}

	/* check event from PCH */
2366
	if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
2367 2368
		u32 pch_iir = I915_READ(SDEIIR);

2369
		cpt_irq_handler(dev_priv, pch_iir);
2370 2371 2372 2373 2374 2375

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2376 2377 2378 2379 2380 2381 2382 2383
/*
 * To handle irqs with the minimum potential races with fresh interrupts, we:
 * 1 - Disable Master Interrupt Control.
 * 2 - Find the source(s) of the interrupt.
 * 3 - Clear the Interrupt Identity bits (IIR).
 * 4 - Process the interrupt(s) that had bits set in the IIRs.
 * 5 - Re-enable Master Interrupt Control.
 */
2384
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2385
{
2386
	struct drm_device *dev = arg;
2387
	struct drm_i915_private *dev_priv = to_i915(dev);
2388
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2389
	irqreturn_t ret = IRQ_NONE;
2390

2391 2392 2393
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2394 2395 2396
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

2397 2398 2399
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2400
	POSTING_READ(DEIER);
2401

2402 2403 2404 2405 2406
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2407
	if (!HAS_PCH_NOP(dev_priv)) {
2408 2409 2410 2411
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2412

2413 2414
	/* Find, clear, then process each source of interrupt */

2415
	gt_iir = I915_READ(GTIIR);
2416
	if (gt_iir) {
2417 2418
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2419
		if (INTEL_GEN(dev_priv) >= 6)
2420
			snb_gt_irq_handler(dev_priv, gt_iir);
2421
		else
2422
			ilk_gt_irq_handler(dev_priv, gt_iir);
2423 2424
	}

2425 2426
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2427 2428
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2429 2430
		if (INTEL_GEN(dev_priv) >= 7)
			ivb_display_irq_handler(dev_priv, de_iir);
2431
		else
2432
			ilk_display_irq_handler(dev_priv, de_iir);
2433 2434
	}

2435
	if (INTEL_GEN(dev_priv) >= 6) {
2436 2437 2438 2439
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
2440
			gen6_rps_irq_handler(dev_priv, pm_iir);
2441
		}
2442
	}
2443 2444 2445

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2446
	if (!HAS_PCH_NOP(dev_priv)) {
2447 2448 2449
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2450

2451 2452 2453
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	enable_rpm_wakeref_asserts(dev_priv);

2454 2455 2456
	return ret;
}

2457 2458
static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2459
				const u32 hpd[HPD_NUM_PINS])
2460
{
2461
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2462

2463 2464
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2465

2466
	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2467
			   dig_hotplug_reg, hpd,
2468
			   bxt_port_hotplug_long_detect);
2469

2470
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2471 2472
}

2473 2474
static irqreturn_t
gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2475 2476
{
	irqreturn_t ret = IRQ_NONE;
2477
	u32 iir;
2478
	enum pipe pipe;
J
Jesse Barnes 已提交
2479

2480
	if (master_ctl & GEN8_DE_MISC_IRQ) {
2481 2482 2483
		iir = I915_READ(GEN8_DE_MISC_IIR);
		if (iir) {
			I915_WRITE(GEN8_DE_MISC_IIR, iir);
2484
			ret = IRQ_HANDLED;
2485
			if (iir & GEN8_DE_MISC_GSE)
2486
				intel_opregion_asle_intr(dev_priv);
2487 2488
			else
				DRM_ERROR("Unexpected DE Misc interrupt\n");
2489
		}
2490 2491
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2492 2493
	}

2494
	if (master_ctl & GEN8_DE_PORT_IRQ) {
2495 2496 2497
		iir = I915_READ(GEN8_DE_PORT_IIR);
		if (iir) {
			u32 tmp_mask;
2498
			bool found = false;
2499

2500
			I915_WRITE(GEN8_DE_PORT_IIR, iir);
2501
			ret = IRQ_HANDLED;
J
Jesse Barnes 已提交
2502

2503 2504 2505 2506 2507 2508 2509
			tmp_mask = GEN8_AUX_CHANNEL_A;
			if (INTEL_INFO(dev_priv)->gen >= 9)
				tmp_mask |= GEN9_AUX_CHANNEL_B |
					    GEN9_AUX_CHANNEL_C |
					    GEN9_AUX_CHANNEL_D;

			if (iir & tmp_mask) {
2510
				dp_aux_irq_handler(dev_priv);
2511 2512 2513
				found = true;
			}

2514
			if (IS_GEN9_LP(dev_priv)) {
2515 2516
				tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
				if (tmp_mask) {
2517 2518
					bxt_hpd_irq_handler(dev_priv, tmp_mask,
							    hpd_bxt);
2519 2520 2521 2522 2523
					found = true;
				}
			} else if (IS_BROADWELL(dev_priv)) {
				tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
				if (tmp_mask) {
2524 2525
					ilk_hpd_irq_handler(dev_priv,
							    tmp_mask, hpd_bdw);
2526 2527
					found = true;
				}
2528 2529
			}

2530
			if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
2531
				gmbus_irq_handler(dev_priv);
S
Shashank Sharma 已提交
2532 2533 2534
				found = true;
			}

2535
			if (!found)
2536
				DRM_ERROR("Unexpected DE Port interrupt\n");
2537
		}
2538 2539
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2540 2541
	}

2542
	for_each_pipe(dev_priv, pipe) {
2543
		u32 flip_done, fault_errors;
2544

2545 2546
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2547

2548 2549 2550 2551 2552
		iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (!iir) {
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
			continue;
		}
2553

2554 2555
		ret = IRQ_HANDLED;
		I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2556

2557 2558 2559
		if (iir & GEN8_PIPE_VBLANK &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2560

2561 2562 2563 2564 2565
		flip_done = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			flip_done &= GEN9_PIPE_PLANE1_FLIP_DONE;
		else
			flip_done &= GEN8_PIPE_PRIMARY_FLIP_DONE;
2566

2567
		if (flip_done)
2568
			intel_finish_page_flip_cs(dev_priv, pipe);
2569

2570
		if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
2571
			hsw_pipe_crc_irq_handler(dev_priv, pipe);
2572

2573 2574
		if (iir & GEN8_PIPE_FIFO_UNDERRUN)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2575

2576 2577 2578 2579 2580
		fault_errors = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
		else
			fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2581

2582
		if (fault_errors)
2583
			DRM_ERROR("Fault errors on pipe %c: 0x%08x\n",
2584 2585
				  pipe_name(pipe),
				  fault_errors);
2586 2587
	}

2588
	if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2589
	    master_ctl & GEN8_DE_PCH_IRQ) {
2590 2591 2592 2593 2594
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
2595 2596 2597
		iir = I915_READ(SDEIIR);
		if (iir) {
			I915_WRITE(SDEIIR, iir);
2598
			ret = IRQ_HANDLED;
2599

2600
			if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv))
2601
				spt_irq_handler(dev_priv, iir);
2602
			else
2603
				cpt_irq_handler(dev_priv, iir);
2604 2605 2606 2607 2608 2609 2610
		} else {
			/*
			 * Like on previous PCH there seems to be something
			 * fishy going on with forwarding PCH interrupts.
			 */
			DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
		}
2611 2612
	}

2613 2614 2615 2616 2617 2618
	return ret;
}

static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
2619
	struct drm_i915_private *dev_priv = to_i915(dev);
2620
	u32 master_ctl;
2621
	u32 gt_iir[4] = {};
2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637
	irqreturn_t ret;

	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);

	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	/* Find, clear, then process each source of interrupt */
2638 2639
	ret = gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
	gen8_gt_irq_handler(dev_priv, gt_iir);
2640 2641
	ret |= gen8_de_irq_handler(dev_priv, master_ctl);

2642 2643
	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ_FW(GEN8_MASTER_IRQ);
2644

2645 2646
	enable_rpm_wakeref_asserts(dev_priv);

2647 2648 2649
	return ret;
}

2650
static void i915_error_wake_up(struct drm_i915_private *dev_priv)
2651 2652 2653 2654 2655 2656 2657 2658 2659
{
	/*
	 * Notify all waiters for GPU completion events that reset state has
	 * been changed, and that they need to restart their wait after
	 * checking for potential errors (and bail out to drop locks if there is
	 * a gpu reset pending so that i915_error_work_func can acquire them).
	 */

	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2660
	wake_up_all(&dev_priv->gpu_error.wait_queue);
2661 2662 2663 2664 2665

	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
	wake_up_all(&dev_priv->pending_flip_queue);
}

2666
/**
2667
 * i915_reset_and_wakeup - do process context error handling work
2668
 * @dev_priv: i915 device private
2669 2670 2671 2672
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
2673
static void i915_reset_and_wakeup(struct drm_i915_private *dev_priv)
2674
{
2675
	struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
2676 2677 2678
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2679

2680
	kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);
2681

2682 2683 2684
	DRM_DEBUG_DRIVER("resetting chip\n");
	kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);

2685
	/*
2686 2687 2688 2689 2690
	 * In most cases it's guaranteed that we get here with an RPM
	 * reference held, for example because there is a pending GPU
	 * request that won't finish until the reset is done. This
	 * isn't the case at least when we get here by doing a
	 * simulated reset via debugs, so get an RPM reference.
2691
	 */
2692 2693
	intel_runtime_pm_get(dev_priv);
	intel_prepare_reset(dev_priv);
2694

2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705
	do {
		/*
		 * All state reset _must_ be completed before we update the
		 * reset counter, for otherwise waiters might miss the reset
		 * pending state and not properly drop locks, resulting in
		 * deadlocks with the reset work.
		 */
		if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
			i915_reset(dev_priv);
			mutex_unlock(&dev_priv->drm.struct_mutex);
		}
2706

2707 2708 2709 2710 2711
		/* We need to wait for anyone holding the lock to wakeup */
	} while (wait_on_bit_timeout(&dev_priv->gpu_error.flags,
				     I915_RESET_IN_PROGRESS,
				     TASK_UNINTERRUPTIBLE,
				     HZ));
2712

2713
	intel_finish_reset(dev_priv);
2714
	intel_runtime_pm_put(dev_priv);
2715

2716
	if (!test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
2717 2718
		kobject_uevent_env(kobj,
				   KOBJ_CHANGE, reset_done_event);
2719

2720 2721 2722 2723 2724
	/*
	 * Note: The wake_up also serves as a memory barrier so that
	 * waiters see the updated value of the dev_priv->gpu_error.
	 */
	wake_up_all(&dev_priv->gpu_error.reset_queue);
2725 2726
}

2727 2728 2729 2730
static inline void
i915_err_print_instdone(struct drm_i915_private *dev_priv,
			struct intel_instdone *instdone)
{
2731 2732 2733
	int slice;
	int subslice;

2734 2735 2736 2737 2738 2739 2740 2741 2742 2743
	pr_err("  INSTDONE: 0x%08x\n", instdone->instdone);

	if (INTEL_GEN(dev_priv) <= 3)
		return;

	pr_err("  SC_INSTDONE: 0x%08x\n", instdone->slice_common);

	if (INTEL_GEN(dev_priv) <= 6)
		return;

2744 2745 2746 2747 2748 2749 2750
	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
		pr_err("  SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
		       slice, subslice, instdone->sampler[slice][subslice]);

	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
		pr_err("  ROW_INSTDONE[%d][%d]: 0x%08x\n",
		       slice, subslice, instdone->row[slice][subslice]);
2751 2752
}

2753
static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
2754
{
2755
	u32 eir;
2756

2757 2758
	if (!IS_GEN2(dev_priv))
		I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER));
2759

2760 2761 2762 2763
	if (INTEL_GEN(dev_priv) < 4)
		I915_WRITE(IPEIR, I915_READ(IPEIR));
	else
		I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965));
2764

2765
	I915_WRITE(EIR, I915_READ(EIR));
2766 2767 2768 2769 2770 2771
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
2772
		DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
2773 2774 2775
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2776 2777 2778
}

/**
2779
 * i915_handle_error - handle a gpu error
2780
 * @dev_priv: i915 device private
2781
 * @engine_mask: mask representing engines that are hung
2782 2783
 * @fmt: Error message format string
 *
2784
 * Do some basic checking of register state at error time and
2785 2786 2787 2788 2789
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
2790 2791
void i915_handle_error(struct drm_i915_private *dev_priv,
		       u32 engine_mask,
2792
		       const char *fmt, ...)
2793
{
2794 2795
	va_list args;
	char error_msg[80];
2796

2797 2798 2799 2800
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

2801
	i915_capture_error_state(dev_priv, engine_mask, error_msg);
2802
	i915_clear_error_registers(dev_priv);
2803

2804 2805
	if (!engine_mask)
		return;
2806

2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823
	if (test_and_set_bit(I915_RESET_IN_PROGRESS,
			     &dev_priv->gpu_error.flags))
		return;

	/*
	 * Wakeup waiting processes so that the reset function
	 * i915_reset_and_wakeup doesn't deadlock trying to grab
	 * various locks. By bumping the reset counter first, the woken
	 * processes will see a reset in progress and back off,
	 * releasing their locks and then wait for the reset completion.
	 * We must do this for _all_ gpu waiters that might hold locks
	 * that the reset work needs to acquire.
	 *
	 * Note: The wake_up also provides a memory barrier to ensure that the
	 * waiters see the updated value of the reset flags.
	 */
	i915_error_wake_up(dev_priv);
2824

2825
	i915_reset_and_wakeup(dev_priv);
2826 2827
}

2828 2829 2830
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2831
static int i8xx_enable_vblank(struct drm_device *dev, unsigned int pipe)
2832
{
2833
	struct drm_i915_private *dev_priv = to_i915(dev);
2834
	unsigned long irqflags;
2835

2836
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2837
	i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2838
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2839

2840 2841 2842
	return 0;
}

2843
static int i965_enable_vblank(struct drm_device *dev, unsigned int pipe)
2844
{
2845
	struct drm_i915_private *dev_priv = to_i915(dev);
2846 2847 2848
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2849 2850
	i915_enable_pipestat(dev_priv, pipe,
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
2851 2852 2853 2854 2855
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2856
static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2857
{
2858
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
2859
	unsigned long irqflags;
2860
	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
2861
		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
J
Jesse Barnes 已提交
2862 2863

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2864
	ilk_enable_display_irq(dev_priv, bit);
J
Jesse Barnes 已提交
2865 2866 2867 2868 2869
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2870
static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
2871
{
2872
	struct drm_i915_private *dev_priv = to_i915(dev);
2873 2874 2875
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2876
	bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2877
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2878

2879 2880 2881
	return 0;
}

2882 2883 2884
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2885
static void i8xx_disable_vblank(struct drm_device *dev, unsigned int pipe)
2886
{
2887
	struct drm_i915_private *dev_priv = to_i915(dev);
2888
	unsigned long irqflags;
2889

2890
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2891
	i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2892 2893 2894
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2895
static void i965_disable_vblank(struct drm_device *dev, unsigned int pipe)
2896
{
2897
	struct drm_i915_private *dev_priv = to_i915(dev);
2898 2899 2900
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2901 2902
	i915_disable_pipestat(dev_priv, pipe,
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2903 2904 2905
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2906
static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2907
{
2908
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
2909
	unsigned long irqflags;
2910
	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
2911
		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
J
Jesse Barnes 已提交
2912 2913

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2914
	ilk_disable_display_irq(dev_priv, bit);
J
Jesse Barnes 已提交
2915 2916 2917
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2918
static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
2919
{
2920
	struct drm_i915_private *dev_priv = to_i915(dev);
2921 2922 2923
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2924
	bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2925 2926 2927
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2928
static void ibx_irq_reset(struct drm_i915_private *dev_priv)
P
Paulo Zanoni 已提交
2929
{
2930
	if (HAS_PCH_NOP(dev_priv))
P
Paulo Zanoni 已提交
2931 2932
		return;

2933
	GEN5_IRQ_RESET(SDE);
2934

2935
	if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
2936
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
2937
}
2938

P
Paulo Zanoni 已提交
2939 2940 2941 2942 2943 2944 2945 2946 2947 2948
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
2949
	struct drm_i915_private *dev_priv = to_i915(dev);
P
Paulo Zanoni 已提交
2950

2951
	if (HAS_PCH_NOP(dev_priv))
P
Paulo Zanoni 已提交
2952 2953 2954
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
2955 2956 2957 2958
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

2959
static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv)
2960
{
2961
	GEN5_IRQ_RESET(GT);
2962
	if (INTEL_GEN(dev_priv) >= 6)
2963
		GEN5_IRQ_RESET(GEN6_PM);
2964 2965
}

2966 2967 2968 2969
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
	enum pipe pipe;

2970 2971 2972 2973 2974
	if (IS_CHERRYVIEW(dev_priv))
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
	else
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);

2975
	i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
2976 2977
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

2978 2979 2980 2981 2982 2983
	for_each_pipe(dev_priv, pipe) {
		I915_WRITE(PIPESTAT(pipe),
			   PIPE_FIFO_UNDERRUN_STATUS |
			   PIPESTAT_INT_STATUS_MASK);
		dev_priv->pipestat_irq_mask[pipe] = 0;
	}
2984 2985

	GEN5_IRQ_RESET(VLV_);
2986
	dev_priv->irq_mask = ~0;
2987 2988
}

2989 2990 2991
static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
2992
	u32 enable_mask;
2993
	enum pipe pipe;
2994
	u32 val;
2995 2996 2997 2998 2999 3000 3001 3002

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		i915_enable_pipestat(dev_priv, pipe, pipestat_mask);

3003 3004 3005
	enable_mask = I915_DISPLAY_PORT_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3006
	if (IS_CHERRYVIEW(dev_priv))
3007
		enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
3008 3009 3010

	WARN_ON(dev_priv->irq_mask != ~0);

3011 3012 3013 3014 3015 3016
	val = (I915_LPE_PIPE_A_INTERRUPT |
		I915_LPE_PIPE_B_INTERRUPT |
		I915_LPE_PIPE_C_INTERRUPT);

	enable_mask |= val;

3017 3018 3019
	dev_priv->irq_mask = ~enable_mask;

	GEN5_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask);
3020 3021 3022 3023 3024 3025
}

/* drm_dma.h hooks
*/
static void ironlake_irq_reset(struct drm_device *dev)
{
3026
	struct drm_i915_private *dev_priv = to_i915(dev);
3027 3028 3029 3030

	I915_WRITE(HWSTAM, 0xffffffff);

	GEN5_IRQ_RESET(DE);
3031
	if (IS_GEN7(dev_priv))
3032 3033
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);

3034
	gen5_gt_irq_reset(dev_priv);
3035

3036
	ibx_irq_reset(dev_priv);
3037 3038
}

J
Jesse Barnes 已提交
3039 3040
static void valleyview_irq_preinstall(struct drm_device *dev)
{
3041
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
3042

3043 3044 3045
	I915_WRITE(VLV_MASTER_IER, 0);
	POSTING_READ(VLV_MASTER_IER);

3046
	gen5_gt_irq_reset(dev_priv);
J
Jesse Barnes 已提交
3047

3048
	spin_lock_irq(&dev_priv->irq_lock);
3049 3050
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3051
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3052 3053
}

3054 3055 3056 3057 3058 3059 3060 3061
static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
{
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
}

P
Paulo Zanoni 已提交
3062
static void gen8_irq_reset(struct drm_device *dev)
3063
{
3064
	struct drm_i915_private *dev_priv = to_i915(dev);
3065 3066 3067 3068 3069
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3070
	gen8_gt_irq_reset(dev_priv);
3071

3072
	for_each_pipe(dev_priv, pipe)
3073 3074
		if (intel_display_power_is_enabled(dev_priv,
						   POWER_DOMAIN_PIPE(pipe)))
3075
			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3076

3077 3078 3079
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3080

3081
	if (HAS_PCH_SPLIT(dev_priv))
3082
		ibx_irq_reset(dev_priv);
3083
}
3084

3085 3086
void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
3087
{
3088
	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3089
	enum pipe pipe;
3090

3091
	spin_lock_irq(&dev_priv->irq_lock);
3092 3093 3094 3095
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
				  dev_priv->de_irq_mask[pipe],
				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
3096
	spin_unlock_irq(&dev_priv->irq_lock);
3097 3098
}

3099 3100 3101
void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
{
3102 3103
	enum pipe pipe;

3104
	spin_lock_irq(&dev_priv->irq_lock);
3105 3106
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3107 3108 3109
	spin_unlock_irq(&dev_priv->irq_lock);

	/* make sure we're done processing display irqs */
3110
	synchronize_irq(dev_priv->drm.irq);
3111 3112
}

3113 3114
static void cherryview_irq_preinstall(struct drm_device *dev)
{
3115
	struct drm_i915_private *dev_priv = to_i915(dev);
3116 3117 3118 3119

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3120
	gen8_gt_irq_reset(dev_priv);
3121 3122 3123

	GEN5_IRQ_RESET(GEN8_PCU_);

3124
	spin_lock_irq(&dev_priv->irq_lock);
3125 3126
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3127
	spin_unlock_irq(&dev_priv->irq_lock);
3128 3129
}

3130
static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
3131 3132 3133 3134 3135
				  const u32 hpd[HPD_NUM_PINS])
{
	struct intel_encoder *encoder;
	u32 enabled_irqs = 0;

3136
	for_each_intel_encoder(&dev_priv->drm, encoder)
3137 3138 3139 3140 3141 3142
		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
			enabled_irqs |= hpd[encoder->hpd_pin];

	return enabled_irqs;
}

3143
static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
3144
{
3145
	u32 hotplug;
3146 3147 3148

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
3149 3150
	 * duration to 2ms (which is the minimum in the Display Port spec).
	 * The pulse duration bits are reserved on LPT+.
3151
	 */
3152
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3153 3154 3155
	hotplug &= ~(PORTB_PULSE_DURATION_MASK |
		     PORTC_PULSE_DURATION_MASK |
		     PORTD_PULSE_DURATION_MASK);
3156
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3157 3158
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3159 3160 3161 3162
	/*
	 * When CPU and PCH are on the same package, port A
	 * HPD must be enabled in both north and south.
	 */
3163
	if (HAS_PCH_LPT_LP(dev_priv))
3164
		hotplug |= PORTA_HOTPLUG_ENABLE;
3165
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3166
}
X
Xiong Zhang 已提交
3167

3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184
static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	if (HAS_PCH_IBX(dev_priv)) {
		hotplug_irqs = SDE_HOTPLUG_MASK;
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx);
	} else {
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt);
	}

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	ibx_hpd_detection_setup(dev_priv);
}

3185
static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3186
{
3187
	u32 hotplug;
3188 3189 3190

	/* Enable digital hotplug on the PCH */
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3191 3192 3193 3194
	hotplug |= PORTA_HOTPLUG_ENABLE |
		   PORTB_HOTPLUG_ENABLE |
		   PORTC_HOTPLUG_ENABLE |
		   PORTD_HOTPLUG_ENABLE;
3195 3196 3197 3198 3199
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);

	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
	hotplug |= PORTE_HOTPLUG_ENABLE;
	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3200 3201
}

3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213
static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt);

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	spt_hpd_detection_setup(dev_priv);
}

3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229
static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug;

	/*
	 * Enable digital hotplug on the CPU, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
	 * The pulse duration bits are reserved on HSW+.
	 */
	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE |
		   DIGITAL_PORTA_PULSE_DURATION_2ms;
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
}

3230
static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3231
{
3232
	u32 hotplug_irqs, enabled_irqs;
3233

3234
	if (INTEL_GEN(dev_priv) >= 8) {
3235
		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3236
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw);
3237 3238

		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3239
	} else if (INTEL_GEN(dev_priv) >= 7) {
3240
		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3241
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb);
3242 3243

		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3244 3245
	} else {
		hotplug_irqs = DE_DP_A_HOTPLUG;
3246
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk);
3247

3248 3249
		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
	}
3250

3251
	ilk_hpd_detection_setup(dev_priv);
3252

3253
	ibx_hpd_irq_setup(dev_priv);
3254 3255
}

3256 3257
static void __bxt_hpd_detection_setup(struct drm_i915_private *dev_priv,
				      u32 enabled_irqs)
3258
{
3259
	u32 hotplug;
3260

3261
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3262 3263 3264
	hotplug |= PORTA_HOTPLUG_ENABLE |
		   PORTB_HOTPLUG_ENABLE |
		   PORTC_HOTPLUG_ENABLE;
3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283

	DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n",
		      hotplug, enabled_irqs);
	hotplug &= ~BXT_DDI_HPD_INVERT_MASK;

	/*
	 * For BXT invert bit has to be set based on AOB design
	 * for HPD detection logic, update it based on VBT fields.
	 */
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
		hotplug |= BXT_DDIA_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
		hotplug |= BXT_DDIB_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
		hotplug |= BXT_DDIC_HPD_INVERT;

3284
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3285 3286
}

3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303
static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
{
	__bxt_hpd_detection_setup(dev_priv, BXT_DE_PORT_HOTPLUG_MASK);
}

static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt);
	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;

	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);

	__bxt_hpd_detection_setup(dev_priv, enabled_irqs);
}

P
Paulo Zanoni 已提交
3304 3305
static void ibx_irq_postinstall(struct drm_device *dev)
{
3306
	struct drm_i915_private *dev_priv = to_i915(dev);
3307
	u32 mask;
3308

3309
	if (HAS_PCH_NOP(dev_priv))
D
Daniel Vetter 已提交
3310 3311
		return;

3312
	if (HAS_PCH_IBX(dev_priv))
3313
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3314
	else
3315
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3316

3317
	gen5_assert_iir_is_zero(dev_priv, SDEIIR);
P
Paulo Zanoni 已提交
3318
	I915_WRITE(SDEIMR, ~mask);
3319 3320 3321

	if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
	    HAS_PCH_LPT(dev_priv))
3322
		ibx_hpd_detection_setup(dev_priv);
3323 3324
	else
		spt_hpd_detection_setup(dev_priv);
P
Paulo Zanoni 已提交
3325 3326
}

3327 3328
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
3329
	struct drm_i915_private *dev_priv = to_i915(dev);
3330 3331 3332 3333 3334
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3335
	if (HAS_L3_DPF(dev_priv)) {
3336
		/* L3 parity interrupt is always unmasked. */
3337 3338
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev_priv);
		gt_irqs |= GT_PARITY_ERROR(dev_priv);
3339 3340 3341
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
3342
	if (IS_GEN5(dev_priv)) {
3343
		gt_irqs |= ILK_BSD_USER_INTERRUPT;
3344 3345 3346 3347
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3348
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3349

3350
	if (INTEL_GEN(dev_priv) >= 6) {
3351 3352 3353 3354
		/*
		 * RPS interrupts will get enabled/disabled on demand when RPS
		 * itself is enabled/disabled.
		 */
3355
		if (HAS_VEBOX(dev_priv)) {
3356
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3357 3358
			dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT;
		}
3359

3360 3361
		dev_priv->pm_imr = 0xffffffff;
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs);
3362 3363 3364
	}
}

3365
static int ironlake_irq_postinstall(struct drm_device *dev)
3366
{
3367
	struct drm_i915_private *dev_priv = to_i915(dev);
3368 3369
	u32 display_mask, extra_mask;

3370
	if (INTEL_GEN(dev_priv) >= 7) {
3371 3372 3373
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3374
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3375
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3376 3377
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
			      DE_DP_A_HOTPLUG_IVB);
3378 3379 3380
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3381 3382 3383
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3384 3385 3386
		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
			      DE_DP_A_HOTPLUG);
3387
	}
3388

3389
	dev_priv->irq_mask = ~display_mask;
3390

3391 3392
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3393 3394
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3395
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3396

3397
	gen5_gt_irq_postinstall(dev);
3398

3399 3400
	ilk_hpd_detection_setup(dev_priv);

P
Paulo Zanoni 已提交
3401
	ibx_irq_postinstall(dev);
3402

3403
	if (IS_IRONLAKE_M(dev_priv)) {
3404 3405 3406
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3407 3408
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
3409
		spin_lock_irq(&dev_priv->irq_lock);
3410
		ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
3411
		spin_unlock_irq(&dev_priv->irq_lock);
3412 3413
	}

3414 3415 3416
	return 0;
}

3417 3418
void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
3419
	lockdep_assert_held(&dev_priv->irq_lock);
3420 3421 3422 3423 3424 3425

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

3426 3427
	if (intel_irqs_enabled(dev_priv)) {
		vlv_display_irq_reset(dev_priv);
3428
		vlv_display_irq_postinstall(dev_priv);
3429
	}
3430 3431 3432 3433
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
3434
	lockdep_assert_held(&dev_priv->irq_lock);
3435 3436 3437 3438 3439 3440

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

3441
	if (intel_irqs_enabled(dev_priv))
3442
		vlv_display_irq_reset(dev_priv);
3443 3444
}

3445 3446 3447

static int valleyview_irq_postinstall(struct drm_device *dev)
{
3448
	struct drm_i915_private *dev_priv = to_i915(dev);
3449

3450
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3451

3452
	spin_lock_irq(&dev_priv->irq_lock);
3453 3454
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3455 3456
	spin_unlock_irq(&dev_priv->irq_lock);

J
Jesse Barnes 已提交
3457
	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3458
	POSTING_READ(VLV_MASTER_IER);
3459 3460 3461 3462

	return 0;
}

3463 3464 3465 3466 3467
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3468 3469 3470
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3471
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3472 3473 3474
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3475
		0,
3476 3477
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3478 3479
		};

3480 3481 3482
	if (HAS_L3_DPF(dev_priv))
		gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;

3483 3484
	dev_priv->pm_ier = 0x0;
	dev_priv->pm_imr = ~dev_priv->pm_ier;
3485 3486
	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3487 3488
	/*
	 * RPS interrupts will get enabled/disabled on demand when RPS itself
3489
	 * is enabled/disabled. Same wil be the case for GuC interrupts.
3490
	 */
3491
	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_imr, dev_priv->pm_ier);
3492
	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3493 3494 3495 3496
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
3497 3498
	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
	uint32_t de_pipe_enables;
3499 3500
	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
	u32 de_port_enables;
3501
	u32 de_misc_masked = GEN8_DE_MISC_GSE;
3502
	enum pipe pipe;
3503

3504
	if (INTEL_INFO(dev_priv)->gen >= 9) {
3505 3506
		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3507 3508
		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
				  GEN9_AUX_CHANNEL_D;
3509
		if (IS_GEN9_LP(dev_priv))
3510 3511
			de_port_masked |= BXT_DE_PORT_GMBUS;
	} else {
3512 3513
		de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
				  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3514
	}
3515 3516 3517 3518

	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
					   GEN8_PIPE_FIFO_UNDERRUN;

3519
	de_port_enables = de_port_masked;
3520
	if (IS_GEN9_LP(dev_priv))
3521 3522
		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
	else if (IS_BROADWELL(dev_priv))
3523 3524
		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;

3525 3526 3527
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3528

3529
	for_each_pipe(dev_priv, pipe)
3530
		if (intel_display_power_is_enabled(dev_priv,
3531 3532 3533 3534
				POWER_DOMAIN_PIPE(pipe)))
			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
					  dev_priv->de_irq_mask[pipe],
					  de_pipe_enables);
3535

3536
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3537
	GEN5_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
3538 3539 3540

	if (IS_GEN9_LP(dev_priv))
		bxt_hpd_detection_setup(dev_priv);
3541 3542
	else if (IS_BROADWELL(dev_priv))
		ilk_hpd_detection_setup(dev_priv);
3543 3544 3545 3546
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
3547
	struct drm_i915_private *dev_priv = to_i915(dev);
3548

3549
	if (HAS_PCH_SPLIT(dev_priv))
3550
		ibx_irq_pre_postinstall(dev);
P
Paulo Zanoni 已提交
3551

3552 3553 3554
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

3555
	if (HAS_PCH_SPLIT(dev_priv))
3556
		ibx_irq_postinstall(dev);
3557

3558
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3559 3560 3561 3562 3563
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3564 3565
static int cherryview_irq_postinstall(struct drm_device *dev)
{
3566
	struct drm_i915_private *dev_priv = to_i915(dev);
3567 3568 3569

	gen8_gt_irq_postinstall(dev_priv);

3570
	spin_lock_irq(&dev_priv->irq_lock);
3571 3572
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3573 3574
	spin_unlock_irq(&dev_priv->irq_lock);

3575
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3576 3577 3578 3579 3580
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3581 3582
static void gen8_irq_uninstall(struct drm_device *dev)
{
3583
	struct drm_i915_private *dev_priv = to_i915(dev);
3584 3585 3586 3587

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3588
	gen8_irq_reset(dev);
3589 3590
}

J
Jesse Barnes 已提交
3591 3592
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3593
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
3594 3595 3596 3597

	if (!dev_priv)
		return;

3598
	I915_WRITE(VLV_MASTER_IER, 0);
3599
	POSTING_READ(VLV_MASTER_IER);
3600

3601
	gen5_gt_irq_reset(dev_priv);
3602

J
Jesse Barnes 已提交
3603
	I915_WRITE(HWSTAM, 0xffffffff);
3604

3605
	spin_lock_irq(&dev_priv->irq_lock);
3606 3607
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3608
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3609 3610
}

3611 3612
static void cherryview_irq_uninstall(struct drm_device *dev)
{
3613
	struct drm_i915_private *dev_priv = to_i915(dev);
3614 3615 3616 3617 3618 3619 3620

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3621
	gen8_gt_irq_reset(dev_priv);
3622

3623
	GEN5_IRQ_RESET(GEN8_PCU_);
3624

3625
	spin_lock_irq(&dev_priv->irq_lock);
3626 3627
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3628
	spin_unlock_irq(&dev_priv->irq_lock);
3629 3630
}

3631
static void ironlake_irq_uninstall(struct drm_device *dev)
3632
{
3633
	struct drm_i915_private *dev_priv = to_i915(dev);
3634 3635 3636 3637

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3638
	ironlake_irq_reset(dev);
3639 3640
}

3641
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3642
{
3643
	struct drm_i915_private *dev_priv = to_i915(dev);
3644
	int pipe;
3645

3646
	for_each_pipe(dev_priv, pipe)
3647
		I915_WRITE(PIPESTAT(pipe), 0);
3648 3649 3650
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3651 3652 3653 3654
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3655
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3656 3657 3658 3659 3660 3661 3662 3663 3664

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3665
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
C
Chris Wilson 已提交
3666 3667 3668 3669 3670 3671 3672 3673
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3674 3675
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3676
	spin_lock_irq(&dev_priv->irq_lock);
3677 3678
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3679
	spin_unlock_irq(&dev_priv->irq_lock);
3680

C
Chris Wilson 已提交
3681 3682 3683
	return 0;
}

3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714
/*
 * Returns true when a page flip has completed.
 */
static bool i8xx_handle_vblank(struct drm_i915_private *dev_priv,
			       int plane, int pipe, u32 iir)
{
	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

	if (!intel_pipe_handle_vblank(dev_priv, pipe))
		return false;

	if ((iir & flip_pending) == 0)
		goto check_page_flip;

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ16(ISR) & flip_pending)
		goto check_page_flip;

	intel_finish_page_flip_cs(dev_priv, pipe);
	return true;

check_page_flip:
	intel_check_page_flip(dev_priv, pipe);
	return false;
}

3715
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
3716
{
3717
	struct drm_device *dev = arg;
3718
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3719 3720 3721 3722 3723 3724
	u16 iir, new_iir;
	u32 pipe_stats[2];
	int pipe;
	u16 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3725
	irqreturn_t ret;
C
Chris Wilson 已提交
3726

3727 3728 3729
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

3730 3731 3732 3733
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	ret = IRQ_NONE;
C
Chris Wilson 已提交
3734 3735
	iir = I915_READ16(IIR);
	if (iir == 0)
3736
		goto out;
C
Chris Wilson 已提交
3737 3738 3739 3740 3741 3742 3743

	while (iir & ~flip_mask) {
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3744
		spin_lock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3745
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3746
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
C
Chris Wilson 已提交
3747

3748
		for_each_pipe(dev_priv, pipe) {
3749
			i915_reg_t reg = PIPESTAT(pipe);
C
Chris Wilson 已提交
3750 3751 3752 3753 3754
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
3755
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
3756 3757
				I915_WRITE(reg, pipe_stats[pipe]);
		}
3758
		spin_unlock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3759 3760 3761 3762 3763

		I915_WRITE16(IIR, iir & ~flip_mask);
		new_iir = I915_READ16(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
3764
			notify_ring(dev_priv->engine[RCS]);
C
Chris Wilson 已提交
3765

3766
		for_each_pipe(dev_priv, pipe) {
3767 3768 3769 3770 3771 3772 3773
			int plane = pipe;
			if (HAS_FBC(dev_priv))
				plane = !plane;

			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
			    i8xx_handle_vblank(dev_priv, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
C
Chris Wilson 已提交
3774

3775
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3776
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
3777

3778 3779 3780
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3781
		}
C
Chris Wilson 已提交
3782 3783 3784

		iir = new_iir;
	}
3785 3786 3787 3788
	ret = IRQ_HANDLED;

out:
	enable_rpm_wakeref_asserts(dev_priv);
C
Chris Wilson 已提交
3789

3790
	return ret;
C
Chris Wilson 已提交
3791 3792 3793 3794
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
3795
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3796 3797
	int pipe;

3798
	for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3799 3800 3801 3802 3803 3804 3805 3806 3807
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

3808 3809
static void i915_irq_preinstall(struct drm_device * dev)
{
3810
	struct drm_i915_private *dev_priv = to_i915(dev);
3811 3812
	int pipe;

3813
	if (I915_HAS_HOTPLUG(dev_priv)) {
3814
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3815 3816 3817
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3818
	I915_WRITE16(HWSTAM, 0xeffe);
3819
	for_each_pipe(dev_priv, pipe)
3820 3821 3822 3823 3824 3825 3826 3827
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
3828
	struct drm_i915_private *dev_priv = to_i915(dev);
3829
	u32 enable_mask;
3830

3831 3832 3833 3834 3835 3836 3837 3838
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3839
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3840 3841 3842 3843 3844 3845 3846

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_USER_INTERRUPT;

3847
	if (I915_HAS_HOTPLUG(dev_priv)) {
3848
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3849 3850
		POSTING_READ(PORT_HOTPLUG_EN);

3851 3852 3853 3854 3855 3856 3857 3858 3859 3860
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

3861
	i915_enable_asle_pipestat(dev_priv);
3862

3863 3864
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3865
	spin_lock_irq(&dev_priv->irq_lock);
3866 3867
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3868
	spin_unlock_irq(&dev_priv->irq_lock);
3869

3870 3871 3872
	return 0;
}

3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903
/*
 * Returns true when a page flip has completed.
 */
static bool i915_handle_vblank(struct drm_i915_private *dev_priv,
			       int plane, int pipe, u32 iir)
{
	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

	if (!intel_pipe_handle_vblank(dev_priv, pipe))
		return false;

	if ((iir & flip_pending) == 0)
		goto check_page_flip;

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ(ISR) & flip_pending)
		goto check_page_flip;

	intel_finish_page_flip_cs(dev_priv, pipe);
	return true;

check_page_flip:
	intel_check_page_flip(dev_priv, pipe);
	return false;
}

3904
static irqreturn_t i915_irq_handler(int irq, void *arg)
3905
{
3906
	struct drm_device *dev = arg;
3907
	struct drm_i915_private *dev_priv = to_i915(dev);
3908
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3909 3910 3911 3912
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
	int pipe, ret = IRQ_NONE;
3913

3914 3915 3916
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

3917 3918 3919
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

3920
	iir = I915_READ(IIR);
3921 3922
	do {
		bool irq_received = (iir & ~flip_mask) != 0;
3923
		bool blc_event = false;
3924 3925 3926 3927 3928 3929

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3930
		spin_lock(&dev_priv->irq_lock);
3931
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3932
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3933

3934
		for_each_pipe(dev_priv, pipe) {
3935
			i915_reg_t reg = PIPESTAT(pipe);
3936 3937
			pipe_stats[pipe] = I915_READ(reg);

3938
			/* Clear the PIPE*STAT regs before the IIR */
3939 3940
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
3941
				irq_received = true;
3942 3943
			}
		}
3944
		spin_unlock(&dev_priv->irq_lock);
3945 3946 3947 3948 3949

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
3950
		if (I915_HAS_HOTPLUG(dev_priv) &&
3951 3952 3953
		    iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
3954
				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
3955
		}
3956

3957
		I915_WRITE(IIR, iir & ~flip_mask);
3958 3959 3960
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
3961
			notify_ring(dev_priv->engine[RCS]);
3962

3963
		for_each_pipe(dev_priv, pipe) {
3964 3965 3966 3967 3968 3969 3970
			int plane = pipe;
			if (HAS_FBC(dev_priv))
				plane = !plane;

			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
			    i915_handle_vblank(dev_priv, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3971 3972 3973

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
3974 3975

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3976
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
3977

3978 3979 3980
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3981 3982 3983
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
3984
			intel_opregion_asle_intr(dev_priv);
3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
4001
		ret = IRQ_HANDLED;
4002
		iir = new_iir;
4003
	} while (iir & ~flip_mask);
4004

4005 4006
	enable_rpm_wakeref_asserts(dev_priv);

4007 4008 4009 4010 4011
	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
4012
	struct drm_i915_private *dev_priv = to_i915(dev);
4013 4014
	int pipe;

4015
	if (I915_HAS_HOTPLUG(dev_priv)) {
4016
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4017 4018 4019
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4020
	I915_WRITE16(HWSTAM, 0xffff);
4021
	for_each_pipe(dev_priv, pipe) {
4022
		/* Clear enable bits; then clear status bits */
4023
		I915_WRITE(PIPESTAT(pipe), 0);
4024 4025
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
4026 4027 4028 4029 4030 4031 4032 4033
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
4034
	struct drm_i915_private *dev_priv = to_i915(dev);
4035 4036
	int pipe;

4037
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4038
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4039 4040

	I915_WRITE(HWSTAM, 0xeffe);
4041
	for_each_pipe(dev_priv, pipe)
4042 4043 4044 4045 4046 4047 4048 4049
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
4050
	struct drm_i915_private *dev_priv = to_i915(dev);
4051
	u32 enable_mask;
4052 4053 4054
	u32 error_mask;

	/* Unmask the interrupts that we always want on. */
4055
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4056
			       I915_DISPLAY_PORT_INTERRUPT |
4057 4058 4059 4060 4061 4062 4063
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
4064 4065
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4066 4067
	enable_mask |= I915_USER_INTERRUPT;

4068
	if (IS_G4X(dev_priv))
4069
		enable_mask |= I915_BSD_USER_INTERRUPT;
4070

4071 4072
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4073
	spin_lock_irq(&dev_priv->irq_lock);
4074 4075 4076
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4077
	spin_unlock_irq(&dev_priv->irq_lock);
4078 4079 4080 4081 4082

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
4083
	if (IS_G4X(dev_priv)) {
4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4098
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4099 4100
	POSTING_READ(PORT_HOTPLUG_EN);

4101
	i915_enable_asle_pipestat(dev_priv);
4102 4103 4104 4105

	return 0;
}

4106
static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
4107 4108 4109
{
	u32 hotplug_en;

4110
	lockdep_assert_held(&dev_priv->irq_lock);
4111

4112 4113
	/* Note HDMI and DP share hotplug bits */
	/* enable bits are the same for all generations */
4114
	hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
4115 4116 4117 4118
	/* Programming the CRT detection parameters tends
	   to generate a spurious hotplug event about three
	   seconds later.  So just do it once.
	*/
4119
	if (IS_G4X(dev_priv))
4120 4121 4122 4123
		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;

	/* Ignore TV since it's buggy */
4124
	i915_hotplug_interrupt_update_locked(dev_priv,
4125 4126 4127 4128
					     HOTPLUG_INT_EN_MASK |
					     CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
					     CRT_HOTPLUG_ACTIVATION_PERIOD_64,
					     hotplug_en);
4129 4130
}

4131
static irqreturn_t i965_irq_handler(int irq, void *arg)
4132
{
4133
	struct drm_device *dev = arg;
4134
	struct drm_i915_private *dev_priv = to_i915(dev);
4135 4136 4137
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	int ret = IRQ_NONE, pipe;
4138 4139 4140
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4141

4142 4143 4144
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4145 4146 4147
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

4148 4149 4150
	iir = I915_READ(IIR);

	for (;;) {
4151
		bool irq_received = (iir & ~flip_mask) != 0;
4152 4153
		bool blc_event = false;

4154 4155 4156 4157 4158
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4159
		spin_lock(&dev_priv->irq_lock);
4160
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4161
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4162

4163
		for_each_pipe(dev_priv, pipe) {
4164
			i915_reg_t reg = PIPESTAT(pipe);
4165 4166 4167 4168 4169 4170 4171
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4172
				irq_received = true;
4173 4174
			}
		}
4175
		spin_unlock(&dev_priv->irq_lock);
4176 4177 4178 4179 4180 4181 4182

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4183 4184 4185
		if (iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
4186
				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4187
		}
4188

4189
		I915_WRITE(IIR, iir & ~flip_mask);
4190 4191 4192
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4193
			notify_ring(dev_priv->engine[RCS]);
4194
		if (iir & I915_BSD_USER_INTERRUPT)
4195
			notify_ring(dev_priv->engine[VCS]);
4196

4197
		for_each_pipe(dev_priv, pipe) {
4198 4199 4200
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
			    i915_handle_vblank(dev_priv, pipe, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4201 4202 4203

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4204 4205

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4206
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
4207

4208 4209
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4210
		}
4211 4212

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
4213
			intel_opregion_asle_intr(dev_priv);
4214

4215
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4216
			gmbus_irq_handler(dev_priv);
4217

4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

4236 4237
	enable_rpm_wakeref_asserts(dev_priv);

4238 4239 4240 4241 4242
	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4243
	struct drm_i915_private *dev_priv = to_i915(dev);
4244 4245 4246 4247 4248
	int pipe;

	if (!dev_priv)
		return;

4249
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4250
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4251 4252

	I915_WRITE(HWSTAM, 0xffffffff);
4253
	for_each_pipe(dev_priv, pipe)
4254 4255 4256 4257
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

4258
	for_each_pipe(dev_priv, pipe)
4259 4260 4261 4262 4263
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4264 4265 4266 4267 4268 4269 4270
/**
 * intel_irq_init - initializes irq support
 * @dev_priv: i915 device instance
 *
 * This function initializes all the irq support including work items, timers
 * and all the vtables. It does not setup the interrupt itself though.
 */
4271
void intel_irq_init(struct drm_i915_private *dev_priv)
4272
{
4273
	struct drm_device *dev = &dev_priv->drm;
4274

4275 4276
	intel_hpd_init_work(dev_priv);

4277
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4278
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4279

4280
	if (HAS_GUC_SCHED(dev_priv))
4281 4282
		dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT;

4283
	/* Let's track the enabled rps events */
4284
	if (IS_VALLEYVIEW(dev_priv))
4285
		/* WaGsvRC0ResidencyMethod:vlv */
4286
		dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4287 4288
	else
		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4289

4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301
	dev_priv->rps.pm_intr_keep = 0;

	/*
	 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
	 * if GEN6_PM_UP_EI_EXPIRED is masked.
	 *
	 * TODO: verify if this can be reproduced on VLV,CHV.
	 */
	if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
		dev_priv->rps.pm_intr_keep |= GEN6_PM_RP_UP_EI_EXPIRED;

	if (INTEL_INFO(dev_priv)->gen >= 8)
4302
		dev_priv->rps.pm_intr_keep |= GEN8_PMINTR_REDIRECT_TO_GUC;
4303

4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327
	/*
	 * The REDIRECT_TO_GUC bit of the PMINTRMSK register directs all
	 * (unmasked) PM interrupts to the GuC. All other bits of this
	 * register *disable* generation of a specific interrupt.
	 *
	 * 'pm_intr_keep' indicates bits that are NOT to be set when
	 * writing to the PM interrupt mask register, i.e. interrupts
	 * that must not be disabled.
	 *
	 * If the GuC is handling these interrupts, then we must not let
	 * the PM code disable ANY interrupt that the GuC is expecting.
	 * So for each ENABLED (0) bit in this register, we must SET the
	 * bit in pm_intr_keep so that it's left enabled for the GuC.
	 * GuC needs ARAT expired interrupt unmasked hence it is set in
	 * pm_intr_keep.
	 *
	 * Here we CLEAR REDIRECT_TO_GUC bit in pm_intr_keep, which will
	 * result in the register bit being left SET!
	 */
	if (HAS_GUC_SCHED(dev_priv)) {
		dev_priv->rps.pm_intr_keep |= ARAT_EXPIRED_INTRMSK;
		dev_priv->rps.pm_intr_keep &= ~GEN8_PMINTR_REDIRECT_TO_GUC;
	}

4328
	if (IS_GEN2(dev_priv)) {
4329
		/* Gen2 doesn't have a hardware frame counter */
4330
		dev->max_vblank_count = 0;
4331
	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4332
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4333
		dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4334 4335 4336
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4337 4338
	}

4339 4340 4341 4342 4343
	/*
	 * Opt out of the vblank disable timer on everything except gen2.
	 * Gen2 doesn't have a hardware frame counter and so depends on
	 * vblank interrupts to produce sane vblank seuquence numbers.
	 */
4344
	if (!IS_GEN2(dev_priv))
4345 4346
		dev->vblank_disable_immediate = true;

4347 4348 4349 4350 4351 4352 4353 4354 4355 4356
	/* Most platforms treat the display irq block as an always-on
	 * power domain. vlv/chv can disable it at runtime and need
	 * special care to avoid writing any of the display block registers
	 * outside of the power domain. We defer setting up the display irqs
	 * in this case to the runtime pm.
	 */
	dev_priv->display_irqs_enabled = true;
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
		dev_priv->display_irqs_enabled = false;

L
Lyude 已提交
4357 4358
	dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;

4359 4360
	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4361

4362
	if (IS_CHERRYVIEW(dev_priv)) {
4363 4364 4365 4366
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
4367 4368
		dev->driver->enable_vblank = i965_enable_vblank;
		dev->driver->disable_vblank = i965_disable_vblank;
4369
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4370
	} else if (IS_VALLEYVIEW(dev_priv)) {
J
Jesse Barnes 已提交
4371 4372 4373 4374
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
4375 4376
		dev->driver->enable_vblank = i965_enable_vblank;
		dev->driver->disable_vblank = i965_disable_vblank;
4377
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4378
	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4379
		dev->driver->irq_handler = gen8_irq_handler;
4380
		dev->driver->irq_preinstall = gen8_irq_reset;
4381 4382 4383 4384
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
4385
		if (IS_GEN9_LP(dev_priv))
4386
			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4387
		else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv))
4388 4389
			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		else
4390
			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4391
	} else if (HAS_PCH_SPLIT(dev_priv)) {
4392
		dev->driver->irq_handler = ironlake_irq_handler;
4393
		dev->driver->irq_preinstall = ironlake_irq_reset;
4394 4395 4396 4397
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4398
		dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4399
	} else {
4400
		if (IS_GEN2(dev_priv)) {
C
Chris Wilson 已提交
4401 4402 4403 4404
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4405 4406
			dev->driver->enable_vblank = i8xx_enable_vblank;
			dev->driver->disable_vblank = i8xx_disable_vblank;
4407
		} else if (IS_GEN3(dev_priv)) {
4408 4409 4410 4411
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
4412 4413
			dev->driver->enable_vblank = i8xx_enable_vblank;
			dev->driver->disable_vblank = i8xx_disable_vblank;
C
Chris Wilson 已提交
4414
		} else {
4415 4416 4417 4418
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
4419 4420
			dev->driver->enable_vblank = i965_enable_vblank;
			dev->driver->disable_vblank = i965_disable_vblank;
C
Chris Wilson 已提交
4421
		}
4422 4423
		if (I915_HAS_HOTPLUG(dev_priv))
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4424 4425
	}
}
4426

4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437
/**
 * intel_irq_install - enables the hardware interrupt
 * @dev_priv: i915 device instance
 *
 * This function enables the hardware interrupt handling, but leaves the hotplug
 * handling still disabled. It is called after intel_irq_init().
 *
 * In the driver load and resume code we need working interrupts in a few places
 * but don't want to deal with the hassle of concurrent probe and hotplug
 * workers. Hence the split into this two-stage approach.
 */
4438 4439 4440 4441 4442 4443 4444 4445 4446
int intel_irq_install(struct drm_i915_private *dev_priv)
{
	/*
	 * We enable some interrupt sources in our postinstall hooks, so mark
	 * interrupts as enabled _before_ actually enabling them to avoid
	 * special cases in our ordering checks.
	 */
	dev_priv->pm.irqs_enabled = true;

4447
	return drm_irq_install(&dev_priv->drm, dev_priv->drm.pdev->irq);
4448 4449
}

4450 4451 4452 4453 4454 4455 4456
/**
 * intel_irq_uninstall - finilizes all irq handling
 * @dev_priv: i915 device instance
 *
 * This stops interrupt and hotplug handling and unregisters and frees all
 * resources acquired in the init functions.
 */
4457 4458
void intel_irq_uninstall(struct drm_i915_private *dev_priv)
{
4459
	drm_irq_uninstall(&dev_priv->drm);
4460 4461 4462 4463
	intel_hpd_cancel_work(dev_priv);
	dev_priv->pm.irqs_enabled = false;
}

4464 4465 4466 4467 4468 4469 4470
/**
 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
 * @dev_priv: i915 device instance
 *
 * This function is used to disable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4471
void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4472
{
4473
	dev_priv->drm.driver->irq_uninstall(&dev_priv->drm);
4474
	dev_priv->pm.irqs_enabled = false;
4475
	synchronize_irq(dev_priv->drm.irq);
4476 4477
}

4478 4479 4480 4481 4482 4483 4484
/**
 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
 * @dev_priv: i915 device instance
 *
 * This function is used to enable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4485
void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4486
{
4487
	dev_priv->pm.irqs_enabled = true;
4488 4489
	dev_priv->drm.driver->irq_preinstall(&dev_priv->drm);
	dev_priv->drm.driver->irq_postinstall(&dev_priv->drm);
4490
}