i915_irq.c 123.9 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47
/**
 * DOC: interrupt handling
 *
 * These functions provide the basic support for enabling and disabling the
 * interrupt handling support. There's a lot more functionality in i915_irq.c
 * and related files, but that will be described in separate chapters.
 */

48 49 50 51
static const u32 hpd_ilk[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
};

52 53 54 55
static const u32 hpd_ivb[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
};

56 57 58 59
static const u32 hpd_bdw[HPD_NUM_PINS] = {
	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
};

60
static const u32 hpd_ibx[HPD_NUM_PINS] = {
61 62 63 64 65 66 67
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

68
static const u32 hpd_cpt[HPD_NUM_PINS] = {
69
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71 72 73 74 75
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

X
Xiong Zhang 已提交
76
static const u32 hpd_spt[HPD_NUM_PINS] = {
77
	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
X
Xiong Zhang 已提交
78 79 80 81 82 83
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
};

84
static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85 86 87 88 89 90 91 92
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

93
static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94 95 96 97 98 99 100 101
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

102
static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103 104 105 106 107 108 109 110
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

111 112
/* BXT hpd list */
static const u32 hpd_bxt[HPD_NUM_PINS] = {
113
	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114 115 116 117
	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
};

118
/* IIR can theoretically queue up two events. Be paranoid. */
119
#define GEN8_IRQ_RESET_NDX(type, which) do { \
120 121 122 123 124 125 126 127 128
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

129
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
130
	I915_WRITE(type##IMR, 0xffffffff); \
131
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
132
	I915_WRITE(type##IER, 0); \
133 134 135 136
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
137 138
} while (0)

139 140 141
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
142 143
static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
				    i915_reg_t reg)
144 145 146 147 148 149 150
{
	u32 val = I915_READ(reg);

	if (val == 0)
		return;

	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
151
	     i915_mmio_reg_offset(reg), val);
152 153 154 155 156
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
}
157

P
Paulo Zanoni 已提交
158
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
159
	gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
160
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
161 162
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
P
Paulo Zanoni 已提交
163 164 165
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
166
	gen5_assert_iir_is_zero(dev_priv, type##IIR); \
P
Paulo Zanoni 已提交
167
	I915_WRITE(type##IER, (ier_val)); \
168 169
	I915_WRITE(type##IMR, (imr_val)); \
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
170 171
} while (0)

172
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
173
static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
174

175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
/* For display hotplug interrupt */
static inline void
i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
				     uint32_t mask,
				     uint32_t bits)
{
	uint32_t val;

	assert_spin_locked(&dev_priv->irq_lock);
	WARN_ON(bits & ~mask);

	val = I915_READ(PORT_HOTPLUG_EN);
	val &= ~mask;
	val |= bits;
	I915_WRITE(PORT_HOTPLUG_EN, val);
}

/**
 * i915_hotplug_interrupt_update - update hotplug interrupt enable
 * @dev_priv: driver private
 * @mask: bits to update
 * @bits: bits to enable
 * NOTE: the HPD enable bits are modified both inside and outside
 * of an interrupt context. To avoid that read-modify-write cycles
 * interfer, these bits are protected by a spinlock. Since this
 * function is usually not called from a context where the lock is
 * held already, this function acquires the lock itself. A non-locking
 * version is also available.
 */
void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
				   uint32_t mask,
				   uint32_t bits)
{
	spin_lock_irq(&dev_priv->irq_lock);
	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
	spin_unlock_irq(&dev_priv->irq_lock);
}

213 214 215 216 217 218
/**
 * ilk_update_display_irq - update DEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
219 220 221
void ilk_update_display_irq(struct drm_i915_private *dev_priv,
			    uint32_t interrupt_mask,
			    uint32_t enabled_irq_mask)
222
{
223 224
	uint32_t new_val;

225 226
	assert_spin_locked(&dev_priv->irq_lock);

227 228
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

229
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
230 231
		return;

232 233 234 235 236 237
	new_val = dev_priv->irq_mask;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->irq_mask) {
		dev_priv->irq_mask = new_val;
238
		I915_WRITE(DEIMR, dev_priv->irq_mask);
239
		POSTING_READ(DEIMR);
240 241 242
	}
}

P
Paulo Zanoni 已提交
243 244 245 246 247 248 249 250 251 252 253 254
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
	assert_spin_locked(&dev_priv->irq_lock);

255 256
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

257
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
258 259
		return;

P
Paulo Zanoni 已提交
260 261 262 263 264
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
}

265
void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
266 267
{
	ilk_update_gt_irq(dev_priv, mask, mask);
268
	POSTING_READ_FW(GTIMR);
P
Paulo Zanoni 已提交
269 270
}

271
void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
272 273 274 275
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

276
static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
277 278 279 280
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
}

281
static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
282 283 284 285
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
}

286
static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
287 288 289 290
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
}

P
Paulo Zanoni 已提交
291
/**
292 293 294 295 296
 * snb_update_pm_irq - update GEN6_PMIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
P
Paulo Zanoni 已提交
297 298 299 300
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
301
	uint32_t new_val;
P
Paulo Zanoni 已提交
302

303 304
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

P
Paulo Zanoni 已提交
305 306
	assert_spin_locked(&dev_priv->irq_lock);

307
	new_val = dev_priv->pm_imr;
308 309 310
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

311 312 313
	if (new_val != dev_priv->pm_imr) {
		dev_priv->pm_imr = new_val;
		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_imr);
314
		POSTING_READ(gen6_pm_imr(dev_priv));
315
	}
P
Paulo Zanoni 已提交
316 317
}

318
void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
P
Paulo Zanoni 已提交
319
{
320 321 322
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

P
Paulo Zanoni 已提交
323 324 325
	snb_update_pm_irq(dev_priv, mask, mask);
}

326
static void __gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
P
Paulo Zanoni 已提交
327 328 329 330
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

331
void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
332 333 334 335
{
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

336
	__gen6_mask_pm_irq(dev_priv, mask);
337 338
}

339
void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 reset_mask)
I
Imre Deak 已提交
340
{
341
	i915_reg_t reg = gen6_pm_iir(dev_priv);
I
Imre Deak 已提交
342

343 344 345 346
	assert_spin_locked(&dev_priv->irq_lock);

	I915_WRITE(reg, reset_mask);
	I915_WRITE(reg, reset_mask);
I
Imre Deak 已提交
347
	POSTING_READ(reg);
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
}

void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mask)
{
	assert_spin_locked(&dev_priv->irq_lock);

	dev_priv->pm_ier |= enable_mask;
	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
	gen6_unmask_pm_irq(dev_priv, enable_mask);
	/* unmask_pm_irq provides an implicit barrier (POSTING_READ) */
}

void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_mask)
{
	assert_spin_locked(&dev_priv->irq_lock);

	dev_priv->pm_ier &= ~disable_mask;
	__gen6_mask_pm_irq(dev_priv, disable_mask);
	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
	/* though a barrier is missing here, but don't really need a one */
}

void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	gen6_reset_pm_iir(dev_priv, dev_priv->pm_rps_events);
374
	dev_priv->rps.pm_iir = 0;
I
Imre Deak 已提交
375 376 377
	spin_unlock_irq(&dev_priv->irq_lock);
}

378
void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
379
{
380 381 382
	if (READ_ONCE(dev_priv->rps.interrupts_enabled))
		return;

383
	spin_lock_irq(&dev_priv->irq_lock);
384 385
	WARN_ON_ONCE(dev_priv->rps.pm_iir);
	WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
I
Imre Deak 已提交
386
	dev_priv->rps.interrupts_enabled = true;
387
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
388

389 390 391
	spin_unlock_irq(&dev_priv->irq_lock);
}

392 393
u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
{
394
	return (mask & ~dev_priv->rps.pm_intr_keep);
395 396
}

397
void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
398
{
399 400 401
	if (!READ_ONCE(dev_priv->rps.interrupts_enabled))
		return;

I
Imre Deak 已提交
402 403
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->rps.interrupts_enabled = false;
404

405
	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u));
406

407
	gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
408 409

	spin_unlock_irq(&dev_priv->irq_lock);
410
	synchronize_irq(dev_priv->drm.irq);
411 412 413 414 415 416 417 418

	/* Now that we will not be generating any more work, flush any
	 * outsanding tasks. As we are called on the RPS idle path,
	 * we will reset the GPU to minimum frequencies, so the current
	 * state of the worker can be discarded.
	 */
	cancel_work_sync(&dev_priv->rps.work);
	gen6_reset_rps_interrupts(dev_priv);
419 420
}

421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452
void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	gen6_reset_pm_iir(dev_priv, dev_priv->pm_guc_events);
	spin_unlock_irq(&dev_priv->irq_lock);
}

void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	if (!dev_priv->guc.interrupts_enabled) {
		WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) &
				       dev_priv->pm_guc_events);
		dev_priv->guc.interrupts_enabled = true;
		gen6_enable_pm_irq(dev_priv, dev_priv->pm_guc_events);
	}
	spin_unlock_irq(&dev_priv->irq_lock);
}

void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->guc.interrupts_enabled = false;

	gen6_disable_pm_irq(dev_priv, dev_priv->pm_guc_events);

	spin_unlock_irq(&dev_priv->irq_lock);
	synchronize_irq(dev_priv->drm.irq);

	gen9_reset_guc_interrupts(dev_priv);
}

453
/**
454 455 456 457 458
 * bdw_update_port_irq - update DE port interrupt
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
				uint32_t interrupt_mask,
				uint32_t enabled_irq_mask)
{
	uint32_t new_val;
	uint32_t old_val;

	assert_spin_locked(&dev_priv->irq_lock);

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	old_val = I915_READ(GEN8_DE_PORT_IMR);

	new_val = old_val;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != old_val) {
		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		POSTING_READ(GEN8_DE_PORT_IMR);
	}
}

485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516
/**
 * bdw_update_pipe_irq - update DE pipe interrupt
 * @dev_priv: driver private
 * @pipe: pipe whose interrupt to update
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
			 enum pipe pipe,
			 uint32_t interrupt_mask,
			 uint32_t enabled_irq_mask)
{
	uint32_t new_val;

	assert_spin_locked(&dev_priv->irq_lock);

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	new_val = dev_priv->de_irq_mask[pipe];
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->de_irq_mask[pipe]) {
		dev_priv->de_irq_mask[pipe] = new_val;
		I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
	}
}

517 518 519 520 521 522
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
523 524 525
void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
				  uint32_t interrupt_mask,
				  uint32_t enabled_irq_mask)
526 527 528 529 530
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

531 532
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

533 534
	assert_spin_locked(&dev_priv->irq_lock);

535
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
536 537
		return;

538 539 540
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
541

D
Daniel Vetter 已提交
542
static void
543 544
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
545
{
546
	i915_reg_t reg = PIPESTAT(pipe);
547
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
548

549
	assert_spin_locked(&dev_priv->irq_lock);
550
	WARN_ON(!intel_irqs_enabled(dev_priv));
551

552 553 554 555
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
556 557 558
		return;

	if ((pipestat & enable_mask) == enable_mask)
559 560
		return;

561 562
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

563
	/* Enable the interrupt, clear any pending status */
564
	pipestat |= enable_mask | status_mask;
565 566
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
567 568
}

D
Daniel Vetter 已提交
569
static void
570 571
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
572
{
573
	i915_reg_t reg = PIPESTAT(pipe);
574
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
575

576
	assert_spin_locked(&dev_priv->irq_lock);
577
	WARN_ON(!intel_irqs_enabled(dev_priv));
578

579 580 581 582
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
583 584
		return;

585 586 587
	if ((pipestat & enable_mask) == 0)
		return;

588 589
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

590
	pipestat &= ~enable_mask;
591 592
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
593 594
}

595 596 597 598 599
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
600 601
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
602 603 604
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
605 606 607 608 609 610
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
611 612 613 614 615 616 617 618 619 620 621 622

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

623 624 625 626 627 628
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

629
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
630
		enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
631 632 633
							   status_mask);
	else
		enable_mask = status_mask << 16;
634 635 636 637 638 639 640 641 642
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

643
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
644
		enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
645 646 647
							   status_mask);
	else
		enable_mask = status_mask << 16;
648 649 650
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

651
/**
652
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
653
 * @dev_priv: i915 device private
654
 */
655
static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
656
{
657
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev_priv))
658 659
		return;

660
	spin_lock_irq(&dev_priv->irq_lock);
661

662
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
663
	if (INTEL_GEN(dev_priv) >= 4)
664
		i915_enable_pipestat(dev_priv, PIPE_A,
665
				     PIPE_LEGACY_BLC_EVENT_STATUS);
666

667
	spin_unlock_irq(&dev_priv->irq_lock);
668 669
}

670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719
/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via PIPECONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on PIPECONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

720 721 722
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
723
static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
724
{
725
	struct drm_i915_private *dev_priv = to_i915(dev);
726
	i915_reg_t high_frame, low_frame;
727
	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
728 729
	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
								pipe);
730
	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
731

732 733 734 735 736
	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vbl_start = mode->crtc_vblank_start;
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
737

738 739 740 741 742 743
	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

744 745
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
746

747 748 749 750 751 752
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
753
		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
754
		low   = I915_READ(low_frame);
755
		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
756 757
	} while (high1 != high2);

758
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
759
	pixel = low & PIPE_PIXEL_MASK;
760
	low >>= PIPE_FRAME_LOW_SHIFT;
761 762 763 764 765 766

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
767
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
768 769
}

770
static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
771
{
772
	struct drm_i915_private *dev_priv = to_i915(dev);
773

774
	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
775 776
}

777
/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
778 779 780
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
781
	struct drm_i915_private *dev_priv = to_i915(dev);
782
	const struct drm_display_mode *mode = &crtc->base.hwmode;
783
	enum pipe pipe = crtc->pipe;
784
	int position, vtotal;
785

786
	vtotal = mode->crtc_vtotal;
787 788 789
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

790
	if (IS_GEN2(dev_priv))
791
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
792
	else
793
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
794

795 796 797 798 799 800 801 802 803 804 805 806
	/*
	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
	 * read it just before the start of vblank.  So try it again
	 * so we don't accidentally end up spanning a vblank frame
	 * increment, causing the pipe_update_end() code to squak at us.
	 *
	 * The nature of this problem means we can't simply check the ISR
	 * bit and return the vblank start value; nor can we use the scanline
	 * debug register in the transcoder as it appears to have the same
	 * problem.  We may need to extend this to include other platforms,
	 * but so far testing only shows the problem on HSW.
	 */
807
	if (HAS_DDI(dev_priv) && !position) {
808 809 810 811 812 813 814 815 816 817 818 819 820
		int i, temp;

		for (i = 0; i < 100; i++) {
			udelay(1);
			temp = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) &
				DSL_LINEMASK_GEN3;
			if (temp != position) {
				position = temp;
				break;
			}
		}
	}

821
	/*
822 823
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
824
	 */
825
	return (position + crtc->scanline_offset) % vtotal;
826 827
}

828
static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
829
				    unsigned int flags, int *vpos, int *hpos,
830 831
				    ktime_t *stime, ktime_t *etime,
				    const struct drm_display_mode *mode)
832
{
833
	struct drm_i915_private *dev_priv = to_i915(dev);
834 835
	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
								pipe);
836
	int position;
837
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
838 839
	bool in_vbl = true;
	int ret = 0;
840
	unsigned long irqflags;
841

842
	if (WARN_ON(!mode->crtc_clock)) {
843
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
844
				 "pipe %c\n", pipe_name(pipe));
845 846 847
		return 0;
	}

848
	htotal = mode->crtc_htotal;
849
	hsync_start = mode->crtc_hsync_start;
850 851 852
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
853

854 855 856 857 858 859
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

860 861
	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

862 863 864 865 866 867
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
868

869 870 871 872 873 874
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

875
	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
876 877 878
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
879
		position = __intel_get_crtc_scanline(intel_crtc);
880 881 882 883 884
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
885
		position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
886

887 888 889 890
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
891

892 893 894 895 896 897 898 899 900 901 902 903
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

904 905 906 907 908 909 910 911 912 913
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
914 915
	}

916 917 918 919 920 921 922 923
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

924 925 926 927 928 929 930 931 932 933 934 935
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
936

937
	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
938 939 940 941 942 943
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
944 945 946

	/* In vblank? */
	if (in_vbl)
947
		ret |= DRM_SCANOUTPOS_IN_VBLANK;
948 949 950 951

	return ret;
}

952 953
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
954
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
955 956 957 958 959 960 961 962 963 964
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

965
static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
966 967 968 969
			      int *max_error,
			      struct timeval *vblank_time,
			      unsigned flags)
{
970
	struct drm_i915_private *dev_priv = to_i915(dev);
971
	struct intel_crtc *crtc;
972

973
	if (pipe >= INTEL_INFO(dev_priv)->num_pipes) {
974
		DRM_ERROR("Invalid crtc %u\n", pipe);
975 976 977 978
		return -EINVAL;
	}

	/* Get drm_crtc to timestamp: */
979
	crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
980
	if (crtc == NULL) {
981
		DRM_ERROR("Invalid crtc %u\n", pipe);
982 983 984
		return -EINVAL;
	}

985
	if (!crtc->base.hwmode.crtc_clock) {
986
		DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
987 988
		return -EBUSY;
	}
989 990

	/* Helper routine in DRM core does all the work: */
991 992
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
						     vblank_time, flags,
993
						     &crtc->base.hwmode);
994 995
}

996
static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
997
{
998
	u32 busy_up, busy_down, max_avg, min_avg;
999 1000
	u8 new_delay;

1001
	spin_lock(&mchdev_lock);
1002

1003 1004
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

1005
	new_delay = dev_priv->ips.cur_delay;
1006

1007
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
1008 1009
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
1010 1011 1012 1013
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
1014
	if (busy_up > max_avg) {
1015 1016 1017 1018
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
1019
	} else if (busy_down < min_avg) {
1020 1021 1022 1023
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
1024 1025
	}

1026
	if (ironlake_set_drps(dev_priv, new_delay))
1027
		dev_priv->ips.cur_delay = new_delay;
1028

1029
	spin_unlock(&mchdev_lock);
1030

1031 1032 1033
	return;
}

1034
static void notify_ring(struct intel_engine_cs *engine)
1035
{
1036 1037
	struct drm_i915_gem_request *rq = NULL;
	struct intel_wait *wait;
1038

1039
	atomic_inc(&engine->irq_count);
1040
	set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071

	rcu_read_lock();

	spin_lock(&engine->breadcrumbs.lock);
	wait = engine->breadcrumbs.first_wait;
	if (wait) {
		/* We use a callback from the dma-fence to submit
		 * requests after waiting on our own requests. To
		 * ensure minimum delay in queuing the next request to
		 * hardware, signal the fence now rather than wait for
		 * the signaler to be woken up. We still wake up the
		 * waiter in order to handle the irq-seqno coherency
		 * issues (we may receive the interrupt before the
		 * seqno is written, see __i915_request_irq_complete())
		 * and to handle coalescing of multiple seqno updates
		 * and many waiters.
		 */
		if (i915_seqno_passed(intel_engine_get_seqno(engine),
				      wait->seqno))
			rq = wait->request;

		wake_up_process(wait->tsk);
	}
	spin_unlock(&engine->breadcrumbs.lock);

	if (rq)
		dma_fence_signal(&rq->fence);

	rcu_read_unlock();

	trace_intel_engine_notify(engine, wait);
1072 1073
}

1074 1075
static void vlv_c0_read(struct drm_i915_private *dev_priv,
			struct intel_rps_ei *ei)
1076
{
1077 1078 1079 1080
	ei->cz_clock = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
}
1081

1082 1083 1084 1085 1086 1087
static bool vlv_c0_above(struct drm_i915_private *dev_priv,
			 const struct intel_rps_ei *old,
			 const struct intel_rps_ei *now,
			 int threshold)
{
	u64 time, c0;
1088
	unsigned int mul = 100;
1089

1090 1091
	if (old->cz_clock == 0)
		return false;
1092

1093 1094 1095
	if (I915_READ(VLV_COUNTER_CONTROL) & VLV_COUNT_RANGE_HIGH)
		mul <<= 8;

1096
	time = now->cz_clock - old->cz_clock;
1097
	time *= threshold * dev_priv->czclk_freq;
1098

1099 1100 1101
	/* Workload can be split between render + media, e.g. SwapBuffers
	 * being blitted in X after being rendered in mesa. To account for
	 * this we need to combine both engines into our activity counter.
1102
	 */
1103 1104
	c0 = now->render_c0 - old->render_c0;
	c0 += now->media_c0 - old->media_c0;
1105
	c0 *= mul * VLV_CZ_CLOCK_TO_MILLI_SEC;
1106

1107
	return c0 >= time;
1108 1109
}

1110
void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1111
{
1112 1113 1114
	vlv_c0_read(dev_priv, &dev_priv->rps.down_ei);
	dev_priv->rps.up_ei = dev_priv->rps.down_ei;
}
1115

1116 1117 1118 1119
static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
{
	struct intel_rps_ei now;
	u32 events = 0;
1120

1121
	if ((pm_iir & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED)) == 0)
1122
		return 0;
1123

1124 1125 1126
	vlv_c0_read(dev_priv, &now);
	if (now.cz_clock == 0)
		return 0;
1127

1128 1129 1130
	if (pm_iir & GEN6_PM_RP_DOWN_EI_EXPIRED) {
		if (!vlv_c0_above(dev_priv,
				  &dev_priv->rps.down_ei, &now,
1131
				  dev_priv->rps.down_threshold))
1132 1133 1134
			events |= GEN6_PM_RP_DOWN_THRESHOLD;
		dev_priv->rps.down_ei = now;
	}
1135

1136 1137 1138
	if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
		if (vlv_c0_above(dev_priv,
				 &dev_priv->rps.up_ei, &now,
1139
				 dev_priv->rps.up_threshold))
1140 1141
			events |= GEN6_PM_RP_UP_THRESHOLD;
		dev_priv->rps.up_ei = now;
1142 1143
	}

1144
	return events;
1145 1146
}

1147 1148
static bool any_waiters(struct drm_i915_private *dev_priv)
{
1149
	struct intel_engine_cs *engine;
1150
	enum intel_engine_id id;
1151

1152
	for_each_engine(engine, dev_priv, id)
1153
		if (intel_engine_has_waiter(engine))
1154 1155 1156 1157 1158
			return true;

	return false;
}

1159
static void gen6_pm_rps_work(struct work_struct *work)
1160
{
1161 1162
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
1163 1164
	bool client_boost;
	int new_delay, adj, min, max;
P
Paulo Zanoni 已提交
1165
	u32 pm_iir;
1166

1167
	spin_lock_irq(&dev_priv->irq_lock);
I
Imre Deak 已提交
1168 1169 1170 1171 1172
	/* Speed up work cancelation during disabling rps interrupts. */
	if (!dev_priv->rps.interrupts_enabled) {
		spin_unlock_irq(&dev_priv->irq_lock);
		return;
	}
1173

1174 1175
	pm_iir = dev_priv->rps.pm_iir;
	dev_priv->rps.pm_iir = 0;
1176
	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
1177
	gen6_unmask_pm_irq(dev_priv, dev_priv->pm_rps_events);
1178 1179
	client_boost = dev_priv->rps.client_boost;
	dev_priv->rps.client_boost = false;
1180
	spin_unlock_irq(&dev_priv->irq_lock);
1181

1182
	/* Make sure we didn't queue anything we're not going to process. */
1183
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1184

1185
	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1186
		return;
1187

1188
	mutex_lock(&dev_priv->rps.hw_lock);
1189

1190 1191
	pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);

1192
	adj = dev_priv->rps.last_adj;
1193
	new_delay = dev_priv->rps.cur_freq;
1194 1195
	min = dev_priv->rps.min_freq_softlimit;
	max = dev_priv->rps.max_freq_softlimit;
1196 1197 1198 1199
	if (client_boost || any_waiters(dev_priv))
		max = dev_priv->rps.max_freq;
	if (client_boost && new_delay < dev_priv->rps.boost_freq) {
		new_delay = dev_priv->rps.boost_freq;
1200 1201
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1202 1203
		if (adj > 0)
			adj *= 2;
1204 1205
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1206 1207 1208

		if (new_delay >= dev_priv->rps.max_freq_softlimit)
			adj = 0;
1209
	} else if (client_boost || any_waiters(dev_priv)) {
1210
		adj = 0;
1211
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1212 1213
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1214
		else if (dev_priv->rps.cur_freq > dev_priv->rps.min_freq_softlimit)
1215
			new_delay = dev_priv->rps.min_freq_softlimit;
1216 1217 1218 1219
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
1220 1221
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1222 1223 1224

		if (new_delay <= dev_priv->rps.min_freq_softlimit)
			adj = 0;
1225
	} else { /* unknown event */
1226
		adj = 0;
1227
	}
1228

1229 1230
	dev_priv->rps.last_adj = adj;

1231 1232 1233
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1234
	new_delay += adj;
1235
	new_delay = clamp_t(int, new_delay, min, max);
1236

1237 1238 1239 1240
	if (intel_set_rps(dev_priv, new_delay)) {
		DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
		dev_priv->rps.last_adj = 0;
	}
1241

1242
	mutex_unlock(&dev_priv->rps.hw_lock);
1243 1244
}

1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1257 1258
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, l3_parity.error_work);
1259
	u32 error_status, row, bank, subbank;
1260
	char *parity_event[6];
1261
	uint32_t misccpctl;
1262
	uint8_t slice = 0;
1263 1264 1265 1266 1267

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
1268
	mutex_lock(&dev_priv->drm.struct_mutex);
1269

1270 1271 1272 1273
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1274 1275 1276 1277
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1278
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1279
		i915_reg_t reg;
1280

1281
		slice--;
1282
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv)))
1283
			break;
1284

1285
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1286

1287
		reg = GEN7_L3CDERRST1(slice);
1288

1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1304
		kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
1305
				   KOBJ_CHANGE, parity_event);
1306

1307 1308
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1309

1310 1311 1312 1313 1314
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1315

1316
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1317

1318 1319
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
1320
	spin_lock_irq(&dev_priv->irq_lock);
1321
	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1322
	spin_unlock_irq(&dev_priv->irq_lock);
1323

1324
	mutex_unlock(&dev_priv->drm.struct_mutex);
1325 1326
}

1327 1328
static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv,
					       u32 iir)
1329
{
1330
	if (!HAS_L3_DPF(dev_priv))
1331 1332
		return;

1333
	spin_lock(&dev_priv->irq_lock);
1334
	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1335
	spin_unlock(&dev_priv->irq_lock);
1336

1337
	iir &= GT_PARITY_ERROR(dev_priv);
1338 1339 1340 1341 1342 1343
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1344
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1345 1346
}

1347
static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
1348 1349
			       u32 gt_iir)
{
1350
	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1351
		notify_ring(dev_priv->engine[RCS]);
1352
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
1353
		notify_ring(dev_priv->engine[VCS]);
1354 1355
}

1356
static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
1357 1358
			       u32 gt_iir)
{
1359
	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1360
		notify_ring(dev_priv->engine[RCS]);
1361
	if (gt_iir & GT_BSD_USER_INTERRUPT)
1362
		notify_ring(dev_priv->engine[VCS]);
1363
	if (gt_iir & GT_BLT_USER_INTERRUPT)
1364
		notify_ring(dev_priv->engine[BCS]);
1365

1366 1367
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
1368 1369
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1370

1371 1372
	if (gt_iir & GT_PARITY_ERROR(dev_priv))
		ivybridge_parity_error_irq_handler(dev_priv, gt_iir);
1373 1374
}

1375
static __always_inline void
1376
gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
1377 1378
{
	if (iir & (GT_RENDER_USER_INTERRUPT << test_shift))
1379
		notify_ring(engine);
1380 1381 1382 1383 1384

	if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) {
		set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
		tasklet_hi_schedule(&engine->irq_tasklet);
	}
1385 1386
}

1387 1388 1389
static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv,
				   u32 master_ctl,
				   u32 gt_iir[4])
1390 1391 1392 1393
{
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1394 1395 1396
		gt_iir[0] = I915_READ_FW(GEN8_GT_IIR(0));
		if (gt_iir[0]) {
			I915_WRITE_FW(GEN8_GT_IIR(0), gt_iir[0]);
1397 1398 1399 1400 1401
			ret = IRQ_HANDLED;
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1402
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1403 1404 1405
		gt_iir[1] = I915_READ_FW(GEN8_GT_IIR(1));
		if (gt_iir[1]) {
			I915_WRITE_FW(GEN8_GT_IIR(1), gt_iir[1]);
1406
			ret = IRQ_HANDLED;
1407
		} else
1408
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
1409 1410
	}

1411
	if (master_ctl & GEN8_GT_VECS_IRQ) {
1412 1413 1414
		gt_iir[3] = I915_READ_FW(GEN8_GT_IIR(3));
		if (gt_iir[3]) {
			I915_WRITE_FW(GEN8_GT_IIR(3), gt_iir[3]);
1415 1416 1417 1418 1419
			ret = IRQ_HANDLED;
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

1420
	if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
1421
		gt_iir[2] = I915_READ_FW(GEN8_GT_IIR(2));
1422 1423
		if (gt_iir[2] & (dev_priv->pm_rps_events |
				 dev_priv->pm_guc_events)) {
1424
			I915_WRITE_FW(GEN8_GT_IIR(2),
1425 1426
				      gt_iir[2] & (dev_priv->pm_rps_events |
						   dev_priv->pm_guc_events));
1427
			ret = IRQ_HANDLED;
1428 1429 1430 1431
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1432 1433 1434
	return ret;
}

1435 1436 1437 1438
static void gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
				u32 gt_iir[4])
{
	if (gt_iir[0]) {
1439
		gen8_cs_irq_handler(dev_priv->engine[RCS],
1440
				    gt_iir[0], GEN8_RCS_IRQ_SHIFT);
1441
		gen8_cs_irq_handler(dev_priv->engine[BCS],
1442 1443 1444 1445
				    gt_iir[0], GEN8_BCS_IRQ_SHIFT);
	}

	if (gt_iir[1]) {
1446
		gen8_cs_irq_handler(dev_priv->engine[VCS],
1447
				    gt_iir[1], GEN8_VCS1_IRQ_SHIFT);
1448
		gen8_cs_irq_handler(dev_priv->engine[VCS2],
1449 1450 1451 1452
				    gt_iir[1], GEN8_VCS2_IRQ_SHIFT);
	}

	if (gt_iir[3])
1453
		gen8_cs_irq_handler(dev_priv->engine[VECS],
1454 1455 1456 1457
				    gt_iir[3], GEN8_VECS_IRQ_SHIFT);

	if (gt_iir[2] & dev_priv->pm_rps_events)
		gen6_rps_irq_handler(dev_priv, gt_iir[2]);
1458 1459 1460

	if (gt_iir[2] & dev_priv->pm_guc_events)
		gen9_guc_irq_handler(dev_priv, gt_iir[2]);
1461 1462
}

1463 1464 1465 1466
static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
1467
		return val & PORTA_HOTPLUG_LONG_DETECT;
1468 1469 1470 1471 1472 1473 1474 1475 1476
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1477 1478 1479 1480 1481 1482 1483 1484 1485 1486
static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_E:
		return val & PORTE_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502
static bool spt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & PORTA_HOTPLUG_LONG_DETECT;
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	case PORT_D:
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1503 1504 1505 1506 1507 1508 1509 1510 1511 1512
static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1513
static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1514 1515 1516
{
	switch (port) {
	case PORT_B:
1517
		return val & PORTB_HOTPLUG_LONG_DETECT;
1518
	case PORT_C:
1519
		return val & PORTC_HOTPLUG_LONG_DETECT;
1520
	case PORT_D:
1521 1522 1523
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
1524 1525 1526
	}
}

1527
static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1528 1529 1530
{
	switch (port) {
	case PORT_B:
1531
		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1532
	case PORT_C:
1533
		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1534
	case PORT_D:
1535 1536 1537
		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
	default:
		return false;
1538 1539 1540
	}
}

1541 1542 1543 1544 1545 1546 1547
/*
 * Get a bit mask of pins that have triggered, and which ones may be long.
 * This can be called multiple times with the same masks to accumulate
 * hotplug detection results from several registers.
 *
 * Note that the caller is expected to zero out the masks initially.
 */
1548
static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1549
			     u32 hotplug_trigger, u32 dig_hotplug_reg,
1550 1551
			     const u32 hpd[HPD_NUM_PINS],
			     bool long_pulse_detect(enum port port, u32 val))
1552
{
1553
	enum port port;
1554 1555 1556
	int i;

	for_each_hpd_pin(i) {
1557 1558
		if ((hpd[i] & hotplug_trigger) == 0)
			continue;
1559

1560 1561
		*pin_mask |= BIT(i);

1562 1563 1564
		if (!intel_hpd_pin_to_port(i, &port))
			continue;

1565
		if (long_pulse_detect(port, dig_hotplug_reg))
1566
			*long_mask |= BIT(i);
1567 1568 1569 1570 1571 1572 1573
	}

	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
			 hotplug_trigger, dig_hotplug_reg, *pin_mask);

}

1574
static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1575
{
1576
	wake_up_all(&dev_priv->gmbus_wait_queue);
1577 1578
}

1579
static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1580
{
1581
	wake_up_all(&dev_priv->gmbus_wait_queue);
1582 1583
}

1584
#if defined(CONFIG_DEBUG_FS)
1585 1586
static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
					 enum pipe pipe,
1587 1588 1589
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1590 1591 1592
{
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
T
Tomeu Vizoso 已提交
1593 1594 1595
	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
	struct drm_driver *driver = dev_priv->drm.driver;
	uint32_t crcs[5];
1596
	int head, tail;
1597

1598
	spin_lock(&pipe_crc->lock);
T
Tomeu Vizoso 已提交
1599 1600 1601 1602 1603 1604
	if (pipe_crc->source) {
		if (!pipe_crc->entries) {
			spin_unlock(&pipe_crc->lock);
			DRM_DEBUG_KMS("spurious interrupt\n");
			return;
		}
1605

T
Tomeu Vizoso 已提交
1606 1607
		head = pipe_crc->head;
		tail = pipe_crc->tail;
1608

T
Tomeu Vizoso 已提交
1609 1610 1611 1612 1613
		if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
			spin_unlock(&pipe_crc->lock);
			DRM_ERROR("CRC buffer overflowing\n");
			return;
		}
1614

T
Tomeu Vizoso 已提交
1615
		entry = &pipe_crc->entries[head];
1616

T
Tomeu Vizoso 已提交
1617 1618 1619 1620 1621 1622
		entry->frame = driver->get_vblank_counter(&dev_priv->drm, pipe);
		entry->crc[0] = crc0;
		entry->crc[1] = crc1;
		entry->crc[2] = crc2;
		entry->crc[3] = crc3;
		entry->crc[4] = crc4;
1623

T
Tomeu Vizoso 已提交
1624 1625
		head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
		pipe_crc->head = head;
1626

T
Tomeu Vizoso 已提交
1627
		spin_unlock(&pipe_crc->lock);
1628

T
Tomeu Vizoso 已提交
1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650
		wake_up_interruptible(&pipe_crc->wq);
	} else {
		/*
		 * For some not yet identified reason, the first CRC is
		 * bonkers. So let's just wait for the next vblank and read
		 * out the buggy result.
		 *
		 * On CHV sometimes the second CRC is bonkers as well, so
		 * don't trust that one either.
		 */
		if (pipe_crc->skipped == 0 ||
		    (IS_CHERRYVIEW(dev_priv) && pipe_crc->skipped == 1)) {
			pipe_crc->skipped++;
			spin_unlock(&pipe_crc->lock);
			return;
		}
		spin_unlock(&pipe_crc->lock);
		crcs[0] = crc0;
		crcs[1] = crc1;
		crcs[2] = crc2;
		crcs[3] = crc3;
		crcs[4] = crc4;
1651 1652 1653
		drm_crtc_add_crc_entry(&crtc->base, true,
				       drm_accurate_vblank_count(&crtc->base),
				       crcs);
T
Tomeu Vizoso 已提交
1654
	}
1655
}
1656 1657
#else
static inline void
1658 1659
display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
			     enum pipe pipe,
1660 1661 1662 1663 1664
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1665

1666 1667
static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				     enum pipe pipe)
D
Daniel Vetter 已提交
1668
{
1669
	display_pipe_crc_irq_handler(dev_priv, pipe,
1670 1671
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1672 1673
}

1674 1675
static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				     enum pipe pipe)
1676
{
1677
	display_pipe_crc_irq_handler(dev_priv, pipe,
1678 1679 1680 1681 1682
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1683
}
1684

1685 1686
static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				      enum pipe pipe)
1687
{
1688 1689
	uint32_t res1, res2;

1690
	if (INTEL_GEN(dev_priv) >= 3)
1691 1692 1693 1694
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

1695
	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1696 1697 1698
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1699

1700
	display_pipe_crc_irq_handler(dev_priv, pipe,
1701 1702 1703 1704
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1705
}
1706

1707 1708 1709 1710
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1711
{
1712
	if (pm_iir & dev_priv->pm_rps_events) {
1713
		spin_lock(&dev_priv->irq_lock);
1714
		gen6_mask_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
I
Imre Deak 已提交
1715 1716
		if (dev_priv->rps.interrupts_enabled) {
			dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1717
			schedule_work(&dev_priv->rps.work);
I
Imre Deak 已提交
1718
		}
1719
		spin_unlock(&dev_priv->irq_lock);
1720 1721
	}

1722 1723 1724
	if (INTEL_INFO(dev_priv)->gen >= 8)
		return;

1725
	if (HAS_VEBOX(dev_priv)) {
1726
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1727
			notify_ring(dev_priv->engine[VECS]);
B
Ben Widawsky 已提交
1728

1729 1730
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
B
Ben Widawsky 已提交
1731
	}
1732 1733
}

1734 1735 1736
static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
{
	if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT) {
1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749
		/* Sample the log buffer flush related bits & clear them out now
		 * itself from the message identity register to minimize the
		 * probability of losing a flush interrupt, when there are back
		 * to back flush interrupts.
		 * There can be a new flush interrupt, for different log buffer
		 * type (like for ISR), whilst Host is handling one (for DPC).
		 * Since same bit is used in message register for ISR & DPC, it
		 * could happen that GuC sets the bit for 2nd interrupt but Host
		 * clears out the bit on handling the 1st interrupt.
		 */
		u32 msg, flush;

		msg = I915_READ(SOFT_SCRATCH(15));
1750 1751
		flush = msg & (INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED |
			       INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER);
1752 1753 1754 1755 1756 1757 1758
		if (flush) {
			/* Clear the message bits that are handled */
			I915_WRITE(SOFT_SCRATCH(15), msg & ~flush);

			/* Handle flush interrupt in bottom half */
			queue_work(dev_priv->guc.log.flush_wq,
				   &dev_priv->guc.log.flush_work);
1759 1760

			dev_priv->guc.log.flush_interrupt_count++;
1761 1762 1763 1764 1765
		} else {
			/* Not clearing of unhandled event bits won't result in
			 * re-triggering of the interrupt.
			 */
		}
1766 1767 1768
	}
}

1769
static bool intel_pipe_handle_vblank(struct drm_i915_private *dev_priv,
1770
				     enum pipe pipe)
1771
{
1772 1773
	bool ret;

1774
	ret = drm_handle_vblank(&dev_priv->drm, pipe);
1775
	if (ret)
1776
		intel_finish_page_flip_mmio(dev_priv, pipe);
1777 1778

	return ret;
1779 1780
}

1781 1782
static void valleyview_pipestat_irq_ack(struct drm_i915_private *dev_priv,
					u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1783 1784 1785
{
	int pipe;

1786
	spin_lock(&dev_priv->irq_lock);
1787 1788 1789 1790 1791 1792

	if (!dev_priv->display_irqs_enabled) {
		spin_unlock(&dev_priv->irq_lock);
		return;
	}

1793
	for_each_pipe(dev_priv, pipe) {
1794
		i915_reg_t reg;
1795
		u32 mask, iir_bit = 0;
1796

1797 1798 1799 1800 1801 1802 1803
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
1804 1805 1806

		/* fifo underruns are filterered in the underrun handler. */
		mask = PIPE_FIFO_UNDERRUN_STATUS;
1807 1808 1809 1810 1811 1812 1813 1814

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1815 1816 1817
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1818 1819 1820 1821 1822
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1823 1824 1825
			continue;

		reg = PIPESTAT(pipe);
1826 1827
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1828 1829 1830 1831

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1832 1833
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1834 1835
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1836
	spin_unlock(&dev_priv->irq_lock);
1837 1838
}

1839
static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1840 1841 1842
					    u32 pipe_stats[I915_MAX_PIPES])
{
	enum pipe pipe;
1843

1844
	for_each_pipe(dev_priv, pipe) {
1845 1846 1847
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
1848

1849
		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV)
1850
			intel_finish_page_flip_cs(dev_priv, pipe);
1851 1852

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1853
			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1854

1855 1856
		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1857 1858 1859
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1860
		gmbus_irq_handler(dev_priv);
1861 1862
}

1863
static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
1864 1865 1866
{
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);

1867 1868
	if (hotplug_status)
		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1869

1870 1871 1872
	return hotplug_status;
}

1873
static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1874 1875 1876
				 u32 hotplug_status)
{
	u32 pin_mask = 0, long_mask = 0;
1877

1878 1879
	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
	    IS_CHERRYVIEW(dev_priv)) {
1880
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1881

1882 1883 1884 1885 1886
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
					   hotplug_trigger, hpd_status_g4x,
					   i9xx_port_hotplug_long_detect);

1887
			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1888
		}
1889 1890

		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1891
			dp_aux_irq_handler(dev_priv);
1892 1893
	} else {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1894

1895 1896
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1897
					   hotplug_trigger, hpd_status_i915,
1898
					   i9xx_port_hotplug_long_detect);
1899
			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1900
		}
1901
	}
1902 1903
}

1904
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1905
{
1906
	struct drm_device *dev = arg;
1907
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
1908 1909
	irqreturn_t ret = IRQ_NONE;

1910 1911 1912
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1913 1914 1915
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1916
	do {
1917
		u32 iir, gt_iir, pm_iir;
1918
		u32 pipe_stats[I915_MAX_PIPES] = {};
1919
		u32 hotplug_status = 0;
1920
		u32 ier = 0;
1921

J
Jesse Barnes 已提交
1922 1923
		gt_iir = I915_READ(GTIIR);
		pm_iir = I915_READ(GEN6_PMIIR);
1924
		iir = I915_READ(VLV_IIR);
J
Jesse Barnes 已提交
1925 1926

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1927
			break;
J
Jesse Barnes 已提交
1928 1929 1930

		ret = IRQ_HANDLED;

1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
		 *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
		 * bits this time around.
		 */
1944
		I915_WRITE(VLV_MASTER_IER, 0);
1945 1946
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
1947 1948 1949 1950 1951 1952

		if (gt_iir)
			I915_WRITE(GTIIR, gt_iir);
		if (pm_iir)
			I915_WRITE(GEN6_PMIIR, pm_iir);

1953
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1954
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1955

1956 1957
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
1958
		valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1959 1960 1961 1962 1963 1964 1965

		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);
1966

1967
		I915_WRITE(VLV_IER, ier);
1968 1969
		I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
		POSTING_READ(VLV_MASTER_IER);
1970

1971
		if (gt_iir)
1972
			snb_gt_irq_handler(dev_priv, gt_iir);
1973 1974 1975
		if (pm_iir)
			gen6_rps_irq_handler(dev_priv, pm_iir);

1976
		if (hotplug_status)
1977
			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1978

1979
		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1980
	} while (0);
J
Jesse Barnes 已提交
1981

1982 1983
	enable_rpm_wakeref_asserts(dev_priv);

J
Jesse Barnes 已提交
1984 1985 1986
	return ret;
}

1987 1988
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1989
	struct drm_device *dev = arg;
1990
	struct drm_i915_private *dev_priv = to_i915(dev);
1991 1992
	irqreturn_t ret = IRQ_NONE;

1993 1994 1995
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1996 1997 1998
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1999
	do {
2000
		u32 master_ctl, iir;
2001
		u32 gt_iir[4] = {};
2002
		u32 pipe_stats[I915_MAX_PIPES] = {};
2003
		u32 hotplug_status = 0;
2004 2005
		u32 ier = 0;

2006 2007
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
2008

2009 2010
		if (master_ctl == 0 && iir == 0)
			break;
2011

2012 2013
		ret = IRQ_HANDLED;

2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
		 *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
		 * bits this time around.
		 */
2027
		I915_WRITE(GEN8_MASTER_IRQ, 0);
2028 2029
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
2030

2031
		gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
2032

2033
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
2034
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
2035

2036 2037
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
2038
		valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
2039

2040 2041 2042 2043 2044 2045 2046
		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);

2047
		I915_WRITE(VLV_IER, ier);
2048
		I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2049
		POSTING_READ(GEN8_MASTER_IRQ);
2050

2051 2052
		gen8_gt_irq_handler(dev_priv, gt_iir);

2053
		if (hotplug_status)
2054
			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
2055

2056
		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
2057
	} while (0);
2058

2059 2060
	enable_rpm_wakeref_asserts(dev_priv);

2061 2062 2063
	return ret;
}

2064 2065
static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2066 2067 2068 2069
				const u32 hpd[HPD_NUM_PINS])
{
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

2070 2071 2072 2073 2074 2075
	/*
	 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
	 * unless we touch the hotplug register, even if hotplug_trigger is
	 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
	 * errors.
	 */
2076
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2077 2078 2079 2080 2081 2082 2083 2084
	if (!hotplug_trigger) {
		u32 mask = PORTA_HOTPLUG_STATUS_MASK |
			PORTD_HOTPLUG_STATUS_MASK |
			PORTC_HOTPLUG_STATUS_MASK |
			PORTB_HOTPLUG_STATUS_MASK;
		dig_hotplug_reg &= ~mask;
	}

2085
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2086 2087
	if (!hotplug_trigger)
		return;
2088 2089 2090 2091 2092

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   pch_port_hotplug_long_detect);

2093
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2094 2095
}

2096
static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2097
{
2098
	int pipe;
2099
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
2100

2101
	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx);
2102

2103 2104 2105
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
2106
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
2107 2108
				 port_name(port));
	}
2109

2110
	if (pch_iir & SDE_AUX_MASK)
2111
		dp_aux_irq_handler(dev_priv);
2112

2113
	if (pch_iir & SDE_GMBUS)
2114
		gmbus_irq_handler(dev_priv);
2115 2116 2117 2118 2119 2120 2121 2122 2123 2124

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2125
	if (pch_iir & SDE_FDI_MASK)
2126
		for_each_pipe(dev_priv, pipe)
2127 2128 2129
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2130 2131 2132 2133 2134 2135 2136 2137

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
2138
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
2139 2140

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
2141
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
2142 2143
}

2144
static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
2145 2146
{
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
2147
	enum pipe pipe;
2148

2149 2150 2151
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

2152
	for_each_pipe(dev_priv, pipe) {
2153 2154
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2155

D
Daniel Vetter 已提交
2156
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
2157 2158
			if (IS_IVYBRIDGE(dev_priv))
				ivb_pipe_crc_irq_handler(dev_priv, pipe);
D
Daniel Vetter 已提交
2159
			else
2160
				hsw_pipe_crc_irq_handler(dev_priv, pipe);
D
Daniel Vetter 已提交
2161 2162
		}
	}
2163

2164 2165 2166
	I915_WRITE(GEN7_ERR_INT, err_int);
}

2167
static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
2168 2169 2170
{
	u32 serr_int = I915_READ(SERR_INT);

2171 2172 2173
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2174
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
2175
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
2176 2177

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
2178
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
2179 2180

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
2181
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
2182 2183

	I915_WRITE(SERR_INT, serr_int);
2184 2185
}

2186
static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2187 2188
{
	int pipe;
2189
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
2190

2191
	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt);
2192

2193 2194 2195 2196 2197 2198
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
2199 2200

	if (pch_iir & SDE_AUX_MASK_CPT)
2201
		dp_aux_irq_handler(dev_priv);
2202 2203

	if (pch_iir & SDE_GMBUS_CPT)
2204
		gmbus_irq_handler(dev_priv);
2205 2206 2207 2208 2209 2210 2211 2212

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
2213
		for_each_pipe(dev_priv, pipe)
2214 2215 2216
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2217 2218

	if (pch_iir & SDE_ERROR_CPT)
2219
		cpt_serr_int_handler(dev_priv);
2220 2221
}

2222
static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236
{
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		~SDE_PORTE_HOTPLUG_SPT;
	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
	u32 pin_mask = 0, long_mask = 0;

	if (hotplug_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
				   dig_hotplug_reg, hpd_spt,
2237
				   spt_port_hotplug_long_detect);
2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251
	}

	if (hotplug2_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
				   dig_hotplug_reg, hpd_spt,
				   spt_port_hotplug2_long_detect);
	}

	if (pin_mask)
2252
		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2253 2254

	if (pch_iir & SDE_GMBUS_CPT)
2255
		gmbus_irq_handler(dev_priv);
2256 2257
}

2258 2259
static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270
				const u32 hpd[HPD_NUM_PINS])
{
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   ilk_port_hotplug_long_detect);

2271
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2272 2273
}

2274 2275
static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
				    u32 de_iir)
2276
{
2277
	enum pipe pipe;
2278 2279
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;

2280
	if (hotplug_trigger)
2281
		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk);
2282 2283

	if (de_iir & DE_AUX_CHANNEL_A)
2284
		dp_aux_irq_handler(dev_priv);
2285 2286

	if (de_iir & DE_GSE)
2287
		intel_opregion_asle_intr(dev_priv);
2288 2289 2290 2291

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2292
	for_each_pipe(dev_priv, pipe) {
2293 2294 2295
		if (de_iir & DE_PIPE_VBLANK(pipe) &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2296

2297
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2298
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2299

2300
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
2301
			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2302

2303
		/* plane/pipes map 1:1 on ilk+ */
2304
		if (de_iir & DE_PLANE_FLIP_DONE(pipe))
2305
			intel_finish_page_flip_cs(dev_priv, pipe);
2306 2307 2308 2309 2310 2311
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

2312 2313
		if (HAS_PCH_CPT(dev_priv))
			cpt_irq_handler(dev_priv, pch_iir);
2314
		else
2315
			ibx_irq_handler(dev_priv, pch_iir);
2316 2317 2318 2319 2320

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

2321 2322
	if (IS_GEN5(dev_priv) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev_priv);
2323 2324
}

2325 2326
static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
				    u32 de_iir)
2327
{
2328
	enum pipe pipe;
2329 2330
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;

2331
	if (hotplug_trigger)
2332
		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb);
2333 2334

	if (de_iir & DE_ERR_INT_IVB)
2335
		ivb_err_int_handler(dev_priv);
2336 2337

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
2338
		dp_aux_irq_handler(dev_priv);
2339 2340

	if (de_iir & DE_GSE_IVB)
2341
		intel_opregion_asle_intr(dev_priv);
2342

2343
	for_each_pipe(dev_priv, pipe) {
2344 2345 2346
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2347 2348

		/* plane/pipes map 1:1 on ilk+ */
2349
		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe))
2350
			intel_finish_page_flip_cs(dev_priv, pipe);
2351 2352 2353
	}

	/* check event from PCH */
2354
	if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
2355 2356
		u32 pch_iir = I915_READ(SDEIIR);

2357
		cpt_irq_handler(dev_priv, pch_iir);
2358 2359 2360 2361 2362 2363

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2364 2365 2366 2367 2368 2369 2370 2371
/*
 * To handle irqs with the minimum potential races with fresh interrupts, we:
 * 1 - Disable Master Interrupt Control.
 * 2 - Find the source(s) of the interrupt.
 * 3 - Clear the Interrupt Identity bits (IIR).
 * 4 - Process the interrupt(s) that had bits set in the IIRs.
 * 5 - Re-enable Master Interrupt Control.
 */
2372
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2373
{
2374
	struct drm_device *dev = arg;
2375
	struct drm_i915_private *dev_priv = to_i915(dev);
2376
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2377
	irqreturn_t ret = IRQ_NONE;
2378

2379 2380 2381
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2382 2383 2384
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

2385 2386 2387
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2388
	POSTING_READ(DEIER);
2389

2390 2391 2392 2393 2394
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2395
	if (!HAS_PCH_NOP(dev_priv)) {
2396 2397 2398 2399
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2400

2401 2402
	/* Find, clear, then process each source of interrupt */

2403
	gt_iir = I915_READ(GTIIR);
2404
	if (gt_iir) {
2405 2406
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2407
		if (INTEL_GEN(dev_priv) >= 6)
2408
			snb_gt_irq_handler(dev_priv, gt_iir);
2409
		else
2410
			ilk_gt_irq_handler(dev_priv, gt_iir);
2411 2412
	}

2413 2414
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2415 2416
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2417 2418
		if (INTEL_GEN(dev_priv) >= 7)
			ivb_display_irq_handler(dev_priv, de_iir);
2419
		else
2420
			ilk_display_irq_handler(dev_priv, de_iir);
2421 2422
	}

2423
	if (INTEL_GEN(dev_priv) >= 6) {
2424 2425 2426 2427
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
2428
			gen6_rps_irq_handler(dev_priv, pm_iir);
2429
		}
2430
	}
2431 2432 2433

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2434
	if (!HAS_PCH_NOP(dev_priv)) {
2435 2436 2437
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2438

2439 2440 2441
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	enable_rpm_wakeref_asserts(dev_priv);

2442 2443 2444
	return ret;
}

2445 2446
static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2447
				const u32 hpd[HPD_NUM_PINS])
2448
{
2449
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2450

2451 2452
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2453

2454
	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2455
			   dig_hotplug_reg, hpd,
2456
			   bxt_port_hotplug_long_detect);
2457

2458
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2459 2460
}

2461 2462
static irqreturn_t
gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2463 2464
{
	irqreturn_t ret = IRQ_NONE;
2465
	u32 iir;
2466
	enum pipe pipe;
J
Jesse Barnes 已提交
2467

2468
	if (master_ctl & GEN8_DE_MISC_IRQ) {
2469 2470 2471
		iir = I915_READ(GEN8_DE_MISC_IIR);
		if (iir) {
			I915_WRITE(GEN8_DE_MISC_IIR, iir);
2472
			ret = IRQ_HANDLED;
2473
			if (iir & GEN8_DE_MISC_GSE)
2474
				intel_opregion_asle_intr(dev_priv);
2475 2476
			else
				DRM_ERROR("Unexpected DE Misc interrupt\n");
2477
		}
2478 2479
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2480 2481
	}

2482
	if (master_ctl & GEN8_DE_PORT_IRQ) {
2483 2484 2485
		iir = I915_READ(GEN8_DE_PORT_IIR);
		if (iir) {
			u32 tmp_mask;
2486
			bool found = false;
2487

2488
			I915_WRITE(GEN8_DE_PORT_IIR, iir);
2489
			ret = IRQ_HANDLED;
J
Jesse Barnes 已提交
2490

2491 2492 2493 2494 2495 2496 2497
			tmp_mask = GEN8_AUX_CHANNEL_A;
			if (INTEL_INFO(dev_priv)->gen >= 9)
				tmp_mask |= GEN9_AUX_CHANNEL_B |
					    GEN9_AUX_CHANNEL_C |
					    GEN9_AUX_CHANNEL_D;

			if (iir & tmp_mask) {
2498
				dp_aux_irq_handler(dev_priv);
2499 2500 2501
				found = true;
			}

2502
			if (IS_GEN9_LP(dev_priv)) {
2503 2504
				tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
				if (tmp_mask) {
2505 2506
					bxt_hpd_irq_handler(dev_priv, tmp_mask,
							    hpd_bxt);
2507 2508 2509 2510 2511
					found = true;
				}
			} else if (IS_BROADWELL(dev_priv)) {
				tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
				if (tmp_mask) {
2512 2513
					ilk_hpd_irq_handler(dev_priv,
							    tmp_mask, hpd_bdw);
2514 2515
					found = true;
				}
2516 2517
			}

2518
			if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
2519
				gmbus_irq_handler(dev_priv);
S
Shashank Sharma 已提交
2520 2521 2522
				found = true;
			}

2523
			if (!found)
2524
				DRM_ERROR("Unexpected DE Port interrupt\n");
2525
		}
2526 2527
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2528 2529
	}

2530
	for_each_pipe(dev_priv, pipe) {
2531
		u32 flip_done, fault_errors;
2532

2533 2534
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2535

2536 2537 2538 2539 2540
		iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (!iir) {
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
			continue;
		}
2541

2542 2543
		ret = IRQ_HANDLED;
		I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2544

2545 2546 2547
		if (iir & GEN8_PIPE_VBLANK &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2548

2549 2550 2551 2552 2553
		flip_done = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			flip_done &= GEN9_PIPE_PLANE1_FLIP_DONE;
		else
			flip_done &= GEN8_PIPE_PRIMARY_FLIP_DONE;
2554

2555
		if (flip_done)
2556
			intel_finish_page_flip_cs(dev_priv, pipe);
2557

2558
		if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
2559
			hsw_pipe_crc_irq_handler(dev_priv, pipe);
2560

2561 2562
		if (iir & GEN8_PIPE_FIFO_UNDERRUN)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2563

2564 2565 2566 2567 2568
		fault_errors = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
		else
			fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2569

2570
		if (fault_errors)
2571
			DRM_ERROR("Fault errors on pipe %c: 0x%08x\n",
2572 2573
				  pipe_name(pipe),
				  fault_errors);
2574 2575
	}

2576
	if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2577
	    master_ctl & GEN8_DE_PCH_IRQ) {
2578 2579 2580 2581 2582
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
2583 2584 2585
		iir = I915_READ(SDEIIR);
		if (iir) {
			I915_WRITE(SDEIIR, iir);
2586
			ret = IRQ_HANDLED;
2587

2588
			if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv))
2589
				spt_irq_handler(dev_priv, iir);
2590
			else
2591
				cpt_irq_handler(dev_priv, iir);
2592 2593 2594 2595 2596 2597 2598
		} else {
			/*
			 * Like on previous PCH there seems to be something
			 * fishy going on with forwarding PCH interrupts.
			 */
			DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
		}
2599 2600
	}

2601 2602 2603 2604 2605 2606
	return ret;
}

static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
2607
	struct drm_i915_private *dev_priv = to_i915(dev);
2608
	u32 master_ctl;
2609
	u32 gt_iir[4] = {};
2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625
	irqreturn_t ret;

	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);

	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	/* Find, clear, then process each source of interrupt */
2626 2627
	ret = gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
	gen8_gt_irq_handler(dev_priv, gt_iir);
2628 2629
	ret |= gen8_de_irq_handler(dev_priv, master_ctl);

2630 2631
	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ_FW(GEN8_MASTER_IRQ);
2632

2633 2634
	enable_rpm_wakeref_asserts(dev_priv);

2635 2636 2637
	return ret;
}

2638
static void i915_error_wake_up(struct drm_i915_private *dev_priv)
2639 2640 2641 2642 2643 2644 2645 2646 2647
{
	/*
	 * Notify all waiters for GPU completion events that reset state has
	 * been changed, and that they need to restart their wait after
	 * checking for potential errors (and bail out to drop locks if there is
	 * a gpu reset pending so that i915_error_work_func can acquire them).
	 */

	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2648
	wake_up_all(&dev_priv->gpu_error.wait_queue);
2649 2650 2651 2652 2653

	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
	wake_up_all(&dev_priv->pending_flip_queue);
}

2654
/**
2655
 * i915_reset_and_wakeup - do process context error handling work
2656
 * @dev_priv: i915 device private
2657 2658 2659 2660
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
2661
static void i915_reset_and_wakeup(struct drm_i915_private *dev_priv)
2662
{
2663
	struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
2664 2665 2666
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2667

2668
	kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);
2669

2670 2671 2672
	DRM_DEBUG_DRIVER("resetting chip\n");
	kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);

2673
	/*
2674 2675 2676 2677 2678
	 * In most cases it's guaranteed that we get here with an RPM
	 * reference held, for example because there is a pending GPU
	 * request that won't finish until the reset is done. This
	 * isn't the case at least when we get here by doing a
	 * simulated reset via debugs, so get an RPM reference.
2679
	 */
2680 2681
	intel_runtime_pm_get(dev_priv);
	intel_prepare_reset(dev_priv);
2682

2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693
	do {
		/*
		 * All state reset _must_ be completed before we update the
		 * reset counter, for otherwise waiters might miss the reset
		 * pending state and not properly drop locks, resulting in
		 * deadlocks with the reset work.
		 */
		if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
			i915_reset(dev_priv);
			mutex_unlock(&dev_priv->drm.struct_mutex);
		}
2694

2695 2696 2697 2698 2699
		/* We need to wait for anyone holding the lock to wakeup */
	} while (wait_on_bit_timeout(&dev_priv->gpu_error.flags,
				     I915_RESET_IN_PROGRESS,
				     TASK_UNINTERRUPTIBLE,
				     HZ));
2700

2701
	intel_finish_reset(dev_priv);
2702
	intel_runtime_pm_put(dev_priv);
2703

2704
	if (!test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
2705 2706
		kobject_uevent_env(kobj,
				   KOBJ_CHANGE, reset_done_event);
2707

2708 2709 2710 2711 2712
	/*
	 * Note: The wake_up also serves as a memory barrier so that
	 * waiters see the updated value of the dev_priv->gpu_error.
	 */
	wake_up_all(&dev_priv->gpu_error.reset_queue);
2713 2714
}

2715 2716 2717 2718
static inline void
i915_err_print_instdone(struct drm_i915_private *dev_priv,
			struct intel_instdone *instdone)
{
2719 2720 2721
	int slice;
	int subslice;

2722 2723 2724 2725 2726 2727 2728 2729 2730 2731
	pr_err("  INSTDONE: 0x%08x\n", instdone->instdone);

	if (INTEL_GEN(dev_priv) <= 3)
		return;

	pr_err("  SC_INSTDONE: 0x%08x\n", instdone->slice_common);

	if (INTEL_GEN(dev_priv) <= 6)
		return;

2732 2733 2734 2735 2736 2737 2738
	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
		pr_err("  SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
		       slice, subslice, instdone->sampler[slice][subslice]);

	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
		pr_err("  ROW_INSTDONE[%d][%d]: 0x%08x\n",
		       slice, subslice, instdone->row[slice][subslice]);
2739 2740
}

2741
static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
2742
{
2743
	u32 eir;
2744

2745 2746
	if (!IS_GEN2(dev_priv))
		I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER));
2747

2748 2749 2750 2751
	if (INTEL_GEN(dev_priv) < 4)
		I915_WRITE(IPEIR, I915_READ(IPEIR));
	else
		I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965));
2752

2753
	I915_WRITE(EIR, I915_READ(EIR));
2754 2755 2756 2757 2758 2759
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
2760
		DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
2761 2762 2763
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2764 2765 2766
}

/**
2767
 * i915_handle_error - handle a gpu error
2768
 * @dev_priv: i915 device private
2769
 * @engine_mask: mask representing engines that are hung
2770 2771
 * @fmt: Error message format string
 *
2772
 * Do some basic checking of register state at error time and
2773 2774 2775 2776 2777
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
2778 2779
void i915_handle_error(struct drm_i915_private *dev_priv,
		       u32 engine_mask,
2780
		       const char *fmt, ...)
2781
{
2782 2783
	va_list args;
	char error_msg[80];
2784

2785 2786 2787 2788
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

2789
	i915_capture_error_state(dev_priv, engine_mask, error_msg);
2790
	i915_clear_error_registers(dev_priv);
2791

2792 2793
	if (!engine_mask)
		return;
2794

2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811
	if (test_and_set_bit(I915_RESET_IN_PROGRESS,
			     &dev_priv->gpu_error.flags))
		return;

	/*
	 * Wakeup waiting processes so that the reset function
	 * i915_reset_and_wakeup doesn't deadlock trying to grab
	 * various locks. By bumping the reset counter first, the woken
	 * processes will see a reset in progress and back off,
	 * releasing their locks and then wait for the reset completion.
	 * We must do this for _all_ gpu waiters that might hold locks
	 * that the reset work needs to acquire.
	 *
	 * Note: The wake_up also provides a memory barrier to ensure that the
	 * waiters see the updated value of the reset flags.
	 */
	i915_error_wake_up(dev_priv);
2812

2813
	i915_reset_and_wakeup(dev_priv);
2814 2815
}

2816 2817 2818
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2819
static int i8xx_enable_vblank(struct drm_device *dev, unsigned int pipe)
2820
{
2821
	struct drm_i915_private *dev_priv = to_i915(dev);
2822
	unsigned long irqflags;
2823

2824
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2825
	i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2826
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2827

2828 2829 2830
	return 0;
}

2831
static int i965_enable_vblank(struct drm_device *dev, unsigned int pipe)
2832
{
2833
	struct drm_i915_private *dev_priv = to_i915(dev);
2834 2835 2836
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2837 2838
	i915_enable_pipestat(dev_priv, pipe,
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
2839 2840 2841 2842 2843
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2844
static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2845
{
2846
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
2847
	unsigned long irqflags;
2848
	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
2849
		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
J
Jesse Barnes 已提交
2850 2851

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2852
	ilk_enable_display_irq(dev_priv, bit);
J
Jesse Barnes 已提交
2853 2854 2855 2856 2857
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2858
static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
2859
{
2860
	struct drm_i915_private *dev_priv = to_i915(dev);
2861 2862 2863
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2864
	bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2865
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2866

2867 2868 2869
	return 0;
}

2870 2871 2872
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2873
static void i8xx_disable_vblank(struct drm_device *dev, unsigned int pipe)
2874
{
2875
	struct drm_i915_private *dev_priv = to_i915(dev);
2876
	unsigned long irqflags;
2877

2878
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2879
	i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2880 2881 2882
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2883
static void i965_disable_vblank(struct drm_device *dev, unsigned int pipe)
2884
{
2885
	struct drm_i915_private *dev_priv = to_i915(dev);
2886 2887 2888
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2889 2890
	i915_disable_pipestat(dev_priv, pipe,
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2891 2892 2893
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2894
static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2895
{
2896
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
2897
	unsigned long irqflags;
2898
	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
2899
		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
J
Jesse Barnes 已提交
2900 2901

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2902
	ilk_disable_display_irq(dev_priv, bit);
J
Jesse Barnes 已提交
2903 2904 2905
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2906
static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
2907
{
2908
	struct drm_i915_private *dev_priv = to_i915(dev);
2909 2910 2911
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2912
	bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2913 2914 2915
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2916
static void ibx_irq_reset(struct drm_i915_private *dev_priv)
P
Paulo Zanoni 已提交
2917
{
2918
	if (HAS_PCH_NOP(dev_priv))
P
Paulo Zanoni 已提交
2919 2920
		return;

2921
	GEN5_IRQ_RESET(SDE);
2922

2923
	if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
2924
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
2925
}
2926

P
Paulo Zanoni 已提交
2927 2928 2929 2930 2931 2932 2933 2934 2935 2936
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
2937
	struct drm_i915_private *dev_priv = to_i915(dev);
P
Paulo Zanoni 已提交
2938

2939
	if (HAS_PCH_NOP(dev_priv))
P
Paulo Zanoni 已提交
2940 2941 2942
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
2943 2944 2945 2946
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

2947
static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv)
2948
{
2949
	GEN5_IRQ_RESET(GT);
2950
	if (INTEL_GEN(dev_priv) >= 6)
2951
		GEN5_IRQ_RESET(GEN6_PM);
2952 2953
}

2954 2955 2956 2957
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
	enum pipe pipe;

2958 2959 2960 2961 2962
	if (IS_CHERRYVIEW(dev_priv))
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
	else
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);

2963
	i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
2964 2965
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

2966 2967 2968 2969 2970 2971
	for_each_pipe(dev_priv, pipe) {
		I915_WRITE(PIPESTAT(pipe),
			   PIPE_FIFO_UNDERRUN_STATUS |
			   PIPESTAT_INT_STATUS_MASK);
		dev_priv->pipestat_irq_mask[pipe] = 0;
	}
2972 2973

	GEN5_IRQ_RESET(VLV_);
2974
	dev_priv->irq_mask = ~0;
2975 2976
}

2977 2978 2979
static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
2980
	u32 enable_mask;
2981 2982 2983 2984 2985 2986 2987 2988 2989
	enum pipe pipe;

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		i915_enable_pipestat(dev_priv, pipe, pipestat_mask);

2990 2991 2992
	enable_mask = I915_DISPLAY_PORT_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
2993
	if (IS_CHERRYVIEW(dev_priv))
2994
		enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
2995 2996 2997

	WARN_ON(dev_priv->irq_mask != ~0);

2998 2999 3000
	dev_priv->irq_mask = ~enable_mask;

	GEN5_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask);
3001 3002 3003 3004 3005 3006
}

/* drm_dma.h hooks
*/
static void ironlake_irq_reset(struct drm_device *dev)
{
3007
	struct drm_i915_private *dev_priv = to_i915(dev);
3008 3009 3010 3011

	I915_WRITE(HWSTAM, 0xffffffff);

	GEN5_IRQ_RESET(DE);
3012
	if (IS_GEN7(dev_priv))
3013 3014
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);

3015
	gen5_gt_irq_reset(dev_priv);
3016

3017
	ibx_irq_reset(dev_priv);
3018 3019
}

J
Jesse Barnes 已提交
3020 3021
static void valleyview_irq_preinstall(struct drm_device *dev)
{
3022
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
3023

3024 3025 3026
	I915_WRITE(VLV_MASTER_IER, 0);
	POSTING_READ(VLV_MASTER_IER);

3027
	gen5_gt_irq_reset(dev_priv);
J
Jesse Barnes 已提交
3028

3029
	spin_lock_irq(&dev_priv->irq_lock);
3030 3031
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3032
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3033 3034
}

3035 3036 3037 3038 3039 3040 3041 3042
static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
{
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
}

P
Paulo Zanoni 已提交
3043
static void gen8_irq_reset(struct drm_device *dev)
3044
{
3045
	struct drm_i915_private *dev_priv = to_i915(dev);
3046 3047 3048 3049 3050
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3051
	gen8_gt_irq_reset(dev_priv);
3052

3053
	for_each_pipe(dev_priv, pipe)
3054 3055
		if (intel_display_power_is_enabled(dev_priv,
						   POWER_DOMAIN_PIPE(pipe)))
3056
			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3057

3058 3059 3060
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3061

3062
	if (HAS_PCH_SPLIT(dev_priv))
3063
		ibx_irq_reset(dev_priv);
3064
}
3065

3066 3067
void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
3068
{
3069
	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3070
	enum pipe pipe;
3071

3072
	spin_lock_irq(&dev_priv->irq_lock);
3073 3074 3075 3076
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
				  dev_priv->de_irq_mask[pipe],
				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
3077
	spin_unlock_irq(&dev_priv->irq_lock);
3078 3079
}

3080 3081 3082
void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
{
3083 3084
	enum pipe pipe;

3085
	spin_lock_irq(&dev_priv->irq_lock);
3086 3087
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3088 3089 3090
	spin_unlock_irq(&dev_priv->irq_lock);

	/* make sure we're done processing display irqs */
3091
	synchronize_irq(dev_priv->drm.irq);
3092 3093
}

3094 3095
static void cherryview_irq_preinstall(struct drm_device *dev)
{
3096
	struct drm_i915_private *dev_priv = to_i915(dev);
3097 3098 3099 3100

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3101
	gen8_gt_irq_reset(dev_priv);
3102 3103 3104

	GEN5_IRQ_RESET(GEN8_PCU_);

3105
	spin_lock_irq(&dev_priv->irq_lock);
3106 3107
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3108
	spin_unlock_irq(&dev_priv->irq_lock);
3109 3110
}

3111
static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
3112 3113 3114 3115 3116
				  const u32 hpd[HPD_NUM_PINS])
{
	struct intel_encoder *encoder;
	u32 enabled_irqs = 0;

3117
	for_each_intel_encoder(&dev_priv->drm, encoder)
3118 3119 3120 3121 3122 3123
		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
			enabled_irqs |= hpd[encoder->hpd_pin];

	return enabled_irqs;
}

3124
static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
3125
{
3126
	u32 hotplug;
3127 3128 3129

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
3130 3131
	 * duration to 2ms (which is the minimum in the Display Port spec).
	 * The pulse duration bits are reserved on LPT+.
3132
	 */
3133
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3134 3135 3136
	hotplug &= ~(PORTB_PULSE_DURATION_MASK |
		     PORTC_PULSE_DURATION_MASK |
		     PORTD_PULSE_DURATION_MASK);
3137
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3138 3139
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3140 3141 3142 3143
	/*
	 * When CPU and PCH are on the same package, port A
	 * HPD must be enabled in both north and south.
	 */
3144
	if (HAS_PCH_LPT_LP(dev_priv))
3145
		hotplug |= PORTA_HOTPLUG_ENABLE;
3146
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3147
}
X
Xiong Zhang 已提交
3148

3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165
static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	if (HAS_PCH_IBX(dev_priv)) {
		hotplug_irqs = SDE_HOTPLUG_MASK;
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx);
	} else {
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt);
	}

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	ibx_hpd_detection_setup(dev_priv);
}

3166
static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3167
{
3168
	u32 hotplug;
3169 3170 3171

	/* Enable digital hotplug on the PCH */
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3172 3173 3174 3175
	hotplug |= PORTA_HOTPLUG_ENABLE |
		   PORTB_HOTPLUG_ENABLE |
		   PORTC_HOTPLUG_ENABLE |
		   PORTD_HOTPLUG_ENABLE;
3176 3177 3178 3179 3180
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);

	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
	hotplug |= PORTE_HOTPLUG_ENABLE;
	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3181 3182
}

3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194
static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt);

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	spt_hpd_detection_setup(dev_priv);
}

3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210
static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug;

	/*
	 * Enable digital hotplug on the CPU, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
	 * The pulse duration bits are reserved on HSW+.
	 */
	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE |
		   DIGITAL_PORTA_PULSE_DURATION_2ms;
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
}

3211
static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3212
{
3213
	u32 hotplug_irqs, enabled_irqs;
3214

3215
	if (INTEL_GEN(dev_priv) >= 8) {
3216
		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3217
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw);
3218 3219

		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3220
	} else if (INTEL_GEN(dev_priv) >= 7) {
3221
		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3222
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb);
3223 3224

		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3225 3226
	} else {
		hotplug_irqs = DE_DP_A_HOTPLUG;
3227
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk);
3228

3229 3230
		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
	}
3231

3232
	ilk_hpd_detection_setup(dev_priv);
3233

3234
	ibx_hpd_irq_setup(dev_priv);
3235 3236
}

3237 3238
static void __bxt_hpd_detection_setup(struct drm_i915_private *dev_priv,
				      u32 enabled_irqs)
3239
{
3240
	u32 hotplug;
3241

3242
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3243 3244 3245
	hotplug |= PORTA_HOTPLUG_ENABLE |
		   PORTB_HOTPLUG_ENABLE |
		   PORTC_HOTPLUG_ENABLE;
3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264

	DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n",
		      hotplug, enabled_irqs);
	hotplug &= ~BXT_DDI_HPD_INVERT_MASK;

	/*
	 * For BXT invert bit has to be set based on AOB design
	 * for HPD detection logic, update it based on VBT fields.
	 */
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
		hotplug |= BXT_DDIA_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
		hotplug |= BXT_DDIB_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
		hotplug |= BXT_DDIC_HPD_INVERT;

3265
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3266 3267
}

3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284
static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
{
	__bxt_hpd_detection_setup(dev_priv, BXT_DE_PORT_HOTPLUG_MASK);
}

static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt);
	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;

	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);

	__bxt_hpd_detection_setup(dev_priv, enabled_irqs);
}

P
Paulo Zanoni 已提交
3285 3286
static void ibx_irq_postinstall(struct drm_device *dev)
{
3287
	struct drm_i915_private *dev_priv = to_i915(dev);
3288
	u32 mask;
3289

3290
	if (HAS_PCH_NOP(dev_priv))
D
Daniel Vetter 已提交
3291 3292
		return;

3293
	if (HAS_PCH_IBX(dev_priv))
3294
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3295
	else
3296
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3297

3298
	gen5_assert_iir_is_zero(dev_priv, SDEIIR);
P
Paulo Zanoni 已提交
3299
	I915_WRITE(SDEIMR, ~mask);
3300 3301 3302

	if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
	    HAS_PCH_LPT(dev_priv))
3303
		ibx_hpd_detection_setup(dev_priv);
3304 3305
	else
		spt_hpd_detection_setup(dev_priv);
P
Paulo Zanoni 已提交
3306 3307
}

3308 3309
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
3310
	struct drm_i915_private *dev_priv = to_i915(dev);
3311 3312 3313 3314 3315
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3316
	if (HAS_L3_DPF(dev_priv)) {
3317
		/* L3 parity interrupt is always unmasked. */
3318 3319
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev_priv);
		gt_irqs |= GT_PARITY_ERROR(dev_priv);
3320 3321 3322
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
3323
	if (IS_GEN5(dev_priv)) {
3324
		gt_irqs |= ILK_BSD_USER_INTERRUPT;
3325 3326 3327 3328
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3329
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3330

3331
	if (INTEL_GEN(dev_priv) >= 6) {
3332 3333 3334 3335
		/*
		 * RPS interrupts will get enabled/disabled on demand when RPS
		 * itself is enabled/disabled.
		 */
3336
		if (HAS_VEBOX(dev_priv)) {
3337
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3338 3339
			dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT;
		}
3340

3341 3342
		dev_priv->pm_imr = 0xffffffff;
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs);
3343 3344 3345
	}
}

3346
static int ironlake_irq_postinstall(struct drm_device *dev)
3347
{
3348
	struct drm_i915_private *dev_priv = to_i915(dev);
3349 3350
	u32 display_mask, extra_mask;

3351
	if (INTEL_GEN(dev_priv) >= 7) {
3352 3353 3354
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3355
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3356
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3357 3358
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
			      DE_DP_A_HOTPLUG_IVB);
3359 3360 3361
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3362 3363 3364
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3365 3366 3367
		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
			      DE_DP_A_HOTPLUG);
3368
	}
3369

3370
	dev_priv->irq_mask = ~display_mask;
3371

3372 3373
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3374 3375
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3376
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3377

3378
	gen5_gt_irq_postinstall(dev);
3379

3380 3381
	ilk_hpd_detection_setup(dev_priv);

P
Paulo Zanoni 已提交
3382
	ibx_irq_postinstall(dev);
3383

3384
	if (IS_IRONLAKE_M(dev_priv)) {
3385 3386 3387
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3388 3389
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
3390
		spin_lock_irq(&dev_priv->irq_lock);
3391
		ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
3392
		spin_unlock_irq(&dev_priv->irq_lock);
3393 3394
	}

3395 3396 3397
	return 0;
}

3398 3399 3400 3401 3402 3403 3404 3405 3406
void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

3407 3408
	if (intel_irqs_enabled(dev_priv)) {
		vlv_display_irq_reset(dev_priv);
3409
		vlv_display_irq_postinstall(dev_priv);
3410
	}
3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

3422
	if (intel_irqs_enabled(dev_priv))
3423
		vlv_display_irq_reset(dev_priv);
3424 3425
}

3426 3427 3428

static int valleyview_irq_postinstall(struct drm_device *dev)
{
3429
	struct drm_i915_private *dev_priv = to_i915(dev);
3430

3431
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3432

3433
	spin_lock_irq(&dev_priv->irq_lock);
3434 3435
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3436 3437
	spin_unlock_irq(&dev_priv->irq_lock);

J
Jesse Barnes 已提交
3438
	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3439
	POSTING_READ(VLV_MASTER_IER);
3440 3441 3442 3443

	return 0;
}

3444 3445 3446 3447 3448
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3449 3450 3451
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3452
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3453 3454 3455
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3456
		0,
3457 3458
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3459 3460
		};

3461 3462 3463
	if (HAS_L3_DPF(dev_priv))
		gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;

3464 3465
	dev_priv->pm_ier = 0x0;
	dev_priv->pm_imr = ~dev_priv->pm_ier;
3466 3467
	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3468 3469
	/*
	 * RPS interrupts will get enabled/disabled on demand when RPS itself
3470
	 * is enabled/disabled. Same wil be the case for GuC interrupts.
3471
	 */
3472
	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_imr, dev_priv->pm_ier);
3473
	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3474 3475 3476 3477
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
3478 3479
	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
	uint32_t de_pipe_enables;
3480 3481
	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
	u32 de_port_enables;
3482
	u32 de_misc_masked = GEN8_DE_MISC_GSE;
3483
	enum pipe pipe;
3484

3485
	if (INTEL_INFO(dev_priv)->gen >= 9) {
3486 3487
		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3488 3489
		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
				  GEN9_AUX_CHANNEL_D;
3490
		if (IS_GEN9_LP(dev_priv))
3491 3492
			de_port_masked |= BXT_DE_PORT_GMBUS;
	} else {
3493 3494
		de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
				  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3495
	}
3496 3497 3498 3499

	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
					   GEN8_PIPE_FIFO_UNDERRUN;

3500
	de_port_enables = de_port_masked;
3501
	if (IS_GEN9_LP(dev_priv))
3502 3503
		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
	else if (IS_BROADWELL(dev_priv))
3504 3505
		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;

3506 3507 3508
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3509

3510
	for_each_pipe(dev_priv, pipe)
3511
		if (intel_display_power_is_enabled(dev_priv,
3512 3513 3514 3515
				POWER_DOMAIN_PIPE(pipe)))
			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
					  dev_priv->de_irq_mask[pipe],
					  de_pipe_enables);
3516

3517
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3518
	GEN5_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
3519 3520 3521

	if (IS_GEN9_LP(dev_priv))
		bxt_hpd_detection_setup(dev_priv);
3522 3523
	else if (IS_BROADWELL(dev_priv))
		ilk_hpd_detection_setup(dev_priv);
3524 3525 3526 3527
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
3528
	struct drm_i915_private *dev_priv = to_i915(dev);
3529

3530
	if (HAS_PCH_SPLIT(dev_priv))
3531
		ibx_irq_pre_postinstall(dev);
P
Paulo Zanoni 已提交
3532

3533 3534 3535
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

3536
	if (HAS_PCH_SPLIT(dev_priv))
3537
		ibx_irq_postinstall(dev);
3538

3539
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3540 3541 3542 3543 3544
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3545 3546
static int cherryview_irq_postinstall(struct drm_device *dev)
{
3547
	struct drm_i915_private *dev_priv = to_i915(dev);
3548 3549 3550

	gen8_gt_irq_postinstall(dev_priv);

3551
	spin_lock_irq(&dev_priv->irq_lock);
3552 3553
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3554 3555
	spin_unlock_irq(&dev_priv->irq_lock);

3556
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3557 3558 3559 3560 3561
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3562 3563
static void gen8_irq_uninstall(struct drm_device *dev)
{
3564
	struct drm_i915_private *dev_priv = to_i915(dev);
3565 3566 3567 3568

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3569
	gen8_irq_reset(dev);
3570 3571
}

J
Jesse Barnes 已提交
3572 3573
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3574
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
3575 3576 3577 3578

	if (!dev_priv)
		return;

3579
	I915_WRITE(VLV_MASTER_IER, 0);
3580
	POSTING_READ(VLV_MASTER_IER);
3581

3582
	gen5_gt_irq_reset(dev_priv);
3583

J
Jesse Barnes 已提交
3584
	I915_WRITE(HWSTAM, 0xffffffff);
3585

3586
	spin_lock_irq(&dev_priv->irq_lock);
3587 3588
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3589
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3590 3591
}

3592 3593
static void cherryview_irq_uninstall(struct drm_device *dev)
{
3594
	struct drm_i915_private *dev_priv = to_i915(dev);
3595 3596 3597 3598 3599 3600 3601

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3602
	gen8_gt_irq_reset(dev_priv);
3603

3604
	GEN5_IRQ_RESET(GEN8_PCU_);
3605

3606
	spin_lock_irq(&dev_priv->irq_lock);
3607 3608
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3609
	spin_unlock_irq(&dev_priv->irq_lock);
3610 3611
}

3612
static void ironlake_irq_uninstall(struct drm_device *dev)
3613
{
3614
	struct drm_i915_private *dev_priv = to_i915(dev);
3615 3616 3617 3618

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3619
	ironlake_irq_reset(dev);
3620 3621
}

3622
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3623
{
3624
	struct drm_i915_private *dev_priv = to_i915(dev);
3625
	int pipe;
3626

3627
	for_each_pipe(dev_priv, pipe)
3628
		I915_WRITE(PIPESTAT(pipe), 0);
3629 3630 3631
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3632 3633 3634 3635
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3636
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3637 3638 3639 3640 3641 3642 3643 3644 3645

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3646
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
C
Chris Wilson 已提交
3647 3648 3649 3650 3651 3652 3653 3654
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3655 3656
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3657
	spin_lock_irq(&dev_priv->irq_lock);
3658 3659
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3660
	spin_unlock_irq(&dev_priv->irq_lock);
3661

C
Chris Wilson 已提交
3662 3663 3664
	return 0;
}

3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695
/*
 * Returns true when a page flip has completed.
 */
static bool i8xx_handle_vblank(struct drm_i915_private *dev_priv,
			       int plane, int pipe, u32 iir)
{
	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

	if (!intel_pipe_handle_vblank(dev_priv, pipe))
		return false;

	if ((iir & flip_pending) == 0)
		goto check_page_flip;

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ16(ISR) & flip_pending)
		goto check_page_flip;

	intel_finish_page_flip_cs(dev_priv, pipe);
	return true;

check_page_flip:
	intel_check_page_flip(dev_priv, pipe);
	return false;
}

3696
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
3697
{
3698
	struct drm_device *dev = arg;
3699
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3700 3701 3702 3703 3704 3705
	u16 iir, new_iir;
	u32 pipe_stats[2];
	int pipe;
	u16 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3706
	irqreturn_t ret;
C
Chris Wilson 已提交
3707

3708 3709 3710
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

3711 3712 3713 3714
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	ret = IRQ_NONE;
C
Chris Wilson 已提交
3715 3716
	iir = I915_READ16(IIR);
	if (iir == 0)
3717
		goto out;
C
Chris Wilson 已提交
3718 3719 3720 3721 3722 3723 3724

	while (iir & ~flip_mask) {
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3725
		spin_lock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3726
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3727
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
C
Chris Wilson 已提交
3728

3729
		for_each_pipe(dev_priv, pipe) {
3730
			i915_reg_t reg = PIPESTAT(pipe);
C
Chris Wilson 已提交
3731 3732 3733 3734 3735
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
3736
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
3737 3738
				I915_WRITE(reg, pipe_stats[pipe]);
		}
3739
		spin_unlock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3740 3741 3742 3743 3744

		I915_WRITE16(IIR, iir & ~flip_mask);
		new_iir = I915_READ16(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
3745
			notify_ring(dev_priv->engine[RCS]);
C
Chris Wilson 已提交
3746

3747
		for_each_pipe(dev_priv, pipe) {
3748 3749 3750 3751 3752 3753 3754
			int plane = pipe;
			if (HAS_FBC(dev_priv))
				plane = !plane;

			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
			    i8xx_handle_vblank(dev_priv, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
C
Chris Wilson 已提交
3755

3756
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3757
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
3758

3759 3760 3761
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3762
		}
C
Chris Wilson 已提交
3763 3764 3765

		iir = new_iir;
	}
3766 3767 3768 3769
	ret = IRQ_HANDLED;

out:
	enable_rpm_wakeref_asserts(dev_priv);
C
Chris Wilson 已提交
3770

3771
	return ret;
C
Chris Wilson 已提交
3772 3773 3774 3775
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
3776
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3777 3778
	int pipe;

3779
	for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3780 3781 3782 3783 3784 3785 3786 3787 3788
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

3789 3790
static void i915_irq_preinstall(struct drm_device * dev)
{
3791
	struct drm_i915_private *dev_priv = to_i915(dev);
3792 3793
	int pipe;

3794
	if (I915_HAS_HOTPLUG(dev_priv)) {
3795
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3796 3797 3798
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3799
	I915_WRITE16(HWSTAM, 0xeffe);
3800
	for_each_pipe(dev_priv, pipe)
3801 3802 3803 3804 3805 3806 3807 3808
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
3809
	struct drm_i915_private *dev_priv = to_i915(dev);
3810
	u32 enable_mask;
3811

3812 3813 3814 3815 3816 3817 3818 3819
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3820
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3821 3822 3823 3824 3825 3826 3827

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_USER_INTERRUPT;

3828
	if (I915_HAS_HOTPLUG(dev_priv)) {
3829
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3830 3831
		POSTING_READ(PORT_HOTPLUG_EN);

3832 3833 3834 3835 3836 3837 3838 3839 3840 3841
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

3842
	i915_enable_asle_pipestat(dev_priv);
3843

3844 3845
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3846
	spin_lock_irq(&dev_priv->irq_lock);
3847 3848
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3849
	spin_unlock_irq(&dev_priv->irq_lock);
3850

3851 3852 3853
	return 0;
}

3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884
/*
 * Returns true when a page flip has completed.
 */
static bool i915_handle_vblank(struct drm_i915_private *dev_priv,
			       int plane, int pipe, u32 iir)
{
	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

	if (!intel_pipe_handle_vblank(dev_priv, pipe))
		return false;

	if ((iir & flip_pending) == 0)
		goto check_page_flip;

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ(ISR) & flip_pending)
		goto check_page_flip;

	intel_finish_page_flip_cs(dev_priv, pipe);
	return true;

check_page_flip:
	intel_check_page_flip(dev_priv, pipe);
	return false;
}

3885
static irqreturn_t i915_irq_handler(int irq, void *arg)
3886
{
3887
	struct drm_device *dev = arg;
3888
	struct drm_i915_private *dev_priv = to_i915(dev);
3889
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3890 3891 3892 3893
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
	int pipe, ret = IRQ_NONE;
3894

3895 3896 3897
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

3898 3899 3900
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

3901
	iir = I915_READ(IIR);
3902 3903
	do {
		bool irq_received = (iir & ~flip_mask) != 0;
3904
		bool blc_event = false;
3905 3906 3907 3908 3909 3910

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3911
		spin_lock(&dev_priv->irq_lock);
3912
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3913
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3914

3915
		for_each_pipe(dev_priv, pipe) {
3916
			i915_reg_t reg = PIPESTAT(pipe);
3917 3918
			pipe_stats[pipe] = I915_READ(reg);

3919
			/* Clear the PIPE*STAT regs before the IIR */
3920 3921
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
3922
				irq_received = true;
3923 3924
			}
		}
3925
		spin_unlock(&dev_priv->irq_lock);
3926 3927 3928 3929 3930

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
3931
		if (I915_HAS_HOTPLUG(dev_priv) &&
3932 3933 3934
		    iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
3935
				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
3936
		}
3937

3938
		I915_WRITE(IIR, iir & ~flip_mask);
3939 3940 3941
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
3942
			notify_ring(dev_priv->engine[RCS]);
3943

3944
		for_each_pipe(dev_priv, pipe) {
3945 3946 3947 3948 3949 3950 3951
			int plane = pipe;
			if (HAS_FBC(dev_priv))
				plane = !plane;

			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
			    i915_handle_vblank(dev_priv, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3952 3953 3954

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
3955 3956

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3957
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
3958

3959 3960 3961
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3962 3963 3964
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
3965
			intel_opregion_asle_intr(dev_priv);
3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
3982
		ret = IRQ_HANDLED;
3983
		iir = new_iir;
3984
	} while (iir & ~flip_mask);
3985

3986 3987
	enable_rpm_wakeref_asserts(dev_priv);

3988 3989 3990 3991 3992
	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
3993
	struct drm_i915_private *dev_priv = to_i915(dev);
3994 3995
	int pipe;

3996
	if (I915_HAS_HOTPLUG(dev_priv)) {
3997
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3998 3999 4000
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

4001
	I915_WRITE16(HWSTAM, 0xffff);
4002
	for_each_pipe(dev_priv, pipe) {
4003
		/* Clear enable bits; then clear status bits */
4004
		I915_WRITE(PIPESTAT(pipe), 0);
4005 4006
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
4007 4008 4009 4010 4011 4012 4013 4014
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
4015
	struct drm_i915_private *dev_priv = to_i915(dev);
4016 4017
	int pipe;

4018
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4019
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4020 4021

	I915_WRITE(HWSTAM, 0xeffe);
4022
	for_each_pipe(dev_priv, pipe)
4023 4024 4025 4026 4027 4028 4029 4030
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
4031
	struct drm_i915_private *dev_priv = to_i915(dev);
4032
	u32 enable_mask;
4033 4034 4035
	u32 error_mask;

	/* Unmask the interrupts that we always want on. */
4036
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4037
			       I915_DISPLAY_PORT_INTERRUPT |
4038 4039 4040 4041 4042 4043 4044
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
4045 4046
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4047 4048
	enable_mask |= I915_USER_INTERRUPT;

4049
	if (IS_G4X(dev_priv))
4050
		enable_mask |= I915_BSD_USER_INTERRUPT;
4051

4052 4053
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4054
	spin_lock_irq(&dev_priv->irq_lock);
4055 4056 4057
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4058
	spin_unlock_irq(&dev_priv->irq_lock);
4059 4060 4061 4062 4063

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
4064
	if (IS_G4X(dev_priv)) {
4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4079
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4080 4081
	POSTING_READ(PORT_HOTPLUG_EN);

4082
	i915_enable_asle_pipestat(dev_priv);
4083 4084 4085 4086

	return 0;
}

4087
static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
4088 4089 4090
{
	u32 hotplug_en;

4091 4092
	assert_spin_locked(&dev_priv->irq_lock);

4093 4094
	/* Note HDMI and DP share hotplug bits */
	/* enable bits are the same for all generations */
4095
	hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
4096 4097 4098 4099
	/* Programming the CRT detection parameters tends
	   to generate a spurious hotplug event about three
	   seconds later.  So just do it once.
	*/
4100
	if (IS_G4X(dev_priv))
4101 4102 4103 4104
		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;

	/* Ignore TV since it's buggy */
4105
	i915_hotplug_interrupt_update_locked(dev_priv,
4106 4107 4108 4109
					     HOTPLUG_INT_EN_MASK |
					     CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
					     CRT_HOTPLUG_ACTIVATION_PERIOD_64,
					     hotplug_en);
4110 4111
}

4112
static irqreturn_t i965_irq_handler(int irq, void *arg)
4113
{
4114
	struct drm_device *dev = arg;
4115
	struct drm_i915_private *dev_priv = to_i915(dev);
4116 4117 4118
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	int ret = IRQ_NONE, pipe;
4119 4120 4121
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4122

4123 4124 4125
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4126 4127 4128
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

4129 4130 4131
	iir = I915_READ(IIR);

	for (;;) {
4132
		bool irq_received = (iir & ~flip_mask) != 0;
4133 4134
		bool blc_event = false;

4135 4136 4137 4138 4139
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4140
		spin_lock(&dev_priv->irq_lock);
4141
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4142
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4143

4144
		for_each_pipe(dev_priv, pipe) {
4145
			i915_reg_t reg = PIPESTAT(pipe);
4146 4147 4148 4149 4150 4151 4152
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4153
				irq_received = true;
4154 4155
			}
		}
4156
		spin_unlock(&dev_priv->irq_lock);
4157 4158 4159 4160 4161 4162 4163

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4164 4165 4166
		if (iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
4167
				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4168
		}
4169

4170
		I915_WRITE(IIR, iir & ~flip_mask);
4171 4172 4173
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4174
			notify_ring(dev_priv->engine[RCS]);
4175
		if (iir & I915_BSD_USER_INTERRUPT)
4176
			notify_ring(dev_priv->engine[VCS]);
4177

4178
		for_each_pipe(dev_priv, pipe) {
4179 4180 4181
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
			    i915_handle_vblank(dev_priv, pipe, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4182 4183 4184

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4185 4186

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4187
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
4188

4189 4190
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4191
		}
4192 4193

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
4194
			intel_opregion_asle_intr(dev_priv);
4195

4196
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4197
			gmbus_irq_handler(dev_priv);
4198

4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

4217 4218
	enable_rpm_wakeref_asserts(dev_priv);

4219 4220 4221 4222 4223
	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4224
	struct drm_i915_private *dev_priv = to_i915(dev);
4225 4226 4227 4228 4229
	int pipe;

	if (!dev_priv)
		return;

4230
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4231
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4232 4233

	I915_WRITE(HWSTAM, 0xffffffff);
4234
	for_each_pipe(dev_priv, pipe)
4235 4236 4237 4238
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

4239
	for_each_pipe(dev_priv, pipe)
4240 4241 4242 4243 4244
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4245 4246 4247 4248 4249 4250 4251
/**
 * intel_irq_init - initializes irq support
 * @dev_priv: i915 device instance
 *
 * This function initializes all the irq support including work items, timers
 * and all the vtables. It does not setup the interrupt itself though.
 */
4252
void intel_irq_init(struct drm_i915_private *dev_priv)
4253
{
4254
	struct drm_device *dev = &dev_priv->drm;
4255

4256 4257
	intel_hpd_init_work(dev_priv);

4258
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4259
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4260

4261
	if (HAS_GUC_SCHED(dev_priv))
4262 4263
		dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT;

4264
	/* Let's track the enabled rps events */
4265
	if (IS_VALLEYVIEW(dev_priv))
4266
		/* WaGsvRC0ResidencyMethod:vlv */
4267
		dev_priv->pm_rps_events = GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED;
4268 4269
	else
		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4270

4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282
	dev_priv->rps.pm_intr_keep = 0;

	/*
	 * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
	 * if GEN6_PM_UP_EI_EXPIRED is masked.
	 *
	 * TODO: verify if this can be reproduced on VLV,CHV.
	 */
	if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
		dev_priv->rps.pm_intr_keep |= GEN6_PM_RP_UP_EI_EXPIRED;

	if (INTEL_INFO(dev_priv)->gen >= 8)
4283
		dev_priv->rps.pm_intr_keep |= GEN8_PMINTR_REDIRECT_TO_GUC;
4284

4285
	if (IS_GEN2(dev_priv)) {
4286
		/* Gen2 doesn't have a hardware frame counter */
4287
		dev->max_vblank_count = 0;
4288
		dev->driver->get_vblank_counter = drm_vblank_no_hw_counter;
4289
	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4290
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4291
		dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4292 4293 4294
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4295 4296
	}

4297 4298 4299 4300 4301
	/*
	 * Opt out of the vblank disable timer on everything except gen2.
	 * Gen2 doesn't have a hardware frame counter and so depends on
	 * vblank interrupts to produce sane vblank seuquence numbers.
	 */
4302
	if (!IS_GEN2(dev_priv))
4303 4304
		dev->vblank_disable_immediate = true;

4305 4306 4307 4308 4309 4310 4311 4312 4313 4314
	/* Most platforms treat the display irq block as an always-on
	 * power domain. vlv/chv can disable it at runtime and need
	 * special care to avoid writing any of the display block registers
	 * outside of the power domain. We defer setting up the display irqs
	 * in this case to the runtime pm.
	 */
	dev_priv->display_irqs_enabled = true;
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
		dev_priv->display_irqs_enabled = false;

L
Lyude 已提交
4315 4316
	dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;

4317 4318
	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4319

4320
	if (IS_CHERRYVIEW(dev_priv)) {
4321 4322 4323 4324
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
4325 4326
		dev->driver->enable_vblank = i965_enable_vblank;
		dev->driver->disable_vblank = i965_disable_vblank;
4327
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4328
	} else if (IS_VALLEYVIEW(dev_priv)) {
J
Jesse Barnes 已提交
4329 4330 4331 4332
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
4333 4334
		dev->driver->enable_vblank = i965_enable_vblank;
		dev->driver->disable_vblank = i965_disable_vblank;
4335
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4336
	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4337
		dev->driver->irq_handler = gen8_irq_handler;
4338
		dev->driver->irq_preinstall = gen8_irq_reset;
4339 4340 4341 4342
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
4343
		if (IS_GEN9_LP(dev_priv))
4344
			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4345
		else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv))
4346 4347
			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		else
4348
			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4349
	} else if (HAS_PCH_SPLIT(dev_priv)) {
4350
		dev->driver->irq_handler = ironlake_irq_handler;
4351
		dev->driver->irq_preinstall = ironlake_irq_reset;
4352 4353 4354 4355
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4356
		dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4357
	} else {
4358
		if (IS_GEN2(dev_priv)) {
C
Chris Wilson 已提交
4359 4360 4361 4362
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4363 4364
			dev->driver->enable_vblank = i8xx_enable_vblank;
			dev->driver->disable_vblank = i8xx_disable_vblank;
4365
		} else if (IS_GEN3(dev_priv)) {
4366 4367 4368 4369
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
4370 4371
			dev->driver->enable_vblank = i8xx_enable_vblank;
			dev->driver->disable_vblank = i8xx_disable_vblank;
C
Chris Wilson 已提交
4372
		} else {
4373 4374 4375 4376
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
4377 4378
			dev->driver->enable_vblank = i965_enable_vblank;
			dev->driver->disable_vblank = i965_disable_vblank;
C
Chris Wilson 已提交
4379
		}
4380 4381
		if (I915_HAS_HOTPLUG(dev_priv))
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4382 4383
	}
}
4384

4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395
/**
 * intel_irq_install - enables the hardware interrupt
 * @dev_priv: i915 device instance
 *
 * This function enables the hardware interrupt handling, but leaves the hotplug
 * handling still disabled. It is called after intel_irq_init().
 *
 * In the driver load and resume code we need working interrupts in a few places
 * but don't want to deal with the hassle of concurrent probe and hotplug
 * workers. Hence the split into this two-stage approach.
 */
4396 4397 4398 4399 4400 4401 4402 4403 4404
int intel_irq_install(struct drm_i915_private *dev_priv)
{
	/*
	 * We enable some interrupt sources in our postinstall hooks, so mark
	 * interrupts as enabled _before_ actually enabling them to avoid
	 * special cases in our ordering checks.
	 */
	dev_priv->pm.irqs_enabled = true;

4405
	return drm_irq_install(&dev_priv->drm, dev_priv->drm.pdev->irq);
4406 4407
}

4408 4409 4410 4411 4412 4413 4414
/**
 * intel_irq_uninstall - finilizes all irq handling
 * @dev_priv: i915 device instance
 *
 * This stops interrupt and hotplug handling and unregisters and frees all
 * resources acquired in the init functions.
 */
4415 4416
void intel_irq_uninstall(struct drm_i915_private *dev_priv)
{
4417
	drm_irq_uninstall(&dev_priv->drm);
4418 4419 4420 4421
	intel_hpd_cancel_work(dev_priv);
	dev_priv->pm.irqs_enabled = false;
}

4422 4423 4424 4425 4426 4427 4428
/**
 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
 * @dev_priv: i915 device instance
 *
 * This function is used to disable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4429
void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4430
{
4431
	dev_priv->drm.driver->irq_uninstall(&dev_priv->drm);
4432
	dev_priv->pm.irqs_enabled = false;
4433
	synchronize_irq(dev_priv->drm.irq);
4434 4435
}

4436 4437 4438 4439 4440 4441 4442
/**
 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
 * @dev_priv: i915 device instance
 *
 * This function is used to enable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4443
void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4444
{
4445
	dev_priv->pm.irqs_enabled = true;
4446 4447
	dev_priv->drm.driver->irq_preinstall(&dev_priv->drm);
	dev_priv->drm.driver->irq_postinstall(&dev_priv->drm);
4448
}