i915_irq.c 123.0 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47
/**
 * DOC: interrupt handling
 *
 * These functions provide the basic support for enabling and disabling the
 * interrupt handling support. There's a lot more functionality in i915_irq.c
 * and related files, but that will be described in separate chapters.
 */

48 49 50 51
static const u32 hpd_ilk[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
};

52 53 54 55
static const u32 hpd_ivb[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
};

56 57 58 59
static const u32 hpd_bdw[HPD_NUM_PINS] = {
	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
};

60
static const u32 hpd_ibx[HPD_NUM_PINS] = {
61 62 63 64 65 66 67
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

68
static const u32 hpd_cpt[HPD_NUM_PINS] = {
69
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71 72 73 74 75
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

X
Xiong Zhang 已提交
76
static const u32 hpd_spt[HPD_NUM_PINS] = {
77
	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
X
Xiong Zhang 已提交
78 79 80 81 82 83
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
};

84
static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85 86 87 88 89 90 91 92
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

93
static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94 95 96 97 98 99 100 101
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

102
static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103 104 105 106 107 108 109 110
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

111 112
/* BXT hpd list */
static const u32 hpd_bxt[HPD_NUM_PINS] = {
113
	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114 115 116 117
	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
};

118
/* IIR can theoretically queue up two events. Be paranoid. */
119
#define GEN8_IRQ_RESET_NDX(type, which) do { \
120 121 122 123 124 125 126 127 128
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

129
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
130
	I915_WRITE(type##IMR, 0xffffffff); \
131
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
132
	I915_WRITE(type##IER, 0); \
133 134 135 136
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
137 138
} while (0)

139 140 141
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
142 143
static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
				    i915_reg_t reg)
144 145 146 147 148 149 150
{
	u32 val = I915_READ(reg);

	if (val == 0)
		return;

	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
151
	     i915_mmio_reg_offset(reg), val);
152 153 154 155 156
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
}
157

P
Paulo Zanoni 已提交
158
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
159
	gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
160
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
161 162
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
P
Paulo Zanoni 已提交
163 164 165
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
166
	gen5_assert_iir_is_zero(dev_priv, type##IIR); \
P
Paulo Zanoni 已提交
167
	I915_WRITE(type##IER, (ier_val)); \
168 169
	I915_WRITE(type##IMR, (imr_val)); \
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
170 171
} while (0)

172
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
173
static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
174

175 176 177 178 179 180 181 182
/* For display hotplug interrupt */
static inline void
i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
				     uint32_t mask,
				     uint32_t bits)
{
	uint32_t val;

183
	lockdep_assert_held(&dev_priv->irq_lock);
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
	WARN_ON(bits & ~mask);

	val = I915_READ(PORT_HOTPLUG_EN);
	val &= ~mask;
	val |= bits;
	I915_WRITE(PORT_HOTPLUG_EN, val);
}

/**
 * i915_hotplug_interrupt_update - update hotplug interrupt enable
 * @dev_priv: driver private
 * @mask: bits to update
 * @bits: bits to enable
 * NOTE: the HPD enable bits are modified both inside and outside
 * of an interrupt context. To avoid that read-modify-write cycles
 * interfer, these bits are protected by a spinlock. Since this
 * function is usually not called from a context where the lock is
 * held already, this function acquires the lock itself. A non-locking
 * version is also available.
 */
void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
				   uint32_t mask,
				   uint32_t bits)
{
	spin_lock_irq(&dev_priv->irq_lock);
	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
	spin_unlock_irq(&dev_priv->irq_lock);
}

213 214 215 216 217 218
/**
 * ilk_update_display_irq - update DEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
219 220 221
void ilk_update_display_irq(struct drm_i915_private *dev_priv,
			    uint32_t interrupt_mask,
			    uint32_t enabled_irq_mask)
222
{
223 224
	uint32_t new_val;

225
	lockdep_assert_held(&dev_priv->irq_lock);
226

227 228
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

229
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
230 231
		return;

232 233 234 235 236 237
	new_val = dev_priv->irq_mask;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->irq_mask) {
		dev_priv->irq_mask = new_val;
238
		I915_WRITE(DEIMR, dev_priv->irq_mask);
239
		POSTING_READ(DEIMR);
240 241 242
	}
}

P
Paulo Zanoni 已提交
243 244 245 246 247 248 249 250 251 252
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
253
	lockdep_assert_held(&dev_priv->irq_lock);
P
Paulo Zanoni 已提交
254

255 256
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

257
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
258 259
		return;

P
Paulo Zanoni 已提交
260 261 262 263 264
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
}

265
void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
266 267
{
	ilk_update_gt_irq(dev_priv, mask, mask);
268
	POSTING_READ_FW(GTIMR);
P
Paulo Zanoni 已提交
269 270
}

271
void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
272 273 274 275
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

276
static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
277 278 279 280
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
}

281
static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
282 283 284 285
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
}

286
static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
287 288 289 290
{
	return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
}

P
Paulo Zanoni 已提交
291
/**
292 293 294 295 296
 * snb_update_pm_irq - update GEN6_PMIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
P
Paulo Zanoni 已提交
297 298 299 300
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
301
	uint32_t new_val;
P
Paulo Zanoni 已提交
302

303 304
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

305
	lockdep_assert_held(&dev_priv->irq_lock);
P
Paulo Zanoni 已提交
306

307
	new_val = dev_priv->pm_imr;
308 309 310
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

311 312 313
	if (new_val != dev_priv->pm_imr) {
		dev_priv->pm_imr = new_val;
		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_imr);
314
		POSTING_READ(gen6_pm_imr(dev_priv));
315
	}
P
Paulo Zanoni 已提交
316 317
}

318
void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
P
Paulo Zanoni 已提交
319
{
320 321 322
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

P
Paulo Zanoni 已提交
323 324 325
	snb_update_pm_irq(dev_priv, mask, mask);
}

326
static void __gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
P
Paulo Zanoni 已提交
327 328 329 330
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

331
void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
332 333 334 335
{
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

336
	__gen6_mask_pm_irq(dev_priv, mask);
337 338
}

339
void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 reset_mask)
I
Imre Deak 已提交
340
{
341
	i915_reg_t reg = gen6_pm_iir(dev_priv);
I
Imre Deak 已提交
342

343
	lockdep_assert_held(&dev_priv->irq_lock);
344 345 346

	I915_WRITE(reg, reset_mask);
	I915_WRITE(reg, reset_mask);
I
Imre Deak 已提交
347
	POSTING_READ(reg);
348 349 350 351
}

void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mask)
{
352
	lockdep_assert_held(&dev_priv->irq_lock);
353 354 355 356 357 358 359 360 361

	dev_priv->pm_ier |= enable_mask;
	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
	gen6_unmask_pm_irq(dev_priv, enable_mask);
	/* unmask_pm_irq provides an implicit barrier (POSTING_READ) */
}

void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_mask)
{
362
	lockdep_assert_held(&dev_priv->irq_lock);
363 364 365 366 367 368 369 370 371 372 373

	dev_priv->pm_ier &= ~disable_mask;
	__gen6_mask_pm_irq(dev_priv, disable_mask);
	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
	/* though a barrier is missing here, but don't really need a one */
}

void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	gen6_reset_pm_iir(dev_priv, dev_priv->pm_rps_events);
374
	dev_priv->rps.pm_iir = 0;
I
Imre Deak 已提交
375 376 377
	spin_unlock_irq(&dev_priv->irq_lock);
}

378
void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
379
{
380 381 382
	if (READ_ONCE(dev_priv->rps.interrupts_enabled))
		return;

383
	spin_lock_irq(&dev_priv->irq_lock);
384 385
	WARN_ON_ONCE(dev_priv->rps.pm_iir);
	WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
I
Imre Deak 已提交
386
	dev_priv->rps.interrupts_enabled = true;
387
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
388

389 390 391
	spin_unlock_irq(&dev_priv->irq_lock);
}

392
void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
393
{
394 395 396
	if (!READ_ONCE(dev_priv->rps.interrupts_enabled))
		return;

I
Imre Deak 已提交
397 398
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->rps.interrupts_enabled = false;
399

400
	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u));
401

402
	gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
403 404

	spin_unlock_irq(&dev_priv->irq_lock);
405
	synchronize_irq(dev_priv->drm.irq);
406 407 408 409 410 411 412 413

	/* Now that we will not be generating any more work, flush any
	 * outsanding tasks. As we are called on the RPS idle path,
	 * we will reset the GPU to minimum frequencies, so the current
	 * state of the worker can be discarded.
	 */
	cancel_work_sync(&dev_priv->rps.work);
	gen6_reset_rps_interrupts(dev_priv);
414 415
}

416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	gen6_reset_pm_iir(dev_priv, dev_priv->pm_guc_events);
	spin_unlock_irq(&dev_priv->irq_lock);
}

void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	if (!dev_priv->guc.interrupts_enabled) {
		WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) &
				       dev_priv->pm_guc_events);
		dev_priv->guc.interrupts_enabled = true;
		gen6_enable_pm_irq(dev_priv, dev_priv->pm_guc_events);
	}
	spin_unlock_irq(&dev_priv->irq_lock);
}

void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->guc.interrupts_enabled = false;

	gen6_disable_pm_irq(dev_priv, dev_priv->pm_guc_events);

	spin_unlock_irq(&dev_priv->irq_lock);
	synchronize_irq(dev_priv->drm.irq);

	gen9_reset_guc_interrupts(dev_priv);
}

448
/**
449 450 451 452 453
 * bdw_update_port_irq - update DE port interrupt
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
454 455 456 457 458 459 460
static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
				uint32_t interrupt_mask,
				uint32_t enabled_irq_mask)
{
	uint32_t new_val;
	uint32_t old_val;

461
	lockdep_assert_held(&dev_priv->irq_lock);
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	old_val = I915_READ(GEN8_DE_PORT_IMR);

	new_val = old_val;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != old_val) {
		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		POSTING_READ(GEN8_DE_PORT_IMR);
	}
}

480 481 482 483 484 485 486 487 488 489 490 491 492 493
/**
 * bdw_update_pipe_irq - update DE pipe interrupt
 * @dev_priv: driver private
 * @pipe: pipe whose interrupt to update
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
			 enum pipe pipe,
			 uint32_t interrupt_mask,
			 uint32_t enabled_irq_mask)
{
	uint32_t new_val;

494
	lockdep_assert_held(&dev_priv->irq_lock);
495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	new_val = dev_priv->de_irq_mask[pipe];
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->de_irq_mask[pipe]) {
		dev_priv->de_irq_mask[pipe] = new_val;
		I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
	}
}

512 513 514 515 516 517
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
518 519 520
void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
				  uint32_t interrupt_mask,
				  uint32_t enabled_irq_mask)
521 522 523 524 525
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

526 527
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

528
	lockdep_assert_held(&dev_priv->irq_lock);
529

530
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
531 532
		return;

533 534 535
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
536

D
Daniel Vetter 已提交
537
static void
538 539
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
540
{
541
	i915_reg_t reg = PIPESTAT(pipe);
542
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
543

544
	lockdep_assert_held(&dev_priv->irq_lock);
545
	WARN_ON(!intel_irqs_enabled(dev_priv));
546

547 548 549 550
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
551 552 553
		return;

	if ((pipestat & enable_mask) == enable_mask)
554 555
		return;

556 557
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

558
	/* Enable the interrupt, clear any pending status */
559
	pipestat |= enable_mask | status_mask;
560 561
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
562 563
}

D
Daniel Vetter 已提交
564
static void
565 566
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
567
{
568
	i915_reg_t reg = PIPESTAT(pipe);
569
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
570

571
	lockdep_assert_held(&dev_priv->irq_lock);
572
	WARN_ON(!intel_irqs_enabled(dev_priv));
573

574 575 576 577
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
578 579
		return;

580 581 582
	if ((pipestat & enable_mask) == 0)
		return;

583 584
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

585
	pipestat &= ~enable_mask;
586 587
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
588 589
}

590 591 592 593 594
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
595 596
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
597 598 599
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
600 601 602 603 604 605
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
606 607 608 609 610 611 612 613 614 615 616 617

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

618 619 620 621 622 623
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

624
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
625
		enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
626 627 628
							   status_mask);
	else
		enable_mask = status_mask << 16;
629 630 631 632 633 634 635 636 637
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

638
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
639
		enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
640 641 642
							   status_mask);
	else
		enable_mask = status_mask << 16;
643 644 645
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

646
/**
647
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
648
 * @dev_priv: i915 device private
649
 */
650
static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
651
{
652
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev_priv))
653 654
		return;

655
	spin_lock_irq(&dev_priv->irq_lock);
656

657
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
658
	if (INTEL_GEN(dev_priv) >= 4)
659
		i915_enable_pipestat(dev_priv, PIPE_A,
660
				     PIPE_LEGACY_BLC_EVENT_STATUS);
661

662
	spin_unlock_irq(&dev_priv->irq_lock);
663 664
}

665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714
/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via PIPECONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on PIPECONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

715 716 717
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
718
static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
719
{
720
	struct drm_i915_private *dev_priv = to_i915(dev);
721
	i915_reg_t high_frame, low_frame;
722
	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
723 724
	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
								pipe);
725
	const struct drm_display_mode *mode = &intel_crtc->base.hwmode;
726
	unsigned long irqflags;
727

728 729 730 731 732
	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vbl_start = mode->crtc_vblank_start;
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
733

734 735 736 737 738 739
	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

740 741
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
742

743 744
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);

745 746 747 748 749 750
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
751 752 753
		high1 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK;
		low   = I915_READ_FW(low_frame);
		high2 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK;
754 755
	} while (high1 != high2);

756 757
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

758
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
759
	pixel = low & PIPE_PIXEL_MASK;
760
	low >>= PIPE_FRAME_LOW_SHIFT;
761 762 763 764 765 766

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
767
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
768 769
}

770
static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
771
{
772
	struct drm_i915_private *dev_priv = to_i915(dev);
773

774
	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
775 776
}

777
/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
778 779 780
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
781
	struct drm_i915_private *dev_priv = to_i915(dev);
782
	const struct drm_display_mode *mode = &crtc->base.hwmode;
783
	enum pipe pipe = crtc->pipe;
784
	int position, vtotal;
785

786 787 788
	if (!crtc->active)
		return -1;

789
	vtotal = mode->crtc_vtotal;
790 791 792
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

793
	if (IS_GEN2(dev_priv))
794
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
795
	else
796
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
797

798 799 800 801 802 803 804 805 806 807 808 809
	/*
	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
	 * read it just before the start of vblank.  So try it again
	 * so we don't accidentally end up spanning a vblank frame
	 * increment, causing the pipe_update_end() code to squak at us.
	 *
	 * The nature of this problem means we can't simply check the ISR
	 * bit and return the vblank start value; nor can we use the scanline
	 * debug register in the transcoder as it appears to have the same
	 * problem.  We may need to extend this to include other platforms,
	 * but so far testing only shows the problem on HSW.
	 */
810
	if (HAS_DDI(dev_priv) && !position) {
811 812 813 814
		int i, temp;

		for (i = 0; i < 100; i++) {
			udelay(1);
815
			temp = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
816 817 818 819 820 821 822
			if (temp != position) {
				position = temp;
				break;
			}
		}
	}

823
	/*
824 825
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
826
	 */
827
	return (position + crtc->scanline_offset) % vtotal;
828 829
}

830
static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
831
				    unsigned int flags, int *vpos, int *hpos,
832 833
				    ktime_t *stime, ktime_t *etime,
				    const struct drm_display_mode *mode)
834
{
835
	struct drm_i915_private *dev_priv = to_i915(dev);
836 837
	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
								pipe);
838
	int position;
839
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
840 841
	bool in_vbl = true;
	int ret = 0;
842
	unsigned long irqflags;
843

844
	if (WARN_ON(!mode->crtc_clock)) {
845
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
846
				 "pipe %c\n", pipe_name(pipe));
847 848 849
		return 0;
	}

850
	htotal = mode->crtc_htotal;
851
	hsync_start = mode->crtc_hsync_start;
852 853 854
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
855

856 857 858 859 860 861
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

862 863
	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

864 865 866 867 868 869
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
870

871 872 873 874 875 876
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

877
	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
878 879 880
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
881
		position = __intel_get_crtc_scanline(intel_crtc);
882 883 884 885 886
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
887
		position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
888

889 890 891 892
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
893

894 895 896 897 898 899 900 901 902 903 904 905
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

906 907 908 909 910 911 912 913 914 915
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
916 917
	}

918 919 920 921 922 923 924 925
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

926 927 928 929 930 931 932 933 934 935 936 937
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
938

939
	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
940 941 942 943 944 945
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
946 947 948

	/* In vblank? */
	if (in_vbl)
949
		ret |= DRM_SCANOUTPOS_IN_VBLANK;
950 951 952 953

	return ret;
}

954 955
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
956
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
957 958 959 960 961 962 963 964 965 966
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

967
static bool i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
968 969
			      int *max_error,
			      struct timeval *vblank_time,
970
			      bool in_vblank_irq)
971
{
972
	struct drm_i915_private *dev_priv = to_i915(dev);
973
	struct intel_crtc *crtc;
974

975
	if (pipe >= INTEL_INFO(dev_priv)->num_pipes) {
976
		DRM_ERROR("Invalid crtc %u\n", pipe);
977
		return false;
978 979 980
	}

	/* Get drm_crtc to timestamp: */
981
	crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
982
	if (crtc == NULL) {
983
		DRM_ERROR("Invalid crtc %u\n", pipe);
984
		return false;
985 986
	}

987
	if (!crtc->base.hwmode.crtc_clock) {
988
		DRM_DEBUG_KMS("crtc %u is disabled\n", pipe);
989
		return false;
990
	}
991 992

	/* Helper routine in DRM core does all the work: */
993
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
994
						     vblank_time, in_vblank_irq,
995
						     &crtc->base.hwmode);
996 997
}

998
static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
999
{
1000
	u32 busy_up, busy_down, max_avg, min_avg;
1001 1002
	u8 new_delay;

1003
	spin_lock(&mchdev_lock);
1004

1005 1006
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

1007
	new_delay = dev_priv->ips.cur_delay;
1008

1009
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
1010 1011
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
1012 1013 1014 1015
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
1016
	if (busy_up > max_avg) {
1017 1018 1019 1020
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
1021
	} else if (busy_down < min_avg) {
1022 1023 1024 1025
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
1026 1027
	}

1028
	if (ironlake_set_drps(dev_priv, new_delay))
1029
		dev_priv->ips.cur_delay = new_delay;
1030

1031
	spin_unlock(&mchdev_lock);
1032

1033 1034 1035
	return;
}

1036
static void notify_ring(struct intel_engine_cs *engine)
1037
{
1038 1039
	struct drm_i915_gem_request *rq = NULL;
	struct intel_wait *wait;
1040

1041
	atomic_inc(&engine->irq_count);
1042
	set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
1043

1044 1045
	spin_lock(&engine->breadcrumbs.irq_lock);
	wait = engine->breadcrumbs.irq_wait;
1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
	if (wait) {
		/* We use a callback from the dma-fence to submit
		 * requests after waiting on our own requests. To
		 * ensure minimum delay in queuing the next request to
		 * hardware, signal the fence now rather than wait for
		 * the signaler to be woken up. We still wake up the
		 * waiter in order to handle the irq-seqno coherency
		 * issues (we may receive the interrupt before the
		 * seqno is written, see __i915_request_irq_complete())
		 * and to handle coalescing of multiple seqno updates
		 * and many waiters.
		 */
		if (i915_seqno_passed(intel_engine_get_seqno(engine),
1059 1060 1061
				      wait->seqno) &&
		    !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
			      &wait->request->fence.flags))
1062
			rq = i915_gem_request_get(wait->request);
1063 1064

		wake_up_process(wait->tsk);
1065 1066
	} else {
		__intel_engine_disarm_breadcrumbs(engine);
1067
	}
1068
	spin_unlock(&engine->breadcrumbs.irq_lock);
1069

1070
	if (rq) {
1071
		dma_fence_signal(&rq->fence);
1072 1073
		i915_gem_request_put(rq);
	}
1074 1075

	trace_intel_engine_notify(engine, wait);
1076 1077
}

1078 1079
static void vlv_c0_read(struct drm_i915_private *dev_priv,
			struct intel_rps_ei *ei)
1080
{
1081
	ei->ktime = ktime_get_raw();
1082 1083 1084
	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
}
1085

1086
void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1087
{
1088
	memset(&dev_priv->rps.ei, 0, sizeof(dev_priv->rps.ei));
1089
}
1090

1091 1092
static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
{
1093
	const struct intel_rps_ei *prev = &dev_priv->rps.ei;
1094 1095
	struct intel_rps_ei now;
	u32 events = 0;
1096

1097
	if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1098
		return 0;
1099

1100
	vlv_c0_read(dev_priv, &now);
1101

1102
	if (prev->ktime) {
1103
		u64 time, c0;
1104
		u32 render, media;
1105

1106
		time = ktime_us_delta(now.ktime, prev->ktime);
1107

1108 1109 1110 1111 1112 1113 1114
		time *= dev_priv->czclk_freq;

		/* Workload can be split between render + media,
		 * e.g. SwapBuffers being blitted in X after being rendered in
		 * mesa. To account for this we need to combine both engines
		 * into our activity counter.
		 */
1115 1116 1117
		render = now.render_c0 - prev->render_c0;
		media = now.media_c0 - prev->media_c0;
		c0 = max(render, media);
1118
		c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1119 1120 1121 1122 1123

		if (c0 > time * dev_priv->rps.up_threshold)
			events = GEN6_PM_RP_UP_THRESHOLD;
		else if (c0 < time * dev_priv->rps.down_threshold)
			events = GEN6_PM_RP_DOWN_THRESHOLD;
1124 1125
	}

1126
	dev_priv->rps.ei = now;
1127
	return events;
1128 1129
}

1130 1131
static bool any_waiters(struct drm_i915_private *dev_priv)
{
1132
	struct intel_engine_cs *engine;
1133
	enum intel_engine_id id;
1134

1135
	for_each_engine(engine, dev_priv, id)
1136
		if (intel_engine_has_waiter(engine))
1137 1138 1139 1140 1141
			return true;

	return false;
}

1142
static void gen6_pm_rps_work(struct work_struct *work)
1143
{
1144 1145
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
1146
	bool client_boost = false;
1147
	int new_delay, adj, min, max;
1148
	u32 pm_iir = 0;
1149

1150
	spin_lock_irq(&dev_priv->irq_lock);
1151 1152 1153
	if (dev_priv->rps.interrupts_enabled) {
		pm_iir = fetch_and_zero(&dev_priv->rps.pm_iir);
		client_boost = fetch_and_zero(&dev_priv->rps.client_boost);
I
Imre Deak 已提交
1154
	}
1155
	spin_unlock_irq(&dev_priv->irq_lock);
1156

1157
	/* Make sure we didn't queue anything we're not going to process. */
1158
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1159
	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1160
		goto out;
1161

1162
	mutex_lock(&dev_priv->rps.hw_lock);
1163

1164 1165
	pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);

1166
	adj = dev_priv->rps.last_adj;
1167
	new_delay = dev_priv->rps.cur_freq;
1168 1169
	min = dev_priv->rps.min_freq_softlimit;
	max = dev_priv->rps.max_freq_softlimit;
1170 1171 1172 1173
	if (client_boost || any_waiters(dev_priv))
		max = dev_priv->rps.max_freq;
	if (client_boost && new_delay < dev_priv->rps.boost_freq) {
		new_delay = dev_priv->rps.boost_freq;
1174 1175
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1176 1177
		if (adj > 0)
			adj *= 2;
1178 1179
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1180 1181 1182

		if (new_delay >= dev_priv->rps.max_freq_softlimit)
			adj = 0;
1183
	} else if (client_boost || any_waiters(dev_priv)) {
1184
		adj = 0;
1185
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1186 1187
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1188
		else if (dev_priv->rps.cur_freq > dev_priv->rps.min_freq_softlimit)
1189
			new_delay = dev_priv->rps.min_freq_softlimit;
1190 1191 1192 1193
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
1194 1195
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1196 1197 1198

		if (new_delay <= dev_priv->rps.min_freq_softlimit)
			adj = 0;
1199
	} else { /* unknown event */
1200
		adj = 0;
1201
	}
1202

1203 1204
	dev_priv->rps.last_adj = adj;

1205 1206 1207
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1208
	new_delay += adj;
1209
	new_delay = clamp_t(int, new_delay, min, max);
1210

1211 1212 1213 1214
	if (intel_set_rps(dev_priv, new_delay)) {
		DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
		dev_priv->rps.last_adj = 0;
	}
1215

1216
	mutex_unlock(&dev_priv->rps.hw_lock);
1217 1218 1219 1220 1221 1222 1223

out:
	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
	spin_lock_irq(&dev_priv->irq_lock);
	if (dev_priv->rps.interrupts_enabled)
		gen6_unmask_pm_irq(dev_priv, dev_priv->pm_rps_events);
	spin_unlock_irq(&dev_priv->irq_lock);
1224 1225
}

1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1238 1239
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, l3_parity.error_work);
1240
	u32 error_status, row, bank, subbank;
1241
	char *parity_event[6];
1242
	uint32_t misccpctl;
1243
	uint8_t slice = 0;
1244 1245 1246 1247 1248

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
1249
	mutex_lock(&dev_priv->drm.struct_mutex);
1250

1251 1252 1253 1254
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1255 1256 1257 1258
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1259
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1260
		i915_reg_t reg;
1261

1262
		slice--;
1263
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv)))
1264
			break;
1265

1266
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1267

1268
		reg = GEN7_L3CDERRST1(slice);
1269

1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1285
		kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
1286
				   KOBJ_CHANGE, parity_event);
1287

1288 1289
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1290

1291 1292 1293 1294 1295
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1296

1297
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1298

1299 1300
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
1301
	spin_lock_irq(&dev_priv->irq_lock);
1302
	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1303
	spin_unlock_irq(&dev_priv->irq_lock);
1304

1305
	mutex_unlock(&dev_priv->drm.struct_mutex);
1306 1307
}

1308 1309
static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv,
					       u32 iir)
1310
{
1311
	if (!HAS_L3_DPF(dev_priv))
1312 1313
		return;

1314
	spin_lock(&dev_priv->irq_lock);
1315
	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1316
	spin_unlock(&dev_priv->irq_lock);
1317

1318
	iir &= GT_PARITY_ERROR(dev_priv);
1319 1320 1321 1322 1323 1324
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1325
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1326 1327
}

1328
static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
1329 1330
			       u32 gt_iir)
{
1331
	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1332
		notify_ring(dev_priv->engine[RCS]);
1333
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
1334
		notify_ring(dev_priv->engine[VCS]);
1335 1336
}

1337
static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
1338 1339
			       u32 gt_iir)
{
1340
	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1341
		notify_ring(dev_priv->engine[RCS]);
1342
	if (gt_iir & GT_BSD_USER_INTERRUPT)
1343
		notify_ring(dev_priv->engine[VCS]);
1344
	if (gt_iir & GT_BLT_USER_INTERRUPT)
1345
		notify_ring(dev_priv->engine[BCS]);
1346

1347 1348
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
1349 1350
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1351

1352 1353
	if (gt_iir & GT_PARITY_ERROR(dev_priv))
		ivybridge_parity_error_irq_handler(dev_priv, gt_iir);
1354 1355
}

1356
static __always_inline void
1357
gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
1358
{
1359
	bool tasklet = false;
1360 1361 1362

	if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) {
		set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
1363
		tasklet = true;
1364
	}
1365 1366 1367 1368 1369 1370 1371 1372

	if (iir & (GT_RENDER_USER_INTERRUPT << test_shift)) {
		notify_ring(engine);
		tasklet |= i915.enable_guc_submission;
	}

	if (tasklet)
		tasklet_hi_schedule(&engine->irq_tasklet);
1373 1374
}

1375 1376 1377
static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv,
				   u32 master_ctl,
				   u32 gt_iir[4])
1378 1379 1380 1381
{
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1382 1383 1384
		gt_iir[0] = I915_READ_FW(GEN8_GT_IIR(0));
		if (gt_iir[0]) {
			I915_WRITE_FW(GEN8_GT_IIR(0), gt_iir[0]);
1385 1386 1387 1388 1389
			ret = IRQ_HANDLED;
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1390
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1391 1392 1393
		gt_iir[1] = I915_READ_FW(GEN8_GT_IIR(1));
		if (gt_iir[1]) {
			I915_WRITE_FW(GEN8_GT_IIR(1), gt_iir[1]);
1394
			ret = IRQ_HANDLED;
1395
		} else
1396
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
1397 1398
	}

1399
	if (master_ctl & GEN8_GT_VECS_IRQ) {
1400 1401 1402
		gt_iir[3] = I915_READ_FW(GEN8_GT_IIR(3));
		if (gt_iir[3]) {
			I915_WRITE_FW(GEN8_GT_IIR(3), gt_iir[3]);
1403 1404 1405 1406 1407
			ret = IRQ_HANDLED;
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

1408
	if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
1409
		gt_iir[2] = I915_READ_FW(GEN8_GT_IIR(2));
1410 1411
		if (gt_iir[2] & (dev_priv->pm_rps_events |
				 dev_priv->pm_guc_events)) {
1412
			I915_WRITE_FW(GEN8_GT_IIR(2),
1413 1414
				      gt_iir[2] & (dev_priv->pm_rps_events |
						   dev_priv->pm_guc_events));
1415
			ret = IRQ_HANDLED;
1416 1417 1418 1419
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1420 1421 1422
	return ret;
}

1423 1424 1425 1426
static void gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
				u32 gt_iir[4])
{
	if (gt_iir[0]) {
1427
		gen8_cs_irq_handler(dev_priv->engine[RCS],
1428
				    gt_iir[0], GEN8_RCS_IRQ_SHIFT);
1429
		gen8_cs_irq_handler(dev_priv->engine[BCS],
1430 1431 1432 1433
				    gt_iir[0], GEN8_BCS_IRQ_SHIFT);
	}

	if (gt_iir[1]) {
1434
		gen8_cs_irq_handler(dev_priv->engine[VCS],
1435
				    gt_iir[1], GEN8_VCS1_IRQ_SHIFT);
1436
		gen8_cs_irq_handler(dev_priv->engine[VCS2],
1437 1438 1439 1440
				    gt_iir[1], GEN8_VCS2_IRQ_SHIFT);
	}

	if (gt_iir[3])
1441
		gen8_cs_irq_handler(dev_priv->engine[VECS],
1442 1443 1444 1445
				    gt_iir[3], GEN8_VECS_IRQ_SHIFT);

	if (gt_iir[2] & dev_priv->pm_rps_events)
		gen6_rps_irq_handler(dev_priv, gt_iir[2]);
1446 1447 1448

	if (gt_iir[2] & dev_priv->pm_guc_events)
		gen9_guc_irq_handler(dev_priv, gt_iir[2]);
1449 1450
}

1451 1452 1453 1454
static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
1455
		return val & PORTA_HOTPLUG_LONG_DETECT;
1456 1457 1458 1459 1460 1461 1462 1463 1464
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1465 1466 1467 1468 1469 1470 1471 1472 1473 1474
static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_E:
		return val & PORTE_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490
static bool spt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & PORTA_HOTPLUG_LONG_DETECT;
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	case PORT_D:
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1491 1492 1493 1494 1495 1496 1497 1498 1499 1500
static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1501
static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1502 1503 1504
{
	switch (port) {
	case PORT_B:
1505
		return val & PORTB_HOTPLUG_LONG_DETECT;
1506
	case PORT_C:
1507
		return val & PORTC_HOTPLUG_LONG_DETECT;
1508
	case PORT_D:
1509 1510 1511
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
1512 1513 1514
	}
}

1515
static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1516 1517 1518
{
	switch (port) {
	case PORT_B:
1519
		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1520
	case PORT_C:
1521
		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1522
	case PORT_D:
1523 1524 1525
		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
	default:
		return false;
1526 1527 1528
	}
}

1529 1530 1531 1532 1533 1534 1535
/*
 * Get a bit mask of pins that have triggered, and which ones may be long.
 * This can be called multiple times with the same masks to accumulate
 * hotplug detection results from several registers.
 *
 * Note that the caller is expected to zero out the masks initially.
 */
1536
static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1537
			     u32 hotplug_trigger, u32 dig_hotplug_reg,
1538 1539
			     const u32 hpd[HPD_NUM_PINS],
			     bool long_pulse_detect(enum port port, u32 val))
1540
{
1541
	enum port port;
1542 1543 1544
	int i;

	for_each_hpd_pin(i) {
1545 1546
		if ((hpd[i] & hotplug_trigger) == 0)
			continue;
1547

1548 1549
		*pin_mask |= BIT(i);

1550 1551 1552
		if (!intel_hpd_pin_to_port(i, &port))
			continue;

1553
		if (long_pulse_detect(port, dig_hotplug_reg))
1554
			*long_mask |= BIT(i);
1555 1556 1557 1558 1559 1560 1561
	}

	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
			 hotplug_trigger, dig_hotplug_reg, *pin_mask);

}

1562
static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1563
{
1564
	wake_up_all(&dev_priv->gmbus_wait_queue);
1565 1566
}

1567
static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1568
{
1569
	wake_up_all(&dev_priv->gmbus_wait_queue);
1570 1571
}

1572
#if defined(CONFIG_DEBUG_FS)
1573 1574
static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
					 enum pipe pipe,
1575 1576 1577
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1578 1579 1580
{
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
T
Tomeu Vizoso 已提交
1581 1582 1583
	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
	struct drm_driver *driver = dev_priv->drm.driver;
	uint32_t crcs[5];
1584
	int head, tail;
1585

1586
	spin_lock(&pipe_crc->lock);
T
Tomeu Vizoso 已提交
1587 1588 1589 1590 1591 1592
	if (pipe_crc->source) {
		if (!pipe_crc->entries) {
			spin_unlock(&pipe_crc->lock);
			DRM_DEBUG_KMS("spurious interrupt\n");
			return;
		}
1593

T
Tomeu Vizoso 已提交
1594 1595
		head = pipe_crc->head;
		tail = pipe_crc->tail;
1596

T
Tomeu Vizoso 已提交
1597 1598 1599 1600 1601
		if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
			spin_unlock(&pipe_crc->lock);
			DRM_ERROR("CRC buffer overflowing\n");
			return;
		}
1602

T
Tomeu Vizoso 已提交
1603
		entry = &pipe_crc->entries[head];
1604

T
Tomeu Vizoso 已提交
1605 1606 1607 1608 1609 1610
		entry->frame = driver->get_vblank_counter(&dev_priv->drm, pipe);
		entry->crc[0] = crc0;
		entry->crc[1] = crc1;
		entry->crc[2] = crc2;
		entry->crc[3] = crc3;
		entry->crc[4] = crc4;
1611

T
Tomeu Vizoso 已提交
1612 1613
		head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
		pipe_crc->head = head;
1614

T
Tomeu Vizoso 已提交
1615
		spin_unlock(&pipe_crc->lock);
1616

T
Tomeu Vizoso 已提交
1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
		wake_up_interruptible(&pipe_crc->wq);
	} else {
		/*
		 * For some not yet identified reason, the first CRC is
		 * bonkers. So let's just wait for the next vblank and read
		 * out the buggy result.
		 *
		 * On CHV sometimes the second CRC is bonkers as well, so
		 * don't trust that one either.
		 */
		if (pipe_crc->skipped == 0 ||
		    (IS_CHERRYVIEW(dev_priv) && pipe_crc->skipped == 1)) {
			pipe_crc->skipped++;
			spin_unlock(&pipe_crc->lock);
			return;
		}
		spin_unlock(&pipe_crc->lock);
		crcs[0] = crc0;
		crcs[1] = crc1;
		crcs[2] = crc2;
		crcs[3] = crc3;
		crcs[4] = crc4;
1639 1640 1641
		drm_crtc_add_crc_entry(&crtc->base, true,
				       drm_accurate_vblank_count(&crtc->base),
				       crcs);
T
Tomeu Vizoso 已提交
1642
	}
1643
}
1644 1645
#else
static inline void
1646 1647
display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
			     enum pipe pipe,
1648 1649 1650 1651 1652
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1653

1654 1655
static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				     enum pipe pipe)
D
Daniel Vetter 已提交
1656
{
1657
	display_pipe_crc_irq_handler(dev_priv, pipe,
1658 1659
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1660 1661
}

1662 1663
static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				     enum pipe pipe)
1664
{
1665
	display_pipe_crc_irq_handler(dev_priv, pipe,
1666 1667 1668 1669 1670
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1671
}
1672

1673 1674
static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				      enum pipe pipe)
1675
{
1676 1677
	uint32_t res1, res2;

1678
	if (INTEL_GEN(dev_priv) >= 3)
1679 1680 1681 1682
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

1683
	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1684 1685 1686
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1687

1688
	display_pipe_crc_irq_handler(dev_priv, pipe,
1689 1690 1691 1692
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1693
}
1694

1695 1696 1697 1698
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1699
{
1700
	if (pm_iir & dev_priv->pm_rps_events) {
1701
		spin_lock(&dev_priv->irq_lock);
1702
		gen6_mask_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
I
Imre Deak 已提交
1703 1704
		if (dev_priv->rps.interrupts_enabled) {
			dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1705
			schedule_work(&dev_priv->rps.work);
I
Imre Deak 已提交
1706
		}
1707
		spin_unlock(&dev_priv->irq_lock);
1708 1709
	}

1710 1711 1712
	if (INTEL_INFO(dev_priv)->gen >= 8)
		return;

1713
	if (HAS_VEBOX(dev_priv)) {
1714
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1715
			notify_ring(dev_priv->engine[VECS]);
B
Ben Widawsky 已提交
1716

1717 1718
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
B
Ben Widawsky 已提交
1719
	}
1720 1721
}

1722 1723 1724
static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
{
	if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT) {
1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737
		/* Sample the log buffer flush related bits & clear them out now
		 * itself from the message identity register to minimize the
		 * probability of losing a flush interrupt, when there are back
		 * to back flush interrupts.
		 * There can be a new flush interrupt, for different log buffer
		 * type (like for ISR), whilst Host is handling one (for DPC).
		 * Since same bit is used in message register for ISR & DPC, it
		 * could happen that GuC sets the bit for 2nd interrupt but Host
		 * clears out the bit on handling the 1st interrupt.
		 */
		u32 msg, flush;

		msg = I915_READ(SOFT_SCRATCH(15));
1738 1739
		flush = msg & (INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED |
			       INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER);
1740 1741 1742 1743 1744
		if (flush) {
			/* Clear the message bits that are handled */
			I915_WRITE(SOFT_SCRATCH(15), msg & ~flush);

			/* Handle flush interrupt in bottom half */
1745 1746
			queue_work(dev_priv->guc.log.runtime.flush_wq,
				   &dev_priv->guc.log.runtime.flush_work);
1747 1748

			dev_priv->guc.log.flush_interrupt_count++;
1749 1750 1751 1752 1753
		} else {
			/* Not clearing of unhandled event bits won't result in
			 * re-triggering of the interrupt.
			 */
		}
1754 1755 1756
	}
}

1757
static bool intel_pipe_handle_vblank(struct drm_i915_private *dev_priv,
1758
				     enum pipe pipe)
1759
{
1760 1761
	bool ret;

1762
	ret = drm_handle_vblank(&dev_priv->drm, pipe);
1763
	if (ret)
1764
		intel_finish_page_flip_mmio(dev_priv, pipe);
1765 1766

	return ret;
1767 1768
}

1769 1770
static void valleyview_pipestat_irq_ack(struct drm_i915_private *dev_priv,
					u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1771 1772 1773
{
	int pipe;

1774
	spin_lock(&dev_priv->irq_lock);
1775 1776 1777 1778 1779 1780

	if (!dev_priv->display_irqs_enabled) {
		spin_unlock(&dev_priv->irq_lock);
		return;
	}

1781
	for_each_pipe(dev_priv, pipe) {
1782
		i915_reg_t reg;
1783
		u32 mask, iir_bit = 0;
1784

1785 1786 1787 1788 1789 1790 1791
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
1792 1793 1794

		/* fifo underruns are filterered in the underrun handler. */
		mask = PIPE_FIFO_UNDERRUN_STATUS;
1795 1796 1797 1798 1799 1800 1801 1802

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1803 1804 1805
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1806 1807 1808 1809 1810
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1811 1812 1813
			continue;

		reg = PIPESTAT(pipe);
1814 1815
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1816 1817 1818 1819

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1820 1821
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1822 1823
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1824
	spin_unlock(&dev_priv->irq_lock);
1825 1826
}

1827
static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1828 1829 1830
					    u32 pipe_stats[I915_MAX_PIPES])
{
	enum pipe pipe;
1831

1832
	for_each_pipe(dev_priv, pipe) {
1833 1834 1835
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
1836

1837
		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV)
1838
			intel_finish_page_flip_cs(dev_priv, pipe);
1839 1840

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1841
			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1842

1843 1844
		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1845 1846 1847
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1848
		gmbus_irq_handler(dev_priv);
1849 1850
}

1851
static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
1852 1853 1854
{
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);

1855 1856
	if (hotplug_status)
		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1857

1858 1859 1860
	return hotplug_status;
}

1861
static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1862 1863 1864
				 u32 hotplug_status)
{
	u32 pin_mask = 0, long_mask = 0;
1865

1866 1867
	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
	    IS_CHERRYVIEW(dev_priv)) {
1868
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1869

1870 1871 1872 1873 1874
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
					   hotplug_trigger, hpd_status_g4x,
					   i9xx_port_hotplug_long_detect);

1875
			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1876
		}
1877 1878

		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1879
			dp_aux_irq_handler(dev_priv);
1880 1881
	} else {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1882

1883 1884
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1885
					   hotplug_trigger, hpd_status_i915,
1886
					   i9xx_port_hotplug_long_detect);
1887
			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1888
		}
1889
	}
1890 1891
}

1892
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1893
{
1894
	struct drm_device *dev = arg;
1895
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
1896 1897
	irqreturn_t ret = IRQ_NONE;

1898 1899 1900
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1901 1902 1903
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1904
	do {
1905
		u32 iir, gt_iir, pm_iir;
1906
		u32 pipe_stats[I915_MAX_PIPES] = {};
1907
		u32 hotplug_status = 0;
1908
		u32 ier = 0;
1909

J
Jesse Barnes 已提交
1910 1911
		gt_iir = I915_READ(GTIIR);
		pm_iir = I915_READ(GEN6_PMIIR);
1912
		iir = I915_READ(VLV_IIR);
J
Jesse Barnes 已提交
1913 1914

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1915
			break;
J
Jesse Barnes 已提交
1916 1917 1918

		ret = IRQ_HANDLED;

1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
		 *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
		 * bits this time around.
		 */
1932
		I915_WRITE(VLV_MASTER_IER, 0);
1933 1934
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
1935 1936 1937 1938 1939 1940

		if (gt_iir)
			I915_WRITE(GTIIR, gt_iir);
		if (pm_iir)
			I915_WRITE(GEN6_PMIIR, pm_iir);

1941
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1942
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1943

1944 1945
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
1946
		valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1947

1948 1949 1950 1951
		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
			   I915_LPE_PIPE_B_INTERRUPT))
			intel_lpe_audio_irq_handler(dev_priv);

1952 1953 1954 1955 1956 1957
		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);
1958

1959
		I915_WRITE(VLV_IER, ier);
1960 1961
		I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
		POSTING_READ(VLV_MASTER_IER);
1962

1963
		if (gt_iir)
1964
			snb_gt_irq_handler(dev_priv, gt_iir);
1965 1966 1967
		if (pm_iir)
			gen6_rps_irq_handler(dev_priv, pm_iir);

1968
		if (hotplug_status)
1969
			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1970

1971
		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1972
	} while (0);
J
Jesse Barnes 已提交
1973

1974 1975
	enable_rpm_wakeref_asserts(dev_priv);

J
Jesse Barnes 已提交
1976 1977 1978
	return ret;
}

1979 1980
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1981
	struct drm_device *dev = arg;
1982
	struct drm_i915_private *dev_priv = to_i915(dev);
1983 1984
	irqreturn_t ret = IRQ_NONE;

1985 1986 1987
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1988 1989 1990
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1991
	do {
1992
		u32 master_ctl, iir;
1993
		u32 gt_iir[4] = {};
1994
		u32 pipe_stats[I915_MAX_PIPES] = {};
1995
		u32 hotplug_status = 0;
1996 1997
		u32 ier = 0;

1998 1999
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
2000

2001 2002
		if (master_ctl == 0 && iir == 0)
			break;
2003

2004 2005
		ret = IRQ_HANDLED;

2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
		 *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
		 * bits this time around.
		 */
2019
		I915_WRITE(GEN8_MASTER_IRQ, 0);
2020 2021
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
2022

2023
		gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
2024

2025
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
2026
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
2027

2028 2029
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
2030
		valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
2031

2032 2033 2034 2035 2036
		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
			   I915_LPE_PIPE_B_INTERRUPT |
			   I915_LPE_PIPE_C_INTERRUPT))
			intel_lpe_audio_irq_handler(dev_priv);

2037 2038 2039 2040 2041 2042 2043
		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);

2044
		I915_WRITE(VLV_IER, ier);
2045
		I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2046
		POSTING_READ(GEN8_MASTER_IRQ);
2047

2048 2049
		gen8_gt_irq_handler(dev_priv, gt_iir);

2050
		if (hotplug_status)
2051
			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
2052

2053
		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
2054
	} while (0);
2055

2056 2057
	enable_rpm_wakeref_asserts(dev_priv);

2058 2059 2060
	return ret;
}

2061 2062
static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2063 2064 2065 2066
				const u32 hpd[HPD_NUM_PINS])
{
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

2067 2068 2069 2070 2071 2072
	/*
	 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
	 * unless we touch the hotplug register, even if hotplug_trigger is
	 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
	 * errors.
	 */
2073
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2074 2075 2076 2077 2078 2079 2080 2081
	if (!hotplug_trigger) {
		u32 mask = PORTA_HOTPLUG_STATUS_MASK |
			PORTD_HOTPLUG_STATUS_MASK |
			PORTC_HOTPLUG_STATUS_MASK |
			PORTB_HOTPLUG_STATUS_MASK;
		dig_hotplug_reg &= ~mask;
	}

2082
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2083 2084
	if (!hotplug_trigger)
		return;
2085 2086 2087 2088 2089

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   pch_port_hotplug_long_detect);

2090
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2091 2092
}

2093
static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2094
{
2095
	int pipe;
2096
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
2097

2098
	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx);
2099

2100 2101 2102
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
2103
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
2104 2105
				 port_name(port));
	}
2106

2107
	if (pch_iir & SDE_AUX_MASK)
2108
		dp_aux_irq_handler(dev_priv);
2109

2110
	if (pch_iir & SDE_GMBUS)
2111
		gmbus_irq_handler(dev_priv);
2112 2113 2114 2115 2116 2117 2118 2119 2120 2121

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2122
	if (pch_iir & SDE_FDI_MASK)
2123
		for_each_pipe(dev_priv, pipe)
2124 2125 2126
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2127 2128 2129 2130 2131 2132 2133 2134

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
2135
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
2136 2137

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
2138
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
2139 2140
}

2141
static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
2142 2143
{
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
2144
	enum pipe pipe;
2145

2146 2147 2148
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

2149
	for_each_pipe(dev_priv, pipe) {
2150 2151
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2152

D
Daniel Vetter 已提交
2153
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
2154 2155
			if (IS_IVYBRIDGE(dev_priv))
				ivb_pipe_crc_irq_handler(dev_priv, pipe);
D
Daniel Vetter 已提交
2156
			else
2157
				hsw_pipe_crc_irq_handler(dev_priv, pipe);
D
Daniel Vetter 已提交
2158 2159
		}
	}
2160

2161 2162 2163
	I915_WRITE(GEN7_ERR_INT, err_int);
}

2164
static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
2165 2166 2167
{
	u32 serr_int = I915_READ(SERR_INT);

2168 2169 2170
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2171
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
2172
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
2173 2174

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
2175
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
2176 2177

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
2178
		intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
2179 2180

	I915_WRITE(SERR_INT, serr_int);
2181 2182
}

2183
static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2184 2185
{
	int pipe;
2186
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
2187

2188
	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt);
2189

2190 2191 2192 2193 2194 2195
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
2196 2197

	if (pch_iir & SDE_AUX_MASK_CPT)
2198
		dp_aux_irq_handler(dev_priv);
2199 2200

	if (pch_iir & SDE_GMBUS_CPT)
2201
		gmbus_irq_handler(dev_priv);
2202 2203 2204 2205 2206 2207 2208 2209

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
2210
		for_each_pipe(dev_priv, pipe)
2211 2212 2213
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2214 2215

	if (pch_iir & SDE_ERROR_CPT)
2216
		cpt_serr_int_handler(dev_priv);
2217 2218
}

2219
static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233
{
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		~SDE_PORTE_HOTPLUG_SPT;
	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
	u32 pin_mask = 0, long_mask = 0;

	if (hotplug_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
				   dig_hotplug_reg, hpd_spt,
2234
				   spt_port_hotplug_long_detect);
2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248
	}

	if (hotplug2_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
				   dig_hotplug_reg, hpd_spt,
				   spt_port_hotplug2_long_detect);
	}

	if (pin_mask)
2249
		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2250 2251

	if (pch_iir & SDE_GMBUS_CPT)
2252
		gmbus_irq_handler(dev_priv);
2253 2254
}

2255 2256
static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267
				const u32 hpd[HPD_NUM_PINS])
{
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   ilk_port_hotplug_long_detect);

2268
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2269 2270
}

2271 2272
static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
				    u32 de_iir)
2273
{
2274
	enum pipe pipe;
2275 2276
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;

2277
	if (hotplug_trigger)
2278
		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk);
2279 2280

	if (de_iir & DE_AUX_CHANNEL_A)
2281
		dp_aux_irq_handler(dev_priv);
2282 2283

	if (de_iir & DE_GSE)
2284
		intel_opregion_asle_intr(dev_priv);
2285 2286 2287 2288

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2289
	for_each_pipe(dev_priv, pipe) {
2290 2291 2292
		if (de_iir & DE_PIPE_VBLANK(pipe) &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2293

2294
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2295
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2296

2297
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
2298
			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2299

2300
		/* plane/pipes map 1:1 on ilk+ */
2301
		if (de_iir & DE_PLANE_FLIP_DONE(pipe))
2302
			intel_finish_page_flip_cs(dev_priv, pipe);
2303 2304 2305 2306 2307 2308
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

2309 2310
		if (HAS_PCH_CPT(dev_priv))
			cpt_irq_handler(dev_priv, pch_iir);
2311
		else
2312
			ibx_irq_handler(dev_priv, pch_iir);
2313 2314 2315 2316 2317

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

2318 2319
	if (IS_GEN5(dev_priv) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev_priv);
2320 2321
}

2322 2323
static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
				    u32 de_iir)
2324
{
2325
	enum pipe pipe;
2326 2327
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;

2328
	if (hotplug_trigger)
2329
		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb);
2330 2331

	if (de_iir & DE_ERR_INT_IVB)
2332
		ivb_err_int_handler(dev_priv);
2333 2334

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
2335
		dp_aux_irq_handler(dev_priv);
2336 2337

	if (de_iir & DE_GSE_IVB)
2338
		intel_opregion_asle_intr(dev_priv);
2339

2340
	for_each_pipe(dev_priv, pipe) {
2341 2342 2343
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2344 2345

		/* plane/pipes map 1:1 on ilk+ */
2346
		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe))
2347
			intel_finish_page_flip_cs(dev_priv, pipe);
2348 2349 2350
	}

	/* check event from PCH */
2351
	if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
2352 2353
		u32 pch_iir = I915_READ(SDEIIR);

2354
		cpt_irq_handler(dev_priv, pch_iir);
2355 2356 2357 2358 2359 2360

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2361 2362 2363 2364 2365 2366 2367 2368
/*
 * To handle irqs with the minimum potential races with fresh interrupts, we:
 * 1 - Disable Master Interrupt Control.
 * 2 - Find the source(s) of the interrupt.
 * 3 - Clear the Interrupt Identity bits (IIR).
 * 4 - Process the interrupt(s) that had bits set in the IIRs.
 * 5 - Re-enable Master Interrupt Control.
 */
2369
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2370
{
2371
	struct drm_device *dev = arg;
2372
	struct drm_i915_private *dev_priv = to_i915(dev);
2373
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2374
	irqreturn_t ret = IRQ_NONE;
2375

2376 2377 2378
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2379 2380 2381
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

2382 2383 2384
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2385
	POSTING_READ(DEIER);
2386

2387 2388 2389 2390 2391
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2392
	if (!HAS_PCH_NOP(dev_priv)) {
2393 2394 2395 2396
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2397

2398 2399
	/* Find, clear, then process each source of interrupt */

2400
	gt_iir = I915_READ(GTIIR);
2401
	if (gt_iir) {
2402 2403
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2404
		if (INTEL_GEN(dev_priv) >= 6)
2405
			snb_gt_irq_handler(dev_priv, gt_iir);
2406
		else
2407
			ilk_gt_irq_handler(dev_priv, gt_iir);
2408 2409
	}

2410 2411
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2412 2413
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2414 2415
		if (INTEL_GEN(dev_priv) >= 7)
			ivb_display_irq_handler(dev_priv, de_iir);
2416
		else
2417
			ilk_display_irq_handler(dev_priv, de_iir);
2418 2419
	}

2420
	if (INTEL_GEN(dev_priv) >= 6) {
2421 2422 2423 2424
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
2425
			gen6_rps_irq_handler(dev_priv, pm_iir);
2426
		}
2427
	}
2428 2429 2430

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2431
	if (!HAS_PCH_NOP(dev_priv)) {
2432 2433 2434
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2435

2436 2437 2438
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	enable_rpm_wakeref_asserts(dev_priv);

2439 2440 2441
	return ret;
}

2442 2443
static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2444
				const u32 hpd[HPD_NUM_PINS])
2445
{
2446
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2447

2448 2449
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2450

2451
	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2452
			   dig_hotplug_reg, hpd,
2453
			   bxt_port_hotplug_long_detect);
2454

2455
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2456 2457
}

2458 2459
static irqreturn_t
gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2460 2461
{
	irqreturn_t ret = IRQ_NONE;
2462
	u32 iir;
2463
	enum pipe pipe;
J
Jesse Barnes 已提交
2464

2465
	if (master_ctl & GEN8_DE_MISC_IRQ) {
2466 2467 2468
		iir = I915_READ(GEN8_DE_MISC_IIR);
		if (iir) {
			I915_WRITE(GEN8_DE_MISC_IIR, iir);
2469
			ret = IRQ_HANDLED;
2470
			if (iir & GEN8_DE_MISC_GSE)
2471
				intel_opregion_asle_intr(dev_priv);
2472 2473
			else
				DRM_ERROR("Unexpected DE Misc interrupt\n");
2474
		}
2475 2476
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2477 2478
	}

2479
	if (master_ctl & GEN8_DE_PORT_IRQ) {
2480 2481 2482
		iir = I915_READ(GEN8_DE_PORT_IIR);
		if (iir) {
			u32 tmp_mask;
2483
			bool found = false;
2484

2485
			I915_WRITE(GEN8_DE_PORT_IIR, iir);
2486
			ret = IRQ_HANDLED;
J
Jesse Barnes 已提交
2487

2488 2489 2490 2491 2492 2493 2494
			tmp_mask = GEN8_AUX_CHANNEL_A;
			if (INTEL_INFO(dev_priv)->gen >= 9)
				tmp_mask |= GEN9_AUX_CHANNEL_B |
					    GEN9_AUX_CHANNEL_C |
					    GEN9_AUX_CHANNEL_D;

			if (iir & tmp_mask) {
2495
				dp_aux_irq_handler(dev_priv);
2496 2497 2498
				found = true;
			}

2499
			if (IS_GEN9_LP(dev_priv)) {
2500 2501
				tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
				if (tmp_mask) {
2502 2503
					bxt_hpd_irq_handler(dev_priv, tmp_mask,
							    hpd_bxt);
2504 2505 2506 2507 2508
					found = true;
				}
			} else if (IS_BROADWELL(dev_priv)) {
				tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
				if (tmp_mask) {
2509 2510
					ilk_hpd_irq_handler(dev_priv,
							    tmp_mask, hpd_bdw);
2511 2512
					found = true;
				}
2513 2514
			}

2515
			if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
2516
				gmbus_irq_handler(dev_priv);
S
Shashank Sharma 已提交
2517 2518 2519
				found = true;
			}

2520
			if (!found)
2521
				DRM_ERROR("Unexpected DE Port interrupt\n");
2522
		}
2523 2524
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2525 2526
	}

2527
	for_each_pipe(dev_priv, pipe) {
2528
		u32 flip_done, fault_errors;
2529

2530 2531
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2532

2533 2534 2535 2536 2537
		iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (!iir) {
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
			continue;
		}
2538

2539 2540
		ret = IRQ_HANDLED;
		I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2541

2542 2543 2544
		if (iir & GEN8_PIPE_VBLANK &&
		    intel_pipe_handle_vblank(dev_priv, pipe))
			intel_check_page_flip(dev_priv, pipe);
2545

2546 2547 2548 2549 2550
		flip_done = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			flip_done &= GEN9_PIPE_PLANE1_FLIP_DONE;
		else
			flip_done &= GEN8_PIPE_PRIMARY_FLIP_DONE;
2551

2552
		if (flip_done)
2553
			intel_finish_page_flip_cs(dev_priv, pipe);
2554

2555
		if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
2556
			hsw_pipe_crc_irq_handler(dev_priv, pipe);
2557

2558 2559
		if (iir & GEN8_PIPE_FIFO_UNDERRUN)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2560

2561 2562 2563 2564 2565
		fault_errors = iir;
		if (INTEL_INFO(dev_priv)->gen >= 9)
			fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
		else
			fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2566

2567
		if (fault_errors)
2568
			DRM_ERROR("Fault errors on pipe %c: 0x%08x\n",
2569 2570
				  pipe_name(pipe),
				  fault_errors);
2571 2572
	}

2573
	if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2574
	    master_ctl & GEN8_DE_PCH_IRQ) {
2575 2576 2577 2578 2579
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
2580 2581 2582
		iir = I915_READ(SDEIIR);
		if (iir) {
			I915_WRITE(SDEIIR, iir);
2583
			ret = IRQ_HANDLED;
2584

2585
			if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv))
2586
				spt_irq_handler(dev_priv, iir);
2587
			else
2588
				cpt_irq_handler(dev_priv, iir);
2589 2590 2591 2592 2593 2594 2595
		} else {
			/*
			 * Like on previous PCH there seems to be something
			 * fishy going on with forwarding PCH interrupts.
			 */
			DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
		}
2596 2597
	}

2598 2599 2600 2601 2602 2603
	return ret;
}

static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
2604
	struct drm_i915_private *dev_priv = to_i915(dev);
2605
	u32 master_ctl;
2606
	u32 gt_iir[4] = {};
2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622
	irqreturn_t ret;

	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);

	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	/* Find, clear, then process each source of interrupt */
2623 2624
	ret = gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
	gen8_gt_irq_handler(dev_priv, gt_iir);
2625 2626
	ret |= gen8_de_irq_handler(dev_priv, master_ctl);

2627 2628
	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ_FW(GEN8_MASTER_IRQ);
2629

2630 2631
	enable_rpm_wakeref_asserts(dev_priv);

2632 2633 2634
	return ret;
}

2635
/**
2636
 * i915_reset_and_wakeup - do process context error handling work
2637
 * @dev_priv: i915 device private
2638 2639 2640 2641
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
2642
static void i915_reset_and_wakeup(struct drm_i915_private *dev_priv)
2643
{
2644
	struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
2645 2646 2647
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2648

2649
	kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);
2650

2651 2652 2653 2654
	DRM_DEBUG_DRIVER("resetting chip\n");
	kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);

	intel_prepare_reset(dev_priv);
2655

2656 2657 2658
	set_bit(I915_RESET_HANDOFF, &dev_priv->gpu_error.flags);
	wake_up_all(&dev_priv->gpu_error.wait_queue);

2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669
	do {
		/*
		 * All state reset _must_ be completed before we update the
		 * reset counter, for otherwise waiters might miss the reset
		 * pending state and not properly drop locks, resulting in
		 * deadlocks with the reset work.
		 */
		if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
			i915_reset(dev_priv);
			mutex_unlock(&dev_priv->drm.struct_mutex);
		}
2670

2671 2672
		/* We need to wait for anyone holding the lock to wakeup */
	} while (wait_on_bit_timeout(&dev_priv->gpu_error.flags,
2673
				     I915_RESET_HANDOFF,
2674 2675
				     TASK_UNINTERRUPTIBLE,
				     HZ));
2676

2677
	intel_finish_reset(dev_priv);
2678

2679
	if (!test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
2680 2681
		kobject_uevent_env(kobj,
				   KOBJ_CHANGE, reset_done_event);
2682

2683 2684 2685 2686
	/*
	 * Note: The wake_up also serves as a memory barrier so that
	 * waiters see the updated value of the dev_priv->gpu_error.
	 */
2687
	clear_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags);
2688
	wake_up_all(&dev_priv->gpu_error.reset_queue);
2689 2690
}

2691 2692 2693 2694
static inline void
i915_err_print_instdone(struct drm_i915_private *dev_priv,
			struct intel_instdone *instdone)
{
2695 2696 2697
	int slice;
	int subslice;

2698 2699 2700 2701 2702 2703 2704 2705 2706 2707
	pr_err("  INSTDONE: 0x%08x\n", instdone->instdone);

	if (INTEL_GEN(dev_priv) <= 3)
		return;

	pr_err("  SC_INSTDONE: 0x%08x\n", instdone->slice_common);

	if (INTEL_GEN(dev_priv) <= 6)
		return;

2708 2709 2710 2711 2712 2713 2714
	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
		pr_err("  SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
		       slice, subslice, instdone->sampler[slice][subslice]);

	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
		pr_err("  ROW_INSTDONE[%d][%d]: 0x%08x\n",
		       slice, subslice, instdone->row[slice][subslice]);
2715 2716
}

2717
static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
2718
{
2719
	u32 eir;
2720

2721 2722
	if (!IS_GEN2(dev_priv))
		I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER));
2723

2724 2725 2726 2727
	if (INTEL_GEN(dev_priv) < 4)
		I915_WRITE(IPEIR, I915_READ(IPEIR));
	else
		I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965));
2728

2729
	I915_WRITE(EIR, I915_READ(EIR));
2730 2731 2732 2733 2734 2735
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
2736
		DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
2737 2738 2739
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2740 2741 2742
}

/**
2743
 * i915_handle_error - handle a gpu error
2744
 * @dev_priv: i915 device private
2745
 * @engine_mask: mask representing engines that are hung
2746 2747
 * @fmt: Error message format string
 *
2748
 * Do some basic checking of register state at error time and
2749 2750 2751 2752 2753
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
2754 2755
void i915_handle_error(struct drm_i915_private *dev_priv,
		       u32 engine_mask,
2756
		       const char *fmt, ...)
2757
{
2758 2759
	va_list args;
	char error_msg[80];
2760

2761 2762 2763 2764
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

2765 2766 2767 2768 2769 2770 2771 2772 2773
	/*
	 * In most cases it's guaranteed that we get here with an RPM
	 * reference held, for example because there is a pending GPU
	 * request that won't finish until the reset is done. This
	 * isn't the case at least when we get here by doing a
	 * simulated reset via debugfs, so get an RPM reference.
	 */
	intel_runtime_pm_get(dev_priv);

2774
	i915_capture_error_state(dev_priv, engine_mask, error_msg);
2775
	i915_clear_error_registers(dev_priv);
2776

2777
	if (!engine_mask)
2778
		goto out;
2779

2780
	if (test_and_set_bit(I915_RESET_BACKOFF,
2781
			     &dev_priv->gpu_error.flags))
2782
		goto out;
2783

2784
	i915_reset_and_wakeup(dev_priv);
2785 2786 2787

out:
	intel_runtime_pm_put(dev_priv);
2788 2789
}

2790 2791 2792
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2793
static int i8xx_enable_vblank(struct drm_device *dev, unsigned int pipe)
2794
{
2795
	struct drm_i915_private *dev_priv = to_i915(dev);
2796
	unsigned long irqflags;
2797

2798
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2799
	i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2800
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2801

2802 2803 2804
	return 0;
}

2805
static int i965_enable_vblank(struct drm_device *dev, unsigned int pipe)
2806
{
2807
	struct drm_i915_private *dev_priv = to_i915(dev);
2808 2809 2810
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2811 2812
	i915_enable_pipestat(dev_priv, pipe,
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
2813 2814 2815 2816 2817
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2818
static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2819
{
2820
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
2821
	unsigned long irqflags;
2822
	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
2823
		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
J
Jesse Barnes 已提交
2824 2825

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2826
	ilk_enable_display_irq(dev_priv, bit);
J
Jesse Barnes 已提交
2827 2828 2829 2830 2831
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2832
static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
2833
{
2834
	struct drm_i915_private *dev_priv = to_i915(dev);
2835 2836 2837
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2838
	bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2839
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2840

2841 2842 2843
	return 0;
}

2844 2845 2846
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2847
static void i8xx_disable_vblank(struct drm_device *dev, unsigned int pipe)
2848
{
2849
	struct drm_i915_private *dev_priv = to_i915(dev);
2850
	unsigned long irqflags;
2851

2852
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2853
	i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2854 2855 2856
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2857
static void i965_disable_vblank(struct drm_device *dev, unsigned int pipe)
2858
{
2859
	struct drm_i915_private *dev_priv = to_i915(dev);
2860 2861 2862
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2863 2864
	i915_disable_pipestat(dev_priv, pipe,
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2865 2866 2867
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2868
static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2869
{
2870
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
2871
	unsigned long irqflags;
2872
	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
2873
		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
J
Jesse Barnes 已提交
2874 2875

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2876
	ilk_disable_display_irq(dev_priv, bit);
J
Jesse Barnes 已提交
2877 2878 2879
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2880
static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
2881
{
2882
	struct drm_i915_private *dev_priv = to_i915(dev);
2883 2884 2885
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2886
	bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2887 2888 2889
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2890
static void ibx_irq_reset(struct drm_i915_private *dev_priv)
P
Paulo Zanoni 已提交
2891
{
2892
	if (HAS_PCH_NOP(dev_priv))
P
Paulo Zanoni 已提交
2893 2894
		return;

2895
	GEN5_IRQ_RESET(SDE);
2896

2897
	if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
2898
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
2899
}
2900

P
Paulo Zanoni 已提交
2901 2902 2903 2904 2905 2906 2907 2908 2909 2910
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
2911
	struct drm_i915_private *dev_priv = to_i915(dev);
P
Paulo Zanoni 已提交
2912

2913
	if (HAS_PCH_NOP(dev_priv))
P
Paulo Zanoni 已提交
2914 2915 2916
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
2917 2918 2919 2920
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

2921
static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv)
2922
{
2923
	GEN5_IRQ_RESET(GT);
2924
	if (INTEL_GEN(dev_priv) >= 6)
2925
		GEN5_IRQ_RESET(GEN6_PM);
2926 2927
}

2928 2929 2930 2931
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
	enum pipe pipe;

2932 2933 2934 2935 2936
	if (IS_CHERRYVIEW(dev_priv))
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
	else
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);

2937
	i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
2938 2939
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

2940 2941 2942 2943 2944 2945
	for_each_pipe(dev_priv, pipe) {
		I915_WRITE(PIPESTAT(pipe),
			   PIPE_FIFO_UNDERRUN_STATUS |
			   PIPESTAT_INT_STATUS_MASK);
		dev_priv->pipestat_irq_mask[pipe] = 0;
	}
2946 2947

	GEN5_IRQ_RESET(VLV_);
2948
	dev_priv->irq_mask = ~0;
2949 2950
}

2951 2952 2953
static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
2954
	u32 enable_mask;
2955
	enum pipe pipe;
2956
	u32 val;
2957 2958 2959 2960 2961 2962 2963 2964

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		i915_enable_pipestat(dev_priv, pipe, pipestat_mask);

2965 2966 2967
	enable_mask = I915_DISPLAY_PORT_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
2968
	if (IS_CHERRYVIEW(dev_priv))
2969
		enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
2970 2971 2972

	WARN_ON(dev_priv->irq_mask != ~0);

2973 2974 2975 2976 2977 2978
	val = (I915_LPE_PIPE_A_INTERRUPT |
		I915_LPE_PIPE_B_INTERRUPT |
		I915_LPE_PIPE_C_INTERRUPT);

	enable_mask |= val;

2979 2980 2981
	dev_priv->irq_mask = ~enable_mask;

	GEN5_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask);
2982 2983 2984 2985 2986 2987
}

/* drm_dma.h hooks
*/
static void ironlake_irq_reset(struct drm_device *dev)
{
2988
	struct drm_i915_private *dev_priv = to_i915(dev);
2989 2990 2991 2992

	I915_WRITE(HWSTAM, 0xffffffff);

	GEN5_IRQ_RESET(DE);
2993
	if (IS_GEN7(dev_priv))
2994 2995
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);

2996
	gen5_gt_irq_reset(dev_priv);
2997

2998
	ibx_irq_reset(dev_priv);
2999 3000
}

J
Jesse Barnes 已提交
3001 3002
static void valleyview_irq_preinstall(struct drm_device *dev)
{
3003
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
3004

3005 3006 3007
	I915_WRITE(VLV_MASTER_IER, 0);
	POSTING_READ(VLV_MASTER_IER);

3008
	gen5_gt_irq_reset(dev_priv);
J
Jesse Barnes 已提交
3009

3010
	spin_lock_irq(&dev_priv->irq_lock);
3011 3012
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3013
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3014 3015
}

3016 3017 3018 3019 3020 3021 3022 3023
static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
{
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
}

P
Paulo Zanoni 已提交
3024
static void gen8_irq_reset(struct drm_device *dev)
3025
{
3026
	struct drm_i915_private *dev_priv = to_i915(dev);
3027 3028 3029 3030 3031
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3032
	gen8_gt_irq_reset(dev_priv);
3033

3034
	for_each_pipe(dev_priv, pipe)
3035 3036
		if (intel_display_power_is_enabled(dev_priv,
						   POWER_DOMAIN_PIPE(pipe)))
3037
			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3038

3039 3040 3041
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3042

3043
	if (HAS_PCH_SPLIT(dev_priv))
3044
		ibx_irq_reset(dev_priv);
3045
}
3046

3047 3048
void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
3049
{
3050
	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3051
	enum pipe pipe;
3052

3053
	spin_lock_irq(&dev_priv->irq_lock);
3054 3055 3056 3057
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
				  dev_priv->de_irq_mask[pipe],
				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
3058
	spin_unlock_irq(&dev_priv->irq_lock);
3059 3060
}

3061 3062 3063
void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
				     unsigned int pipe_mask)
{
3064 3065
	enum pipe pipe;

3066
	spin_lock_irq(&dev_priv->irq_lock);
3067 3068
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3069 3070 3071
	spin_unlock_irq(&dev_priv->irq_lock);

	/* make sure we're done processing display irqs */
3072
	synchronize_irq(dev_priv->drm.irq);
3073 3074
}

3075 3076
static void cherryview_irq_preinstall(struct drm_device *dev)
{
3077
	struct drm_i915_private *dev_priv = to_i915(dev);
3078 3079 3080 3081

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3082
	gen8_gt_irq_reset(dev_priv);
3083 3084 3085

	GEN5_IRQ_RESET(GEN8_PCU_);

3086
	spin_lock_irq(&dev_priv->irq_lock);
3087 3088
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3089
	spin_unlock_irq(&dev_priv->irq_lock);
3090 3091
}

3092
static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
3093 3094 3095 3096 3097
				  const u32 hpd[HPD_NUM_PINS])
{
	struct intel_encoder *encoder;
	u32 enabled_irqs = 0;

3098
	for_each_intel_encoder(&dev_priv->drm, encoder)
3099 3100 3101 3102 3103 3104
		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
			enabled_irqs |= hpd[encoder->hpd_pin];

	return enabled_irqs;
}

3105
static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
3106
{
3107
	u32 hotplug;
3108 3109 3110

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
3111 3112
	 * duration to 2ms (which is the minimum in the Display Port spec).
	 * The pulse duration bits are reserved on LPT+.
3113
	 */
3114
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3115 3116 3117
	hotplug &= ~(PORTB_PULSE_DURATION_MASK |
		     PORTC_PULSE_DURATION_MASK |
		     PORTD_PULSE_DURATION_MASK);
3118
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3119 3120
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3121 3122 3123 3124
	/*
	 * When CPU and PCH are on the same package, port A
	 * HPD must be enabled in both north and south.
	 */
3125
	if (HAS_PCH_LPT_LP(dev_priv))
3126
		hotplug |= PORTA_HOTPLUG_ENABLE;
3127
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3128
}
X
Xiong Zhang 已提交
3129

3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146
static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	if (HAS_PCH_IBX(dev_priv)) {
		hotplug_irqs = SDE_HOTPLUG_MASK;
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx);
	} else {
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt);
	}

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	ibx_hpd_detection_setup(dev_priv);
}

3147
static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3148
{
3149
	u32 hotplug;
3150 3151 3152

	/* Enable digital hotplug on the PCH */
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3153 3154 3155 3156
	hotplug |= PORTA_HOTPLUG_ENABLE |
		   PORTB_HOTPLUG_ENABLE |
		   PORTC_HOTPLUG_ENABLE |
		   PORTD_HOTPLUG_ENABLE;
3157 3158 3159 3160 3161
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);

	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
	hotplug |= PORTE_HOTPLUG_ENABLE;
	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3162 3163
}

3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175
static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt);

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	spt_hpd_detection_setup(dev_priv);
}

3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191
static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug;

	/*
	 * Enable digital hotplug on the CPU, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
	 * The pulse duration bits are reserved on HSW+.
	 */
	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE |
		   DIGITAL_PORTA_PULSE_DURATION_2ms;
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
}

3192
static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3193
{
3194
	u32 hotplug_irqs, enabled_irqs;
3195

3196
	if (INTEL_GEN(dev_priv) >= 8) {
3197
		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3198
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw);
3199 3200

		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3201
	} else if (INTEL_GEN(dev_priv) >= 7) {
3202
		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3203
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb);
3204 3205

		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3206 3207
	} else {
		hotplug_irqs = DE_DP_A_HOTPLUG;
3208
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk);
3209

3210 3211
		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
	}
3212

3213
	ilk_hpd_detection_setup(dev_priv);
3214

3215
	ibx_hpd_irq_setup(dev_priv);
3216 3217
}

3218 3219
static void __bxt_hpd_detection_setup(struct drm_i915_private *dev_priv,
				      u32 enabled_irqs)
3220
{
3221
	u32 hotplug;
3222

3223
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3224 3225 3226
	hotplug |= PORTA_HOTPLUG_ENABLE |
		   PORTB_HOTPLUG_ENABLE |
		   PORTC_HOTPLUG_ENABLE;
3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245

	DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n",
		      hotplug, enabled_irqs);
	hotplug &= ~BXT_DDI_HPD_INVERT_MASK;

	/*
	 * For BXT invert bit has to be set based on AOB design
	 * for HPD detection logic, update it based on VBT fields.
	 */
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
		hotplug |= BXT_DDIA_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
		hotplug |= BXT_DDIB_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
		hotplug |= BXT_DDIC_HPD_INVERT;

3246
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3247 3248
}

3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265
static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
{
	__bxt_hpd_detection_setup(dev_priv, BXT_DE_PORT_HOTPLUG_MASK);
}

static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt);
	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;

	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);

	__bxt_hpd_detection_setup(dev_priv, enabled_irqs);
}

P
Paulo Zanoni 已提交
3266 3267
static void ibx_irq_postinstall(struct drm_device *dev)
{
3268
	struct drm_i915_private *dev_priv = to_i915(dev);
3269
	u32 mask;
3270

3271
	if (HAS_PCH_NOP(dev_priv))
D
Daniel Vetter 已提交
3272 3273
		return;

3274
	if (HAS_PCH_IBX(dev_priv))
3275
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3276
	else
3277
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3278

3279
	gen5_assert_iir_is_zero(dev_priv, SDEIIR);
P
Paulo Zanoni 已提交
3280
	I915_WRITE(SDEIMR, ~mask);
3281 3282 3283

	if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
	    HAS_PCH_LPT(dev_priv))
3284
		ibx_hpd_detection_setup(dev_priv);
3285 3286
	else
		spt_hpd_detection_setup(dev_priv);
P
Paulo Zanoni 已提交
3287 3288
}

3289 3290
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
3291
	struct drm_i915_private *dev_priv = to_i915(dev);
3292 3293 3294 3295 3296
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3297
	if (HAS_L3_DPF(dev_priv)) {
3298
		/* L3 parity interrupt is always unmasked. */
3299 3300
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev_priv);
		gt_irqs |= GT_PARITY_ERROR(dev_priv);
3301 3302 3303
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
3304
	if (IS_GEN5(dev_priv)) {
3305
		gt_irqs |= ILK_BSD_USER_INTERRUPT;
3306 3307 3308 3309
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3310
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3311

3312
	if (INTEL_GEN(dev_priv) >= 6) {
3313 3314 3315 3316
		/*
		 * RPS interrupts will get enabled/disabled on demand when RPS
		 * itself is enabled/disabled.
		 */
3317
		if (HAS_VEBOX(dev_priv)) {
3318
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3319 3320
			dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT;
		}
3321

3322 3323
		dev_priv->pm_imr = 0xffffffff;
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs);
3324 3325 3326
	}
}

3327
static int ironlake_irq_postinstall(struct drm_device *dev)
3328
{
3329
	struct drm_i915_private *dev_priv = to_i915(dev);
3330 3331
	u32 display_mask, extra_mask;

3332
	if (INTEL_GEN(dev_priv) >= 7) {
3333 3334 3335
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3336
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3337
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3338 3339
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
			      DE_DP_A_HOTPLUG_IVB);
3340 3341 3342
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3343 3344 3345
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3346 3347 3348
		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
			      DE_DP_A_HOTPLUG);
3349
	}
3350

3351
	dev_priv->irq_mask = ~display_mask;
3352

3353 3354
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3355 3356
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3357
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3358

3359
	gen5_gt_irq_postinstall(dev);
3360

3361 3362
	ilk_hpd_detection_setup(dev_priv);

P
Paulo Zanoni 已提交
3363
	ibx_irq_postinstall(dev);
3364

3365
	if (IS_IRONLAKE_M(dev_priv)) {
3366 3367 3368
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3369 3370
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
3371
		spin_lock_irq(&dev_priv->irq_lock);
3372
		ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
3373
		spin_unlock_irq(&dev_priv->irq_lock);
3374 3375
	}

3376 3377 3378
	return 0;
}

3379 3380
void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
3381
	lockdep_assert_held(&dev_priv->irq_lock);
3382 3383 3384 3385 3386 3387

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

3388 3389
	if (intel_irqs_enabled(dev_priv)) {
		vlv_display_irq_reset(dev_priv);
3390
		vlv_display_irq_postinstall(dev_priv);
3391
	}
3392 3393 3394 3395
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
3396
	lockdep_assert_held(&dev_priv->irq_lock);
3397 3398 3399 3400 3401 3402

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

3403
	if (intel_irqs_enabled(dev_priv))
3404
		vlv_display_irq_reset(dev_priv);
3405 3406
}

3407 3408 3409

static int valleyview_irq_postinstall(struct drm_device *dev)
{
3410
	struct drm_i915_private *dev_priv = to_i915(dev);
3411

3412
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3413

3414
	spin_lock_irq(&dev_priv->irq_lock);
3415 3416
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3417 3418
	spin_unlock_irq(&dev_priv->irq_lock);

J
Jesse Barnes 已提交
3419
	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3420
	POSTING_READ(VLV_MASTER_IER);
3421 3422 3423 3424

	return 0;
}

3425 3426 3427 3428 3429
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3430 3431 3432
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3433
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3434 3435 3436
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3437
		0,
3438 3439
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3440 3441
		};

3442 3443 3444
	if (HAS_L3_DPF(dev_priv))
		gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;

3445 3446
	dev_priv->pm_ier = 0x0;
	dev_priv->pm_imr = ~dev_priv->pm_ier;
3447 3448
	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3449 3450
	/*
	 * RPS interrupts will get enabled/disabled on demand when RPS itself
3451
	 * is enabled/disabled. Same wil be the case for GuC interrupts.
3452
	 */
3453
	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_imr, dev_priv->pm_ier);
3454
	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3455 3456 3457 3458
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
3459 3460
	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
	uint32_t de_pipe_enables;
3461 3462
	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
	u32 de_port_enables;
3463
	u32 de_misc_masked = GEN8_DE_MISC_GSE;
3464
	enum pipe pipe;
3465

3466
	if (INTEL_INFO(dev_priv)->gen >= 9) {
3467 3468
		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3469 3470
		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
				  GEN9_AUX_CHANNEL_D;
3471
		if (IS_GEN9_LP(dev_priv))
3472 3473
			de_port_masked |= BXT_DE_PORT_GMBUS;
	} else {
3474 3475
		de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
				  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3476
	}
3477 3478 3479 3480

	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
					   GEN8_PIPE_FIFO_UNDERRUN;

3481
	de_port_enables = de_port_masked;
3482
	if (IS_GEN9_LP(dev_priv))
3483 3484
		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
	else if (IS_BROADWELL(dev_priv))
3485 3486
		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;

3487 3488 3489
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3490

3491
	for_each_pipe(dev_priv, pipe)
3492
		if (intel_display_power_is_enabled(dev_priv,
3493 3494 3495 3496
				POWER_DOMAIN_PIPE(pipe)))
			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
					  dev_priv->de_irq_mask[pipe],
					  de_pipe_enables);
3497

3498
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3499
	GEN5_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
3500 3501 3502

	if (IS_GEN9_LP(dev_priv))
		bxt_hpd_detection_setup(dev_priv);
3503 3504
	else if (IS_BROADWELL(dev_priv))
		ilk_hpd_detection_setup(dev_priv);
3505 3506 3507 3508
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
3509
	struct drm_i915_private *dev_priv = to_i915(dev);
3510

3511
	if (HAS_PCH_SPLIT(dev_priv))
3512
		ibx_irq_pre_postinstall(dev);
P
Paulo Zanoni 已提交
3513

3514 3515 3516
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

3517
	if (HAS_PCH_SPLIT(dev_priv))
3518
		ibx_irq_postinstall(dev);
3519

3520
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3521 3522 3523 3524 3525
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3526 3527
static int cherryview_irq_postinstall(struct drm_device *dev)
{
3528
	struct drm_i915_private *dev_priv = to_i915(dev);
3529 3530 3531

	gen8_gt_irq_postinstall(dev_priv);

3532
	spin_lock_irq(&dev_priv->irq_lock);
3533 3534
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3535 3536
	spin_unlock_irq(&dev_priv->irq_lock);

3537
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3538 3539 3540 3541 3542
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3543 3544
static void gen8_irq_uninstall(struct drm_device *dev)
{
3545
	struct drm_i915_private *dev_priv = to_i915(dev);
3546 3547 3548 3549

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3550
	gen8_irq_reset(dev);
3551 3552
}

J
Jesse Barnes 已提交
3553 3554
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3555
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
3556 3557 3558 3559

	if (!dev_priv)
		return;

3560
	I915_WRITE(VLV_MASTER_IER, 0);
3561
	POSTING_READ(VLV_MASTER_IER);
3562

3563
	gen5_gt_irq_reset(dev_priv);
3564

J
Jesse Barnes 已提交
3565
	I915_WRITE(HWSTAM, 0xffffffff);
3566

3567
	spin_lock_irq(&dev_priv->irq_lock);
3568 3569
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3570
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3571 3572
}

3573 3574
static void cherryview_irq_uninstall(struct drm_device *dev)
{
3575
	struct drm_i915_private *dev_priv = to_i915(dev);
3576 3577 3578 3579 3580 3581 3582

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3583
	gen8_gt_irq_reset(dev_priv);
3584

3585
	GEN5_IRQ_RESET(GEN8_PCU_);
3586

3587
	spin_lock_irq(&dev_priv->irq_lock);
3588 3589
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3590
	spin_unlock_irq(&dev_priv->irq_lock);
3591 3592
}

3593
static void ironlake_irq_uninstall(struct drm_device *dev)
3594
{
3595
	struct drm_i915_private *dev_priv = to_i915(dev);
3596 3597 3598 3599

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3600
	ironlake_irq_reset(dev);
3601 3602
}

3603
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3604
{
3605
	struct drm_i915_private *dev_priv = to_i915(dev);
3606
	int pipe;
3607

3608
	for_each_pipe(dev_priv, pipe)
3609
		I915_WRITE(PIPESTAT(pipe), 0);
3610 3611 3612
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3613 3614 3615 3616
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3617
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3618 3619 3620 3621 3622 3623 3624 3625 3626

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3627
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
C
Chris Wilson 已提交
3628 3629 3630 3631 3632 3633 3634 3635
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3636 3637
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3638
	spin_lock_irq(&dev_priv->irq_lock);
3639 3640
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3641
	spin_unlock_irq(&dev_priv->irq_lock);
3642

C
Chris Wilson 已提交
3643 3644 3645
	return 0;
}

3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676
/*
 * Returns true when a page flip has completed.
 */
static bool i8xx_handle_vblank(struct drm_i915_private *dev_priv,
			       int plane, int pipe, u32 iir)
{
	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

	if (!intel_pipe_handle_vblank(dev_priv, pipe))
		return false;

	if ((iir & flip_pending) == 0)
		goto check_page_flip;

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ16(ISR) & flip_pending)
		goto check_page_flip;

	intel_finish_page_flip_cs(dev_priv, pipe);
	return true;

check_page_flip:
	intel_check_page_flip(dev_priv, pipe);
	return false;
}

3677
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
3678
{
3679
	struct drm_device *dev = arg;
3680
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3681 3682 3683 3684 3685 3686
	u16 iir, new_iir;
	u32 pipe_stats[2];
	int pipe;
	u16 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3687
	irqreturn_t ret;
C
Chris Wilson 已提交
3688

3689 3690 3691
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

3692 3693 3694 3695
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	ret = IRQ_NONE;
C
Chris Wilson 已提交
3696 3697
	iir = I915_READ16(IIR);
	if (iir == 0)
3698
		goto out;
C
Chris Wilson 已提交
3699 3700 3701 3702 3703 3704 3705

	while (iir & ~flip_mask) {
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3706
		spin_lock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3707
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3708
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
C
Chris Wilson 已提交
3709

3710
		for_each_pipe(dev_priv, pipe) {
3711
			i915_reg_t reg = PIPESTAT(pipe);
C
Chris Wilson 已提交
3712 3713 3714 3715 3716
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
3717
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
3718 3719
				I915_WRITE(reg, pipe_stats[pipe]);
		}
3720
		spin_unlock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3721 3722 3723 3724 3725

		I915_WRITE16(IIR, iir & ~flip_mask);
		new_iir = I915_READ16(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
3726
			notify_ring(dev_priv->engine[RCS]);
C
Chris Wilson 已提交
3727

3728
		for_each_pipe(dev_priv, pipe) {
3729 3730 3731 3732 3733 3734 3735
			int plane = pipe;
			if (HAS_FBC(dev_priv))
				plane = !plane;

			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
			    i8xx_handle_vblank(dev_priv, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
C
Chris Wilson 已提交
3736

3737
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3738
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
3739

3740 3741 3742
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3743
		}
C
Chris Wilson 已提交
3744 3745 3746

		iir = new_iir;
	}
3747 3748 3749 3750
	ret = IRQ_HANDLED;

out:
	enable_rpm_wakeref_asserts(dev_priv);
C
Chris Wilson 已提交
3751

3752
	return ret;
C
Chris Wilson 已提交
3753 3754 3755 3756
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
3757
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3758 3759
	int pipe;

3760
	for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3761 3762 3763 3764 3765 3766 3767 3768 3769
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

3770 3771
static void i915_irq_preinstall(struct drm_device * dev)
{
3772
	struct drm_i915_private *dev_priv = to_i915(dev);
3773 3774
	int pipe;

3775
	if (I915_HAS_HOTPLUG(dev_priv)) {
3776
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3777 3778 3779
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3780
	I915_WRITE16(HWSTAM, 0xeffe);
3781
	for_each_pipe(dev_priv, pipe)
3782 3783 3784 3785 3786 3787 3788 3789
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
3790
	struct drm_i915_private *dev_priv = to_i915(dev);
3791
	u32 enable_mask;
3792

3793 3794 3795 3796 3797 3798 3799 3800
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3801
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3802 3803 3804 3805 3806 3807 3808

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_USER_INTERRUPT;

3809
	if (I915_HAS_HOTPLUG(dev_priv)) {
3810
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3811 3812
		POSTING_READ(PORT_HOTPLUG_EN);

3813 3814 3815 3816 3817 3818 3819 3820 3821 3822
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

3823
	i915_enable_asle_pipestat(dev_priv);
3824

3825 3826
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3827
	spin_lock_irq(&dev_priv->irq_lock);
3828 3829
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3830
	spin_unlock_irq(&dev_priv->irq_lock);
3831

3832 3833 3834
	return 0;
}

3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865
/*
 * Returns true when a page flip has completed.
 */
static bool i915_handle_vblank(struct drm_i915_private *dev_priv,
			       int plane, int pipe, u32 iir)
{
	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

	if (!intel_pipe_handle_vblank(dev_priv, pipe))
		return false;

	if ((iir & flip_pending) == 0)
		goto check_page_flip;

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ(ISR) & flip_pending)
		goto check_page_flip;

	intel_finish_page_flip_cs(dev_priv, pipe);
	return true;

check_page_flip:
	intel_check_page_flip(dev_priv, pipe);
	return false;
}

3866
static irqreturn_t i915_irq_handler(int irq, void *arg)
3867
{
3868
	struct drm_device *dev = arg;
3869
	struct drm_i915_private *dev_priv = to_i915(dev);
3870
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3871 3872 3873 3874
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
	int pipe, ret = IRQ_NONE;
3875

3876 3877 3878
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

3879 3880 3881
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

3882
	iir = I915_READ(IIR);
3883 3884
	do {
		bool irq_received = (iir & ~flip_mask) != 0;
3885
		bool blc_event = false;
3886 3887 3888 3889 3890 3891

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3892
		spin_lock(&dev_priv->irq_lock);
3893
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3894
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3895

3896
		for_each_pipe(dev_priv, pipe) {
3897
			i915_reg_t reg = PIPESTAT(pipe);
3898 3899
			pipe_stats[pipe] = I915_READ(reg);

3900
			/* Clear the PIPE*STAT regs before the IIR */
3901 3902
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
3903
				irq_received = true;
3904 3905
			}
		}
3906
		spin_unlock(&dev_priv->irq_lock);
3907 3908 3909 3910 3911

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
3912
		if (I915_HAS_HOTPLUG(dev_priv) &&
3913 3914 3915
		    iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
3916
				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
3917
		}
3918

3919
		I915_WRITE(IIR, iir & ~flip_mask);
3920 3921 3922
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
3923
			notify_ring(dev_priv->engine[RCS]);
3924

3925
		for_each_pipe(dev_priv, pipe) {
3926 3927 3928 3929 3930 3931 3932
			int plane = pipe;
			if (HAS_FBC(dev_priv))
				plane = !plane;

			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
			    i915_handle_vblank(dev_priv, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3933 3934 3935

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
3936 3937

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3938
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
3939

3940 3941 3942
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3943 3944 3945
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
3946
			intel_opregion_asle_intr(dev_priv);
3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
3963
		ret = IRQ_HANDLED;
3964
		iir = new_iir;
3965
	} while (iir & ~flip_mask);
3966

3967 3968
	enable_rpm_wakeref_asserts(dev_priv);

3969 3970 3971 3972 3973
	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
3974
	struct drm_i915_private *dev_priv = to_i915(dev);
3975 3976
	int pipe;

3977
	if (I915_HAS_HOTPLUG(dev_priv)) {
3978
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3979 3980 3981
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3982
	I915_WRITE16(HWSTAM, 0xffff);
3983
	for_each_pipe(dev_priv, pipe) {
3984
		/* Clear enable bits; then clear status bits */
3985
		I915_WRITE(PIPESTAT(pipe), 0);
3986 3987
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
3988 3989 3990 3991 3992 3993 3994 3995
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
3996
	struct drm_i915_private *dev_priv = to_i915(dev);
3997 3998
	int pipe;

3999
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4000
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4001 4002

	I915_WRITE(HWSTAM, 0xeffe);
4003
	for_each_pipe(dev_priv, pipe)
4004 4005 4006 4007 4008 4009 4010 4011
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
4012
	struct drm_i915_private *dev_priv = to_i915(dev);
4013
	u32 enable_mask;
4014 4015 4016
	u32 error_mask;

	/* Unmask the interrupts that we always want on. */
4017
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4018
			       I915_DISPLAY_PORT_INTERRUPT |
4019 4020 4021 4022 4023 4024 4025
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
4026 4027
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4028 4029
	enable_mask |= I915_USER_INTERRUPT;

4030
	if (IS_G4X(dev_priv))
4031
		enable_mask |= I915_BSD_USER_INTERRUPT;
4032

4033 4034
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
4035
	spin_lock_irq(&dev_priv->irq_lock);
4036 4037 4038
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4039
	spin_unlock_irq(&dev_priv->irq_lock);
4040 4041 4042 4043 4044

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
4045
	if (IS_G4X(dev_priv)) {
4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4060
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4061 4062
	POSTING_READ(PORT_HOTPLUG_EN);

4063
	i915_enable_asle_pipestat(dev_priv);
4064 4065 4066 4067

	return 0;
}

4068
static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
4069 4070 4071
{
	u32 hotplug_en;

4072
	lockdep_assert_held(&dev_priv->irq_lock);
4073

4074 4075
	/* Note HDMI and DP share hotplug bits */
	/* enable bits are the same for all generations */
4076
	hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
4077 4078 4079 4080
	/* Programming the CRT detection parameters tends
	   to generate a spurious hotplug event about three
	   seconds later.  So just do it once.
	*/
4081
	if (IS_G4X(dev_priv))
4082 4083 4084 4085
		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;

	/* Ignore TV since it's buggy */
4086
	i915_hotplug_interrupt_update_locked(dev_priv,
4087 4088 4089 4090
					     HOTPLUG_INT_EN_MASK |
					     CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
					     CRT_HOTPLUG_ACTIVATION_PERIOD_64,
					     hotplug_en);
4091 4092
}

4093
static irqreturn_t i965_irq_handler(int irq, void *arg)
4094
{
4095
	struct drm_device *dev = arg;
4096
	struct drm_i915_private *dev_priv = to_i915(dev);
4097 4098 4099
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	int ret = IRQ_NONE, pipe;
4100 4101 4102
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4103

4104 4105 4106
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4107 4108 4109
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

4110 4111 4112
	iir = I915_READ(IIR);

	for (;;) {
4113
		bool irq_received = (iir & ~flip_mask) != 0;
4114 4115
		bool blc_event = false;

4116 4117 4118 4119 4120
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4121
		spin_lock(&dev_priv->irq_lock);
4122
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4123
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4124

4125
		for_each_pipe(dev_priv, pipe) {
4126
			i915_reg_t reg = PIPESTAT(pipe);
4127 4128 4129 4130 4131 4132 4133
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4134
				irq_received = true;
4135 4136
			}
		}
4137
		spin_unlock(&dev_priv->irq_lock);
4138 4139 4140 4141 4142 4143 4144

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4145 4146 4147
		if (iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
4148
				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4149
		}
4150

4151
		I915_WRITE(IIR, iir & ~flip_mask);
4152 4153 4154
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4155
			notify_ring(dev_priv->engine[RCS]);
4156
		if (iir & I915_BSD_USER_INTERRUPT)
4157
			notify_ring(dev_priv->engine[VCS]);
4158

4159
		for_each_pipe(dev_priv, pipe) {
4160 4161 4162
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
			    i915_handle_vblank(dev_priv, pipe, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4163 4164 4165

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4166 4167

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4168
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
4169

4170 4171
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4172
		}
4173 4174

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
4175
			intel_opregion_asle_intr(dev_priv);
4176

4177
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4178
			gmbus_irq_handler(dev_priv);
4179

4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

4198 4199
	enable_rpm_wakeref_asserts(dev_priv);

4200 4201 4202 4203 4204
	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4205
	struct drm_i915_private *dev_priv = to_i915(dev);
4206 4207 4208 4209 4210
	int pipe;

	if (!dev_priv)
		return;

4211
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4212
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4213 4214

	I915_WRITE(HWSTAM, 0xffffffff);
4215
	for_each_pipe(dev_priv, pipe)
4216 4217 4218 4219
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

4220
	for_each_pipe(dev_priv, pipe)
4221 4222 4223 4224 4225
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4226 4227 4228 4229 4230 4231 4232
/**
 * intel_irq_init - initializes irq support
 * @dev_priv: i915 device instance
 *
 * This function initializes all the irq support including work items, timers
 * and all the vtables. It does not setup the interrupt itself though.
 */
4233
void intel_irq_init(struct drm_i915_private *dev_priv)
4234
{
4235
	struct drm_device *dev = &dev_priv->drm;
4236

4237 4238
	intel_hpd_init_work(dev_priv);

4239
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4240
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4241

4242
	if (HAS_GUC_SCHED(dev_priv))
4243 4244
		dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT;

4245
	/* Let's track the enabled rps events */
4246
	if (IS_VALLEYVIEW(dev_priv))
4247
		/* WaGsvRC0ResidencyMethod:vlv */
4248
		dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
4249 4250
	else
		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4251

4252
	dev_priv->rps.pm_intrmsk_mbz = 0;
4253 4254

	/*
4255
	 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
4256 4257 4258 4259
	 * if GEN6_PM_UP_EI_EXPIRED is masked.
	 *
	 * TODO: verify if this can be reproduced on VLV,CHV.
	 */
4260
	if (INTEL_INFO(dev_priv)->gen <= 7)
4261
		dev_priv->rps.pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
4262 4263

	if (INTEL_INFO(dev_priv)->gen >= 8)
4264
		dev_priv->rps.pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
4265

4266
	if (IS_GEN2(dev_priv)) {
4267
		/* Gen2 doesn't have a hardware frame counter */
4268
		dev->max_vblank_count = 0;
4269
	} else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
4270
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4271
		dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4272 4273 4274
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4275 4276
	}

4277 4278 4279 4280 4281
	/*
	 * Opt out of the vblank disable timer on everything except gen2.
	 * Gen2 doesn't have a hardware frame counter and so depends on
	 * vblank interrupts to produce sane vblank seuquence numbers.
	 */
4282
	if (!IS_GEN2(dev_priv))
4283 4284
		dev->vblank_disable_immediate = true;

4285 4286 4287 4288 4289 4290 4291 4292 4293 4294
	/* Most platforms treat the display irq block as an always-on
	 * power domain. vlv/chv can disable it at runtime and need
	 * special care to avoid writing any of the display block registers
	 * outside of the power domain. We defer setting up the display irqs
	 * in this case to the runtime pm.
	 */
	dev_priv->display_irqs_enabled = true;
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
		dev_priv->display_irqs_enabled = false;

L
Lyude 已提交
4295 4296
	dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;

4297 4298
	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4299

4300
	if (IS_CHERRYVIEW(dev_priv)) {
4301 4302 4303 4304
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
4305 4306
		dev->driver->enable_vblank = i965_enable_vblank;
		dev->driver->disable_vblank = i965_disable_vblank;
4307
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4308
	} else if (IS_VALLEYVIEW(dev_priv)) {
J
Jesse Barnes 已提交
4309 4310 4311 4312
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
4313 4314
		dev->driver->enable_vblank = i965_enable_vblank;
		dev->driver->disable_vblank = i965_disable_vblank;
4315
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4316
	} else if (INTEL_INFO(dev_priv)->gen >= 8) {
4317
		dev->driver->irq_handler = gen8_irq_handler;
4318
		dev->driver->irq_preinstall = gen8_irq_reset;
4319 4320 4321 4322
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
4323
		if (IS_GEN9_LP(dev_priv))
4324
			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4325
		else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv))
4326 4327
			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		else
4328
			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4329
	} else if (HAS_PCH_SPLIT(dev_priv)) {
4330
		dev->driver->irq_handler = ironlake_irq_handler;
4331
		dev->driver->irq_preinstall = ironlake_irq_reset;
4332 4333 4334 4335
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4336
		dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4337
	} else {
4338
		if (IS_GEN2(dev_priv)) {
C
Chris Wilson 已提交
4339 4340 4341 4342
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4343 4344
			dev->driver->enable_vblank = i8xx_enable_vblank;
			dev->driver->disable_vblank = i8xx_disable_vblank;
4345
		} else if (IS_GEN3(dev_priv)) {
4346 4347 4348 4349
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
4350 4351
			dev->driver->enable_vblank = i8xx_enable_vblank;
			dev->driver->disable_vblank = i8xx_disable_vblank;
C
Chris Wilson 已提交
4352
		} else {
4353 4354 4355 4356
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
4357 4358
			dev->driver->enable_vblank = i965_enable_vblank;
			dev->driver->disable_vblank = i965_disable_vblank;
C
Chris Wilson 已提交
4359
		}
4360 4361
		if (I915_HAS_HOTPLUG(dev_priv))
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4362 4363
	}
}
4364

4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375
/**
 * intel_irq_install - enables the hardware interrupt
 * @dev_priv: i915 device instance
 *
 * This function enables the hardware interrupt handling, but leaves the hotplug
 * handling still disabled. It is called after intel_irq_init().
 *
 * In the driver load and resume code we need working interrupts in a few places
 * but don't want to deal with the hassle of concurrent probe and hotplug
 * workers. Hence the split into this two-stage approach.
 */
4376 4377 4378 4379 4380 4381 4382 4383 4384
int intel_irq_install(struct drm_i915_private *dev_priv)
{
	/*
	 * We enable some interrupt sources in our postinstall hooks, so mark
	 * interrupts as enabled _before_ actually enabling them to avoid
	 * special cases in our ordering checks.
	 */
	dev_priv->pm.irqs_enabled = true;

4385
	return drm_irq_install(&dev_priv->drm, dev_priv->drm.pdev->irq);
4386 4387
}

4388 4389 4390 4391 4392 4393 4394
/**
 * intel_irq_uninstall - finilizes all irq handling
 * @dev_priv: i915 device instance
 *
 * This stops interrupt and hotplug handling and unregisters and frees all
 * resources acquired in the init functions.
 */
4395 4396
void intel_irq_uninstall(struct drm_i915_private *dev_priv)
{
4397
	drm_irq_uninstall(&dev_priv->drm);
4398 4399 4400 4401
	intel_hpd_cancel_work(dev_priv);
	dev_priv->pm.irqs_enabled = false;
}

4402 4403 4404 4405 4406 4407 4408
/**
 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
 * @dev_priv: i915 device instance
 *
 * This function is used to disable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4409
void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4410
{
4411
	dev_priv->drm.driver->irq_uninstall(&dev_priv->drm);
4412
	dev_priv->pm.irqs_enabled = false;
4413
	synchronize_irq(dev_priv->drm.irq);
4414 4415
}

4416 4417 4418 4419 4420 4421 4422
/**
 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
 * @dev_priv: i915 device instance
 *
 * This function is used to enable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4423
void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4424
{
4425
	dev_priv->pm.irqs_enabled = true;
4426 4427
	dev_priv->drm.driver->irq_preinstall(&dev_priv->drm);
	dev_priv->drm.driver->irq_postinstall(&dev_priv->drm);
4428
}