i915_irq.c 120.9 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47
/**
 * DOC: interrupt handling
 *
 * These functions provide the basic support for enabling and disabling the
 * interrupt handling support. There's a lot more functionality in i915_irq.c
 * and related files, but that will be described in separate chapters.
 */

48 49 50 51
static const u32 hpd_ilk[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG,
};

52 53 54 55
static const u32 hpd_ivb[HPD_NUM_PINS] = {
	[HPD_PORT_A] = DE_DP_A_HOTPLUG_IVB,
};

56 57 58 59
static const u32 hpd_bdw[HPD_NUM_PINS] = {
	[HPD_PORT_A] = GEN8_PORT_DP_A_HOTPLUG,
};

60
static const u32 hpd_ibx[HPD_NUM_PINS] = {
61 62 63 64 65 66 67
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

68
static const u32 hpd_cpt[HPD_NUM_PINS] = {
69
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
70
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
71 72 73 74 75
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

X
Xiong Zhang 已提交
76
static const u32 hpd_spt[HPD_NUM_PINS] = {
77
	[HPD_PORT_A] = SDE_PORTA_HOTPLUG_SPT,
X
Xiong Zhang 已提交
78 79 80 81 82 83
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT,
	[HPD_PORT_E] = SDE_PORTE_HOTPLUG_SPT
};

84
static const u32 hpd_mask_i915[HPD_NUM_PINS] = {
85 86 87 88 89 90 91 92
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

93
static const u32 hpd_status_g4x[HPD_NUM_PINS] = {
94 95 96 97 98 99 100 101
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

102
static const u32 hpd_status_i915[HPD_NUM_PINS] = {
103 104 105 106 107 108 109 110
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

111 112
/* BXT hpd list */
static const u32 hpd_bxt[HPD_NUM_PINS] = {
113
	[HPD_PORT_A] = BXT_DE_PORT_HP_DDIA,
114 115 116 117
	[HPD_PORT_B] = BXT_DE_PORT_HP_DDIB,
	[HPD_PORT_C] = BXT_DE_PORT_HP_DDIC
};

118
/* IIR can theoretically queue up two events. Be paranoid. */
119
#define GEN8_IRQ_RESET_NDX(type, which) do { \
120 121 122 123 124 125 126 127 128
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

129
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
130
	I915_WRITE(type##IMR, 0xffffffff); \
131
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
132
	I915_WRITE(type##IER, 0); \
133 134 135 136
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
137 138
} while (0)

139 140 141
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
142 143
static void gen5_assert_iir_is_zero(struct drm_i915_private *dev_priv,
				    i915_reg_t reg)
144 145 146 147 148 149 150
{
	u32 val = I915_READ(reg);

	if (val == 0)
		return;

	WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n",
151
	     i915_mmio_reg_offset(reg), val);
152 153 154 155 156
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
	I915_WRITE(reg, 0xffffffff);
	POSTING_READ(reg);
}
157

P
Paulo Zanoni 已提交
158
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
159
	gen5_assert_iir_is_zero(dev_priv, GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
160
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
161 162
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
P
Paulo Zanoni 已提交
163 164 165
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
166
	gen5_assert_iir_is_zero(dev_priv, type##IIR); \
P
Paulo Zanoni 已提交
167
	I915_WRITE(type##IER, (ier_val)); \
168 169
	I915_WRITE(type##IMR, (imr_val)); \
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
170 171
} while (0)

172
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
173
static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
174

175 176 177 178 179 180 181 182
/* For display hotplug interrupt */
static inline void
i915_hotplug_interrupt_update_locked(struct drm_i915_private *dev_priv,
				     uint32_t mask,
				     uint32_t bits)
{
	uint32_t val;

183
	lockdep_assert_held(&dev_priv->irq_lock);
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
	WARN_ON(bits & ~mask);

	val = I915_READ(PORT_HOTPLUG_EN);
	val &= ~mask;
	val |= bits;
	I915_WRITE(PORT_HOTPLUG_EN, val);
}

/**
 * i915_hotplug_interrupt_update - update hotplug interrupt enable
 * @dev_priv: driver private
 * @mask: bits to update
 * @bits: bits to enable
 * NOTE: the HPD enable bits are modified both inside and outside
 * of an interrupt context. To avoid that read-modify-write cycles
 * interfer, these bits are protected by a spinlock. Since this
 * function is usually not called from a context where the lock is
 * held already, this function acquires the lock itself. A non-locking
 * version is also available.
 */
void i915_hotplug_interrupt_update(struct drm_i915_private *dev_priv,
				   uint32_t mask,
				   uint32_t bits)
{
	spin_lock_irq(&dev_priv->irq_lock);
	i915_hotplug_interrupt_update_locked(dev_priv, mask, bits);
	spin_unlock_irq(&dev_priv->irq_lock);
}

213 214 215 216 217 218
/**
 * ilk_update_display_irq - update DEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
219 220 221
void ilk_update_display_irq(struct drm_i915_private *dev_priv,
			    uint32_t interrupt_mask,
			    uint32_t enabled_irq_mask)
222
{
223 224
	uint32_t new_val;

225
	lockdep_assert_held(&dev_priv->irq_lock);
226

227 228
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

229
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
230 231
		return;

232 233 234 235 236 237
	new_val = dev_priv->irq_mask;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->irq_mask) {
		dev_priv->irq_mask = new_val;
238
		I915_WRITE(DEIMR, dev_priv->irq_mask);
239
		POSTING_READ(DEIMR);
240 241 242
	}
}

P
Paulo Zanoni 已提交
243 244 245 246 247 248 249 250 251 252
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
253
	lockdep_assert_held(&dev_priv->irq_lock);
P
Paulo Zanoni 已提交
254

255 256
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

257
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
258 259
		return;

P
Paulo Zanoni 已提交
260 261 262 263 264
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
}

265
void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
266 267
{
	ilk_update_gt_irq(dev_priv, mask, mask);
268
	POSTING_READ_FW(GTIMR);
P
Paulo Zanoni 已提交
269 270
}

271
void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
P
Paulo Zanoni 已提交
272 273 274 275
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

276
static i915_reg_t gen6_pm_iir(struct drm_i915_private *dev_priv)
277
{
278
	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
279 280
}

281
static i915_reg_t gen6_pm_imr(struct drm_i915_private *dev_priv)
282
{
283
	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
284 285
}

286
static i915_reg_t gen6_pm_ier(struct drm_i915_private *dev_priv)
287
{
288
	return INTEL_GEN(dev_priv) >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
289 290
}

P
Paulo Zanoni 已提交
291
/**
292 293 294 295 296
 * snb_update_pm_irq - update GEN6_PMIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
P
Paulo Zanoni 已提交
297 298 299 300
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
301
	uint32_t new_val;
P
Paulo Zanoni 已提交
302

303 304
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

305
	lockdep_assert_held(&dev_priv->irq_lock);
P
Paulo Zanoni 已提交
306

307
	new_val = dev_priv->pm_imr;
308 309 310
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

311 312 313
	if (new_val != dev_priv->pm_imr) {
		dev_priv->pm_imr = new_val;
		I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_imr);
314
		POSTING_READ(gen6_pm_imr(dev_priv));
315
	}
P
Paulo Zanoni 已提交
316 317
}

318
void gen6_unmask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
P
Paulo Zanoni 已提交
319
{
320 321 322
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

P
Paulo Zanoni 已提交
323 324 325
	snb_update_pm_irq(dev_priv, mask, mask);
}

326
static void __gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
P
Paulo Zanoni 已提交
327 328 329 330
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

331
void gen6_mask_pm_irq(struct drm_i915_private *dev_priv, u32 mask)
332 333 334 335
{
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

336
	__gen6_mask_pm_irq(dev_priv, mask);
337 338
}

339
void gen6_reset_pm_iir(struct drm_i915_private *dev_priv, u32 reset_mask)
I
Imre Deak 已提交
340
{
341
	i915_reg_t reg = gen6_pm_iir(dev_priv);
I
Imre Deak 已提交
342

343
	lockdep_assert_held(&dev_priv->irq_lock);
344 345 346

	I915_WRITE(reg, reset_mask);
	I915_WRITE(reg, reset_mask);
I
Imre Deak 已提交
347
	POSTING_READ(reg);
348 349 350 351
}

void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, u32 enable_mask)
{
352
	lockdep_assert_held(&dev_priv->irq_lock);
353 354 355 356 357 358 359 360 361

	dev_priv->pm_ier |= enable_mask;
	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
	gen6_unmask_pm_irq(dev_priv, enable_mask);
	/* unmask_pm_irq provides an implicit barrier (POSTING_READ) */
}

void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, u32 disable_mask)
{
362
	lockdep_assert_held(&dev_priv->irq_lock);
363 364 365 366 367 368 369 370 371 372 373

	dev_priv->pm_ier &= ~disable_mask;
	__gen6_mask_pm_irq(dev_priv, disable_mask);
	I915_WRITE(gen6_pm_ier(dev_priv), dev_priv->pm_ier);
	/* though a barrier is missing here, but don't really need a one */
}

void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	gen6_reset_pm_iir(dev_priv, dev_priv->pm_rps_events);
374
	dev_priv->rps.pm_iir = 0;
I
Imre Deak 已提交
375 376 377
	spin_unlock_irq(&dev_priv->irq_lock);
}

378
void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
379
{
380 381 382
	if (READ_ONCE(dev_priv->rps.interrupts_enabled))
		return;

383
	spin_lock_irq(&dev_priv->irq_lock);
384 385
	WARN_ON_ONCE(dev_priv->rps.pm_iir);
	WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
I
Imre Deak 已提交
386
	dev_priv->rps.interrupts_enabled = true;
387
	gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
388

389 390 391
	spin_unlock_irq(&dev_priv->irq_lock);
}

392
void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
393
{
394 395 396
	if (!READ_ONCE(dev_priv->rps.interrupts_enabled))
		return;

I
Imre Deak 已提交
397 398
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->rps.interrupts_enabled = false;
399

400
	I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0u));
401

402
	gen6_disable_pm_irq(dev_priv, dev_priv->pm_rps_events);
403 404

	spin_unlock_irq(&dev_priv->irq_lock);
405
	synchronize_irq(dev_priv->drm.irq);
406 407 408 409 410 411 412 413

	/* Now that we will not be generating any more work, flush any
	 * outsanding tasks. As we are called on the RPS idle path,
	 * we will reset the GPU to minimum frequencies, so the current
	 * state of the worker can be discarded.
	 */
	cancel_work_sync(&dev_priv->rps.work);
	gen6_reset_rps_interrupts(dev_priv);
414 415
}

416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
void gen9_reset_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	gen6_reset_pm_iir(dev_priv, dev_priv->pm_guc_events);
	spin_unlock_irq(&dev_priv->irq_lock);
}

void gen9_enable_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	if (!dev_priv->guc.interrupts_enabled) {
		WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) &
				       dev_priv->pm_guc_events);
		dev_priv->guc.interrupts_enabled = true;
		gen6_enable_pm_irq(dev_priv, dev_priv->pm_guc_events);
	}
	spin_unlock_irq(&dev_priv->irq_lock);
}

void gen9_disable_guc_interrupts(struct drm_i915_private *dev_priv)
{
	spin_lock_irq(&dev_priv->irq_lock);
	dev_priv->guc.interrupts_enabled = false;

	gen6_disable_pm_irq(dev_priv, dev_priv->pm_guc_events);

	spin_unlock_irq(&dev_priv->irq_lock);
	synchronize_irq(dev_priv->drm.irq);

	gen9_reset_guc_interrupts(dev_priv);
}

448
/**
449 450 451 452 453
 * bdw_update_port_irq - update DE port interrupt
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
454 455 456 457 458 459 460
static void bdw_update_port_irq(struct drm_i915_private *dev_priv,
				uint32_t interrupt_mask,
				uint32_t enabled_irq_mask)
{
	uint32_t new_val;
	uint32_t old_val;

461
	lockdep_assert_held(&dev_priv->irq_lock);
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	old_val = I915_READ(GEN8_DE_PORT_IMR);

	new_val = old_val;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != old_val) {
		I915_WRITE(GEN8_DE_PORT_IMR, new_val);
		POSTING_READ(GEN8_DE_PORT_IMR);
	}
}

480 481 482 483 484 485 486 487 488 489 490 491 492 493
/**
 * bdw_update_pipe_irq - update DE pipe interrupt
 * @dev_priv: driver private
 * @pipe: pipe whose interrupt to update
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
void bdw_update_pipe_irq(struct drm_i915_private *dev_priv,
			 enum pipe pipe,
			 uint32_t interrupt_mask,
			 uint32_t enabled_irq_mask)
{
	uint32_t new_val;

494
	lockdep_assert_held(&dev_priv->irq_lock);
495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511

	WARN_ON(enabled_irq_mask & ~interrupt_mask);

	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
		return;

	new_val = dev_priv->de_irq_mask[pipe];
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->de_irq_mask[pipe]) {
		dev_priv->de_irq_mask[pipe] = new_val;
		I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
		POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
	}
}

512 513 514 515 516 517
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
518 519 520
void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
				  uint32_t interrupt_mask,
				  uint32_t enabled_irq_mask)
521 522 523 524 525
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

526 527
	WARN_ON(enabled_irq_mask & ~interrupt_mask);

528
	lockdep_assert_held(&dev_priv->irq_lock);
529

530
	if (WARN_ON(!intel_irqs_enabled(dev_priv)))
531 532
		return;

533 534 535
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
536

D
Daniel Vetter 已提交
537
static void
538 539
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
540
{
541
	i915_reg_t reg = PIPESTAT(pipe);
542
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
543

544
	lockdep_assert_held(&dev_priv->irq_lock);
545
	WARN_ON(!intel_irqs_enabled(dev_priv));
546

547 548 549 550
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
551 552 553
		return;

	if ((pipestat & enable_mask) == enable_mask)
554 555
		return;

556 557
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

558
	/* Enable the interrupt, clear any pending status */
559
	pipestat |= enable_mask | status_mask;
560 561
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
562 563
}

D
Daniel Vetter 已提交
564
static void
565 566
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
567
{
568
	i915_reg_t reg = PIPESTAT(pipe);
569
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
570

571
	lockdep_assert_held(&dev_priv->irq_lock);
572
	WARN_ON(!intel_irqs_enabled(dev_priv));
573

574 575 576 577
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
578 579
		return;

580 581 582
	if ((pipestat & enable_mask) == 0)
		return;

583 584
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

585
	pipestat &= ~enable_mask;
586 587
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
588 589
}

590 591 592 593 594
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
595 596
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
597 598 599
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
600 601 602 603 604 605
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
606 607 608 609 610 611 612 613 614 615 616 617

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

618 619 620 621 622 623
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

624
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
625
		enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
626 627 628
							   status_mask);
	else
		enable_mask = status_mask << 16;
629 630 631 632 633 634 635 636 637
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

638
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
639
		enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
640 641 642
							   status_mask);
	else
		enable_mask = status_mask << 16;
643 644 645
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

646
/**
647
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
648
 * @dev_priv: i915 device private
649
 */
650
static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
651
{
652
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev_priv))
653 654
		return;

655
	spin_lock_irq(&dev_priv->irq_lock);
656

657
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
658
	if (INTEL_GEN(dev_priv) >= 4)
659
		i915_enable_pipestat(dev_priv, PIPE_A,
660
				     PIPE_LEGACY_BLC_EVENT_STATUS);
661

662
	spin_unlock_irq(&dev_priv->irq_lock);
663 664
}

665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714
/*
 * This timing diagram depicts the video signal in and
 * around the vertical blanking period.
 *
 * Assumptions about the fictitious mode used in this example:
 *  vblank_start >= 3
 *  vsync_start = vblank_start + 1
 *  vsync_end = vblank_start + 2
 *  vtotal = vblank_start + 3
 *
 *           start of vblank:
 *           latch double buffered registers
 *           increment frame counter (ctg+)
 *           generate start of vblank interrupt (gen4+)
 *           |
 *           |          frame start:
 *           |          generate frame start interrupt (aka. vblank interrupt) (gmch)
 *           |          may be shifted forward 1-3 extra lines via PIPECONF
 *           |          |
 *           |          |  start of vsync:
 *           |          |  generate vsync interrupt
 *           |          |  |
 * ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx___    ___xxxx
 *       .   \hs/   .      \hs/          \hs/          \hs/   .      \hs/
 * ----va---> <-----------------vb--------------------> <--------va-------------
 *       |          |       <----vs----->                     |
 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
 *       |          |                                         |
 *       last visible pixel                                   first visible pixel
 *                  |                                         increment frame counter (gen3/4)
 *                  pixel counter = vblank_start * htotal     pixel counter = 0 (gen3/4)
 *
 * x  = horizontal active
 * _  = horizontal blanking
 * hs = horizontal sync
 * va = vertical active
 * vb = vertical blanking
 * vs = vertical sync
 * vbs = vblank_start (number)
 *
 * Summary:
 * - most events happen at the start of horizontal sync
 * - frame start happens at the start of horizontal blank, 1-4 lines
 *   (depending on PIPECONF settings) after the start of vblank
 * - gen3/4 pixel and frame counter are synchronized with the start
 *   of horizontal active on the first line of vertical active
 */

715 716 717
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
718
static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
719
{
720
	struct drm_i915_private *dev_priv = to_i915(dev);
721
	i915_reg_t high_frame, low_frame;
722
	u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
723
	const struct drm_display_mode *mode = &dev->vblank[pipe].hwmode;
724
	unsigned long irqflags;
725

726 727 728 729 730
	htotal = mode->crtc_htotal;
	hsync_start = mode->crtc_hsync_start;
	vbl_start = mode->crtc_vblank_start;
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
731

732 733 734 735 736 737
	/* Convert to pixel count */
	vbl_start *= htotal;

	/* Start of vblank event occurs at start of hsync */
	vbl_start -= htotal - hsync_start;

738 739
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
740

741 742
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);

743 744 745 746 747 748
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
749 750 751
		high1 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK;
		low   = I915_READ_FW(low_frame);
		high2 = I915_READ_FW(high_frame) & PIPE_FRAME_HIGH_MASK;
752 753
	} while (high1 != high2);

754 755
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

756
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
757
	pixel = low & PIPE_PIXEL_MASK;
758
	low >>= PIPE_FRAME_LOW_SHIFT;
759 760 761 762 763 764

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
765
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
766 767
}

768
static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
769
{
770
	struct drm_i915_private *dev_priv = to_i915(dev);
771

772
	return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
773 774
}

775
/* I915_READ_FW, only for fast reads of display block, no need for forcewake etc. */
776 777 778
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
779
	struct drm_i915_private *dev_priv = to_i915(dev);
780 781
	const struct drm_display_mode *mode;
	struct drm_vblank_crtc *vblank;
782
	enum pipe pipe = crtc->pipe;
783
	int position, vtotal;
784

785 786 787
	if (!crtc->active)
		return -1;

788 789 790
	vblank = &crtc->base.dev->vblank[drm_crtc_index(&crtc->base)];
	mode = &vblank->hwmode;

791
	vtotal = mode->crtc_vtotal;
792 793 794
	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

795
	if (IS_GEN2(dev_priv))
796
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
797
	else
798
		position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
799

800 801 802 803 804 805 806 807 808 809 810 811
	/*
	 * On HSW, the DSL reg (0x70000) appears to return 0 if we
	 * read it just before the start of vblank.  So try it again
	 * so we don't accidentally end up spanning a vblank frame
	 * increment, causing the pipe_update_end() code to squak at us.
	 *
	 * The nature of this problem means we can't simply check the ISR
	 * bit and return the vblank start value; nor can we use the scanline
	 * debug register in the transcoder as it appears to have the same
	 * problem.  We may need to extend this to include other platforms,
	 * but so far testing only shows the problem on HSW.
	 */
812
	if (HAS_DDI(dev_priv) && !position) {
813 814 815 816
		int i, temp;

		for (i = 0; i < 100; i++) {
			udelay(1);
817
			temp = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
818 819 820 821 822 823 824
			if (temp != position) {
				position = temp;
				break;
			}
		}
	}

825
	/*
826 827
	 * See update_scanline_offset() for the details on the
	 * scanline_offset adjustment.
828
	 */
829
	return (position + crtc->scanline_offset) % vtotal;
830 831
}

832 833 834 835
static bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
				     bool in_vblank_irq, int *vpos, int *hpos,
				     ktime_t *stime, ktime_t *etime,
				     const struct drm_display_mode *mode)
836
{
837
	struct drm_i915_private *dev_priv = to_i915(dev);
838 839
	struct intel_crtc *intel_crtc = intel_get_crtc_for_pipe(dev_priv,
								pipe);
840
	int position;
841
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
842
	bool in_vbl = true;
843
	unsigned long irqflags;
844

845
	if (WARN_ON(!mode->crtc_clock)) {
846
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
847
				 "pipe %c\n", pipe_name(pipe));
848
		return false;
849 850
	}

851
	htotal = mode->crtc_htotal;
852
	hsync_start = mode->crtc_hsync_start;
853 854 855
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
856

857 858 859 860 861 862
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

863 864 865 866 867 868
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
869

870 871 872 873 874 875
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

876
	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
877 878 879
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
880
		position = __intel_get_crtc_scanline(intel_crtc);
881 882 883 884 885
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
886
		position = (I915_READ_FW(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
887

888 889 890 891
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
892

893 894 895 896 897 898 899 900 901 902 903 904
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

905 906 907 908 909 910 911 912 913 914
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
915 916
	}

917 918 919 920 921 922 923 924
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

925 926 927 928 929 930 931 932 933 934 935 936
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
937

938
	if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
939 940 941 942 943 944
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
945

946
	return true;
947 948
}

949 950
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
951
	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
952 953 954 955 956 957 958 959 960 961
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

962
static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
963
{
964
	u32 busy_up, busy_down, max_avg, min_avg;
965 966
	u8 new_delay;

967
	spin_lock(&mchdev_lock);
968

969 970
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

971
	new_delay = dev_priv->ips.cur_delay;
972

973
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
974 975
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
976 977 978 979
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
980
	if (busy_up > max_avg) {
981 982 983 984
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
985
	} else if (busy_down < min_avg) {
986 987 988 989
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
990 991
	}

992
	if (ironlake_set_drps(dev_priv, new_delay))
993
		dev_priv->ips.cur_delay = new_delay;
994

995
	spin_unlock(&mchdev_lock);
996

997 998 999
	return;
}

1000
static void notify_ring(struct intel_engine_cs *engine)
1001
{
1002 1003
	struct drm_i915_gem_request *rq = NULL;
	struct intel_wait *wait;
1004

1005
	atomic_inc(&engine->irq_count);
1006
	set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
1007

1008 1009
	spin_lock(&engine->breadcrumbs.irq_lock);
	wait = engine->breadcrumbs.irq_wait;
1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022
	if (wait) {
		/* We use a callback from the dma-fence to submit
		 * requests after waiting on our own requests. To
		 * ensure minimum delay in queuing the next request to
		 * hardware, signal the fence now rather than wait for
		 * the signaler to be woken up. We still wake up the
		 * waiter in order to handle the irq-seqno coherency
		 * issues (we may receive the interrupt before the
		 * seqno is written, see __i915_request_irq_complete())
		 * and to handle coalescing of multiple seqno updates
		 * and many waiters.
		 */
		if (i915_seqno_passed(intel_engine_get_seqno(engine),
1023 1024 1025
				      wait->seqno) &&
		    !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
			      &wait->request->fence.flags))
1026
			rq = i915_gem_request_get(wait->request);
1027 1028

		wake_up_process(wait->tsk);
1029 1030
	} else {
		__intel_engine_disarm_breadcrumbs(engine);
1031
	}
1032
	spin_unlock(&engine->breadcrumbs.irq_lock);
1033

1034
	if (rq) {
1035
		dma_fence_signal(&rq->fence);
1036 1037
		i915_gem_request_put(rq);
	}
1038 1039

	trace_intel_engine_notify(engine, wait);
1040 1041
}

1042 1043
static void vlv_c0_read(struct drm_i915_private *dev_priv,
			struct intel_rps_ei *ei)
1044
{
1045
	ei->ktime = ktime_get_raw();
1046 1047 1048
	ei->render_c0 = I915_READ(VLV_RENDER_C0_COUNT);
	ei->media_c0 = I915_READ(VLV_MEDIA_C0_COUNT);
}
1049

1050
void gen6_rps_reset_ei(struct drm_i915_private *dev_priv)
1051
{
1052
	memset(&dev_priv->rps.ei, 0, sizeof(dev_priv->rps.ei));
1053
}
1054

1055 1056
static u32 vlv_wa_c0_ei(struct drm_i915_private *dev_priv, u32 pm_iir)
{
1057
	const struct intel_rps_ei *prev = &dev_priv->rps.ei;
1058 1059
	struct intel_rps_ei now;
	u32 events = 0;
1060

1061
	if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1062
		return 0;
1063

1064
	vlv_c0_read(dev_priv, &now);
1065

1066
	if (prev->ktime) {
1067
		u64 time, c0;
1068
		u32 render, media;
1069

1070
		time = ktime_us_delta(now.ktime, prev->ktime);
1071

1072 1073 1074 1075 1076 1077 1078
		time *= dev_priv->czclk_freq;

		/* Workload can be split between render + media,
		 * e.g. SwapBuffers being blitted in X after being rendered in
		 * mesa. To account for this we need to combine both engines
		 * into our activity counter.
		 */
1079 1080 1081
		render = now.render_c0 - prev->render_c0;
		media = now.media_c0 - prev->media_c0;
		c0 = max(render, media);
1082
		c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1083 1084 1085 1086 1087

		if (c0 > time * dev_priv->rps.up_threshold)
			events = GEN6_PM_RP_UP_THRESHOLD;
		else if (c0 < time * dev_priv->rps.down_threshold)
			events = GEN6_PM_RP_DOWN_THRESHOLD;
1088 1089
	}

1090
	dev_priv->rps.ei = now;
1091
	return events;
1092 1093
}

1094
static void gen6_pm_rps_work(struct work_struct *work)
1095
{
1096 1097
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
1098
	bool client_boost = false;
1099
	int new_delay, adj, min, max;
1100
	u32 pm_iir = 0;
1101

1102
	spin_lock_irq(&dev_priv->irq_lock);
1103 1104
	if (dev_priv->rps.interrupts_enabled) {
		pm_iir = fetch_and_zero(&dev_priv->rps.pm_iir);
1105
		client_boost = atomic_read(&dev_priv->rps.num_waiters);
I
Imre Deak 已提交
1106
	}
1107
	spin_unlock_irq(&dev_priv->irq_lock);
1108

1109
	/* Make sure we didn't queue anything we're not going to process. */
1110
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1111
	if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
1112
		goto out;
1113

1114
	mutex_lock(&dev_priv->rps.hw_lock);
1115

1116 1117
	pm_iir |= vlv_wa_c0_ei(dev_priv, pm_iir);

1118
	adj = dev_priv->rps.last_adj;
1119
	new_delay = dev_priv->rps.cur_freq;
1120 1121
	min = dev_priv->rps.min_freq_softlimit;
	max = dev_priv->rps.max_freq_softlimit;
1122
	if (client_boost)
1123 1124 1125
		max = dev_priv->rps.max_freq;
	if (client_boost && new_delay < dev_priv->rps.boost_freq) {
		new_delay = dev_priv->rps.boost_freq;
1126 1127
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1128 1129
		if (adj > 0)
			adj *= 2;
1130 1131
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? 2 : 1;
1132 1133 1134

		if (new_delay >= dev_priv->rps.max_freq_softlimit)
			adj = 0;
1135
	} else if (client_boost) {
1136
		adj = 0;
1137
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1138 1139
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1140
		else if (dev_priv->rps.cur_freq > dev_priv->rps.min_freq_softlimit)
1141
			new_delay = dev_priv->rps.min_freq_softlimit;
1142 1143 1144 1145
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
1146 1147
		else /* CHV needs even encode values */
			adj = IS_CHERRYVIEW(dev_priv) ? -2 : -1;
1148 1149 1150

		if (new_delay <= dev_priv->rps.min_freq_softlimit)
			adj = 0;
1151
	} else { /* unknown event */
1152
		adj = 0;
1153
	}
1154

1155 1156
	dev_priv->rps.last_adj = adj;

1157 1158 1159
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1160
	new_delay += adj;
1161
	new_delay = clamp_t(int, new_delay, min, max);
1162

1163 1164 1165 1166
	if (intel_set_rps(dev_priv, new_delay)) {
		DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n");
		dev_priv->rps.last_adj = 0;
	}
1167

1168
	mutex_unlock(&dev_priv->rps.hw_lock);
1169 1170 1171 1172 1173 1174 1175

out:
	/* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
	spin_lock_irq(&dev_priv->irq_lock);
	if (dev_priv->rps.interrupts_enabled)
		gen6_unmask_pm_irq(dev_priv, dev_priv->pm_rps_events);
	spin_unlock_irq(&dev_priv->irq_lock);
1176 1177
}

1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1190
	struct drm_i915_private *dev_priv =
1191
		container_of(work, typeof(*dev_priv), l3_parity.error_work);
1192
	u32 error_status, row, bank, subbank;
1193
	char *parity_event[6];
1194
	uint32_t misccpctl;
1195
	uint8_t slice = 0;
1196 1197 1198 1199 1200

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
1201
	mutex_lock(&dev_priv->drm.struct_mutex);
1202

1203 1204 1205 1206
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1207 1208 1209 1210
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1211
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1212
		i915_reg_t reg;
1213

1214
		slice--;
1215
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv)))
1216
			break;
1217

1218
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1219

1220
		reg = GEN7_L3CDERRST1(slice);
1221

1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1237
		kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
1238
				   KOBJ_CHANGE, parity_event);
1239

1240 1241
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1242

1243 1244 1245 1246 1247
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1248

1249
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1250

1251 1252
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
1253
	spin_lock_irq(&dev_priv->irq_lock);
1254
	gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1255
	spin_unlock_irq(&dev_priv->irq_lock);
1256

1257
	mutex_unlock(&dev_priv->drm.struct_mutex);
1258 1259
}

1260 1261
static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv,
					       u32 iir)
1262
{
1263
	if (!HAS_L3_DPF(dev_priv))
1264 1265
		return;

1266
	spin_lock(&dev_priv->irq_lock);
1267
	gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
1268
	spin_unlock(&dev_priv->irq_lock);
1269

1270
	iir &= GT_PARITY_ERROR(dev_priv);
1271 1272 1273 1274 1275 1276
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1277
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1278 1279
}

1280
static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
1281 1282
			       u32 gt_iir)
{
1283
	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1284
		notify_ring(dev_priv->engine[RCS]);
1285
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
1286
		notify_ring(dev_priv->engine[VCS]);
1287 1288
}

1289
static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
1290 1291
			       u32 gt_iir)
{
1292
	if (gt_iir & GT_RENDER_USER_INTERRUPT)
1293
		notify_ring(dev_priv->engine[RCS]);
1294
	if (gt_iir & GT_BSD_USER_INTERRUPT)
1295
		notify_ring(dev_priv->engine[VCS]);
1296
	if (gt_iir & GT_BLT_USER_INTERRUPT)
1297
		notify_ring(dev_priv->engine[BCS]);
1298

1299 1300
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
1301 1302
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT))
		DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir);
1303

1304 1305
	if (gt_iir & GT_PARITY_ERROR(dev_priv))
		ivybridge_parity_error_irq_handler(dev_priv, gt_iir);
1306 1307
}

1308
static void
1309
gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
1310
{
1311
	bool tasklet = false;
1312 1313

	if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) {
1314
		if (port_count(&engine->execlist_port[0])) {
1315
			__set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
1316 1317
			tasklet = true;
		}
1318
	}
1319 1320 1321 1322 1323 1324 1325 1326

	if (iir & (GT_RENDER_USER_INTERRUPT << test_shift)) {
		notify_ring(engine);
		tasklet |= i915.enable_guc_submission;
	}

	if (tasklet)
		tasklet_hi_schedule(&engine->irq_tasklet);
1327 1328
}

1329 1330 1331
static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv,
				   u32 master_ctl,
				   u32 gt_iir[4])
1332 1333 1334 1335
{
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1336 1337 1338
		gt_iir[0] = I915_READ_FW(GEN8_GT_IIR(0));
		if (gt_iir[0]) {
			I915_WRITE_FW(GEN8_GT_IIR(0), gt_iir[0]);
1339 1340 1341 1342 1343
			ret = IRQ_HANDLED;
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1344
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1345 1346 1347
		gt_iir[1] = I915_READ_FW(GEN8_GT_IIR(1));
		if (gt_iir[1]) {
			I915_WRITE_FW(GEN8_GT_IIR(1), gt_iir[1]);
1348
			ret = IRQ_HANDLED;
1349
		} else
1350
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
1351 1352
	}

1353
	if (master_ctl & GEN8_GT_VECS_IRQ) {
1354 1355 1356
		gt_iir[3] = I915_READ_FW(GEN8_GT_IIR(3));
		if (gt_iir[3]) {
			I915_WRITE_FW(GEN8_GT_IIR(3), gt_iir[3]);
1357 1358 1359 1360 1361
			ret = IRQ_HANDLED;
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

1362
	if (master_ctl & (GEN8_GT_PM_IRQ | GEN8_GT_GUC_IRQ)) {
1363
		gt_iir[2] = I915_READ_FW(GEN8_GT_IIR(2));
1364 1365
		if (gt_iir[2] & (dev_priv->pm_rps_events |
				 dev_priv->pm_guc_events)) {
1366
			I915_WRITE_FW(GEN8_GT_IIR(2),
1367 1368
				      gt_iir[2] & (dev_priv->pm_rps_events |
						   dev_priv->pm_guc_events));
1369
			ret = IRQ_HANDLED;
1370 1371 1372 1373
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1374 1375 1376
	return ret;
}

1377 1378 1379 1380
static void gen8_gt_irq_handler(struct drm_i915_private *dev_priv,
				u32 gt_iir[4])
{
	if (gt_iir[0]) {
1381
		gen8_cs_irq_handler(dev_priv->engine[RCS],
1382
				    gt_iir[0], GEN8_RCS_IRQ_SHIFT);
1383
		gen8_cs_irq_handler(dev_priv->engine[BCS],
1384 1385 1386 1387
				    gt_iir[0], GEN8_BCS_IRQ_SHIFT);
	}

	if (gt_iir[1]) {
1388
		gen8_cs_irq_handler(dev_priv->engine[VCS],
1389
				    gt_iir[1], GEN8_VCS1_IRQ_SHIFT);
1390
		gen8_cs_irq_handler(dev_priv->engine[VCS2],
1391 1392 1393 1394
				    gt_iir[1], GEN8_VCS2_IRQ_SHIFT);
	}

	if (gt_iir[3])
1395
		gen8_cs_irq_handler(dev_priv->engine[VECS],
1396 1397 1398 1399
				    gt_iir[3], GEN8_VECS_IRQ_SHIFT);

	if (gt_iir[2] & dev_priv->pm_rps_events)
		gen6_rps_irq_handler(dev_priv, gt_iir[2]);
1400 1401 1402

	if (gt_iir[2] & dev_priv->pm_guc_events)
		gen9_guc_irq_handler(dev_priv, gt_iir[2]);
1403 1404
}

1405 1406 1407 1408
static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
1409
		return val & PORTA_HOTPLUG_LONG_DETECT;
1410 1411 1412 1413 1414 1415 1416 1417 1418
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1419 1420 1421 1422 1423 1424 1425 1426 1427 1428
static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_E:
		return val & PORTE_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444
static bool spt_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & PORTA_HOTPLUG_LONG_DETECT;
	case PORT_B:
		return val & PORTB_HOTPLUG_LONG_DETECT;
	case PORT_C:
		return val & PORTC_HOTPLUG_LONG_DETECT;
	case PORT_D:
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
static bool ilk_port_hotplug_long_detect(enum port port, u32 val)
{
	switch (port) {
	case PORT_A:
		return val & DIGITAL_PORTA_HOTPLUG_LONG_DETECT;
	default:
		return false;
	}
}

1455
static bool pch_port_hotplug_long_detect(enum port port, u32 val)
1456 1457 1458
{
	switch (port) {
	case PORT_B:
1459
		return val & PORTB_HOTPLUG_LONG_DETECT;
1460
	case PORT_C:
1461
		return val & PORTC_HOTPLUG_LONG_DETECT;
1462
	case PORT_D:
1463 1464 1465
		return val & PORTD_HOTPLUG_LONG_DETECT;
	default:
		return false;
1466 1467 1468
	}
}

1469
static bool i9xx_port_hotplug_long_detect(enum port port, u32 val)
1470 1471 1472
{
	switch (port) {
	case PORT_B:
1473
		return val & PORTB_HOTPLUG_INT_LONG_PULSE;
1474
	case PORT_C:
1475
		return val & PORTC_HOTPLUG_INT_LONG_PULSE;
1476
	case PORT_D:
1477 1478 1479
		return val & PORTD_HOTPLUG_INT_LONG_PULSE;
	default:
		return false;
1480 1481 1482
	}
}

1483 1484 1485 1486 1487 1488 1489
/*
 * Get a bit mask of pins that have triggered, and which ones may be long.
 * This can be called multiple times with the same masks to accumulate
 * hotplug detection results from several registers.
 *
 * Note that the caller is expected to zero out the masks initially.
 */
1490
static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
1491
			     u32 hotplug_trigger, u32 dig_hotplug_reg,
1492 1493
			     const u32 hpd[HPD_NUM_PINS],
			     bool long_pulse_detect(enum port port, u32 val))
1494
{
1495
	enum port port;
1496 1497 1498
	int i;

	for_each_hpd_pin(i) {
1499 1500
		if ((hpd[i] & hotplug_trigger) == 0)
			continue;
1501

1502 1503
		*pin_mask |= BIT(i);

1504 1505 1506
		if (!intel_hpd_pin_to_port(i, &port))
			continue;

1507
		if (long_pulse_detect(port, dig_hotplug_reg))
1508
			*long_mask |= BIT(i);
1509 1510 1511 1512 1513 1514 1515
	}

	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x, pins 0x%08x\n",
			 hotplug_trigger, dig_hotplug_reg, *pin_mask);

}

1516
static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
1517
{
1518
	wake_up_all(&dev_priv->gmbus_wait_queue);
1519 1520
}

1521
static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
1522
{
1523
	wake_up_all(&dev_priv->gmbus_wait_queue);
1524 1525
}

1526
#if defined(CONFIG_DEBUG_FS)
1527 1528
static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
					 enum pipe pipe,
1529 1530 1531
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1532 1533 1534
{
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
T
Tomeu Vizoso 已提交
1535 1536 1537
	struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
	struct drm_driver *driver = dev_priv->drm.driver;
	uint32_t crcs[5];
1538
	int head, tail;
1539

1540
	spin_lock(&pipe_crc->lock);
T
Tomeu Vizoso 已提交
1541 1542 1543 1544 1545 1546
	if (pipe_crc->source) {
		if (!pipe_crc->entries) {
			spin_unlock(&pipe_crc->lock);
			DRM_DEBUG_KMS("spurious interrupt\n");
			return;
		}
1547

T
Tomeu Vizoso 已提交
1548 1549
		head = pipe_crc->head;
		tail = pipe_crc->tail;
1550

T
Tomeu Vizoso 已提交
1551 1552 1553 1554 1555
		if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
			spin_unlock(&pipe_crc->lock);
			DRM_ERROR("CRC buffer overflowing\n");
			return;
		}
1556

T
Tomeu Vizoso 已提交
1557
		entry = &pipe_crc->entries[head];
1558

T
Tomeu Vizoso 已提交
1559 1560 1561 1562 1563 1564
		entry->frame = driver->get_vblank_counter(&dev_priv->drm, pipe);
		entry->crc[0] = crc0;
		entry->crc[1] = crc1;
		entry->crc[2] = crc2;
		entry->crc[3] = crc3;
		entry->crc[4] = crc4;
1565

T
Tomeu Vizoso 已提交
1566 1567
		head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
		pipe_crc->head = head;
1568

T
Tomeu Vizoso 已提交
1569
		spin_unlock(&pipe_crc->lock);
1570

T
Tomeu Vizoso 已提交
1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592
		wake_up_interruptible(&pipe_crc->wq);
	} else {
		/*
		 * For some not yet identified reason, the first CRC is
		 * bonkers. So let's just wait for the next vblank and read
		 * out the buggy result.
		 *
		 * On CHV sometimes the second CRC is bonkers as well, so
		 * don't trust that one either.
		 */
		if (pipe_crc->skipped == 0 ||
		    (IS_CHERRYVIEW(dev_priv) && pipe_crc->skipped == 1)) {
			pipe_crc->skipped++;
			spin_unlock(&pipe_crc->lock);
			return;
		}
		spin_unlock(&pipe_crc->lock);
		crcs[0] = crc0;
		crcs[1] = crc1;
		crcs[2] = crc2;
		crcs[3] = crc3;
		crcs[4] = crc4;
1593
		drm_crtc_add_crc_entry(&crtc->base, true,
1594
				       drm_crtc_accurate_vblank_count(&crtc->base),
1595
				       crcs);
T
Tomeu Vizoso 已提交
1596
	}
1597
}
1598 1599
#else
static inline void
1600 1601
display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
			     enum pipe pipe,
1602 1603 1604 1605 1606
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1607

1608 1609
static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				     enum pipe pipe)
D
Daniel Vetter 已提交
1610
{
1611
	display_pipe_crc_irq_handler(dev_priv, pipe,
1612 1613
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1614 1615
}

1616 1617
static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				     enum pipe pipe)
1618
{
1619
	display_pipe_crc_irq_handler(dev_priv, pipe,
1620 1621 1622 1623 1624
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1625
}
1626

1627 1628
static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
				      enum pipe pipe)
1629
{
1630 1631
	uint32_t res1, res2;

1632
	if (INTEL_GEN(dev_priv) >= 3)
1633 1634 1635 1636
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

1637
	if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
1638 1639 1640
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1641

1642
	display_pipe_crc_irq_handler(dev_priv, pipe,
1643 1644 1645 1646
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1647
}
1648

1649 1650 1651 1652
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1653
{
1654
	if (pm_iir & dev_priv->pm_rps_events) {
1655
		spin_lock(&dev_priv->irq_lock);
1656
		gen6_mask_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
I
Imre Deak 已提交
1657 1658
		if (dev_priv->rps.interrupts_enabled) {
			dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
1659
			schedule_work(&dev_priv->rps.work);
I
Imre Deak 已提交
1660
		}
1661
		spin_unlock(&dev_priv->irq_lock);
1662 1663
	}

1664
	if (INTEL_GEN(dev_priv) >= 8)
1665 1666
		return;

1667
	if (HAS_VEBOX(dev_priv)) {
1668
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1669
			notify_ring(dev_priv->engine[VECS]);
B
Ben Widawsky 已提交
1670

1671 1672
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
			DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir);
B
Ben Widawsky 已提交
1673
	}
1674 1675
}

1676 1677 1678
static void gen9_guc_irq_handler(struct drm_i915_private *dev_priv, u32 gt_iir)
{
	if (gt_iir & GEN9_GUC_TO_HOST_INT_EVENT) {
1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691
		/* Sample the log buffer flush related bits & clear them out now
		 * itself from the message identity register to minimize the
		 * probability of losing a flush interrupt, when there are back
		 * to back flush interrupts.
		 * There can be a new flush interrupt, for different log buffer
		 * type (like for ISR), whilst Host is handling one (for DPC).
		 * Since same bit is used in message register for ISR & DPC, it
		 * could happen that GuC sets the bit for 2nd interrupt but Host
		 * clears out the bit on handling the 1st interrupt.
		 */
		u32 msg, flush;

		msg = I915_READ(SOFT_SCRATCH(15));
1692 1693
		flush = msg & (INTEL_GUC_RECV_MSG_CRASH_DUMP_POSTED |
			       INTEL_GUC_RECV_MSG_FLUSH_LOG_BUFFER);
1694 1695 1696 1697 1698
		if (flush) {
			/* Clear the message bits that are handled */
			I915_WRITE(SOFT_SCRATCH(15), msg & ~flush);

			/* Handle flush interrupt in bottom half */
1699 1700
			queue_work(dev_priv->guc.log.runtime.flush_wq,
				   &dev_priv->guc.log.runtime.flush_work);
1701 1702

			dev_priv->guc.log.flush_interrupt_count++;
1703 1704 1705 1706 1707
		} else {
			/* Not clearing of unhandled event bits won't result in
			 * re-triggering of the interrupt.
			 */
		}
1708 1709 1710
	}
}

1711 1712
static void valleyview_pipestat_irq_ack(struct drm_i915_private *dev_priv,
					u32 iir, u32 pipe_stats[I915_MAX_PIPES])
1713 1714 1715
{
	int pipe;

1716
	spin_lock(&dev_priv->irq_lock);
1717 1718 1719 1720 1721 1722

	if (!dev_priv->display_irqs_enabled) {
		spin_unlock(&dev_priv->irq_lock);
		return;
	}

1723
	for_each_pipe(dev_priv, pipe) {
1724
		i915_reg_t reg;
1725
		u32 mask, iir_bit = 0;
1726

1727 1728 1729 1730 1731 1732 1733
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
1734 1735 1736

		/* fifo underruns are filterered in the underrun handler. */
		mask = PIPE_FIFO_UNDERRUN_STATUS;
1737 1738 1739 1740 1741 1742 1743 1744

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1745 1746 1747
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1748 1749 1750 1751 1752
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1753 1754 1755
			continue;

		reg = PIPESTAT(pipe);
1756 1757
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1758 1759 1760 1761

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1762 1763
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1764 1765
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1766
	spin_unlock(&dev_priv->irq_lock);
1767 1768
}

1769
static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
1770 1771 1772
					    u32 pipe_stats[I915_MAX_PIPES])
{
	enum pipe pipe;
1773

1774
	for_each_pipe(dev_priv, pipe) {
1775 1776
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
			drm_handle_vblank(&dev_priv->drm, pipe);
1777 1778

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1779
			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
1780

1781 1782
		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
1783 1784 1785
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1786
		gmbus_irq_handler(dev_priv);
1787 1788
}

1789
static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
1790 1791 1792
{
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);

1793 1794
	if (hotplug_status)
		I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1795

1796 1797 1798
	return hotplug_status;
}

1799
static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
1800 1801 1802
				 u32 hotplug_status)
{
	u32 pin_mask = 0, long_mask = 0;
1803

1804 1805
	if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
	    IS_CHERRYVIEW(dev_priv)) {
1806
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
1807

1808 1809 1810 1811 1812
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
					   hotplug_trigger, hpd_status_g4x,
					   i9xx_port_hotplug_long_detect);

1813
			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1814
		}
1815 1816

		if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1817
			dp_aux_irq_handler(dev_priv);
1818 1819
	} else {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1820

1821 1822
		if (hotplug_trigger) {
			intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
1823
					   hotplug_trigger, hpd_status_i915,
1824
					   i9xx_port_hotplug_long_detect);
1825
			intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
1826
		}
1827
	}
1828 1829
}

1830
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1831
{
1832
	struct drm_device *dev = arg;
1833
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
1834 1835
	irqreturn_t ret = IRQ_NONE;

1836 1837 1838
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1839 1840 1841
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1842
	do {
1843
		u32 iir, gt_iir, pm_iir;
1844
		u32 pipe_stats[I915_MAX_PIPES] = {};
1845
		u32 hotplug_status = 0;
1846
		u32 ier = 0;
1847

J
Jesse Barnes 已提交
1848 1849
		gt_iir = I915_READ(GTIIR);
		pm_iir = I915_READ(GEN6_PMIIR);
1850
		iir = I915_READ(VLV_IIR);
J
Jesse Barnes 已提交
1851 1852

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1853
			break;
J
Jesse Barnes 已提交
1854 1855 1856

		ret = IRQ_HANDLED;

1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      (((GT_IIR & GT_IER) || (GEN6_PMIIR & GEN6_PMIER)) &&
		 *       (VLV_MASTER_IER & MASTER_INTERRUPT_ENABLE)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear MASTER_INTERRUPT_ENABLE and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR, GT_IIR, GEN6_PMIIR
		 * bits this time around.
		 */
1870
		I915_WRITE(VLV_MASTER_IER, 0);
1871 1872
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
1873 1874 1875 1876 1877 1878

		if (gt_iir)
			I915_WRITE(GTIIR, gt_iir);
		if (pm_iir)
			I915_WRITE(GEN6_PMIIR, pm_iir);

1879
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1880
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1881

1882 1883
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
1884
		valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1885

1886 1887 1888 1889
		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
			   I915_LPE_PIPE_B_INTERRUPT))
			intel_lpe_audio_irq_handler(dev_priv);

1890 1891 1892 1893 1894 1895
		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);
1896

1897
		I915_WRITE(VLV_IER, ier);
1898 1899
		I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
		POSTING_READ(VLV_MASTER_IER);
1900

1901
		if (gt_iir)
1902
			snb_gt_irq_handler(dev_priv, gt_iir);
1903 1904 1905
		if (pm_iir)
			gen6_rps_irq_handler(dev_priv, pm_iir);

1906
		if (hotplug_status)
1907
			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1908

1909
		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1910
	} while (0);
J
Jesse Barnes 已提交
1911

1912 1913
	enable_rpm_wakeref_asserts(dev_priv);

J
Jesse Barnes 已提交
1914 1915 1916
	return ret;
}

1917 1918
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1919
	struct drm_device *dev = arg;
1920
	struct drm_i915_private *dev_priv = to_i915(dev);
1921 1922
	irqreturn_t ret = IRQ_NONE;

1923 1924 1925
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

1926 1927 1928
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

1929
	do {
1930
		u32 master_ctl, iir;
1931
		u32 gt_iir[4] = {};
1932
		u32 pipe_stats[I915_MAX_PIPES] = {};
1933
		u32 hotplug_status = 0;
1934 1935
		u32 ier = 0;

1936 1937
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
1938

1939 1940
		if (master_ctl == 0 && iir == 0)
			break;
1941

1942 1943
		ret = IRQ_HANDLED;

1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956
		/*
		 * Theory on interrupt generation, based on empirical evidence:
		 *
		 * x = ((VLV_IIR & VLV_IER) ||
		 *      ((GEN8_MASTER_IRQ & ~GEN8_MASTER_IRQ_CONTROL) &&
		 *       (GEN8_MASTER_IRQ & GEN8_MASTER_IRQ_CONTROL)));
		 *
		 * A CPU interrupt will only be raised when 'x' has a 0->1 edge.
		 * Hence we clear GEN8_MASTER_IRQ_CONTROL and VLV_IER to
		 * guarantee the CPU interrupt will be raised again even if we
		 * don't end up clearing all the VLV_IIR and GEN8_MASTER_IRQ_CONTROL
		 * bits this time around.
		 */
1957
		I915_WRITE(GEN8_MASTER_IRQ, 0);
1958 1959
		ier = I915_READ(VLV_IER);
		I915_WRITE(VLV_IER, 0);
1960

1961
		gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
1962

1963
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
1964
			hotplug_status = i9xx_hpd_irq_ack(dev_priv);
1965

1966 1967
		/* Call regardless, as some status bits might not be
		 * signalled in iir */
1968
		valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
1969

1970 1971 1972 1973 1974
		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
			   I915_LPE_PIPE_B_INTERRUPT |
			   I915_LPE_PIPE_C_INTERRUPT))
			intel_lpe_audio_irq_handler(dev_priv);

1975 1976 1977 1978 1979 1980 1981
		/*
		 * VLV_IIR is single buffered, and reflects the level
		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
		 */
		if (iir)
			I915_WRITE(VLV_IIR, iir);

1982
		I915_WRITE(VLV_IER, ier);
1983
		I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
1984
		POSTING_READ(GEN8_MASTER_IRQ);
1985

1986 1987
		gen8_gt_irq_handler(dev_priv, gt_iir);

1988
		if (hotplug_status)
1989
			i9xx_hpd_irq_handler(dev_priv, hotplug_status);
1990

1991
		valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
1992
	} while (0);
1993

1994 1995
	enable_rpm_wakeref_asserts(dev_priv);

1996 1997 1998
	return ret;
}

1999 2000
static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2001 2002 2003 2004
				const u32 hpd[HPD_NUM_PINS])
{
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

2005 2006 2007 2008 2009 2010
	/*
	 * Somehow the PCH doesn't seem to really ack the interrupt to the CPU
	 * unless we touch the hotplug register, even if hotplug_trigger is
	 * zero. Not acking leads to "The master control interrupt lied (SDE)!"
	 * errors.
	 */
2011
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2012 2013 2014 2015 2016 2017 2018 2019
	if (!hotplug_trigger) {
		u32 mask = PORTA_HOTPLUG_STATUS_MASK |
			PORTD_HOTPLUG_STATUS_MASK |
			PORTC_HOTPLUG_STATUS_MASK |
			PORTB_HOTPLUG_STATUS_MASK;
		dig_hotplug_reg &= ~mask;
	}

2020
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2021 2022
	if (!hotplug_trigger)
		return;
2023 2024 2025 2026 2027

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   pch_port_hotplug_long_detect);

2028
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2029 2030
}

2031
static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2032
{
2033
	int pipe;
2034
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
2035

2036
	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx);
2037

2038 2039 2040
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
2041
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
2042 2043
				 port_name(port));
	}
2044

2045
	if (pch_iir & SDE_AUX_MASK)
2046
		dp_aux_irq_handler(dev_priv);
2047

2048
	if (pch_iir & SDE_GMBUS)
2049
		gmbus_irq_handler(dev_priv);
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2060
	if (pch_iir & SDE_FDI_MASK)
2061
		for_each_pipe(dev_priv, pipe)
2062 2063 2064
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2065 2066 2067 2068 2069 2070 2071 2072

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
2073
		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
2074 2075

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
2076
		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
2077 2078
}

2079
static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
2080 2081
{
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
2082
	enum pipe pipe;
2083

2084 2085 2086
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

2087
	for_each_pipe(dev_priv, pipe) {
2088 2089
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2090

D
Daniel Vetter 已提交
2091
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
2092 2093
			if (IS_IVYBRIDGE(dev_priv))
				ivb_pipe_crc_irq_handler(dev_priv, pipe);
D
Daniel Vetter 已提交
2094
			else
2095
				hsw_pipe_crc_irq_handler(dev_priv, pipe);
D
Daniel Vetter 已提交
2096 2097
		}
	}
2098

2099 2100 2101
	I915_WRITE(GEN7_ERR_INT, err_int);
}

2102
static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
2103 2104 2105
{
	u32 serr_int = I915_READ(SERR_INT);

2106 2107 2108
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

2109
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
2110
		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_A);
2111 2112

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
2113
		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_B);
2114 2115

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
2116
		intel_pch_fifo_underrun_irq_handler(dev_priv, PIPE_C);
2117 2118

	I915_WRITE(SERR_INT, serr_int);
2119 2120
}

2121
static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2122 2123
{
	int pipe;
2124
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
2125

2126
	ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt);
2127

2128 2129 2130 2131 2132 2133
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
2134 2135

	if (pch_iir & SDE_AUX_MASK_CPT)
2136
		dp_aux_irq_handler(dev_priv);
2137 2138

	if (pch_iir & SDE_GMBUS_CPT)
2139
		gmbus_irq_handler(dev_priv);
2140 2141 2142 2143 2144 2145 2146 2147

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
2148
		for_each_pipe(dev_priv, pipe)
2149 2150 2151
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
2152 2153

	if (pch_iir & SDE_ERROR_CPT)
2154
		cpt_serr_int_handler(dev_priv);
2155 2156
}

2157
static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171
{
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
		~SDE_PORTE_HOTPLUG_SPT;
	u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
	u32 pin_mask = 0, long_mask = 0;

	if (hotplug_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
		I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
				   dig_hotplug_reg, hpd_spt,
2172
				   spt_port_hotplug_long_detect);
2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186
	}

	if (hotplug2_trigger) {
		u32 dig_hotplug_reg;

		dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG2);
		I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg);

		intel_get_hpd_pins(&pin_mask, &long_mask, hotplug2_trigger,
				   dig_hotplug_reg, hpd_spt,
				   spt_port_hotplug2_long_detect);
	}

	if (pin_mask)
2187
		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2188 2189

	if (pch_iir & SDE_GMBUS_CPT)
2190
		gmbus_irq_handler(dev_priv);
2191 2192
}

2193 2194
static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205
				const u32 hpd[HPD_NUM_PINS])
{
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;

	dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg);

	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
			   dig_hotplug_reg, hpd,
			   ilk_port_hotplug_long_detect);

2206
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2207 2208
}

2209 2210
static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
				    u32 de_iir)
2211
{
2212
	enum pipe pipe;
2213 2214
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;

2215
	if (hotplug_trigger)
2216
		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk);
2217 2218

	if (de_iir & DE_AUX_CHANNEL_A)
2219
		dp_aux_irq_handler(dev_priv);
2220 2221

	if (de_iir & DE_GSE)
2222
		intel_opregion_asle_intr(dev_priv);
2223 2224 2225 2226

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2227
	for_each_pipe(dev_priv, pipe) {
2228 2229
		if (de_iir & DE_PIPE_VBLANK(pipe))
			drm_handle_vblank(&dev_priv->drm, pipe);
2230

2231
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
2232
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2233

2234
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
2235
			i9xx_pipe_crc_irq_handler(dev_priv, pipe);
2236 2237 2238 2239 2240 2241
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

2242 2243
		if (HAS_PCH_CPT(dev_priv))
			cpt_irq_handler(dev_priv, pch_iir);
2244
		else
2245
			ibx_irq_handler(dev_priv, pch_iir);
2246 2247 2248 2249 2250

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

2251 2252
	if (IS_GEN5(dev_priv) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev_priv);
2253 2254
}

2255 2256
static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
				    u32 de_iir)
2257
{
2258
	enum pipe pipe;
2259 2260
	u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;

2261
	if (hotplug_trigger)
2262
		ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb);
2263 2264

	if (de_iir & DE_ERR_INT_IVB)
2265
		ivb_err_int_handler(dev_priv);
2266 2267

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
2268
		dp_aux_irq_handler(dev_priv);
2269 2270

	if (de_iir & DE_GSE_IVB)
2271
		intel_opregion_asle_intr(dev_priv);
2272

2273
	for_each_pipe(dev_priv, pipe) {
2274 2275
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
			drm_handle_vblank(&dev_priv->drm, pipe);
2276 2277 2278
	}

	/* check event from PCH */
2279
	if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
2280 2281
		u32 pch_iir = I915_READ(SDEIIR);

2282
		cpt_irq_handler(dev_priv, pch_iir);
2283 2284 2285 2286 2287 2288

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2289 2290 2291 2292 2293 2294 2295 2296
/*
 * To handle irqs with the minimum potential races with fresh interrupts, we:
 * 1 - Disable Master Interrupt Control.
 * 2 - Find the source(s) of the interrupt.
 * 3 - Clear the Interrupt Identity bits (IIR).
 * 4 - Process the interrupt(s) that had bits set in the IIRs.
 * 5 - Re-enable Master Interrupt Control.
 */
2297
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2298
{
2299
	struct drm_device *dev = arg;
2300
	struct drm_i915_private *dev_priv = to_i915(dev);
2301
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2302
	irqreturn_t ret = IRQ_NONE;
2303

2304 2305 2306
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

2307 2308 2309
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

2310 2311 2312
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2313
	POSTING_READ(DEIER);
2314

2315 2316 2317 2318 2319
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2320
	if (!HAS_PCH_NOP(dev_priv)) {
2321 2322 2323 2324
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2325

2326 2327
	/* Find, clear, then process each source of interrupt */

2328
	gt_iir = I915_READ(GTIIR);
2329
	if (gt_iir) {
2330 2331
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2332
		if (INTEL_GEN(dev_priv) >= 6)
2333
			snb_gt_irq_handler(dev_priv, gt_iir);
2334
		else
2335
			ilk_gt_irq_handler(dev_priv, gt_iir);
2336 2337
	}

2338 2339
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2340 2341
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2342 2343
		if (INTEL_GEN(dev_priv) >= 7)
			ivb_display_irq_handler(dev_priv, de_iir);
2344
		else
2345
			ilk_display_irq_handler(dev_priv, de_iir);
2346 2347
	}

2348
	if (INTEL_GEN(dev_priv) >= 6) {
2349 2350 2351 2352
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
2353
			gen6_rps_irq_handler(dev_priv, pm_iir);
2354
		}
2355
	}
2356 2357 2358

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2359
	if (!HAS_PCH_NOP(dev_priv)) {
2360 2361 2362
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2363

2364 2365 2366
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	enable_rpm_wakeref_asserts(dev_priv);

2367 2368 2369
	return ret;
}

2370 2371
static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
				u32 hotplug_trigger,
2372
				const u32 hpd[HPD_NUM_PINS])
2373
{
2374
	u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
2375

2376 2377
	dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
	I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2378

2379
	intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
2380
			   dig_hotplug_reg, hpd,
2381
			   bxt_port_hotplug_long_detect);
2382

2383
	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
2384 2385
}

2386 2387
static irqreturn_t
gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
2388 2389
{
	irqreturn_t ret = IRQ_NONE;
2390
	u32 iir;
2391
	enum pipe pipe;
J
Jesse Barnes 已提交
2392

2393
	if (master_ctl & GEN8_DE_MISC_IRQ) {
2394 2395 2396
		iir = I915_READ(GEN8_DE_MISC_IIR);
		if (iir) {
			I915_WRITE(GEN8_DE_MISC_IIR, iir);
2397
			ret = IRQ_HANDLED;
2398
			if (iir & GEN8_DE_MISC_GSE)
2399
				intel_opregion_asle_intr(dev_priv);
2400 2401
			else
				DRM_ERROR("Unexpected DE Misc interrupt\n");
2402
		}
2403 2404
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
2405 2406
	}

2407
	if (master_ctl & GEN8_DE_PORT_IRQ) {
2408 2409 2410
		iir = I915_READ(GEN8_DE_PORT_IIR);
		if (iir) {
			u32 tmp_mask;
2411
			bool found = false;
2412

2413
			I915_WRITE(GEN8_DE_PORT_IIR, iir);
2414
			ret = IRQ_HANDLED;
J
Jesse Barnes 已提交
2415

2416
			tmp_mask = GEN8_AUX_CHANNEL_A;
2417
			if (INTEL_GEN(dev_priv) >= 9)
2418 2419 2420 2421 2422
				tmp_mask |= GEN9_AUX_CHANNEL_B |
					    GEN9_AUX_CHANNEL_C |
					    GEN9_AUX_CHANNEL_D;

			if (iir & tmp_mask) {
2423
				dp_aux_irq_handler(dev_priv);
2424 2425 2426
				found = true;
			}

2427
			if (IS_GEN9_LP(dev_priv)) {
2428 2429
				tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
				if (tmp_mask) {
2430 2431
					bxt_hpd_irq_handler(dev_priv, tmp_mask,
							    hpd_bxt);
2432 2433 2434 2435 2436
					found = true;
				}
			} else if (IS_BROADWELL(dev_priv)) {
				tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
				if (tmp_mask) {
2437 2438
					ilk_hpd_irq_handler(dev_priv,
							    tmp_mask, hpd_bdw);
2439 2440
					found = true;
				}
2441 2442
			}

2443
			if (IS_GEN9_LP(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
2444
				gmbus_irq_handler(dev_priv);
S
Shashank Sharma 已提交
2445 2446 2447
				found = true;
			}

2448
			if (!found)
2449
				DRM_ERROR("Unexpected DE Port interrupt\n");
2450
		}
2451 2452
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
2453 2454
	}

2455
	for_each_pipe(dev_priv, pipe) {
2456
		u32 fault_errors;
2457

2458 2459
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2460

2461 2462 2463 2464 2465
		iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (!iir) {
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
			continue;
		}
2466

2467 2468
		ret = IRQ_HANDLED;
		I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
2469

2470 2471
		if (iir & GEN8_PIPE_VBLANK)
			drm_handle_vblank(&dev_priv->drm, pipe);
2472

2473
		if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
2474
			hsw_pipe_crc_irq_handler(dev_priv, pipe);
2475

2476 2477
		if (iir & GEN8_PIPE_FIFO_UNDERRUN)
			intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
2478

2479
		fault_errors = iir;
2480
		if (INTEL_GEN(dev_priv) >= 9)
2481 2482 2483
			fault_errors &= GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
		else
			fault_errors &= GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2484

2485
		if (fault_errors)
2486
			DRM_ERROR("Fault errors on pipe %c: 0x%08x\n",
2487 2488
				  pipe_name(pipe),
				  fault_errors);
2489 2490
	}

2491
	if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
2492
	    master_ctl & GEN8_DE_PCH_IRQ) {
2493 2494 2495 2496 2497
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
2498 2499 2500
		iir = I915_READ(SDEIIR);
		if (iir) {
			I915_WRITE(SDEIIR, iir);
2501
			ret = IRQ_HANDLED;
2502

2503 2504
			if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv) ||
			    HAS_PCH_CNP(dev_priv))
2505
				spt_irq_handler(dev_priv, iir);
2506
			else
2507
				cpt_irq_handler(dev_priv, iir);
2508 2509 2510 2511 2512 2513 2514
		} else {
			/*
			 * Like on previous PCH there seems to be something
			 * fishy going on with forwarding PCH interrupts.
			 */
			DRM_DEBUG_DRIVER("The master control interrupt lied (SDE)!\n");
		}
2515 2516
	}

2517 2518 2519 2520 2521 2522
	return ret;
}

static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
2523
	struct drm_i915_private *dev_priv = to_i915(dev);
2524
	u32 master_ctl;
2525
	u32 gt_iir[4] = {};
2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541
	irqreturn_t ret;

	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

	master_ctl = I915_READ_FW(GEN8_MASTER_IRQ);
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

	I915_WRITE_FW(GEN8_MASTER_IRQ, 0);

	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	/* Find, clear, then process each source of interrupt */
2542 2543
	ret = gen8_gt_irq_ack(dev_priv, master_ctl, gt_iir);
	gen8_gt_irq_handler(dev_priv, gt_iir);
2544 2545
	ret |= gen8_de_irq_handler(dev_priv, master_ctl);

2546 2547
	I915_WRITE_FW(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ_FW(GEN8_MASTER_IRQ);
2548

2549 2550
	enable_rpm_wakeref_asserts(dev_priv);

2551 2552 2553
	return ret;
}

2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593
struct wedge_me {
	struct delayed_work work;
	struct drm_i915_private *i915;
	const char *name;
};

static void wedge_me(struct work_struct *work)
{
	struct wedge_me *w = container_of(work, typeof(*w), work.work);

	dev_err(w->i915->drm.dev,
		"%s timed out, cancelling all in-flight rendering.\n",
		w->name);
	i915_gem_set_wedged(w->i915);
}

static void __init_wedge(struct wedge_me *w,
			 struct drm_i915_private *i915,
			 long timeout,
			 const char *name)
{
	w->i915 = i915;
	w->name = name;

	INIT_DELAYED_WORK_ONSTACK(&w->work, wedge_me);
	schedule_delayed_work(&w->work, timeout);
}

static void __fini_wedge(struct wedge_me *w)
{
	cancel_delayed_work_sync(&w->work);
	destroy_delayed_work_on_stack(&w->work);
	w->i915 = NULL;
}

#define i915_wedge_on_timeout(W, DEV, TIMEOUT)				\
	for (__init_wedge((W), (DEV), (TIMEOUT), __func__);		\
	     (W)->i915;							\
	     __fini_wedge((W)))

2594
/**
2595
 * i915_reset_device - do process context error handling work
2596
 * @dev_priv: i915 device private
2597 2598 2599 2600
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
2601
static void i915_reset_device(struct drm_i915_private *dev_priv)
2602
{
2603
	struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
2604 2605 2606
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2607
	struct wedge_me w;
2608

2609
	kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);
2610

2611 2612 2613
	DRM_DEBUG_DRIVER("resetting chip\n");
	kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);

2614 2615 2616
	/* Use a watchdog to ensure that our reset completes */
	i915_wedge_on_timeout(&w, dev_priv, 5*HZ) {
		intel_prepare_reset(dev_priv);
2617

2618 2619 2620
		/* Signal that locked waiters should reset the GPU */
		set_bit(I915_RESET_HANDOFF, &dev_priv->gpu_error.flags);
		wake_up_all(&dev_priv->gpu_error.wait_queue);
2621

2622 2623
		/* Wait for anyone holding the lock to wakeup, without
		 * blocking indefinitely on struct_mutex.
2624
		 */
2625 2626 2627 2628 2629 2630 2631 2632 2633
		do {
			if (mutex_trylock(&dev_priv->drm.struct_mutex)) {
				i915_reset(dev_priv);
				mutex_unlock(&dev_priv->drm.struct_mutex);
			}
		} while (wait_on_bit_timeout(&dev_priv->gpu_error.flags,
					     I915_RESET_HANDOFF,
					     TASK_UNINTERRUPTIBLE,
					     1));
2634

2635 2636
		intel_finish_reset(dev_priv);
	}
2637

2638
	if (!test_bit(I915_WEDGED, &dev_priv->gpu_error.flags))
2639 2640
		kobject_uevent_env(kobj,
				   KOBJ_CHANGE, reset_done_event);
2641 2642
}

2643 2644 2645 2646
static inline void
i915_err_print_instdone(struct drm_i915_private *dev_priv,
			struct intel_instdone *instdone)
{
2647 2648 2649
	int slice;
	int subslice;

2650 2651 2652 2653 2654 2655 2656 2657 2658 2659
	pr_err("  INSTDONE: 0x%08x\n", instdone->instdone);

	if (INTEL_GEN(dev_priv) <= 3)
		return;

	pr_err("  SC_INSTDONE: 0x%08x\n", instdone->slice_common);

	if (INTEL_GEN(dev_priv) <= 6)
		return;

2660 2661 2662 2663 2664 2665 2666
	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
		pr_err("  SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
		       slice, subslice, instdone->sampler[slice][subslice]);

	for_each_instdone_slice_subslice(dev_priv, slice, subslice)
		pr_err("  ROW_INSTDONE[%d][%d]: 0x%08x\n",
		       slice, subslice, instdone->row[slice][subslice]);
2667 2668
}

2669
static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
2670
{
2671
	u32 eir;
2672

2673 2674
	if (!IS_GEN2(dev_priv))
		I915_WRITE(PGTBL_ER, I915_READ(PGTBL_ER));
2675

2676 2677 2678 2679
	if (INTEL_GEN(dev_priv) < 4)
		I915_WRITE(IPEIR, I915_READ(IPEIR));
	else
		I915_WRITE(IPEIR_I965, I915_READ(IPEIR_I965));
2680

2681
	I915_WRITE(EIR, I915_READ(EIR));
2682 2683 2684 2685 2686 2687
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
2688
		DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
2689 2690 2691
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2692 2693 2694
}

/**
2695
 * i915_handle_error - handle a gpu error
2696
 * @dev_priv: i915 device private
2697
 * @engine_mask: mask representing engines that are hung
2698 2699
 * @fmt: Error message format string
 *
2700
 * Do some basic checking of register state at error time and
2701 2702 2703 2704 2705
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
2706 2707
void i915_handle_error(struct drm_i915_private *dev_priv,
		       u32 engine_mask,
2708
		       const char *fmt, ...)
2709
{
2710 2711
	struct intel_engine_cs *engine;
	unsigned int tmp;
2712 2713
	va_list args;
	char error_msg[80];
2714

2715 2716 2717 2718
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

2719 2720 2721 2722 2723 2724 2725 2726 2727
	/*
	 * In most cases it's guaranteed that we get here with an RPM
	 * reference held, for example because there is a pending GPU
	 * request that won't finish until the reset is done. This
	 * isn't the case at least when we get here by doing a
	 * simulated reset via debugfs, so get an RPM reference.
	 */
	intel_runtime_pm_get(dev_priv);

2728
	i915_capture_error_state(dev_priv, engine_mask, error_msg);
2729
	i915_clear_error_registers(dev_priv);
2730

2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751
	/*
	 * Try engine reset when available. We fall back to full reset if
	 * single reset fails.
	 */
	if (intel_has_reset_engine(dev_priv)) {
		for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
			BUILD_BUG_ON(I915_RESET_HANDOFF >= I915_RESET_ENGINE);
			if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
					     &dev_priv->gpu_error.flags))
				continue;

			if (i915_reset_engine(engine) == 0)
				engine_mask &= ~intel_engine_flag(engine);

			clear_bit(I915_RESET_ENGINE + engine->id,
				  &dev_priv->gpu_error.flags);
			wake_up_bit(&dev_priv->gpu_error.flags,
				    I915_RESET_ENGINE + engine->id);
		}
	}

2752
	if (!engine_mask)
2753
		goto out;
2754

2755
	/* Full reset needs the mutex, stop any other user trying to do so. */
2756 2757 2758 2759
	if (test_and_set_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags)) {
		wait_event(dev_priv->gpu_error.reset_queue,
			   !test_bit(I915_RESET_BACKOFF,
				     &dev_priv->gpu_error.flags));
2760
		goto out;
2761 2762
	}

2763 2764 2765 2766 2767 2768 2769 2770 2771
	/* Prevent any other reset-engine attempt. */
	for_each_engine(engine, dev_priv, tmp) {
		while (test_and_set_bit(I915_RESET_ENGINE + engine->id,
					&dev_priv->gpu_error.flags))
			wait_on_bit(&dev_priv->gpu_error.flags,
				    I915_RESET_ENGINE + engine->id,
				    TASK_UNINTERRUPTIBLE);
	}

2772
	i915_reset_device(dev_priv);
2773

2774 2775 2776 2777 2778
	for_each_engine(engine, dev_priv, tmp) {
		clear_bit(I915_RESET_ENGINE + engine->id,
			  &dev_priv->gpu_error.flags);
	}

2779 2780
	clear_bit(I915_RESET_BACKOFF, &dev_priv->gpu_error.flags);
	wake_up_all(&dev_priv->gpu_error.reset_queue);
2781 2782 2783

out:
	intel_runtime_pm_put(dev_priv);
2784 2785
}

2786 2787 2788
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2789
static int i8xx_enable_vblank(struct drm_device *dev, unsigned int pipe)
2790
{
2791
	struct drm_i915_private *dev_priv = to_i915(dev);
2792
	unsigned long irqflags;
2793

2794
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2795
	i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2796
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2797

2798 2799 2800
	return 0;
}

2801
static int i965_enable_vblank(struct drm_device *dev, unsigned int pipe)
2802
{
2803
	struct drm_i915_private *dev_priv = to_i915(dev);
2804 2805 2806
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2807 2808
	i915_enable_pipestat(dev_priv, pipe,
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
2809 2810 2811 2812 2813
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2814
static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2815
{
2816
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
2817
	unsigned long irqflags;
2818
	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
2819
		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
J
Jesse Barnes 已提交
2820 2821

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2822
	ilk_enable_display_irq(dev_priv, bit);
J
Jesse Barnes 已提交
2823 2824 2825 2826 2827
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2828
static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
2829
{
2830
	struct drm_i915_private *dev_priv = to_i915(dev);
2831 2832 2833
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2834
	bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2835
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2836

2837 2838 2839
	return 0;
}

2840 2841 2842
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2843
static void i8xx_disable_vblank(struct drm_device *dev, unsigned int pipe)
2844
{
2845
	struct drm_i915_private *dev_priv = to_i915(dev);
2846
	unsigned long irqflags;
2847

2848
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2849
	i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_STATUS);
2850 2851 2852
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2853
static void i965_disable_vblank(struct drm_device *dev, unsigned int pipe)
2854
{
2855
	struct drm_i915_private *dev_priv = to_i915(dev);
2856 2857 2858
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2859 2860
	i915_disable_pipestat(dev_priv, pipe,
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2861 2862 2863
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2864
static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
J
Jesse Barnes 已提交
2865
{
2866
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
2867
	unsigned long irqflags;
2868
	uint32_t bit = INTEL_GEN(dev_priv) >= 7 ?
2869
		DE_PIPE_VBLANK_IVB(pipe) : DE_PIPE_VBLANK(pipe);
J
Jesse Barnes 已提交
2870 2871

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2872
	ilk_disable_display_irq(dev_priv, bit);
J
Jesse Barnes 已提交
2873 2874 2875
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2876
static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
2877
{
2878
	struct drm_i915_private *dev_priv = to_i915(dev);
2879 2880 2881
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2882
	bdw_disable_pipe_irq(dev_priv, pipe, GEN8_PIPE_VBLANK);
2883 2884 2885
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2886
static void ibx_irq_reset(struct drm_i915_private *dev_priv)
P
Paulo Zanoni 已提交
2887
{
2888
	if (HAS_PCH_NOP(dev_priv))
P
Paulo Zanoni 已提交
2889 2890
		return;

2891
	GEN5_IRQ_RESET(SDE);
2892

2893
	if (HAS_PCH_CPT(dev_priv) || HAS_PCH_LPT(dev_priv))
2894
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
2895
}
2896

P
Paulo Zanoni 已提交
2897 2898 2899 2900 2901 2902 2903 2904 2905 2906
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
2907
	struct drm_i915_private *dev_priv = to_i915(dev);
P
Paulo Zanoni 已提交
2908

2909
	if (HAS_PCH_NOP(dev_priv))
P
Paulo Zanoni 已提交
2910 2911 2912
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
2913 2914 2915 2916
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

2917
static void gen5_gt_irq_reset(struct drm_i915_private *dev_priv)
2918
{
2919
	GEN5_IRQ_RESET(GT);
2920
	if (INTEL_GEN(dev_priv) >= 6)
2921
		GEN5_IRQ_RESET(GEN6_PM);
2922 2923
}

2924 2925 2926 2927
static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
{
	enum pipe pipe;

2928 2929 2930 2931 2932
	if (IS_CHERRYVIEW(dev_priv))
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
	else
		I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);

2933
	i915_hotplug_interrupt_update_locked(dev_priv, 0xffffffff, 0);
2934 2935
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

2936 2937 2938 2939 2940 2941
	for_each_pipe(dev_priv, pipe) {
		I915_WRITE(PIPESTAT(pipe),
			   PIPE_FIFO_UNDERRUN_STATUS |
			   PIPESTAT_INT_STATUS_MASK);
		dev_priv->pipestat_irq_mask[pipe] = 0;
	}
2942 2943

	GEN5_IRQ_RESET(VLV_);
2944
	dev_priv->irq_mask = ~0;
2945 2946
}

2947 2948 2949
static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
2950
	u32 enable_mask;
2951 2952 2953 2954 2955 2956 2957 2958 2959
	enum pipe pipe;

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	for_each_pipe(dev_priv, pipe)
		i915_enable_pipestat(dev_priv, pipe, pipestat_mask);

2960 2961
	enable_mask = I915_DISPLAY_PORT_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
2962 2963 2964 2965
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_LPE_PIPE_A_INTERRUPT |
		I915_LPE_PIPE_B_INTERRUPT;

2966
	if (IS_CHERRYVIEW(dev_priv))
2967 2968
		enable_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT |
			I915_LPE_PIPE_C_INTERRUPT;
2969 2970 2971

	WARN_ON(dev_priv->irq_mask != ~0);

2972 2973 2974
	dev_priv->irq_mask = ~enable_mask;

	GEN5_IRQ_INIT(VLV_, dev_priv->irq_mask, enable_mask);
2975 2976 2977 2978 2979 2980
}

/* drm_dma.h hooks
*/
static void ironlake_irq_reset(struct drm_device *dev)
{
2981
	struct drm_i915_private *dev_priv = to_i915(dev);
2982 2983 2984 2985

	I915_WRITE(HWSTAM, 0xffffffff);

	GEN5_IRQ_RESET(DE);
2986
	if (IS_GEN7(dev_priv))
2987 2988
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);

2989
	gen5_gt_irq_reset(dev_priv);
2990

2991
	ibx_irq_reset(dev_priv);
2992 2993
}

J
Jesse Barnes 已提交
2994 2995
static void valleyview_irq_preinstall(struct drm_device *dev)
{
2996
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
2997

2998 2999 3000
	I915_WRITE(VLV_MASTER_IER, 0);
	POSTING_READ(VLV_MASTER_IER);

3001
	gen5_gt_irq_reset(dev_priv);
J
Jesse Barnes 已提交
3002

3003
	spin_lock_irq(&dev_priv->irq_lock);
3004 3005
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3006
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3007 3008
}

3009 3010 3011 3012 3013 3014 3015 3016
static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
{
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
}

P
Paulo Zanoni 已提交
3017
static void gen8_irq_reset(struct drm_device *dev)
3018
{
3019
	struct drm_i915_private *dev_priv = to_i915(dev);
3020 3021 3022 3023 3024
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3025
	gen8_gt_irq_reset(dev_priv);
3026

3027
	for_each_pipe(dev_priv, pipe)
3028 3029
		if (intel_display_power_is_enabled(dev_priv,
						   POWER_DOMAIN_PIPE(pipe)))
3030
			GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3031

3032 3033 3034
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3035

3036
	if (HAS_PCH_SPLIT(dev_priv))
3037
		ibx_irq_reset(dev_priv);
3038
}
3039

3040
void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
3041
				     u8 pipe_mask)
3042
{
3043
	uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
3044
	enum pipe pipe;
3045

3046
	spin_lock_irq(&dev_priv->irq_lock);
3047 3048 3049 3050
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
				  dev_priv->de_irq_mask[pipe],
				  ~dev_priv->de_irq_mask[pipe] | extra_ier);
3051
	spin_unlock_irq(&dev_priv->irq_lock);
3052 3053
}

3054
void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
3055
				     u8 pipe_mask)
3056
{
3057 3058
	enum pipe pipe;

3059
	spin_lock_irq(&dev_priv->irq_lock);
3060 3061
	for_each_pipe_masked(dev_priv, pipe, pipe_mask)
		GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3062 3063 3064
	spin_unlock_irq(&dev_priv->irq_lock);

	/* make sure we're done processing display irqs */
3065
	synchronize_irq(dev_priv->drm.irq);
3066 3067
}

3068 3069
static void cherryview_irq_preinstall(struct drm_device *dev)
{
3070
	struct drm_i915_private *dev_priv = to_i915(dev);
3071 3072 3073 3074

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3075
	gen8_gt_irq_reset(dev_priv);
3076 3077 3078

	GEN5_IRQ_RESET(GEN8_PCU_);

3079
	spin_lock_irq(&dev_priv->irq_lock);
3080 3081
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3082
	spin_unlock_irq(&dev_priv->irq_lock);
3083 3084
}

3085
static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
3086 3087 3088 3089 3090
				  const u32 hpd[HPD_NUM_PINS])
{
	struct intel_encoder *encoder;
	u32 enabled_irqs = 0;

3091
	for_each_intel_encoder(&dev_priv->drm, encoder)
3092 3093 3094 3095 3096 3097
		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
			enabled_irqs |= hpd[encoder->hpd_pin];

	return enabled_irqs;
}

3098
static void ibx_hpd_detection_setup(struct drm_i915_private *dev_priv)
3099
{
3100
	u32 hotplug;
3101 3102 3103

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
3104 3105
	 * duration to 2ms (which is the minimum in the Display Port spec).
	 * The pulse duration bits are reserved on LPT+.
3106
	 */
3107
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3108 3109 3110
	hotplug &= ~(PORTB_PULSE_DURATION_MASK |
		     PORTC_PULSE_DURATION_MASK |
		     PORTD_PULSE_DURATION_MASK);
3111
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3112 3113
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3114 3115 3116 3117
	/*
	 * When CPU and PCH are on the same package, port A
	 * HPD must be enabled in both north and south.
	 */
3118
	if (HAS_PCH_LPT_LP(dev_priv))
3119
		hotplug |= PORTA_HOTPLUG_ENABLE;
3120
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3121
}
X
Xiong Zhang 已提交
3122

3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139
static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	if (HAS_PCH_IBX(dev_priv)) {
		hotplug_irqs = SDE_HOTPLUG_MASK;
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx);
	} else {
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt);
	}

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	ibx_hpd_detection_setup(dev_priv);
}

3140
static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
3141
{
3142
	u32 hotplug;
3143 3144 3145

	/* Enable digital hotplug on the PCH */
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3146 3147 3148 3149
	hotplug |= PORTA_HOTPLUG_ENABLE |
		   PORTB_HOTPLUG_ENABLE |
		   PORTC_HOTPLUG_ENABLE |
		   PORTD_HOTPLUG_ENABLE;
3150 3151 3152 3153 3154
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);

	hotplug = I915_READ(PCH_PORT_HOTPLUG2);
	hotplug |= PORTE_HOTPLUG_ENABLE;
	I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
3155 3156
}

3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168
static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt);

	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);

	spt_hpd_detection_setup(dev_priv);
}

3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184
static void ilk_hpd_detection_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug;

	/*
	 * Enable digital hotplug on the CPU, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
	 * The pulse duration bits are reserved on HSW+.
	 */
	hotplug = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
	hotplug &= ~DIGITAL_PORTA_PULSE_DURATION_MASK;
	hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE |
		   DIGITAL_PORTA_PULSE_DURATION_2ms;
	I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
}

3185
static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
3186
{
3187
	u32 hotplug_irqs, enabled_irqs;
3188

3189
	if (INTEL_GEN(dev_priv) >= 8) {
3190
		hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
3191
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw);
3192 3193

		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
3194
	} else if (INTEL_GEN(dev_priv) >= 7) {
3195
		hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
3196
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb);
3197 3198

		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
3199 3200
	} else {
		hotplug_irqs = DE_DP_A_HOTPLUG;
3201
		enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk);
3202

3203 3204
		ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
	}
3205

3206
	ilk_hpd_detection_setup(dev_priv);
3207

3208
	ibx_hpd_irq_setup(dev_priv);
3209 3210
}

3211 3212
static void __bxt_hpd_detection_setup(struct drm_i915_private *dev_priv,
				      u32 enabled_irqs)
3213
{
3214
	u32 hotplug;
3215

3216
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
3217 3218 3219
	hotplug |= PORTA_HOTPLUG_ENABLE |
		   PORTB_HOTPLUG_ENABLE |
		   PORTC_HOTPLUG_ENABLE;
3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238

	DRM_DEBUG_KMS("Invert bit setting: hp_ctl:%x hp_port:%x\n",
		      hotplug, enabled_irqs);
	hotplug &= ~BXT_DDI_HPD_INVERT_MASK;

	/*
	 * For BXT invert bit has to be set based on AOB design
	 * for HPD detection logic, update it based on VBT fields.
	 */
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIA) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_A))
		hotplug |= BXT_DDIA_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIB) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_B))
		hotplug |= BXT_DDIB_HPD_INVERT;
	if ((enabled_irqs & BXT_DE_PORT_HP_DDIC) &&
	    intel_bios_is_port_hpd_inverted(dev_priv, PORT_C))
		hotplug |= BXT_DDIC_HPD_INVERT;

3239
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3240 3241
}

3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258
static void bxt_hpd_detection_setup(struct drm_i915_private *dev_priv)
{
	__bxt_hpd_detection_setup(dev_priv, BXT_DE_PORT_HOTPLUG_MASK);
}

static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
{
	u32 hotplug_irqs, enabled_irqs;

	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt);
	hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;

	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);

	__bxt_hpd_detection_setup(dev_priv, enabled_irqs);
}

P
Paulo Zanoni 已提交
3259 3260
static void ibx_irq_postinstall(struct drm_device *dev)
{
3261
	struct drm_i915_private *dev_priv = to_i915(dev);
3262
	u32 mask;
3263

3264
	if (HAS_PCH_NOP(dev_priv))
D
Daniel Vetter 已提交
3265 3266
		return;

3267
	if (HAS_PCH_IBX(dev_priv))
3268
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3269
	else
3270
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3271

3272
	gen5_assert_iir_is_zero(dev_priv, SDEIIR);
P
Paulo Zanoni 已提交
3273
	I915_WRITE(SDEIMR, ~mask);
3274 3275 3276

	if (HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv) ||
	    HAS_PCH_LPT(dev_priv))
3277
		ibx_hpd_detection_setup(dev_priv);
3278 3279
	else
		spt_hpd_detection_setup(dev_priv);
P
Paulo Zanoni 已提交
3280 3281
}

3282 3283
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
3284
	struct drm_i915_private *dev_priv = to_i915(dev);
3285 3286 3287 3288 3289
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3290
	if (HAS_L3_DPF(dev_priv)) {
3291
		/* L3 parity interrupt is always unmasked. */
3292 3293
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev_priv);
		gt_irqs |= GT_PARITY_ERROR(dev_priv);
3294 3295 3296
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
3297
	if (IS_GEN5(dev_priv)) {
3298
		gt_irqs |= ILK_BSD_USER_INTERRUPT;
3299 3300 3301 3302
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3303
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3304

3305
	if (INTEL_GEN(dev_priv) >= 6) {
3306 3307 3308 3309
		/*
		 * RPS interrupts will get enabled/disabled on demand when RPS
		 * itself is enabled/disabled.
		 */
3310
		if (HAS_VEBOX(dev_priv)) {
3311
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3312 3313
			dev_priv->pm_ier |= PM_VEBOX_USER_INTERRUPT;
		}
3314

3315 3316
		dev_priv->pm_imr = 0xffffffff;
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_imr, pm_irqs);
3317 3318 3319
	}
}

3320
static int ironlake_irq_postinstall(struct drm_device *dev)
3321
{
3322
	struct drm_i915_private *dev_priv = to_i915(dev);
3323 3324
	u32 display_mask, extra_mask;

3325
	if (INTEL_GEN(dev_priv) >= 7) {
3326 3327 3328
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3329
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3330
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3331 3332
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB |
			      DE_DP_A_HOTPLUG_IVB);
3333 3334 3335
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3336 3337 3338
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3339 3340 3341
		extra_mask = (DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
			      DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN |
			      DE_DP_A_HOTPLUG);
3342
	}
3343

3344
	dev_priv->irq_mask = ~display_mask;
3345

3346 3347
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3348 3349
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3350
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3351

3352
	gen5_gt_irq_postinstall(dev);
3353

3354 3355
	ilk_hpd_detection_setup(dev_priv);

P
Paulo Zanoni 已提交
3356
	ibx_irq_postinstall(dev);
3357

3358
	if (IS_IRONLAKE_M(dev_priv)) {
3359 3360 3361
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3362 3363
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
3364
		spin_lock_irq(&dev_priv->irq_lock);
3365
		ilk_enable_display_irq(dev_priv, DE_PCU_EVENT);
3366
		spin_unlock_irq(&dev_priv->irq_lock);
3367 3368
	}

3369 3370 3371
	return 0;
}

3372 3373
void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
3374
	lockdep_assert_held(&dev_priv->irq_lock);
3375 3376 3377 3378 3379 3380

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

3381 3382
	if (intel_irqs_enabled(dev_priv)) {
		vlv_display_irq_reset(dev_priv);
3383
		vlv_display_irq_postinstall(dev_priv);
3384
	}
3385 3386 3387 3388
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
3389
	lockdep_assert_held(&dev_priv->irq_lock);
3390 3391 3392 3393 3394 3395

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

3396
	if (intel_irqs_enabled(dev_priv))
3397
		vlv_display_irq_reset(dev_priv);
3398 3399
}

3400 3401 3402

static int valleyview_irq_postinstall(struct drm_device *dev)
{
3403
	struct drm_i915_private *dev_priv = to_i915(dev);
3404

3405
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3406

3407
	spin_lock_irq(&dev_priv->irq_lock);
3408 3409
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3410 3411
	spin_unlock_irq(&dev_priv->irq_lock);

J
Jesse Barnes 已提交
3412
	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3413
	POSTING_READ(VLV_MASTER_IER);
3414 3415 3416 3417

	return 0;
}

3418 3419 3420 3421 3422
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
3423 3424 3425
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
3426
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3427 3428 3429
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
3430
		0,
3431 3432
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
			GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
3433 3434
		};

3435 3436 3437
	if (HAS_L3_DPF(dev_priv))
		gt_interrupts[0] |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;

3438 3439
	dev_priv->pm_ier = 0x0;
	dev_priv->pm_imr = ~dev_priv->pm_ier;
3440 3441
	GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
	GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3442 3443
	/*
	 * RPS interrupts will get enabled/disabled on demand when RPS itself
3444
	 * is enabled/disabled. Same wil be the case for GuC interrupts.
3445
	 */
3446
	GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_imr, dev_priv->pm_ier);
3447
	GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
3448 3449 3450 3451
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
3452 3453
	uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
	uint32_t de_pipe_enables;
3454 3455
	u32 de_port_masked = GEN8_AUX_CHANNEL_A;
	u32 de_port_enables;
3456
	u32 de_misc_masked = GEN8_DE_MISC_GSE;
3457
	enum pipe pipe;
3458

3459
	if (INTEL_GEN(dev_priv) >= 9) {
3460 3461
		de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
				  GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
3462 3463
		de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
				  GEN9_AUX_CHANNEL_D;
3464
		if (IS_GEN9_LP(dev_priv))
3465 3466
			de_port_masked |= BXT_DE_PORT_GMBUS;
	} else {
3467 3468
		de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
				  GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3469
	}
3470 3471 3472 3473

	de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
					   GEN8_PIPE_FIFO_UNDERRUN;

3474
	de_port_enables = de_port_masked;
3475
	if (IS_GEN9_LP(dev_priv))
3476 3477
		de_port_enables |= BXT_DE_PORT_HOTPLUG_MASK;
	else if (IS_BROADWELL(dev_priv))
3478 3479
		de_port_enables |= GEN8_PORT_DP_A_HOTPLUG;

3480 3481 3482
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3483

3484
	for_each_pipe(dev_priv, pipe)
3485
		if (intel_display_power_is_enabled(dev_priv,
3486 3487 3488 3489
				POWER_DOMAIN_PIPE(pipe)))
			GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
					  dev_priv->de_irq_mask[pipe],
					  de_pipe_enables);
3490

3491
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
3492
	GEN5_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
3493 3494 3495

	if (IS_GEN9_LP(dev_priv))
		bxt_hpd_detection_setup(dev_priv);
3496 3497
	else if (IS_BROADWELL(dev_priv))
		ilk_hpd_detection_setup(dev_priv);
3498 3499 3500 3501
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
3502
	struct drm_i915_private *dev_priv = to_i915(dev);
3503

3504
	if (HAS_PCH_SPLIT(dev_priv))
3505
		ibx_irq_pre_postinstall(dev);
P
Paulo Zanoni 已提交
3506

3507 3508 3509
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

3510
	if (HAS_PCH_SPLIT(dev_priv))
3511
		ibx_irq_postinstall(dev);
3512

3513
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3514 3515 3516 3517 3518
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3519 3520
static int cherryview_irq_postinstall(struct drm_device *dev)
{
3521
	struct drm_i915_private *dev_priv = to_i915(dev);
3522 3523 3524

	gen8_gt_irq_postinstall(dev_priv);

3525
	spin_lock_irq(&dev_priv->irq_lock);
3526 3527
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_postinstall(dev_priv);
3528 3529
	spin_unlock_irq(&dev_priv->irq_lock);

3530
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
3531 3532 3533 3534 3535
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3536 3537
static void gen8_irq_uninstall(struct drm_device *dev)
{
3538
	struct drm_i915_private *dev_priv = to_i915(dev);
3539 3540 3541 3542

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3543
	gen8_irq_reset(dev);
3544 3545
}

J
Jesse Barnes 已提交
3546 3547
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3548
	struct drm_i915_private *dev_priv = to_i915(dev);
J
Jesse Barnes 已提交
3549 3550 3551 3552

	if (!dev_priv)
		return;

3553
	I915_WRITE(VLV_MASTER_IER, 0);
3554
	POSTING_READ(VLV_MASTER_IER);
3555

3556
	gen5_gt_irq_reset(dev_priv);
3557

J
Jesse Barnes 已提交
3558
	I915_WRITE(HWSTAM, 0xffffffff);
3559

3560
	spin_lock_irq(&dev_priv->irq_lock);
3561 3562
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3563
	spin_unlock_irq(&dev_priv->irq_lock);
J
Jesse Barnes 已提交
3564 3565
}

3566 3567
static void cherryview_irq_uninstall(struct drm_device *dev)
{
3568
	struct drm_i915_private *dev_priv = to_i915(dev);
3569 3570 3571 3572 3573 3574 3575

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3576
	gen8_gt_irq_reset(dev_priv);
3577

3578
	GEN5_IRQ_RESET(GEN8_PCU_);
3579

3580
	spin_lock_irq(&dev_priv->irq_lock);
3581 3582
	if (dev_priv->display_irqs_enabled)
		vlv_display_irq_reset(dev_priv);
3583
	spin_unlock_irq(&dev_priv->irq_lock);
3584 3585
}

3586
static void ironlake_irq_uninstall(struct drm_device *dev)
3587
{
3588
	struct drm_i915_private *dev_priv = to_i915(dev);
3589 3590 3591 3592

	if (!dev_priv)
		return;

P
Paulo Zanoni 已提交
3593
	ironlake_irq_reset(dev);
3594 3595
}

3596
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3597
{
3598
	struct drm_i915_private *dev_priv = to_i915(dev);
3599
	int pipe;
3600

3601
	for_each_pipe(dev_priv, pipe)
3602
		I915_WRITE(PIPESTAT(pipe), 0);
3603 3604 3605
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3606 3607 3608 3609
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3610
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3611 3612 3613 3614 3615 3616 3617 3618 3619

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3620
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
C
Chris Wilson 已提交
3621 3622 3623 3624 3625 3626 3627 3628
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3629 3630
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3631
	spin_lock_irq(&dev_priv->irq_lock);
3632 3633
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3634
	spin_unlock_irq(&dev_priv->irq_lock);
3635

C
Chris Wilson 已提交
3636 3637 3638
	return 0;
}

3639 3640 3641
/*
 * Returns true when a page flip has completed.
 */
3642
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
3643
{
3644
	struct drm_device *dev = arg;
3645
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3646 3647 3648
	u16 iir, new_iir;
	u32 pipe_stats[2];
	int pipe;
3649
	irqreturn_t ret;
C
Chris Wilson 已提交
3650

3651 3652 3653
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

3654 3655 3656 3657
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

	ret = IRQ_NONE;
C
Chris Wilson 已提交
3658 3659
	iir = I915_READ16(IIR);
	if (iir == 0)
3660
		goto out;
C
Chris Wilson 已提交
3661

3662
	while (iir) {
C
Chris Wilson 已提交
3663 3664 3665 3666 3667
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3668
		spin_lock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3669
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3670
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
C
Chris Wilson 已提交
3671

3672
		for_each_pipe(dev_priv, pipe) {
3673
			i915_reg_t reg = PIPESTAT(pipe);
C
Chris Wilson 已提交
3674 3675 3676 3677 3678
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
3679
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
3680 3681
				I915_WRITE(reg, pipe_stats[pipe]);
		}
3682
		spin_unlock(&dev_priv->irq_lock);
C
Chris Wilson 已提交
3683

3684
		I915_WRITE16(IIR, iir);
C
Chris Wilson 已提交
3685 3686 3687
		new_iir = I915_READ16(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
3688
			notify_ring(dev_priv->engine[RCS]);
C
Chris Wilson 已提交
3689

3690
		for_each_pipe(dev_priv, pipe) {
3691 3692 3693 3694
			int plane = pipe;
			if (HAS_FBC(dev_priv))
				plane = !plane;

3695 3696
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
				drm_handle_vblank(&dev_priv->drm, pipe);
C
Chris Wilson 已提交
3697

3698
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3699
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
3700

3701 3702 3703
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3704
		}
C
Chris Wilson 已提交
3705 3706 3707

		iir = new_iir;
	}
3708 3709 3710 3711
	ret = IRQ_HANDLED;

out:
	enable_rpm_wakeref_asserts(dev_priv);
C
Chris Wilson 已提交
3712

3713
	return ret;
C
Chris Wilson 已提交
3714 3715 3716 3717
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
3718
	struct drm_i915_private *dev_priv = to_i915(dev);
C
Chris Wilson 已提交
3719 3720
	int pipe;

3721
	for_each_pipe(dev_priv, pipe) {
C
Chris Wilson 已提交
3722 3723 3724 3725 3726 3727 3728 3729 3730
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

3731 3732
static void i915_irq_preinstall(struct drm_device * dev)
{
3733
	struct drm_i915_private *dev_priv = to_i915(dev);
3734 3735
	int pipe;

3736
	if (I915_HAS_HOTPLUG(dev_priv)) {
3737
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3738 3739 3740
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3741
	I915_WRITE16(HWSTAM, 0xeffe);
3742
	for_each_pipe(dev_priv, pipe)
3743 3744 3745 3746 3747 3748 3749 3750
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
3751
	struct drm_i915_private *dev_priv = to_i915(dev);
3752
	u32 enable_mask;
3753

3754 3755 3756 3757 3758 3759 3760 3761
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3762
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3763 3764 3765 3766 3767 3768 3769

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_USER_INTERRUPT;

3770
	if (I915_HAS_HOTPLUG(dev_priv)) {
3771
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3772 3773
		POSTING_READ(PORT_HOTPLUG_EN);

3774 3775 3776 3777 3778 3779 3780 3781 3782 3783
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

3784
	i915_enable_asle_pipestat(dev_priv);
3785

3786 3787
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3788
	spin_lock_irq(&dev_priv->irq_lock);
3789 3790
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3791
	spin_unlock_irq(&dev_priv->irq_lock);
3792

3793 3794 3795
	return 0;
}

3796
static irqreturn_t i915_irq_handler(int irq, void *arg)
3797
{
3798
	struct drm_device *dev = arg;
3799
	struct drm_i915_private *dev_priv = to_i915(dev);
3800
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3801
	int pipe, ret = IRQ_NONE;
3802

3803 3804 3805
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

3806 3807 3808
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

3809
	iir = I915_READ(IIR);
3810
	do {
3811
		bool irq_received = (iir) != 0;
3812
		bool blc_event = false;
3813 3814 3815 3816 3817 3818

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
3819
		spin_lock(&dev_priv->irq_lock);
3820
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3821
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
3822

3823
		for_each_pipe(dev_priv, pipe) {
3824
			i915_reg_t reg = PIPESTAT(pipe);
3825 3826
			pipe_stats[pipe] = I915_READ(reg);

3827
			/* Clear the PIPE*STAT regs before the IIR */
3828 3829
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
3830
				irq_received = true;
3831 3832
			}
		}
3833
		spin_unlock(&dev_priv->irq_lock);
3834 3835 3836 3837 3838

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
3839
		if (I915_HAS_HOTPLUG(dev_priv) &&
3840 3841 3842
		    iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
3843
				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
3844
		}
3845

3846
		I915_WRITE(IIR, iir);
3847 3848 3849
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
3850
			notify_ring(dev_priv->engine[RCS]);
3851

3852
		for_each_pipe(dev_priv, pipe) {
3853 3854 3855 3856
			int plane = pipe;
			if (HAS_FBC(dev_priv))
				plane = !plane;

3857 3858
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
				drm_handle_vblank(&dev_priv->drm, pipe);
3859 3860 3861

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
3862 3863

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3864
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
3865

3866 3867 3868
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv,
								    pipe);
3869 3870 3871
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
3872
			intel_opregion_asle_intr(dev_priv);
3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
3889
		ret = IRQ_HANDLED;
3890
		iir = new_iir;
3891
	} while (iir);
3892

3893 3894
	enable_rpm_wakeref_asserts(dev_priv);

3895 3896 3897 3898 3899
	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
3900
	struct drm_i915_private *dev_priv = to_i915(dev);
3901 3902
	int pipe;

3903
	if (I915_HAS_HOTPLUG(dev_priv)) {
3904
		i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3905 3906 3907
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3908
	I915_WRITE16(HWSTAM, 0xffff);
3909
	for_each_pipe(dev_priv, pipe) {
3910
		/* Clear enable bits; then clear status bits */
3911
		I915_WRITE(PIPESTAT(pipe), 0);
3912 3913
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
3914 3915 3916 3917 3918 3919 3920 3921
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
3922
	struct drm_i915_private *dev_priv = to_i915(dev);
3923 3924
	int pipe;

3925
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3926
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3927 3928

	I915_WRITE(HWSTAM, 0xeffe);
3929
	for_each_pipe(dev_priv, pipe)
3930 3931 3932 3933 3934 3935 3936 3937
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
3938
	struct drm_i915_private *dev_priv = to_i915(dev);
3939
	u32 enable_mask;
3940 3941 3942
	u32 error_mask;

	/* Unmask the interrupts that we always want on. */
3943
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
3944
			       I915_DISPLAY_PORT_INTERRUPT |
3945 3946 3947 3948 3949 3950 3951
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
3952 3953
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
3954 3955
	enable_mask |= I915_USER_INTERRUPT;

3956
	if (IS_G4X(dev_priv))
3957
		enable_mask |= I915_BSD_USER_INTERRUPT;
3958

3959 3960
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
3961
	spin_lock_irq(&dev_priv->irq_lock);
3962 3963 3964
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3965
	spin_unlock_irq(&dev_priv->irq_lock);
3966 3967 3968 3969 3970

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
3971
	if (IS_G4X(dev_priv)) {
3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

3986
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
3987 3988
	POSTING_READ(PORT_HOTPLUG_EN);

3989
	i915_enable_asle_pipestat(dev_priv);
3990 3991 3992 3993

	return 0;
}

3994
static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
3995 3996 3997
{
	u32 hotplug_en;

3998
	lockdep_assert_held(&dev_priv->irq_lock);
3999

4000 4001
	/* Note HDMI and DP share hotplug bits */
	/* enable bits are the same for all generations */
4002
	hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
4003 4004 4005 4006
	/* Programming the CRT detection parameters tends
	   to generate a spurious hotplug event about three
	   seconds later.  So just do it once.
	*/
4007
	if (IS_G4X(dev_priv))
4008 4009 4010 4011
		hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
	hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;

	/* Ignore TV since it's buggy */
4012
	i915_hotplug_interrupt_update_locked(dev_priv,
4013 4014 4015 4016
					     HOTPLUG_INT_EN_MASK |
					     CRT_HOTPLUG_VOLTAGE_COMPARE_MASK |
					     CRT_HOTPLUG_ACTIVATION_PERIOD_64,
					     hotplug_en);
4017 4018
}

4019
static irqreturn_t i965_irq_handler(int irq, void *arg)
4020
{
4021
	struct drm_device *dev = arg;
4022
	struct drm_i915_private *dev_priv = to_i915(dev);
4023 4024 4025 4026
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	int ret = IRQ_NONE, pipe;

4027 4028 4029
	if (!intel_irqs_enabled(dev_priv))
		return IRQ_NONE;

4030 4031 4032
	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
	disable_rpm_wakeref_asserts(dev_priv);

4033 4034 4035
	iir = I915_READ(IIR);

	for (;;) {
4036
		bool irq_received = (iir) != 0;
4037 4038
		bool blc_event = false;

4039 4040 4041 4042 4043
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
4044
		spin_lock(&dev_priv->irq_lock);
4045
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4046
			DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
4047

4048
		for_each_pipe(dev_priv, pipe) {
4049
			i915_reg_t reg = PIPESTAT(pipe);
4050 4051 4052 4053 4054 4055 4056
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4057
				irq_received = true;
4058 4059
			}
		}
4060
		spin_unlock(&dev_priv->irq_lock);
4061 4062 4063 4064 4065 4066 4067

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4068 4069 4070
		if (iir & I915_DISPLAY_PORT_INTERRUPT) {
			u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
			if (hotplug_status)
4071
				i9xx_hpd_irq_handler(dev_priv, hotplug_status);
4072
		}
4073

4074
		I915_WRITE(IIR, iir);
4075 4076 4077
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
4078
			notify_ring(dev_priv->engine[RCS]);
4079
		if (iir & I915_BSD_USER_INTERRUPT)
4080
			notify_ring(dev_priv->engine[VCS]);
4081

4082
		for_each_pipe(dev_priv, pipe) {
4083 4084
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
				drm_handle_vblank(&dev_priv->drm, pipe);
4085 4086 4087

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4088 4089

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4090
				i9xx_pipe_crc_irq_handler(dev_priv, pipe);
4091

4092 4093
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
				intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
4094
		}
4095 4096

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
4097
			intel_opregion_asle_intr(dev_priv);
4098

4099
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4100
			gmbus_irq_handler(dev_priv);
4101

4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

4120 4121
	enable_rpm_wakeref_asserts(dev_priv);

4122 4123 4124 4125 4126
	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4127
	struct drm_i915_private *dev_priv = to_i915(dev);
4128 4129 4130 4131 4132
	int pipe;

	if (!dev_priv)
		return;

4133
	i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
4134
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4135 4136

	I915_WRITE(HWSTAM, 0xffffffff);
4137
	for_each_pipe(dev_priv, pipe)
4138 4139 4140 4141
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

4142
	for_each_pipe(dev_priv, pipe)
4143 4144 4145 4146 4147
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4148 4149 4150 4151 4152 4153 4154
/**
 * intel_irq_init - initializes irq support
 * @dev_priv: i915 device instance
 *
 * This function initializes all the irq support including work items, timers
 * and all the vtables. It does not setup the interrupt itself though.
 */
4155
void intel_irq_init(struct drm_i915_private *dev_priv)
4156
{
4157
	struct drm_device *dev = &dev_priv->drm;
4158
	int i;
4159

4160 4161
	intel_hpd_init_work(dev_priv);

4162
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4163

4164
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4165 4166
	for (i = 0; i < MAX_L3_SLICES; ++i)
		dev_priv->l3_parity.remap_info[i] = NULL;
4167

4168
	if (HAS_GUC_SCHED(dev_priv))
4169 4170
		dev_priv->pm_guc_events = GEN9_GUC_TO_HOST_INT_EVENT;

4171
	/* Let's track the enabled rps events */
4172
	if (IS_VALLEYVIEW(dev_priv))
4173
		/* WaGsvRC0ResidencyMethod:vlv */
4174
		dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
4175 4176
	else
		dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
4177

4178
	dev_priv->rps.pm_intrmsk_mbz = 0;
4179 4180

	/*
4181
	 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
4182 4183 4184 4185
	 * if GEN6_PM_UP_EI_EXPIRED is masked.
	 *
	 * TODO: verify if this can be reproduced on VLV,CHV.
	 */
4186
	if (INTEL_GEN(dev_priv) <= 7)
4187
		dev_priv->rps.pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
4188

4189
	if (INTEL_GEN(dev_priv) >= 8)
4190
		dev_priv->rps.pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
4191

4192
	if (IS_GEN2(dev_priv)) {
4193
		/* Gen2 doesn't have a hardware frame counter */
4194
		dev->max_vblank_count = 0;
4195
	} else if (IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
4196
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4197
		dev->driver->get_vblank_counter = g4x_get_vblank_counter;
4198 4199 4200
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4201 4202
	}

4203 4204 4205 4206 4207
	/*
	 * Opt out of the vblank disable timer on everything except gen2.
	 * Gen2 doesn't have a hardware frame counter and so depends on
	 * vblank interrupts to produce sane vblank seuquence numbers.
	 */
4208
	if (!IS_GEN2(dev_priv))
4209 4210
		dev->vblank_disable_immediate = true;

4211 4212 4213 4214 4215 4216 4217 4218 4219 4220
	/* Most platforms treat the display irq block as an always-on
	 * power domain. vlv/chv can disable it at runtime and need
	 * special care to avoid writing any of the display block registers
	 * outside of the power domain. We defer setting up the display irqs
	 * in this case to the runtime pm.
	 */
	dev_priv->display_irqs_enabled = true;
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
		dev_priv->display_irqs_enabled = false;

L
Lyude 已提交
4221 4222
	dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;

4223
	dev->driver->get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos;
4224
	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4225

4226
	if (IS_CHERRYVIEW(dev_priv)) {
4227 4228 4229 4230
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
4231 4232
		dev->driver->enable_vblank = i965_enable_vblank;
		dev->driver->disable_vblank = i965_disable_vblank;
4233
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4234
	} else if (IS_VALLEYVIEW(dev_priv)) {
J
Jesse Barnes 已提交
4235 4236 4237 4238
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
4239 4240
		dev->driver->enable_vblank = i965_enable_vblank;
		dev->driver->disable_vblank = i965_disable_vblank;
4241
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4242
	} else if (INTEL_GEN(dev_priv) >= 8) {
4243
		dev->driver->irq_handler = gen8_irq_handler;
4244
		dev->driver->irq_preinstall = gen8_irq_reset;
4245 4246 4247 4248
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
4249
		if (IS_GEN9_LP(dev_priv))
4250
			dev_priv->display.hpd_irq_setup = bxt_hpd_irq_setup;
4251 4252
		else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv) ||
			 HAS_PCH_CNP(dev_priv))
4253 4254
			dev_priv->display.hpd_irq_setup = spt_hpd_irq_setup;
		else
4255
			dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4256
	} else if (HAS_PCH_SPLIT(dev_priv)) {
4257
		dev->driver->irq_handler = ironlake_irq_handler;
4258
		dev->driver->irq_preinstall = ironlake_irq_reset;
4259 4260 4261 4262
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4263
		dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
4264
	} else {
4265
		if (IS_GEN2(dev_priv)) {
C
Chris Wilson 已提交
4266 4267 4268 4269
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4270 4271
			dev->driver->enable_vblank = i8xx_enable_vblank;
			dev->driver->disable_vblank = i8xx_disable_vblank;
4272
		} else if (IS_GEN3(dev_priv)) {
4273 4274 4275 4276
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
4277 4278
			dev->driver->enable_vblank = i8xx_enable_vblank;
			dev->driver->disable_vblank = i8xx_disable_vblank;
C
Chris Wilson 已提交
4279
		} else {
4280 4281 4282 4283
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
4284 4285
			dev->driver->enable_vblank = i965_enable_vblank;
			dev->driver->disable_vblank = i965_disable_vblank;
C
Chris Wilson 已提交
4286
		}
4287 4288
		if (I915_HAS_HOTPLUG(dev_priv))
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4289 4290
	}
}
4291

4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305
/**
 * intel_irq_fini - deinitializes IRQ support
 * @i915: i915 device instance
 *
 * This function deinitializes all the IRQ support.
 */
void intel_irq_fini(struct drm_i915_private *i915)
{
	int i;

	for (i = 0; i < MAX_L3_SLICES; ++i)
		kfree(i915->l3_parity.remap_info[i]);
}

4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316
/**
 * intel_irq_install - enables the hardware interrupt
 * @dev_priv: i915 device instance
 *
 * This function enables the hardware interrupt handling, but leaves the hotplug
 * handling still disabled. It is called after intel_irq_init().
 *
 * In the driver load and resume code we need working interrupts in a few places
 * but don't want to deal with the hassle of concurrent probe and hotplug
 * workers. Hence the split into this two-stage approach.
 */
4317 4318 4319 4320 4321 4322 4323 4324 4325
int intel_irq_install(struct drm_i915_private *dev_priv)
{
	/*
	 * We enable some interrupt sources in our postinstall hooks, so mark
	 * interrupts as enabled _before_ actually enabling them to avoid
	 * special cases in our ordering checks.
	 */
	dev_priv->pm.irqs_enabled = true;

4326
	return drm_irq_install(&dev_priv->drm, dev_priv->drm.pdev->irq);
4327 4328
}

4329 4330 4331 4332 4333 4334 4335
/**
 * intel_irq_uninstall - finilizes all irq handling
 * @dev_priv: i915 device instance
 *
 * This stops interrupt and hotplug handling and unregisters and frees all
 * resources acquired in the init functions.
 */
4336 4337
void intel_irq_uninstall(struct drm_i915_private *dev_priv)
{
4338
	drm_irq_uninstall(&dev_priv->drm);
4339 4340 4341 4342
	intel_hpd_cancel_work(dev_priv);
	dev_priv->pm.irqs_enabled = false;
}

4343 4344 4345 4346 4347 4348 4349
/**
 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
 * @dev_priv: i915 device instance
 *
 * This function is used to disable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4350
void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
4351
{
4352
	dev_priv->drm.driver->irq_uninstall(&dev_priv->drm);
4353
	dev_priv->pm.irqs_enabled = false;
4354
	synchronize_irq(dev_priv->drm.irq);
4355 4356
}

4357 4358 4359 4360 4361 4362 4363
/**
 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
 * @dev_priv: i915 device instance
 *
 * This function is used to enable interrupts at runtime, both in the runtime
 * pm and the system suspend/resume code.
 */
4364
void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
4365
{
4366
	dev_priv->pm.irqs_enabled = true;
4367 4368
	dev_priv->drm.driver->irq_preinstall(&dev_priv->drm);
	dev_priv->drm.driver->irq_postinstall(&dev_priv->drm);
4369
}