i915_irq.c 123.7 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29 30
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

31
#include <linux/sysrq.h>
32
#include <linux/slab.h>
33
#include <linux/circ_buf.h>
34 35
#include <drm/drmP.h>
#include <drm/i915_drm.h>
L
Linus Torvalds 已提交
36
#include "i915_drv.h"
C
Chris Wilson 已提交
37
#include "i915_trace.h"
J
Jesse Barnes 已提交
38
#include "intel_drv.h"
L
Linus Torvalds 已提交
39

40 41 42 43 44 45 46 47 48 49
static const u32 hpd_ibx[] = {
	[HPD_CRT] = SDE_CRT_HOTPLUG,
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG
};

static const u32 hpd_cpt[] = {
	[HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
50
	[HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
51 52 53 54 55 56 57 58 59 60 61 62 63 64
	[HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
	[HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
	[HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
};

static const u32 hpd_mask_i915[] = {
	[HPD_CRT] = CRT_HOTPLUG_INT_EN,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
};

65
static const u32 hpd_status_g4x[] = {
66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
	[HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
	[HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
	[HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
	[HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
	[HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
	[HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
};

83
/* IIR can theoretically queue up two events. Be paranoid. */
84
#define GEN8_IRQ_RESET_NDX(type, which) do { \
85 86 87 88 89 90 91 92 93
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IMR(which)); \
	I915_WRITE(GEN8_##type##_IER(which), 0); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
	POSTING_READ(GEN8_##type##_IIR(which)); \
} while (0)

94
#define GEN5_IRQ_RESET(type) do { \
P
Paulo Zanoni 已提交
95
	I915_WRITE(type##IMR, 0xffffffff); \
96
	POSTING_READ(type##IMR); \
P
Paulo Zanoni 已提交
97
	I915_WRITE(type##IER, 0); \
98 99 100 101
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
	I915_WRITE(type##IIR, 0xffffffff); \
	POSTING_READ(type##IIR); \
P
Paulo Zanoni 已提交
102 103
} while (0)

104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
/*
 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
 */
#define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
	u32 val = I915_READ(reg); \
	if (val) { \
		WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
		     (reg), val); \
		I915_WRITE((reg), 0xffffffff); \
		POSTING_READ(reg); \
		I915_WRITE((reg), 0xffffffff); \
		POSTING_READ(reg); \
	} \
} while (0)

P
Paulo Zanoni 已提交
119
#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
120
	GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
P
Paulo Zanoni 已提交
121 122 123 124 125 126
	I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
	I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
	POSTING_READ(GEN8_##type##_IER(which)); \
} while (0)

#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
127
	GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
P
Paulo Zanoni 已提交
128 129 130 131 132
	I915_WRITE(type##IMR, (imr_val)); \
	I915_WRITE(type##IER, (ier_val)); \
	POSTING_READ(type##IER); \
} while (0)

133
/* For display hotplug interrupt */
134
static void
135
ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
136
{
137 138
	assert_spin_locked(&dev_priv->irq_lock);

139
	if (WARN_ON(dev_priv->pm.irqs_disabled))
140 141
		return;

142 143 144
	if ((dev_priv->irq_mask & mask) != 0) {
		dev_priv->irq_mask &= ~mask;
		I915_WRITE(DEIMR, dev_priv->irq_mask);
145
		POSTING_READ(DEIMR);
146 147 148
	}
}

149
static void
150
ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
151
{
152 153
	assert_spin_locked(&dev_priv->irq_lock);

154
	if (WARN_ON(dev_priv->pm.irqs_disabled))
155 156
		return;

157 158 159
	if ((dev_priv->irq_mask & mask) != mask) {
		dev_priv->irq_mask |= mask;
		I915_WRITE(DEIMR, dev_priv->irq_mask);
160
		POSTING_READ(DEIMR);
161 162 163
	}
}

P
Paulo Zanoni 已提交
164 165 166 167 168 169 170 171 172 173 174 175
/**
 * ilk_update_gt_irq - update GTIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
	assert_spin_locked(&dev_priv->irq_lock);

176
	if (WARN_ON(dev_priv->pm.irqs_disabled))
177 178
		return;

P
Paulo Zanoni 已提交
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
	dev_priv->gt_irq_mask &= ~interrupt_mask;
	dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
	POSTING_READ(GTIMR);
}

void ilk_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	ilk_update_gt_irq(dev_priv, mask, mask);
}

void ilk_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	ilk_update_gt_irq(dev_priv, mask, 0);
}

P
Paulo Zanoni 已提交
195 196 197 198 199 200 201 202 203 204
/**
  * snb_update_pm_irq - update GEN6_PMIMR
  * @dev_priv: driver private
  * @interrupt_mask: mask of interrupt bits to update
  * @enabled_irq_mask: mask of interrupt bits to enable
  */
static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
205
	uint32_t new_val;
P
Paulo Zanoni 已提交
206 207 208

	assert_spin_locked(&dev_priv->irq_lock);

209
	if (WARN_ON(dev_priv->pm.irqs_disabled))
210 211
		return;

212
	new_val = dev_priv->pm_irq_mask;
213 214 215
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

216 217 218
	if (new_val != dev_priv->pm_irq_mask) {
		dev_priv->pm_irq_mask = new_val;
		I915_WRITE(GEN6_PMIMR, dev_priv->pm_irq_mask);
219 220
		POSTING_READ(GEN6_PMIMR);
	}
P
Paulo Zanoni 已提交
221 222 223 224 225 226 227 228 229 230 231 232
}

void snb_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	snb_update_pm_irq(dev_priv, mask, mask);
}

void snb_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	snb_update_pm_irq(dev_priv, mask, 0);
}

233 234 235 236 237 238
static bool ivb_can_enable_err_int(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_crtc *crtc;
	enum pipe pipe;

239 240
	assert_spin_locked(&dev_priv->irq_lock);

241 242 243 244 245 246 247 248 249 250
	for_each_pipe(pipe) {
		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);

		if (crtc->cpu_fifo_underrun_disabled)
			return false;
	}

	return true;
}

251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
/**
  * bdw_update_pm_irq - update GT interrupt 2
  * @dev_priv: driver private
  * @interrupt_mask: mask of interrupt bits to update
  * @enabled_irq_mask: mask of interrupt bits to enable
  *
  * Copied from the snb function, updated with relevant register offsets
  */
static void bdw_update_pm_irq(struct drm_i915_private *dev_priv,
			      uint32_t interrupt_mask,
			      uint32_t enabled_irq_mask)
{
	uint32_t new_val;

	assert_spin_locked(&dev_priv->irq_lock);

	if (WARN_ON(dev_priv->pm.irqs_disabled))
		return;

	new_val = dev_priv->pm_irq_mask;
	new_val &= ~interrupt_mask;
	new_val |= (~enabled_irq_mask & interrupt_mask);

	if (new_val != dev_priv->pm_irq_mask) {
		dev_priv->pm_irq_mask = new_val;
		I915_WRITE(GEN8_GT_IMR(2), dev_priv->pm_irq_mask);
		POSTING_READ(GEN8_GT_IMR(2));
	}
}

void bdw_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	bdw_update_pm_irq(dev_priv, mask, mask);
}

void bdw_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
{
	bdw_update_pm_irq(dev_priv, mask, 0);
}

291 292 293 294 295 296
static bool cpt_can_enable_serr_int(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	enum pipe pipe;
	struct intel_crtc *crtc;

297 298
	assert_spin_locked(&dev_priv->irq_lock);

299 300 301 302 303 304 305 306 307 308
	for_each_pipe(pipe) {
		crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);

		if (crtc->pch_fifo_underrun_disabled)
			return false;
	}

	return true;
}

309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
void i9xx_check_fifo_underruns(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_crtc *crtc;
	unsigned long flags;

	spin_lock_irqsave(&dev_priv->irq_lock, flags);

	for_each_intel_crtc(dev, crtc) {
		u32 reg = PIPESTAT(crtc->pipe);
		u32 pipestat;

		if (crtc->cpu_fifo_underrun_disabled)
			continue;

		pipestat = I915_READ(reg) & 0xffff0000;
		if ((pipestat & PIPE_FIFO_UNDERRUN_STATUS) == 0)
			continue;

		I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS);
		POSTING_READ(reg);

		DRM_ERROR("pipe %c underrun\n", pipe_name(crtc->pipe));
	}

	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
}

337 338
static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev,
					     enum pipe pipe, bool enable)
339 340 341
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 reg = PIPESTAT(pipe);
342
	u32 pipestat = I915_READ(reg) & 0xffff0000;
343 344 345

	assert_spin_locked(&dev_priv->irq_lock);

346 347 348 349 350 351 352
	if (enable) {
		I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS);
		POSTING_READ(reg);
	} else {
		if (pipestat & PIPE_FIFO_UNDERRUN_STATUS)
			DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
	}
353 354
}

355 356 357 358 359 360 361 362 363 364 365 366 367 368
static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
						 enum pipe pipe, bool enable)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
					  DE_PIPEB_FIFO_UNDERRUN;

	if (enable)
		ironlake_enable_display_irq(dev_priv, bit);
	else
		ironlake_disable_display_irq(dev_priv, bit);
}

static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
369
						  enum pipe pipe, bool enable)
370 371 372
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	if (enable) {
373 374
		I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe));

375 376 377 378 379 380
		if (!ivb_can_enable_err_int(dev))
			return;

		ironlake_enable_display_irq(dev_priv, DE_ERR_INT_IVB);
	} else {
		ironlake_disable_display_irq(dev_priv, DE_ERR_INT_IVB);
381

382
		if (I915_READ(GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe)) {
383 384
			DRM_ERROR("uncleared fifo underrun on pipe %c\n",
				  pipe_name(pipe));
385
		}
386 387 388
	}
}

389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
static void broadwell_set_fifo_underrun_reporting(struct drm_device *dev,
						  enum pipe pipe, bool enable)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	assert_spin_locked(&dev_priv->irq_lock);

	if (enable)
		dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_FIFO_UNDERRUN;
	else
		dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_FIFO_UNDERRUN;
	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
}

404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419
/**
 * ibx_display_interrupt_update - update SDEIMR
 * @dev_priv: driver private
 * @interrupt_mask: mask of interrupt bits to update
 * @enabled_irq_mask: mask of interrupt bits to enable
 */
static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
					 uint32_t interrupt_mask,
					 uint32_t enabled_irq_mask)
{
	uint32_t sdeimr = I915_READ(SDEIMR);
	sdeimr &= ~interrupt_mask;
	sdeimr |= (~enabled_irq_mask & interrupt_mask);

	assert_spin_locked(&dev_priv->irq_lock);

420
	if (WARN_ON(dev_priv->pm.irqs_disabled))
421 422
		return;

423 424 425 426 427 428 429 430
	I915_WRITE(SDEIMR, sdeimr);
	POSTING_READ(SDEIMR);
}
#define ibx_enable_display_interrupt(dev_priv, bits) \
	ibx_display_interrupt_update((dev_priv), (bits), (bits))
#define ibx_disable_display_interrupt(dev_priv, bits) \
	ibx_display_interrupt_update((dev_priv), (bits), 0)

431 432
static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
					    enum transcoder pch_transcoder,
433 434 435
					    bool enable)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
436 437
	uint32_t bit = (pch_transcoder == TRANSCODER_A) ?
		       SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER;
438 439

	if (enable)
440
		ibx_enable_display_interrupt(dev_priv, bit);
441
	else
442
		ibx_disable_display_interrupt(dev_priv, bit);
443 444 445 446 447 448 449 450 451
}

static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
					    enum transcoder pch_transcoder,
					    bool enable)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (enable) {
452 453 454
		I915_WRITE(SERR_INT,
			   SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder));

455 456 457
		if (!cpt_can_enable_serr_int(dev))
			return;

458
		ibx_enable_display_interrupt(dev_priv, SDE_ERROR_CPT);
459
	} else {
460
		ibx_disable_display_interrupt(dev_priv, SDE_ERROR_CPT);
461

462
		if (I915_READ(SERR_INT) & SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder)) {
463 464
			DRM_ERROR("uncleared pch fifo underrun on pch transcoder %c\n",
				  transcoder_name(pch_transcoder));
465
		}
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
	}
}

/**
 * intel_set_cpu_fifo_underrun_reporting - enable/disable FIFO underrun messages
 * @dev: drm device
 * @pipe: pipe
 * @enable: true if we want to report FIFO underrun errors, false otherwise
 *
 * This function makes us disable or enable CPU fifo underruns for a specific
 * pipe. Notice that on some Gens (e.g. IVB, HSW), disabling FIFO underrun
 * reporting for one pipe may also disable all the other CPU error interruts for
 * the other pipes, due to the fact that there's just one interrupt mask/enable
 * bit for all the pipes.
 *
 * Returns the previous state of underrun reporting.
 */
483 484
static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
						    enum pipe pipe, bool enable)
485 486 487 488 489 490
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
	bool ret;

491 492
	assert_spin_locked(&dev_priv->irq_lock);

493 494 495 496 497 498 499
	ret = !intel_crtc->cpu_fifo_underrun_disabled;

	if (enable == ret)
		goto done;

	intel_crtc->cpu_fifo_underrun_disabled = !enable;

500 501
	if (INTEL_INFO(dev)->gen < 5 || IS_VALLEYVIEW(dev))
		i9xx_set_fifo_underrun_reporting(dev, pipe, enable);
502
	else if (IS_GEN5(dev) || IS_GEN6(dev))
503 504
		ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
	else if (IS_GEN7(dev))
505
		ivybridge_set_fifo_underrun_reporting(dev, pipe, enable);
506 507
	else if (IS_GEN8(dev))
		broadwell_set_fifo_underrun_reporting(dev, pipe, enable);
508 509

done:
510 511 512 513 514 515 516 517 518 519 520 521
	return ret;
}

bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
					   enum pipe pipe, bool enable)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long flags;
	bool ret;

	spin_lock_irqsave(&dev_priv->irq_lock, flags);
	ret = __intel_set_cpu_fifo_underrun_reporting(dev, pipe, enable);
522
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
523

524 525 526
	return ret;
}

527 528 529 530 531 532 533 534 535 536
static bool __cpu_fifo_underrun_reporting_enabled(struct drm_device *dev,
						  enum pipe pipe)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);

	return !intel_crtc->cpu_fifo_underrun_disabled;
}

537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
/**
 * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
 * @dev: drm device
 * @pch_transcoder: the PCH transcoder (same as pipe on IVB and older)
 * @enable: true if we want to report FIFO underrun errors, false otherwise
 *
 * This function makes us disable or enable PCH fifo underruns for a specific
 * PCH transcoder. Notice that on some PCHs (e.g. CPT/PPT), disabling FIFO
 * underrun reporting for one transcoder may also disable all the other PCH
 * error interruts for the other transcoders, due to the fact that there's just
 * one interrupt mask/enable bit for all the transcoders.
 *
 * Returns the previous state of underrun reporting.
 */
bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
					   enum transcoder pch_transcoder,
					   bool enable)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
556 557
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pch_transcoder];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
558 559 560
	unsigned long flags;
	bool ret;

561 562 563 564 565 566 567 568
	/*
	 * NOTE: Pre-LPT has a fixed cpu pipe -> pch transcoder mapping, but LPT
	 * has only one pch transcoder A that all pipes can use. To avoid racy
	 * pch transcoder -> pipe lookups from interrupt code simply store the
	 * underrun statistics in crtc A. Since we never expose this anywhere
	 * nor use it outside of the fifo underrun code here using the "wrong"
	 * crtc on LPT won't cause issues.
	 */
569 570 571 572 573 574 575 576 577 578 579

	spin_lock_irqsave(&dev_priv->irq_lock, flags);

	ret = !intel_crtc->pch_fifo_underrun_disabled;

	if (enable == ret)
		goto done;

	intel_crtc->pch_fifo_underrun_disabled = !enable;

	if (HAS_PCH_IBX(dev))
580
		ibx_set_fifo_underrun_reporting(dev, pch_transcoder, enable);
581 582 583 584 585 586 587 588 589
	else
		cpt_set_fifo_underrun_reporting(dev, pch_transcoder, enable);

done:
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
	return ret;
}


D
Daniel Vetter 已提交
590
static void
591 592
__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		       u32 enable_mask, u32 status_mask)
593
{
594
	u32 reg = PIPESTAT(pipe);
595
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
596

597 598
	assert_spin_locked(&dev_priv->irq_lock);

599 600 601 602
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
603 604 605
		return;

	if ((pipestat & enable_mask) == enable_mask)
606 607
		return;

608 609
	dev_priv->pipestat_irq_mask[pipe] |= status_mask;

610
	/* Enable the interrupt, clear any pending status */
611
	pipestat |= enable_mask | status_mask;
612 613
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
614 615
}

D
Daniel Vetter 已提交
616
static void
617 618
__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		        u32 enable_mask, u32 status_mask)
619
{
620
	u32 reg = PIPESTAT(pipe);
621
	u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
622

623 624
	assert_spin_locked(&dev_priv->irq_lock);

625 626 627 628
	if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
		      status_mask & ~PIPESTAT_INT_STATUS_MASK,
		      "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
		      pipe_name(pipe), enable_mask, status_mask))
629 630
		return;

631 632 633
	if ((pipestat & enable_mask) == 0)
		return;

634 635
	dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;

636
	pipestat &= ~enable_mask;
637 638
	I915_WRITE(reg, pipestat);
	POSTING_READ(reg);
639 640
}

641 642 643 644 645
static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
{
	u32 enable_mask = status_mask << 16;

	/*
646 647
	 * On pipe A we don't support the PSR interrupt yet,
	 * on pipe B and C the same bit MBZ.
648 649 650
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
		return 0;
651 652 653 654 655 656
	/*
	 * On pipe B and C we don't support the PSR interrupt yet, on pipe
	 * A the same bit is for perf counters which we don't use either.
	 */
	if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
		return 0;
657 658 659 660 661 662 663 664 665 666 667 668

	enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
			 SPRITE0_FLIP_DONE_INT_EN_VLV |
			 SPRITE1_FLIP_DONE_INT_EN_VLV);
	if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
	if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
		enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;

	return enable_mask;
}

669 670 671 672 673 674
void
i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		     u32 status_mask)
{
	u32 enable_mask;

675 676 677 678 679
	if (IS_VALLEYVIEW(dev_priv->dev))
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
680 681 682 683 684 685 686 687 688
	__i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

void
i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
		      u32 status_mask)
{
	u32 enable_mask;

689 690 691 692 693
	if (IS_VALLEYVIEW(dev_priv->dev))
		enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
							   status_mask);
	else
		enable_mask = status_mask << 16;
694 695 696
	__i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
}

697
/**
698
 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
699
 */
700
static void i915_enable_asle_pipestat(struct drm_device *dev)
701
{
702
	struct drm_i915_private *dev_priv = dev->dev_private;
703 704
	unsigned long irqflags;

705 706 707
	if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
		return;

708
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
709

710
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
711
	if (INTEL_INFO(dev)->gen >= 4)
712
		i915_enable_pipestat(dev_priv, PIPE_A,
713
				     PIPE_LEGACY_BLC_EVENT_STATUS);
714 715

	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
716 717
}

718 719 720 721 722 723 724 725 726 727 728 729
/**
 * i915_pipe_enabled - check if a pipe is enabled
 * @dev: DRM device
 * @pipe: pipe to check
 *
 * Reading certain registers when the pipe is disabled can hang the chip.
 * Use this routine to make sure the PLL is running and the pipe is active
 * before reading such registers if unsure.
 */
static int
i915_pipe_enabled(struct drm_device *dev, int pipe)
{
730
	struct drm_i915_private *dev_priv = dev->dev_private;
731

732 733 734 735
	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
		/* Locking is horribly broken here, but whatever. */
		struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
		struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
736

737 738 739 740
		return intel_crtc->active;
	} else {
		return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
	}
741 742
}

743 744 745 746 747 748
static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
{
	/* Gen2 doesn't have a hardware frame counter */
	return 0;
}

749 750 751
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
752
static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
753
{
754
	struct drm_i915_private *dev_priv = dev->dev_private;
755 756
	unsigned long high_frame;
	unsigned long low_frame;
757
	u32 high1, high2, low, pixel, vbl_start;
758 759

	if (!i915_pipe_enabled(dev, pipe)) {
760
		DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
761
				"pipe %c\n", pipe_name(pipe));
762 763 764
		return 0;
	}

765 766 767 768 769 770 771 772
	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
		struct intel_crtc *intel_crtc =
			to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
		const struct drm_display_mode *mode =
			&intel_crtc->config.adjusted_mode;

		vbl_start = mode->crtc_vblank_start * mode->crtc_htotal;
	} else {
773
		enum transcoder cpu_transcoder = (enum transcoder) pipe;
774 775 776 777 778 779 780 781
		u32 htotal;

		htotal = ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff) + 1;
		vbl_start = (I915_READ(VBLANK(cpu_transcoder)) & 0x1fff) + 1;

		vbl_start *= htotal;
	}

782 783
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
784

785 786 787 788 789 790
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
791
		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
792
		low   = I915_READ(low_frame);
793
		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
794 795
	} while (high1 != high2);

796
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
797
	pixel = low & PIPE_PIXEL_MASK;
798
	low >>= PIPE_FRAME_LOW_SHIFT;
799 800 801 802 803 804

	/*
	 * The frame counter increments at beginning of active.
	 * Cook up a vblank counter by also checking the pixel
	 * counter against vblank start.
	 */
805
	return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
806 807
}

808
static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
809
{
810
	struct drm_i915_private *dev_priv = dev->dev_private;
811
	int reg = PIPE_FRMCOUNT_GM45(pipe);
812 813

	if (!i915_pipe_enabled(dev, pipe)) {
814
		DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
815
				 "pipe %c\n", pipe_name(pipe));
816 817 818 819 820 821
		return 0;
	}

	return I915_READ(reg);
}

822 823 824
/* raw reads, only for fast reads of display block, no need for forcewake etc. */
#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))

825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_device *dev = crtc->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	const struct drm_display_mode *mode = &crtc->config.adjusted_mode;
	enum pipe pipe = crtc->pipe;
	int vtotal = mode->crtc_vtotal;
	int position;

	if (mode->flags & DRM_MODE_FLAG_INTERLACE)
		vtotal /= 2;

	if (IS_GEN2(dev))
		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
	else
		position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;

	/*
	 * Scanline counter increments at leading edge of hsync, and
	 * it starts counting from vtotal-1 on the first active line.
	 * That means the scanline counter value is always one less
	 * than what we would expect. Ie. just after start of vblank,
	 * which also occurs at start of hsync (on the last active line),
	 * the scanline counter will read vblank_start-1.
	 */
	return (position + 1) % vtotal;
}

853
static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
854 855
				    unsigned int flags, int *vpos, int *hpos,
				    ktime_t *stime, ktime_t *etime)
856
{
857 858 859 860
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
	const struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
861
	int position;
862
	int vbl_start, vbl_end, hsync_start, htotal, vtotal;
863 864
	bool in_vbl = true;
	int ret = 0;
865
	unsigned long irqflags;
866

867
	if (!intel_crtc->active) {
868
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
869
				 "pipe %c\n", pipe_name(pipe));
870 871 872
		return 0;
	}

873
	htotal = mode->crtc_htotal;
874
	hsync_start = mode->crtc_hsync_start;
875 876 877
	vtotal = mode->crtc_vtotal;
	vbl_start = mode->crtc_vblank_start;
	vbl_end = mode->crtc_vblank_end;
878

879 880 881 882 883 884
	if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
		vbl_start = DIV_ROUND_UP(vbl_start, 2);
		vbl_end /= 2;
		vtotal /= 2;
	}

885 886
	ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

887 888 889 890 891 892
	/*
	 * Lock uncore.lock, as we will do multiple timing critical raw
	 * register reads, potentially with preemption disabled, so the
	 * following code must not block on uncore.lock.
	 */
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
893

894 895 896 897 898 899
	/* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */

	/* Get optional system timestamp before query. */
	if (stime)
		*stime = ktime_get();

900
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
901 902 903
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
904
		position = __intel_get_crtc_scanline(intel_crtc);
905 906 907 908 909
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
910
		position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
911

912 913 914 915
		/* convert to pixel counts */
		vbl_start *= htotal;
		vbl_end *= htotal;
		vtotal *= htotal;
916

917 918 919 920 921 922 923 924 925 926 927 928
		/*
		 * In interlaced modes, the pixel counter counts all pixels,
		 * so one field will have htotal more pixels. In order to avoid
		 * the reported position from jumping backwards when the pixel
		 * counter is beyond the length of the shorter field, just
		 * clamp the position the length of the shorter field. This
		 * matches how the scanline counter based position works since
		 * the scanline counter doesn't count the two half lines.
		 */
		if (position >= vtotal)
			position = vtotal - 1;

929 930 931 932 933 934 935 936 937 938
		/*
		 * Start of vblank interrupt is triggered at start of hsync,
		 * just prior to the first active line of vblank. However we
		 * consider lines to start at the leading edge of horizontal
		 * active. So, should we get here before we've crossed into
		 * the horizontal active of the first line in vblank, we would
		 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
		 * always add htotal-hsync_start to the current pixel position.
		 */
		position = (position + htotal - hsync_start) % vtotal;
939 940
	}

941 942 943 944 945 946 947 948
	/* Get optional system timestamp after query. */
	if (etime)
		*etime = ktime_get();

	/* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

949 950 951 952 953 954 955 956 957 958 959 960
	in_vbl = position >= vbl_start && position < vbl_end;

	/*
	 * While in vblank, position will be negative
	 * counting up towards 0 at vbl_end. And outside
	 * vblank, position will be positive counting
	 * up since vbl_end.
	 */
	if (position >= vbl_start)
		position -= vbl_end;
	else
		position += vtotal - vbl_end;
961

962
	if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
963 964 965 966 967 968
		*vpos = position;
		*hpos = 0;
	} else {
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}
969 970 971 972 973 974 975 976

	/* In vblank? */
	if (in_vbl)
		ret |= DRM_SCANOUTPOS_INVBL;

	return ret;
}

977 978 979 980 981 982 983 984 985 986 987 988 989
int intel_get_crtc_scanline(struct intel_crtc *crtc)
{
	struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
	unsigned long irqflags;
	int position;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	position = __intel_get_crtc_scanline(crtc);
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);

	return position;
}

990
static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
991 992 993 994
			      int *max_error,
			      struct timeval *vblank_time,
			      unsigned flags)
{
995
	struct drm_crtc *crtc;
996

997
	if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
998
		DRM_ERROR("Invalid crtc %d\n", pipe);
999 1000 1001 1002
		return -EINVAL;
	}

	/* Get drm_crtc to timestamp: */
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012
	crtc = intel_get_crtc_for_pipe(dev, pipe);
	if (crtc == NULL) {
		DRM_ERROR("Invalid crtc %d\n", pipe);
		return -EINVAL;
	}

	if (!crtc->enabled) {
		DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
		return -EBUSY;
	}
1013 1014

	/* Helper routine in DRM core does all the work: */
1015 1016
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
						     vblank_time, flags,
1017 1018
						     crtc,
						     &to_intel_crtc(crtc)->config.adjusted_mode);
1019 1020
}

1021 1022
static bool intel_hpd_irq_event(struct drm_device *dev,
				struct drm_connector *connector)
1023 1024 1025 1026 1027 1028 1029
{
	enum drm_connector_status old_status;

	WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
	old_status = connector->status;

	connector->status = connector->funcs->detect(connector, false);
1030 1031 1032 1033
	if (old_status == connector->status)
		return false;

	DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
1034 1035
		      connector->base.id,
		      drm_get_connector_name(connector),
1036 1037 1038 1039
		      drm_get_connector_status_name(old_status),
		      drm_get_connector_status_name(connector->status));

	return true;
1040 1041
}

1042 1043 1044
/*
 * Handle hotplug events outside the interrupt handler proper.
 */
1045 1046
#define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)

1047 1048
static void i915_hotplug_work_func(struct work_struct *work)
{
1049 1050
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, hotplug_work);
1051
	struct drm_device *dev = dev_priv->dev;
1052
	struct drm_mode_config *mode_config = &dev->mode_config;
1053 1054 1055 1056 1057
	struct intel_connector *intel_connector;
	struct intel_encoder *intel_encoder;
	struct drm_connector *connector;
	unsigned long irqflags;
	bool hpd_disabled = false;
1058
	bool changed = false;
1059
	u32 hpd_event_bits;
1060

1061 1062 1063 1064
	/* HPD irq before everything is fully set up. */
	if (!dev_priv->enable_hotplug_processing)
		return;

1065
	mutex_lock(&mode_config->mutex);
1066 1067
	DRM_DEBUG_KMS("running encoder hotplug functions\n");

1068
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1069 1070 1071

	hpd_event_bits = dev_priv->hpd_event_bits;
	dev_priv->hpd_event_bits = 0;
1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085
	list_for_each_entry(connector, &mode_config->connector_list, head) {
		intel_connector = to_intel_connector(connector);
		intel_encoder = intel_connector->encoder;
		if (intel_encoder->hpd_pin > HPD_NONE &&
		    dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
		    connector->polled == DRM_CONNECTOR_POLL_HPD) {
			DRM_INFO("HPD interrupt storm detected on connector %s: "
				 "switching from hotplug detection to polling\n",
				drm_get_connector_name(connector));
			dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
			connector->polled = DRM_CONNECTOR_POLL_CONNECT
				| DRM_CONNECTOR_POLL_DISCONNECT;
			hpd_disabled = true;
		}
1086 1087 1088 1089
		if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
			DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
				      drm_get_connector_name(connector), intel_encoder->hpd_pin);
		}
1090 1091 1092 1093
	}
	 /* if there were no outputs to poll, poll was disabled,
	  * therefore make sure it's enabled when disabling HPD on
	  * some connectors */
1094
	if (hpd_disabled) {
1095
		drm_kms_helper_poll_enable(dev);
1096 1097 1098
		mod_timer(&dev_priv->hotplug_reenable_timer,
			  jiffies + msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
	}
1099 1100 1101

	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

1102 1103 1104 1105 1106 1107 1108 1109 1110 1111
	list_for_each_entry(connector, &mode_config->connector_list, head) {
		intel_connector = to_intel_connector(connector);
		intel_encoder = intel_connector->encoder;
		if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
			if (intel_encoder->hot_plug)
				intel_encoder->hot_plug(intel_encoder);
			if (intel_hpd_irq_event(dev, connector))
				changed = true;
		}
	}
1112 1113
	mutex_unlock(&mode_config->mutex);

1114 1115
	if (changed)
		drm_kms_helper_hotplug_event(dev);
1116 1117
}

1118 1119 1120 1121 1122
static void intel_hpd_irq_uninstall(struct drm_i915_private *dev_priv)
{
	del_timer_sync(&dev_priv->hotplug_reenable_timer);
}

1123
static void ironlake_rps_change_irq_handler(struct drm_device *dev)
1124
{
1125
	struct drm_i915_private *dev_priv = dev->dev_private;
1126
	u32 busy_up, busy_down, max_avg, min_avg;
1127 1128
	u8 new_delay;

1129
	spin_lock(&mchdev_lock);
1130

1131 1132
	I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));

1133
	new_delay = dev_priv->ips.cur_delay;
1134

1135
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
1136 1137
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
1138 1139 1140 1141
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
1142
	if (busy_up > max_avg) {
1143 1144 1145 1146
		if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.cur_delay - 1;
		if (new_delay < dev_priv->ips.max_delay)
			new_delay = dev_priv->ips.max_delay;
1147
	} else if (busy_down < min_avg) {
1148 1149 1150 1151
		if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.cur_delay + 1;
		if (new_delay > dev_priv->ips.min_delay)
			new_delay = dev_priv->ips.min_delay;
1152 1153
	}

1154
	if (ironlake_set_drps(dev, new_delay))
1155
		dev_priv->ips.cur_delay = new_delay;
1156

1157
	spin_unlock(&mchdev_lock);
1158

1159 1160 1161
	return;
}

1162 1163 1164
static void notify_ring(struct drm_device *dev,
			struct intel_ring_buffer *ring)
{
1165 1166 1167
	if (ring->obj == NULL)
		return;

1168
	trace_i915_gem_request_complete(ring);
1169

1170
	wake_up_all(&ring->irq_queue);
1171
	i915_queue_hangcheck(dev);
1172 1173
}

1174
static void gen6_pm_rps_work(struct work_struct *work)
1175
{
1176 1177
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, rps.work);
P
Paulo Zanoni 已提交
1178
	u32 pm_iir;
1179
	int new_delay, adj;
1180

1181
	spin_lock_irq(&dev_priv->irq_lock);
1182 1183
	pm_iir = dev_priv->rps.pm_iir;
	dev_priv->rps.pm_iir = 0;
1184 1185 1186 1187 1188 1189
	if (IS_BROADWELL(dev_priv->dev))
		bdw_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
	else {
		/* Make sure not to corrupt PMIMR state used by ringbuffer */
		snb_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
	}
1190
	spin_unlock_irq(&dev_priv->irq_lock);
1191

1192
	/* Make sure we didn't queue anything we're not going to process. */
1193
	WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
1194

1195
	if ((pm_iir & dev_priv->pm_rps_events) == 0)
1196 1197
		return;

1198
	mutex_lock(&dev_priv->rps.hw_lock);
1199

1200
	adj = dev_priv->rps.last_adj;
1201
	if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1202 1203 1204 1205
		if (adj > 0)
			adj *= 2;
		else
			adj = 1;
1206
		new_delay = dev_priv->rps.cur_freq + adj;
1207 1208 1209 1210 1211

		/*
		 * For better performance, jump directly
		 * to RPe if we're below it.
		 */
1212 1213
		if (new_delay < dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1214
	} else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1215 1216
		if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
			new_delay = dev_priv->rps.efficient_freq;
1217
		else
1218
			new_delay = dev_priv->rps.min_freq_softlimit;
1219 1220 1221 1222 1223 1224
		adj = 0;
	} else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
		if (adj < 0)
			adj *= 2;
		else
			adj = -1;
1225
		new_delay = dev_priv->rps.cur_freq + adj;
1226
	} else { /* unknown event */
1227
		new_delay = dev_priv->rps.cur_freq;
1228
	}
1229

1230 1231 1232
	/* sysfs frequency interfaces may have snuck in while servicing the
	 * interrupt
	 */
1233
	new_delay = clamp_t(int, new_delay,
1234 1235
			    dev_priv->rps.min_freq_softlimit,
			    dev_priv->rps.max_freq_softlimit);
1236

1237
	dev_priv->rps.last_adj = new_delay - dev_priv->rps.cur_freq;
1238 1239 1240 1241 1242

	if (IS_VALLEYVIEW(dev_priv->dev))
		valleyview_set_rps(dev_priv->dev, new_delay);
	else
		gen6_set_rps(dev_priv->dev, new_delay);
1243

1244
	mutex_unlock(&dev_priv->rps.hw_lock);
1245 1246
}

1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258

/**
 * ivybridge_parity_work - Workqueue called when a parity error interrupt
 * occurred.
 * @work: workqueue struct
 *
 * Doesn't actually do anything except notify userspace. As a consequence of
 * this event, userspace should try to remap the bad rows since statistically
 * it is likely the same row is more likely to go bad again.
 */
static void ivybridge_parity_work(struct work_struct *work)
{
1259 1260
	struct drm_i915_private *dev_priv =
		container_of(work, struct drm_i915_private, l3_parity.error_work);
1261
	u32 error_status, row, bank, subbank;
1262
	char *parity_event[6];
1263 1264
	uint32_t misccpctl;
	unsigned long flags;
1265
	uint8_t slice = 0;
1266 1267 1268 1269 1270 1271 1272

	/* We must turn off DOP level clock gating to access the L3 registers.
	 * In order to prevent a get/put style interface, acquire struct mutex
	 * any time we access those registers.
	 */
	mutex_lock(&dev_priv->dev->struct_mutex);

1273 1274 1275 1276
	/* If we've screwed up tracking, just let the interrupt fire again */
	if (WARN_ON(!dev_priv->l3_parity.which_slice))
		goto out;

1277 1278 1279 1280
	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

1281 1282
	while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
		u32 reg;
1283

1284 1285 1286
		slice--;
		if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
			break;
1287

1288
		dev_priv->l3_parity.which_slice &= ~(1<<slice);
1289

1290
		reg = GEN7_L3CDERRST1 + (slice * 0x200);
1291

1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
		error_status = I915_READ(reg);
		row = GEN7_PARITY_ERROR_ROW(error_status);
		bank = GEN7_PARITY_ERROR_BANK(error_status);
		subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);

		I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
		POSTING_READ(reg);

		parity_event[0] = I915_L3_PARITY_UEVENT "=1";
		parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
		parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
		parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
		parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
		parity_event[5] = NULL;

1307
		kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
1308
				   KOBJ_CHANGE, parity_event);
1309

1310 1311
		DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
			  slice, row, bank, subbank);
1312

1313 1314 1315 1316 1317
		kfree(parity_event[4]);
		kfree(parity_event[3]);
		kfree(parity_event[2]);
		kfree(parity_event[1]);
	}
1318

1319
	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1320

1321 1322 1323 1324 1325 1326 1327
out:
	WARN_ON(dev_priv->l3_parity.which_slice);
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
	ilk_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);

	mutex_unlock(&dev_priv->dev->struct_mutex);
1328 1329
}

1330
static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
1331
{
1332
	struct drm_i915_private *dev_priv = dev->dev_private;
1333

1334
	if (!HAS_L3_DPF(dev))
1335 1336
		return;

1337
	spin_lock(&dev_priv->irq_lock);
1338
	ilk_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
1339
	spin_unlock(&dev_priv->irq_lock);
1340

1341 1342 1343 1344 1345 1346 1347
	iir &= GT_PARITY_ERROR(dev);
	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
		dev_priv->l3_parity.which_slice |= 1 << 1;

	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
		dev_priv->l3_parity.which_slice |= 1 << 0;

1348
	queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
1349 1350
}

1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361
static void ilk_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
		notify_ring(dev, &dev_priv->ring[RCS]);
	if (gt_iir & ILK_BSD_USER_INTERRUPT)
		notify_ring(dev, &dev_priv->ring[VCS]);
}

1362 1363 1364 1365 1366
static void snb_gt_irq_handler(struct drm_device *dev,
			       struct drm_i915_private *dev_priv,
			       u32 gt_iir)
{

1367 1368
	if (gt_iir &
	    (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1369
		notify_ring(dev, &dev_priv->ring[RCS]);
1370
	if (gt_iir & GT_BSD_USER_INTERRUPT)
1371
		notify_ring(dev, &dev_priv->ring[VCS]);
1372
	if (gt_iir & GT_BLT_USER_INTERRUPT)
1373 1374
		notify_ring(dev, &dev_priv->ring[BCS]);

1375 1376 1377
	if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
		      GT_BSD_CS_ERROR_INTERRUPT |
		      GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
1378 1379
		i915_handle_error(dev, false, "GT error interrupt 0x%08x",
				  gt_iir);
1380
	}
1381

1382 1383
	if (gt_iir & GT_PARITY_ERROR(dev))
		ivybridge_parity_error_irq_handler(dev, gt_iir);
1384 1385
}

1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
static void gen8_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
{
	if ((pm_iir & dev_priv->pm_rps_events) == 0)
		return;

	spin_lock(&dev_priv->irq_lock);
	dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
	bdw_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
	spin_unlock(&dev_priv->irq_lock);

	queue_work(dev_priv->wq, &dev_priv->rps.work);
}

1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
				       struct drm_i915_private *dev_priv,
				       u32 master_ctl)
{
	u32 rcs, bcs, vcs;
	uint32_t tmp = 0;
	irqreturn_t ret = IRQ_NONE;

	if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
		tmp = I915_READ(GEN8_GT_IIR(0));
		if (tmp) {
			ret = IRQ_HANDLED;
			rcs = tmp >> GEN8_RCS_IRQ_SHIFT;
			bcs = tmp >> GEN8_BCS_IRQ_SHIFT;
			if (rcs & GT_RENDER_USER_INTERRUPT)
				notify_ring(dev, &dev_priv->ring[RCS]);
			if (bcs & GT_RENDER_USER_INTERRUPT)
				notify_ring(dev, &dev_priv->ring[BCS]);
			I915_WRITE(GEN8_GT_IIR(0), tmp);
		} else
			DRM_ERROR("The master control interrupt lied (GT0)!\n");
	}

1422
	if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
1423 1424 1425 1426 1427 1428
		tmp = I915_READ(GEN8_GT_IIR(1));
		if (tmp) {
			ret = IRQ_HANDLED;
			vcs = tmp >> GEN8_VCS1_IRQ_SHIFT;
			if (vcs & GT_RENDER_USER_INTERRUPT)
				notify_ring(dev, &dev_priv->ring[VCS]);
1429 1430 1431
			vcs = tmp >> GEN8_VCS2_IRQ_SHIFT;
			if (vcs & GT_RENDER_USER_INTERRUPT)
				notify_ring(dev, &dev_priv->ring[VCS2]);
1432 1433 1434 1435 1436
			I915_WRITE(GEN8_GT_IIR(1), tmp);
		} else
			DRM_ERROR("The master control interrupt lied (GT1)!\n");
	}

1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447
	if (master_ctl & GEN8_GT_PM_IRQ) {
		tmp = I915_READ(GEN8_GT_IIR(2));
		if (tmp & dev_priv->pm_rps_events) {
			ret = IRQ_HANDLED;
			gen8_rps_irq_handler(dev_priv, tmp);
			I915_WRITE(GEN8_GT_IIR(2),
				   tmp & dev_priv->pm_rps_events);
		} else
			DRM_ERROR("The master control interrupt lied (PM)!\n");
	}

1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462
	if (master_ctl & GEN8_GT_VECS_IRQ) {
		tmp = I915_READ(GEN8_GT_IIR(3));
		if (tmp) {
			ret = IRQ_HANDLED;
			vcs = tmp >> GEN8_VECS_IRQ_SHIFT;
			if (vcs & GT_RENDER_USER_INTERRUPT)
				notify_ring(dev, &dev_priv->ring[VECS]);
			I915_WRITE(GEN8_GT_IIR(3), tmp);
		} else
			DRM_ERROR("The master control interrupt lied (GT3)!\n");
	}

	return ret;
}

1463 1464 1465
#define HPD_STORM_DETECT_PERIOD 1000
#define HPD_STORM_THRESHOLD 5

1466
static inline void intel_hpd_irq_handler(struct drm_device *dev,
1467 1468
					 u32 hotplug_trigger,
					 const u32 *hpd)
1469
{
1470
	struct drm_i915_private *dev_priv = dev->dev_private;
1471
	int i;
1472
	bool storm_detected = false;
1473

1474 1475 1476
	if (!hotplug_trigger)
		return;

1477 1478 1479
	DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
			  hotplug_trigger);

1480
	spin_lock(&dev_priv->irq_lock);
1481
	for (i = 1; i < HPD_NUM_PINS; i++) {
1482

1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496
		if (hpd[i] & hotplug_trigger &&
		    dev_priv->hpd_stats[i].hpd_mark == HPD_DISABLED) {
			/*
			 * On GMCH platforms the interrupt mask bits only
			 * prevent irq generation, not the setting of the
			 * hotplug bits itself. So only WARN about unexpected
			 * interrupts on saner platforms.
			 */
			WARN_ONCE(INTEL_INFO(dev)->gen >= 5 && !IS_VALLEYVIEW(dev),
				  "Received HPD interrupt (0x%08x) on pin %d (0x%08x) although disabled\n",
				  hotplug_trigger, i, hpd[i]);

			continue;
		}
1497

1498 1499 1500 1501
		if (!(hpd[i] & hotplug_trigger) ||
		    dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
			continue;

1502
		dev_priv->hpd_event_bits |= (1 << i);
1503 1504 1505 1506 1507
		if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
				   dev_priv->hpd_stats[i].hpd_last_jiffies
				   + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
			dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
			dev_priv->hpd_stats[i].hpd_cnt = 0;
1508
			DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i);
1509 1510
		} else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
			dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
1511
			dev_priv->hpd_event_bits &= ~(1 << i);
1512
			DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
1513
			storm_detected = true;
1514 1515
		} else {
			dev_priv->hpd_stats[i].hpd_cnt++;
1516 1517
			DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i,
				      dev_priv->hpd_stats[i].hpd_cnt);
1518 1519 1520
		}
	}

1521 1522
	if (storm_detected)
		dev_priv->display.hpd_irq_setup(dev);
1523
	spin_unlock(&dev_priv->irq_lock);
1524

1525 1526 1527 1528 1529 1530 1531
	/*
	 * Our hotplug handler can grab modeset locks (by calling down into the
	 * fb helpers). Hence it must not be run on our own dev-priv->wq work
	 * queue for otherwise the flush_work in the pageflip code will
	 * deadlock.
	 */
	schedule_work(&dev_priv->hotplug_work);
1532 1533
}

1534 1535
static void gmbus_irq_handler(struct drm_device *dev)
{
1536
	struct drm_i915_private *dev_priv = dev->dev_private;
1537 1538

	wake_up_all(&dev_priv->gmbus_wait_queue);
1539 1540
}

1541 1542
static void dp_aux_irq_handler(struct drm_device *dev)
{
1543
	struct drm_i915_private *dev_priv = dev->dev_private;
1544 1545

	wake_up_all(&dev_priv->gmbus_wait_queue);
1546 1547
}

1548
#if defined(CONFIG_DEBUG_FS)
1549 1550 1551 1552
static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
					 uint32_t crc0, uint32_t crc1,
					 uint32_t crc2, uint32_t crc3,
					 uint32_t crc4)
1553 1554 1555 1556
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
	struct intel_pipe_crc_entry *entry;
1557
	int head, tail;
1558

1559 1560
	spin_lock(&pipe_crc->lock);

1561
	if (!pipe_crc->entries) {
1562
		spin_unlock(&pipe_crc->lock);
1563 1564 1565 1566
		DRM_ERROR("spurious interrupt\n");
		return;
	}

1567 1568
	head = pipe_crc->head;
	tail = pipe_crc->tail;
1569 1570

	if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
1571
		spin_unlock(&pipe_crc->lock);
1572 1573 1574 1575 1576
		DRM_ERROR("CRC buffer overflowing\n");
		return;
	}

	entry = &pipe_crc->entries[head];
1577

1578
	entry->frame = dev->driver->get_vblank_counter(dev, pipe);
1579 1580 1581 1582 1583
	entry->crc[0] = crc0;
	entry->crc[1] = crc1;
	entry->crc[2] = crc2;
	entry->crc[3] = crc3;
	entry->crc[4] = crc4;
1584 1585

	head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
1586 1587 1588
	pipe_crc->head = head;

	spin_unlock(&pipe_crc->lock);
1589 1590

	wake_up_interruptible(&pipe_crc->wq);
1591
}
1592 1593 1594 1595 1596 1597 1598 1599
#else
static inline void
display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
			     uint32_t crc0, uint32_t crc1,
			     uint32_t crc2, uint32_t crc3,
			     uint32_t crc4) {}
#endif

1600

1601
static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
D
Daniel Vetter 已提交
1602 1603 1604
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1605 1606 1607
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     0, 0, 0, 0);
D
Daniel Vetter 已提交
1608 1609
}

1610
static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1611 1612 1613
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1614 1615 1616 1617 1618 1619
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
				     I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
1620
}
1621

1622
static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
1623 1624
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635
	uint32_t res1, res2;

	if (INTEL_INFO(dev)->gen >= 3)
		res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
	else
		res1 = 0;

	if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
		res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
	else
		res2 = 0;
1636

1637 1638 1639 1640 1641
	display_pipe_crc_irq_handler(dev, pipe,
				     I915_READ(PIPE_CRC_RES_RED(pipe)),
				     I915_READ(PIPE_CRC_RES_GREEN(pipe)),
				     I915_READ(PIPE_CRC_RES_BLUE(pipe)),
				     res1, res2);
1642
}
1643

1644 1645 1646 1647
/* The RPS events need forcewake, so we add them to a work queue and mask their
 * IMR bits until the work is done. Other interrupts can be processed without
 * the work queue. */
static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
1648
{
1649
	if (pm_iir & dev_priv->pm_rps_events) {
1650
		spin_lock(&dev_priv->irq_lock);
1651 1652
		dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
		snb_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
1653
		spin_unlock(&dev_priv->irq_lock);
1654 1655

		queue_work(dev_priv->wq, &dev_priv->rps.work);
1656 1657
	}

1658 1659 1660
	if (HAS_VEBOX(dev_priv->dev)) {
		if (pm_iir & PM_VEBOX_USER_INTERRUPT)
			notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
B
Ben Widawsky 已提交
1661

1662
		if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
1663 1664 1665
			i915_handle_error(dev_priv->dev, false,
					  "VEBOX CS error interrupt 0x%08x",
					  pm_iir);
1666
		}
B
Ben Widawsky 已提交
1667
	}
1668 1669
}

1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682
static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
{
	struct intel_crtc *crtc;

	if (!drm_handle_vblank(dev, pipe))
		return false;

	crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
	wake_up(&crtc->vbl_wait);

	return true;
}

1683 1684 1685
static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1686
	u32 pipe_stats[I915_MAX_PIPES] = { };
1687 1688
	int pipe;

1689
	spin_lock(&dev_priv->irq_lock);
1690
	for_each_pipe(pipe) {
1691
		int reg;
1692
		u32 mask, iir_bit = 0;
1693

1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711
		/*
		 * PIPESTAT bits get signalled even when the interrupt is
		 * disabled with the mask bits, and some of the status bits do
		 * not generate interrupts at all (like the underrun bit). Hence
		 * we need to be careful that we only handle what we want to
		 * handle.
		 */
		mask = 0;
		if (__cpu_fifo_underrun_reporting_enabled(dev, pipe))
			mask |= PIPE_FIFO_UNDERRUN_STATUS;

		switch (pipe) {
		case PIPE_A:
			iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
			break;
		case PIPE_B:
			iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
			break;
1712 1713 1714
		case PIPE_C:
			iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
			break;
1715 1716 1717 1718 1719
		}
		if (iir & iir_bit)
			mask |= dev_priv->pipestat_irq_mask[pipe];

		if (!mask)
1720 1721 1722
			continue;

		reg = PIPESTAT(pipe);
1723 1724
		mask |= PIPESTAT_INT_ENABLE_MASK;
		pipe_stats[pipe] = I915_READ(reg) & mask;
1725 1726 1727 1728

		/*
		 * Clear the PIPE*STAT regs before the IIR
		 */
1729 1730
		if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
					PIPESTAT_INT_STATUS_MASK))
1731 1732
			I915_WRITE(reg, pipe_stats[pipe]);
	}
1733
	spin_unlock(&dev_priv->irq_lock);
1734 1735 1736

	for_each_pipe(pipe) {
		if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
1737
			intel_pipe_handle_vblank(dev, pipe);
1738

1739
		if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip(dev, pipe);
		}

		if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
			i9xx_pipe_crc_irq_handler(dev, pipe);

		if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
		    intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
			DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
	}

	if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
		gmbus_irq_handler(dev);
}

1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782
static void i9xx_hpd_irq_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);

	if (IS_G4X(dev)) {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;

		intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_g4x);
	} else {
		u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;

		intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
	}

	if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)) &&
	    hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
		dp_aux_irq_handler(dev);

	I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
	/*
	 * Make sure hotplug status is cleared before we clear IIR, or else we
	 * may miss hotplug events.
	 */
	POSTING_READ(PORT_HOTPLUG_STAT);
}

1783
static irqreturn_t valleyview_irq_handler(int irq, void *arg)
J
Jesse Barnes 已提交
1784
{
1785
	struct drm_device *dev = arg;
1786
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
	u32 iir, gt_iir, pm_iir;
	irqreturn_t ret = IRQ_NONE;

	while (true) {
		iir = I915_READ(VLV_IIR);
		gt_iir = I915_READ(GTIIR);
		pm_iir = I915_READ(GEN6_PMIIR);

		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
			goto out;

		ret = IRQ_HANDLED;

1800
		snb_gt_irq_handler(dev, dev_priv, gt_iir);
J
Jesse Barnes 已提交
1801

1802
		valleyview_pipestat_irq_handler(dev, iir);
1803

J
Jesse Barnes 已提交
1804
		/* Consume port.  Then clear IIR or we'll miss events */
1805 1806
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
J
Jesse Barnes 已提交
1807

1808
		if (pm_iir)
1809
			gen6_rps_irq_handler(dev_priv, pm_iir);
J
Jesse Barnes 已提交
1810 1811 1812 1813 1814 1815 1816 1817 1818 1819

		I915_WRITE(GTIIR, gt_iir);
		I915_WRITE(GEN6_PMIIR, pm_iir);
		I915_WRITE(VLV_IIR, iir);
	}

out:
	return ret;
}

1820 1821
static irqreturn_t cherryview_irq_handler(int irq, void *arg)
{
1822
	struct drm_device *dev = arg;
1823 1824 1825 1826
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl, iir;
	irqreturn_t ret = IRQ_NONE;

1827 1828 1829
	for (;;) {
		master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
		iir = I915_READ(VLV_IIR);
1830

1831 1832
		if (master_ctl == 0 && iir == 0)
			break;
1833

1834
		I915_WRITE(GEN8_MASTER_IRQ, 0);
1835

1836
		gen8_gt_irq_handler(dev, dev_priv, master_ctl);
1837

1838
		valleyview_pipestat_irq_handler(dev, iir);
1839

1840
		/* Consume port.  Then clear IIR or we'll miss events */
1841
		i9xx_hpd_irq_handler(dev);
1842

1843
		I915_WRITE(VLV_IIR, iir);
1844

1845 1846
		I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
		POSTING_READ(GEN8_MASTER_IRQ);
1847

1848 1849
		ret = IRQ_HANDLED;
	}
1850

1851 1852 1853
	return ret;
}

1854
static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
1855
{
1856
	struct drm_i915_private *dev_priv = dev->dev_private;
1857
	int pipe;
1858
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
1859

1860 1861
	intel_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);

1862 1863 1864
	if (pch_iir & SDE_AUDIO_POWER_MASK) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
			       SDE_AUDIO_POWER_SHIFT);
1865
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
1866 1867
				 port_name(port));
	}
1868

1869 1870 1871
	if (pch_iir & SDE_AUX_MASK)
		dp_aux_irq_handler(dev);

1872
	if (pch_iir & SDE_GMBUS)
1873
		gmbus_irq_handler(dev);
1874 1875 1876 1877 1878 1879 1880 1881 1882 1883

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1884 1885 1886 1887 1888
	if (pch_iir & SDE_FDI_MASK)
		for_each_pipe(pipe)
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
1889 1890 1891 1892 1893 1894 1895 1896

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
1897 1898
		if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
							  false))
1899
			DRM_ERROR("PCH transcoder A FIFO underrun\n");
1900 1901 1902 1903

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
		if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
							  false))
1904
			DRM_ERROR("PCH transcoder B FIFO underrun\n");
1905 1906 1907 1908 1909 1910
}

static void ivb_err_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 err_int = I915_READ(GEN7_ERR_INT);
D
Daniel Vetter 已提交
1911
	enum pipe pipe;
1912

1913 1914 1915
	if (err_int & ERR_INT_POISON)
		DRM_ERROR("Poison interrupt\n");

D
Daniel Vetter 已提交
1916 1917 1918 1919
	for_each_pipe(pipe) {
		if (err_int & ERR_INT_FIFO_UNDERRUN(pipe)) {
			if (intel_set_cpu_fifo_underrun_reporting(dev, pipe,
								  false))
1920 1921
				DRM_ERROR("Pipe %c FIFO underrun\n",
					  pipe_name(pipe));
D
Daniel Vetter 已提交
1922
		}
1923

D
Daniel Vetter 已提交
1924 1925
		if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
			if (IS_IVYBRIDGE(dev))
1926
				ivb_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1927
			else
1928
				hsw_pipe_crc_irq_handler(dev, pipe);
D
Daniel Vetter 已提交
1929 1930
		}
	}
1931

1932 1933 1934 1935 1936 1937 1938 1939
	I915_WRITE(GEN7_ERR_INT, err_int);
}

static void cpt_serr_int_handler(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 serr_int = I915_READ(SERR_INT);

1940 1941 1942
	if (serr_int & SERR_INT_POISON)
		DRM_ERROR("PCH poison interrupt\n");

1943 1944 1945
	if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
		if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
							  false))
1946
			DRM_ERROR("PCH transcoder A FIFO underrun\n");
1947 1948 1949 1950

	if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
		if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
							  false))
1951
			DRM_ERROR("PCH transcoder B FIFO underrun\n");
1952 1953 1954 1955

	if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
		if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
							  false))
1956
			DRM_ERROR("PCH transcoder C FIFO underrun\n");
1957 1958

	I915_WRITE(SERR_INT, serr_int);
1959 1960
}

1961 1962
static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
{
1963
	struct drm_i915_private *dev_priv = dev->dev_private;
1964
	int pipe;
1965
	u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
1966

1967 1968
	intel_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);

1969 1970 1971 1972 1973 1974
	if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
		int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
			       SDE_AUDIO_POWER_SHIFT_CPT);
		DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
				 port_name(port));
	}
1975 1976

	if (pch_iir & SDE_AUX_MASK_CPT)
1977
		dp_aux_irq_handler(dev);
1978 1979

	if (pch_iir & SDE_GMBUS_CPT)
1980
		gmbus_irq_handler(dev);
1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992

	if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
		DRM_DEBUG_DRIVER("Audio CP request interrupt\n");

	if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
		DRM_DEBUG_DRIVER("Audio CP change interrupt\n");

	if (pch_iir & SDE_FDI_MASK_CPT)
		for_each_pipe(pipe)
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
1993 1994 1995

	if (pch_iir & SDE_ERROR_CPT)
		cpt_serr_int_handler(dev);
1996 1997
}

1998 1999 2000
static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2001
	enum pipe pipe;
2002 2003 2004 2005 2006 2007 2008 2009 2010 2011

	if (de_iir & DE_AUX_CHANNEL_A)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE)
		intel_opregion_asle_intr(dev);

	if (de_iir & DE_POISON)
		DRM_ERROR("Poison interrupt\n");

2012 2013
	for_each_pipe(pipe) {
		if (de_iir & DE_PIPE_VBLANK(pipe))
2014
			intel_pipe_handle_vblank(dev, pipe);
2015

2016 2017
		if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
			if (intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
2018 2019
				DRM_ERROR("Pipe %c FIFO underrun\n",
					  pipe_name(pipe));
2020

2021 2022
		if (de_iir & DE_PIPE_CRC_DONE(pipe))
			i9xx_pipe_crc_irq_handler(dev, pipe);
2023

2024 2025 2026 2027 2028
		/* plane/pipes map 1:1 on ilk+ */
		if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
		}
2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047
	}

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT) {
		u32 pch_iir = I915_READ(SDEIIR);

		if (HAS_PCH_CPT(dev))
			cpt_irq_handler(dev, pch_iir);
		else
			ibx_irq_handler(dev, pch_iir);

		/* should clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}

	if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
		ironlake_rps_change_irq_handler(dev);
}

2048 2049 2050
static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2051
	enum pipe pipe;
2052 2053 2054 2055 2056 2057 2058 2059 2060 2061

	if (de_iir & DE_ERR_INT_IVB)
		ivb_err_int_handler(dev);

	if (de_iir & DE_AUX_CHANNEL_A_IVB)
		dp_aux_irq_handler(dev);

	if (de_iir & DE_GSE_IVB)
		intel_opregion_asle_intr(dev);

2062 2063
	for_each_pipe(pipe) {
		if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
2064
			intel_pipe_handle_vblank(dev, pipe);
2065 2066

		/* plane/pipes map 1:1 on ilk+ */
2067 2068 2069
		if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083
		}
	}

	/* check event from PCH */
	if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
		u32 pch_iir = I915_READ(SDEIIR);

		cpt_irq_handler(dev, pch_iir);

		/* clear PCH hotplug event before clear CPU irq */
		I915_WRITE(SDEIIR, pch_iir);
	}
}

2084
static irqreturn_t ironlake_irq_handler(int irq, void *arg)
2085
{
2086
	struct drm_device *dev = arg;
2087
	struct drm_i915_private *dev_priv = dev->dev_private;
2088
	u32 de_iir, gt_iir, de_ier, sde_ier = 0;
2089
	irqreturn_t ret = IRQ_NONE;
2090

2091 2092
	/* We get interrupts on unclaimed registers, so check for this before we
	 * do any I915_{READ,WRITE}. */
2093
	intel_uncore_check_errors(dev);
2094

2095 2096 2097
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
2098
	POSTING_READ(DEIER);
2099

2100 2101 2102 2103 2104
	/* Disable south interrupts. We'll only write to SDEIIR once, so further
	 * interrupts will will be stored on its back queue, and then we'll be
	 * able to process them after we restore SDEIER (as soon as we restore
	 * it, we'll get an interrupt if SDEIIR still has something to process
	 * due to its back queue). */
2105 2106 2107 2108 2109
	if (!HAS_PCH_NOP(dev)) {
		sde_ier = I915_READ(SDEIER);
		I915_WRITE(SDEIER, 0);
		POSTING_READ(SDEIER);
	}
2110

2111
	gt_iir = I915_READ(GTIIR);
2112
	if (gt_iir) {
2113
		if (INTEL_INFO(dev)->gen >= 6)
2114
			snb_gt_irq_handler(dev, dev_priv, gt_iir);
2115 2116
		else
			ilk_gt_irq_handler(dev, dev_priv, gt_iir);
2117 2118
		I915_WRITE(GTIIR, gt_iir);
		ret = IRQ_HANDLED;
2119 2120
	}

2121 2122
	de_iir = I915_READ(DEIIR);
	if (de_iir) {
2123 2124 2125 2126
		if (INTEL_INFO(dev)->gen >= 7)
			ivb_display_irq_handler(dev, de_iir);
		else
			ilk_display_irq_handler(dev, de_iir);
2127 2128
		I915_WRITE(DEIIR, de_iir);
		ret = IRQ_HANDLED;
2129 2130
	}

2131 2132 2133
	if (INTEL_INFO(dev)->gen >= 6) {
		u32 pm_iir = I915_READ(GEN6_PMIIR);
		if (pm_iir) {
2134
			gen6_rps_irq_handler(dev_priv, pm_iir);
2135 2136 2137
			I915_WRITE(GEN6_PMIIR, pm_iir);
			ret = IRQ_HANDLED;
		}
2138
	}
2139 2140 2141

	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);
2142 2143 2144 2145
	if (!HAS_PCH_NOP(dev)) {
		I915_WRITE(SDEIER, sde_ier);
		POSTING_READ(SDEIER);
	}
2146 2147 2148 2149

	return ret;
}

2150 2151 2152 2153 2154 2155 2156
static irqreturn_t gen8_irq_handler(int irq, void *arg)
{
	struct drm_device *dev = arg;
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 master_ctl;
	irqreturn_t ret = IRQ_NONE;
	uint32_t tmp = 0;
2157
	enum pipe pipe;
2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183

	master_ctl = I915_READ(GEN8_MASTER_IRQ);
	master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
	if (!master_ctl)
		return IRQ_NONE;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

	ret = gen8_gt_irq_handler(dev, dev_priv, master_ctl);

	if (master_ctl & GEN8_DE_MISC_IRQ) {
		tmp = I915_READ(GEN8_DE_MISC_IIR);
		if (tmp & GEN8_DE_MISC_GSE)
			intel_opregion_asle_intr(dev);
		else if (tmp)
			DRM_ERROR("Unexpected DE Misc interrupt\n");
		else
			DRM_ERROR("The master control interrupt lied (DE MISC)!\n");

		if (tmp) {
			I915_WRITE(GEN8_DE_MISC_IIR, tmp);
			ret = IRQ_HANDLED;
		}
	}

2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198
	if (master_ctl & GEN8_DE_PORT_IRQ) {
		tmp = I915_READ(GEN8_DE_PORT_IIR);
		if (tmp & GEN8_AUX_CHANNEL_A)
			dp_aux_irq_handler(dev);
		else if (tmp)
			DRM_ERROR("Unexpected DE Port interrupt\n");
		else
			DRM_ERROR("The master control interrupt lied (DE PORT)!\n");

		if (tmp) {
			I915_WRITE(GEN8_DE_PORT_IIR, tmp);
			ret = IRQ_HANDLED;
		}
	}

2199 2200
	for_each_pipe(pipe) {
		uint32_t pipe_iir;
2201

2202 2203
		if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
			continue;
2204

2205 2206
		pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
		if (pipe_iir & GEN8_PIPE_VBLANK)
2207
			intel_pipe_handle_vblank(dev, pipe);
2208

2209
		if (pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE) {
2210 2211
			intel_prepare_page_flip(dev, pipe);
			intel_finish_page_flip_plane(dev, pipe);
2212
		}
2213

2214 2215 2216
		if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
			hsw_pipe_crc_irq_handler(dev, pipe);

2217 2218 2219
		if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN) {
			if (intel_set_cpu_fifo_underrun_reporting(dev, pipe,
								  false))
2220 2221
				DRM_ERROR("Pipe %c FIFO underrun\n",
					  pipe_name(pipe));
2222 2223
		}

2224 2225 2226 2227 2228
		if (pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS) {
			DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
				  pipe_name(pipe),
				  pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
		}
2229 2230 2231 2232 2233

		if (pipe_iir) {
			ret = IRQ_HANDLED;
			I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
		} else
2234 2235 2236
			DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
	}

2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252
	if (!HAS_PCH_NOP(dev) && master_ctl & GEN8_DE_PCH_IRQ) {
		/*
		 * FIXME(BDW): Assume for now that the new interrupt handling
		 * scheme also closed the SDE interrupt handling race we've seen
		 * on older pch-split platforms. But this needs testing.
		 */
		u32 pch_iir = I915_READ(SDEIIR);

		cpt_irq_handler(dev, pch_iir);

		if (pch_iir) {
			I915_WRITE(SDEIIR, pch_iir);
			ret = IRQ_HANDLED;
		}
	}

2253 2254 2255 2256 2257 2258
	I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
	POSTING_READ(GEN8_MASTER_IRQ);

	return ret;
}

2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286
static void i915_error_wake_up(struct drm_i915_private *dev_priv,
			       bool reset_completed)
{
	struct intel_ring_buffer *ring;
	int i;

	/*
	 * Notify all waiters for GPU completion events that reset state has
	 * been changed, and that they need to restart their wait after
	 * checking for potential errors (and bail out to drop locks if there is
	 * a gpu reset pending so that i915_error_work_func can acquire them).
	 */

	/* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
	for_each_ring(ring, dev_priv, i)
		wake_up_all(&ring->irq_queue);

	/* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
	wake_up_all(&dev_priv->pending_flip_queue);

	/*
	 * Signal tasks blocked in i915_gem_wait_for_error that the pending
	 * reset state is cleared.
	 */
	if (reset_completed)
		wake_up_all(&dev_priv->gpu_error.reset_queue);
}

2287 2288 2289 2290 2291 2292 2293 2294 2295
/**
 * i915_error_work_func - do process context error handling work
 * @work: work struct
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
static void i915_error_work_func(struct work_struct *work)
{
2296 2297
	struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
						    work);
2298 2299
	struct drm_i915_private *dev_priv =
		container_of(error, struct drm_i915_private, gpu_error);
2300
	struct drm_device *dev = dev_priv->dev;
2301 2302 2303
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
2304
	int ret;
2305

2306
	kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
2307

2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318
	/*
	 * Note that there's only one work item which does gpu resets, so we
	 * need not worry about concurrent gpu resets potentially incrementing
	 * error->reset_counter twice. We only need to take care of another
	 * racing irq/hangcheck declaring the gpu dead for a second time. A
	 * quick check for that is good enough: schedule_work ensures the
	 * correct ordering between hang detection and this work item, and since
	 * the reset in-progress bit is only ever set by code outside of this
	 * work we don't need to worry about any other races.
	 */
	if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
2319
		DRM_DEBUG_DRIVER("resetting chip\n");
2320
		kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
2321
				   reset_event);
2322

2323 2324 2325 2326 2327 2328 2329 2330
		/*
		 * In most cases it's guaranteed that we get here with an RPM
		 * reference held, for example because there is a pending GPU
		 * request that won't finish until the reset is done. This
		 * isn't the case at least when we get here by doing a
		 * simulated reset via debugs, so get an RPM reference.
		 */
		intel_runtime_pm_get(dev_priv);
2331 2332 2333 2334 2335 2336
		/*
		 * All state reset _must_ be completed before we update the
		 * reset counter, for otherwise waiters might miss the reset
		 * pending state and not properly drop locks, resulting in
		 * deadlocks with the reset work.
		 */
2337 2338
		ret = i915_reset(dev);

2339 2340
		intel_display_handle_reset(dev);

2341 2342
		intel_runtime_pm_put(dev_priv);

2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356
		if (ret == 0) {
			/*
			 * After all the gem state is reset, increment the reset
			 * counter and wake up everyone waiting for the reset to
			 * complete.
			 *
			 * Since unlock operations are a one-sided barrier only,
			 * we need to insert a barrier here to order any seqno
			 * updates before
			 * the counter increment.
			 */
			smp_mb__before_atomic_inc();
			atomic_inc(&dev_priv->gpu_error.reset_counter);

2357
			kobject_uevent_env(&dev->primary->kdev->kobj,
2358
					   KOBJ_CHANGE, reset_done_event);
2359
		} else {
M
Mika Kuoppala 已提交
2360
			atomic_set_mask(I915_WEDGED, &error->reset_counter);
2361
		}
2362

2363 2364 2365 2366 2367
		/*
		 * Note: The wake_up also serves as a memory barrier so that
		 * waiters see the update value of the reset counter atomic_t.
		 */
		i915_error_wake_up(dev_priv, true);
2368
	}
2369 2370
}

2371
static void i915_report_and_clear_eir(struct drm_device *dev)
2372 2373
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2374
	uint32_t instdone[I915_NUM_INSTDONE_REG];
2375
	u32 eir = I915_READ(EIR);
2376
	int pipe, i;
2377

2378 2379
	if (!eir)
		return;
2380

2381
	pr_err("render error detected, EIR: 0x%08x\n", eir);
2382

2383 2384
	i915_get_extra_instdone(dev, instdone);

2385 2386 2387 2388
	if (IS_G4X(dev)) {
		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
			u32 ipeir = I915_READ(IPEIR_I965);

2389 2390
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
2391 2392
			for (i = 0; i < ARRAY_SIZE(instdone); i++)
				pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2393 2394
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2395
			I915_WRITE(IPEIR_I965, ipeir);
2396
			POSTING_READ(IPEIR_I965);
2397 2398 2399
		}
		if (eir & GM45_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2400 2401
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2402
			I915_WRITE(PGTBL_ER, pgtbl_err);
2403
			POSTING_READ(PGTBL_ER);
2404 2405 2406
		}
	}

2407
	if (!IS_GEN2(dev)) {
2408 2409
		if (eir & I915_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
2410 2411
			pr_err("page table error\n");
			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
2412
			I915_WRITE(PGTBL_ER, pgtbl_err);
2413
			POSTING_READ(PGTBL_ER);
2414 2415 2416 2417
		}
	}

	if (eir & I915_ERROR_MEMORY_REFRESH) {
2418
		pr_err("memory refresh error:\n");
2419
		for_each_pipe(pipe)
2420
			pr_err("pipe %c stat: 0x%08x\n",
2421
			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
2422 2423 2424
		/* pipestat has already been acked */
	}
	if (eir & I915_ERROR_INSTRUCTION) {
2425 2426
		pr_err("instruction error\n");
		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
2427 2428
		for (i = 0; i < ARRAY_SIZE(instdone); i++)
			pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
2429
		if (INTEL_INFO(dev)->gen < 4) {
2430 2431
			u32 ipeir = I915_READ(IPEIR);

2432 2433 2434
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
2435
			I915_WRITE(IPEIR, ipeir);
2436
			POSTING_READ(IPEIR);
2437 2438 2439
		} else {
			u32 ipeir = I915_READ(IPEIR_I965);

2440 2441 2442 2443
			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
2444
			I915_WRITE(IPEIR_I965, ipeir);
2445
			POSTING_READ(IPEIR_I965);
2446 2447 2448 2449
		}
	}

	I915_WRITE(EIR, eir);
2450
	POSTING_READ(EIR);
2451 2452 2453 2454 2455 2456 2457 2458 2459 2460
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472
}

/**
 * i915_handle_error - handle an error interrupt
 * @dev: drm device
 *
 * Do some basic checking of regsiter state at error interrupt time and
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
2473 2474
void i915_handle_error(struct drm_device *dev, bool wedged,
		       const char *fmt, ...)
2475 2476
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2477 2478
	va_list args;
	char error_msg[80];
2479

2480 2481 2482 2483 2484
	va_start(args, fmt);
	vscnprintf(error_msg, sizeof(error_msg), fmt, args);
	va_end(args);

	i915_capture_error_state(dev, wedged, error_msg);
2485
	i915_report_and_clear_eir(dev);
2486

2487
	if (wedged) {
2488 2489
		atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
				&dev_priv->gpu_error.reset_counter);
2490

2491
		/*
2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502
		 * Wakeup waiting processes so that the reset work function
		 * i915_error_work_func doesn't deadlock trying to grab various
		 * locks. By bumping the reset counter first, the woken
		 * processes will see a reset in progress and back off,
		 * releasing their locks and then wait for the reset completion.
		 * We must do this for _all_ gpu waiters that might hold locks
		 * that the reset work needs to acquire.
		 *
		 * Note: The wake_up serves as the required memory barrier to
		 * ensure that the waiters see the updated value of the reset
		 * counter atomic_t.
2503
		 */
2504
		i915_error_wake_up(dev_priv, false);
2505 2506
	}

2507 2508 2509 2510 2511 2512 2513
	/*
	 * Our reset work can grab modeset locks (since it needs to reset the
	 * state of outstanding pagelips). Hence it must not be run on our own
	 * dev-priv->wq work queue for otherwise the flush_work in the pageflip
	 * code will deadlock.
	 */
	schedule_work(&dev_priv->gpu_error.work);
2514 2515
}

2516
static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
2517
{
2518
	struct drm_i915_private *dev_priv = dev->dev_private;
2519 2520
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2521
	struct drm_i915_gem_object *obj;
2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532
	struct intel_unpin_work *work;
	unsigned long flags;
	bool stall_detected;

	/* Ignore early vblank irqs */
	if (intel_crtc == NULL)
		return;

	spin_lock_irqsave(&dev->event_lock, flags);
	work = intel_crtc->unpin_work;

2533 2534 2535
	if (work == NULL ||
	    atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
	    !work->enable_stall_check) {
2536 2537 2538 2539 2540 2541
		/* Either the pending flip IRQ arrived, or we're too early. Don't check */
		spin_unlock_irqrestore(&dev->event_lock, flags);
		return;
	}

	/* Potential stall - if we see that the flip has happened, assume a missed interrupt */
2542
	obj = work->pending_flip_obj;
2543
	if (INTEL_INFO(dev)->gen >= 4) {
2544
		int dspsurf = DSPSURF(intel_crtc->plane);
2545
		stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
2546
					i915_gem_obj_ggtt_offset(obj);
2547
	} else {
2548
		int dspaddr = DSPADDR(intel_crtc->plane);
2549
		stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
2550 2551
							crtc->y * crtc->primary->fb->pitches[0] +
							crtc->x * crtc->primary->fb->bits_per_pixel/8);
2552 2553 2554 2555 2556 2557 2558 2559 2560 2561
	}

	spin_unlock_irqrestore(&dev->event_lock, flags);

	if (stall_detected) {
		DRM_DEBUG_DRIVER("Pageflip stall detected\n");
		intel_prepare_page_flip(dev, intel_crtc->plane);
	}
}

2562 2563 2564
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2565
static int i915_enable_vblank(struct drm_device *dev, int pipe)
2566
{
2567
	struct drm_i915_private *dev_priv = dev->dev_private;
2568
	unsigned long irqflags;
2569

2570
	if (!i915_pipe_enabled(dev, pipe))
2571
		return -EINVAL;
2572

2573
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2574
	if (INTEL_INFO(dev)->gen >= 4)
2575
		i915_enable_pipestat(dev_priv, pipe,
2576
				     PIPE_START_VBLANK_INTERRUPT_STATUS);
2577
	else
2578
		i915_enable_pipestat(dev_priv, pipe,
2579
				     PIPE_VBLANK_INTERRUPT_STATUS);
2580 2581

	/* maintain vblank delivery even in deep C-states */
2582
	if (INTEL_INFO(dev)->gen == 3)
2583
		I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
2584
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2585

2586 2587 2588
	return 0;
}

2589
static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
2590
{
2591
	struct drm_i915_private *dev_priv = dev->dev_private;
2592
	unsigned long irqflags;
2593
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2594
						     DE_PIPE_VBLANK(pipe);
2595 2596 2597 2598 2599

	if (!i915_pipe_enabled(dev, pipe))
		return -EINVAL;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2600
	ironlake_enable_display_irq(dev_priv, bit);
2601 2602 2603 2604 2605
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

J
Jesse Barnes 已提交
2606 2607
static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
{
2608
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2609 2610 2611 2612 2613 2614
	unsigned long irqflags;

	if (!i915_pipe_enabled(dev, pipe))
		return -EINVAL;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2615
	i915_enable_pipestat(dev_priv, pipe,
2616
			     PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2617 2618 2619 2620 2621
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

2622 2623 2624 2625 2626 2627 2628 2629 2630
static int gen8_enable_vblank(struct drm_device *dev, int pipe)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	if (!i915_pipe_enabled(dev, pipe))
		return -EINVAL;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2631 2632 2633
	dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2634 2635 2636 2637
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
	return 0;
}

2638 2639 2640
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
2641
static void i915_disable_vblank(struct drm_device *dev, int pipe)
2642
{
2643
	struct drm_i915_private *dev_priv = dev->dev_private;
2644
	unsigned long irqflags;
2645

2646
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2647
	if (INTEL_INFO(dev)->gen == 3)
2648
		I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
2649

2650
	i915_disable_pipestat(dev_priv, pipe,
2651 2652
			      PIPE_VBLANK_INTERRUPT_STATUS |
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
2653 2654 2655
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2656
static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
2657
{
2658
	struct drm_i915_private *dev_priv = dev->dev_private;
2659
	unsigned long irqflags;
2660
	uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
2661
						     DE_PIPE_VBLANK(pipe);
2662 2663

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2664
	ironlake_disable_display_irq(dev_priv, bit);
2665 2666 2667
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

J
Jesse Barnes 已提交
2668 2669
static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
{
2670
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
2671 2672 2673
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2674
	i915_disable_pipestat(dev_priv, pipe,
2675
			      PIPE_START_VBLANK_INTERRUPT_STATUS);
J
Jesse Barnes 已提交
2676 2677 2678
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2679 2680 2681 2682 2683 2684 2685 2686 2687
static void gen8_disable_vblank(struct drm_device *dev, int pipe)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	unsigned long irqflags;

	if (!i915_pipe_enabled(dev, pipe))
		return;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
2688 2689 2690
	dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
	I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
	POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
2691 2692 2693
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

2694 2695
static u32
ring_last_seqno(struct intel_ring_buffer *ring)
2696
{
2697 2698 2699 2700
	return list_entry(ring->request_list.prev,
			  struct drm_i915_gem_request, list)->seqno;
}

2701 2702 2703 2704 2705
static bool
ring_idle(struct intel_ring_buffer *ring, u32 seqno)
{
	return (list_empty(&ring->request_list) ||
		i915_seqno_passed(seqno, ring_last_seqno(ring)));
B
Ben Gamari 已提交
2706 2707
}

2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724
static bool
ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
{
	if (INTEL_INFO(dev)->gen >= 8) {
		/*
		 * FIXME: gen8 semaphore support - currently we don't emit
		 * semaphores on bdw anyway, but this needs to be addressed when
		 * we merge that code.
		 */
		return false;
	} else {
		ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
		return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
				 MI_SEMAPHORE_REGISTER);
	}
}

2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745
static struct intel_ring_buffer *
semaphore_wait_to_signaller_ring(struct intel_ring_buffer *ring, u32 ipehr)
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
	struct intel_ring_buffer *signaller;
	int i;

	if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
		/*
		 * FIXME: gen8 semaphore support - currently we don't emit
		 * semaphores on bdw anyway, but this needs to be addressed when
		 * we merge that code.
		 */
		return NULL;
	} else {
		u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;

		for_each_ring(signaller, dev_priv, i) {
			if(ring == signaller)
				continue;

2746
			if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
2747 2748 2749 2750 2751 2752 2753 2754 2755 2756
				return signaller;
		}
	}

	DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x\n",
		  ring->id, ipehr);

	return NULL;
}

2757 2758
static struct intel_ring_buffer *
semaphore_waits_for(struct intel_ring_buffer *ring, u32 *seqno)
2759 2760
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
2761 2762
	u32 cmd, ipehr, head;
	int i;
2763 2764

	ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
2765
	if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
2766
		return NULL;
2767

2768 2769 2770 2771 2772 2773
	/*
	 * HEAD is likely pointing to the dword after the actual command,
	 * so scan backwards until we find the MBOX. But limit it to just 3
	 * dwords. Note that we don't care about ACTHD here since that might
	 * point at at batch, and semaphores are always emitted into the
	 * ringbuffer itself.
2774
	 */
2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786
	head = I915_READ_HEAD(ring) & HEAD_ADDR;

	for (i = 4; i; --i) {
		/*
		 * Be paranoid and presume the hw has gone off into the wild -
		 * our ring is smaller than what the hardware (and hence
		 * HEAD_ADDR) allows. Also handles wrap-around.
		 */
		head &= ring->size - 1;

		/* This here seems to blow up */
		cmd = ioread32(ring->virtual_start + head);
2787 2788 2789
		if (cmd == ipehr)
			break;

2790 2791
		head -= 4;
	}
2792

2793 2794
	if (!i)
		return NULL;
2795

2796
	*seqno = ioread32(ring->virtual_start + head + 4) + 1;
2797
	return semaphore_wait_to_signaller_ring(ring, ipehr);
2798 2799
}

2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828
static int semaphore_passed(struct intel_ring_buffer *ring)
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
	struct intel_ring_buffer *signaller;
	u32 seqno, ctl;

	ring->hangcheck.deadlock = true;

	signaller = semaphore_waits_for(ring, &seqno);
	if (signaller == NULL || signaller->hangcheck.deadlock)
		return -1;

	/* cursory check for an unkickable deadlock */
	ctl = I915_READ_CTL(signaller);
	if (ctl & RING_WAIT_SEMAPHORE && semaphore_passed(signaller) < 0)
		return -1;

	return i915_seqno_passed(signaller->get_seqno(signaller, false), seqno);
}

static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
{
	struct intel_ring_buffer *ring;
	int i;

	for_each_ring(ring, dev_priv, i)
		ring->hangcheck.deadlock = false;
}

2829
static enum intel_ring_hangcheck_action
2830
ring_stuck(struct intel_ring_buffer *ring, u64 acthd)
2831 2832 2833
{
	struct drm_device *dev = ring->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
2834 2835
	u32 tmp;

2836
	if (ring->hangcheck.acthd != acthd)
2837
		return HANGCHECK_ACTIVE;
2838

2839
	if (IS_GEN2(dev))
2840
		return HANGCHECK_HUNG;
2841 2842 2843 2844 2845 2846 2847

	/* Is the chip hanging on a WAIT_FOR_EVENT?
	 * If so we can simply poke the RB_WAIT bit
	 * and break the hang. This should work on
	 * all but the second generation chipsets.
	 */
	tmp = I915_READ_CTL(ring);
2848
	if (tmp & RING_WAIT) {
2849 2850 2851
		i915_handle_error(dev, false,
				  "Kicking stuck wait on %s",
				  ring->name);
2852
		I915_WRITE_CTL(ring, tmp);
2853
		return HANGCHECK_KICK;
2854 2855 2856 2857 2858
	}

	if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
		switch (semaphore_passed(ring)) {
		default:
2859
			return HANGCHECK_HUNG;
2860
		case 1:
2861 2862 2863
			i915_handle_error(dev, false,
					  "Kicking stuck semaphore on %s",
					  ring->name);
2864
			I915_WRITE_CTL(ring, tmp);
2865
			return HANGCHECK_KICK;
2866
		case 0:
2867
			return HANGCHECK_WAIT;
2868
		}
2869
	}
2870

2871
	return HANGCHECK_HUNG;
2872 2873
}

B
Ben Gamari 已提交
2874 2875
/**
 * This is called when the chip hasn't reported back with completed
2876 2877 2878 2879 2880
 * batchbuffers in a long time. We keep track per ring seqno progress and
 * if there are no progress, hangcheck score for that ring is increased.
 * Further, acthd is inspected to see if the ring is stuck. On stuck case
 * we kick the ring. If we see no progress on three subsequent calls
 * we assume chip is wedged and try to fix it by resetting the chip.
B
Ben Gamari 已提交
2881
 */
2882
static void i915_hangcheck_elapsed(unsigned long data)
B
Ben Gamari 已提交
2883 2884
{
	struct drm_device *dev = (struct drm_device *)data;
2885
	struct drm_i915_private *dev_priv = dev->dev_private;
2886 2887
	struct intel_ring_buffer *ring;
	int i;
2888
	int busy_count = 0, rings_hung = 0;
2889 2890 2891 2892
	bool stuck[I915_NUM_RINGS] = { 0 };
#define BUSY 1
#define KICK 5
#define HUNG 20
2893

2894
	if (!i915.enable_hangcheck)
2895 2896
		return;

2897
	for_each_ring(ring, dev_priv, i) {
2898 2899
		u64 acthd;
		u32 seqno;
2900
		bool busy = true;
2901

2902 2903
		semaphore_clear_deadlocks(dev_priv);

2904 2905
		seqno = ring->get_seqno(ring, false);
		acthd = intel_ring_get_active_head(ring);
2906

2907 2908
		if (ring->hangcheck.seqno == seqno) {
			if (ring_idle(ring, seqno)) {
2909 2910
				ring->hangcheck.action = HANGCHECK_IDLE;

2911 2912
				if (waitqueue_active(&ring->irq_queue)) {
					/* Issue a wake-up to catch stuck h/w. */
2913
					if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
2914 2915 2916 2917 2918 2919
						if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
							DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
								  ring->name);
						else
							DRM_INFO("Fake missed irq on %s\n",
								 ring->name);
2920 2921 2922 2923
						wake_up_all(&ring->irq_queue);
					}
					/* Safeguard against driver failure */
					ring->hangcheck.score += BUSY;
2924 2925
				} else
					busy = false;
2926
			} else {
2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941
				/* We always increment the hangcheck score
				 * if the ring is busy and still processing
				 * the same request, so that no single request
				 * can run indefinitely (such as a chain of
				 * batches). The only time we do not increment
				 * the hangcheck score on this ring, if this
				 * ring is in a legitimate wait for another
				 * ring. In that case the waiting ring is a
				 * victim and we want to be sure we catch the
				 * right culprit. Then every time we do kick
				 * the ring, add a small increment to the
				 * score so that we can catch a batch that is
				 * being repeatedly kicked and so responsible
				 * for stalling the machine.
				 */
2942 2943 2944 2945
				ring->hangcheck.action = ring_stuck(ring,
								    acthd);

				switch (ring->hangcheck.action) {
2946
				case HANGCHECK_IDLE:
2947
				case HANGCHECK_WAIT:
2948
					break;
2949
				case HANGCHECK_ACTIVE:
2950
					ring->hangcheck.score += BUSY;
2951
					break;
2952
				case HANGCHECK_KICK:
2953
					ring->hangcheck.score += KICK;
2954
					break;
2955
				case HANGCHECK_HUNG:
2956
					ring->hangcheck.score += HUNG;
2957 2958 2959
					stuck[i] = true;
					break;
				}
2960
			}
2961
		} else {
2962 2963
			ring->hangcheck.action = HANGCHECK_ACTIVE;

2964 2965 2966 2967 2968
			/* Gradually reduce the count so that we catch DoS
			 * attempts across multiple batches.
			 */
			if (ring->hangcheck.score > 0)
				ring->hangcheck.score--;
2969 2970
		}

2971 2972
		ring->hangcheck.seqno = seqno;
		ring->hangcheck.acthd = acthd;
2973
		busy_count += busy;
2974
	}
2975

2976
	for_each_ring(ring, dev_priv, i) {
2977
		if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
2978 2979 2980
			DRM_INFO("%s on %s\n",
				 stuck[i] ? "stuck" : "no progress",
				 ring->name);
2981
			rings_hung++;
2982 2983 2984
		}
	}

2985
	if (rings_hung)
2986
		return i915_handle_error(dev, true, "Ring hung");
B
Ben Gamari 已提交
2987

2988 2989 2990
	if (busy_count)
		/* Reset timer case chip hangs without another request
		 * being added */
2991 2992 2993 2994 2995 2996
		i915_queue_hangcheck(dev);
}

void i915_queue_hangcheck(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2997
	if (!i915.enable_hangcheck)
2998 2999 3000 3001
		return;

	mod_timer(&dev_priv->gpu_error.hangcheck_timer,
		  round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
B
Ben Gamari 已提交
3002 3003
}

3004
static void ibx_irq_reset(struct drm_device *dev)
P
Paulo Zanoni 已提交
3005 3006 3007 3008 3009 3010
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

3011
	GEN5_IRQ_RESET(SDE);
3012 3013 3014

	if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
		I915_WRITE(SERR_INT, 0xffffffff);
P
Paulo Zanoni 已提交
3015
}
3016

P
Paulo Zanoni 已提交
3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032
/*
 * SDEIER is also touched by the interrupt handler to work around missed PCH
 * interrupts. Hence we can't update it after the interrupt handler is enabled -
 * instead we unconditionally enable all PCH interrupt sources here, but then
 * only unmask them as needed with SDEIMR.
 *
 * This function needs to be called before interrupts are enabled.
 */
static void ibx_irq_pre_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (HAS_PCH_NOP(dev))
		return;

	WARN_ON(I915_READ(SDEIER) != 0);
P
Paulo Zanoni 已提交
3033 3034 3035 3036
	I915_WRITE(SDEIER, 0xffffffff);
	POSTING_READ(SDEIER);
}

3037
static void gen5_gt_irq_reset(struct drm_device *dev)
3038 3039 3040
{
	struct drm_i915_private *dev_priv = dev->dev_private;

3041
	GEN5_IRQ_RESET(GT);
P
Paulo Zanoni 已提交
3042
	if (INTEL_INFO(dev)->gen >= 6)
3043
		GEN5_IRQ_RESET(GEN6_PM);
3044 3045
}

L
Linus Torvalds 已提交
3046 3047
/* drm_dma.h hooks
*/
P
Paulo Zanoni 已提交
3048
static void ironlake_irq_reset(struct drm_device *dev)
3049
{
3050
	struct drm_i915_private *dev_priv = dev->dev_private;
3051

3052
	I915_WRITE(HWSTAM, 0xffffffff);
3053

3054
	GEN5_IRQ_RESET(DE);
3055 3056
	if (IS_GEN7(dev))
		I915_WRITE(GEN7_ERR_INT, 0xffffffff);
3057

3058
	gen5_gt_irq_reset(dev);
3059

3060
	ibx_irq_reset(dev);
3061
}
3062

P
Paulo Zanoni 已提交
3063 3064 3065
static void ironlake_irq_preinstall(struct drm_device *dev)
{
	ironlake_irq_reset(dev);
3066 3067
}

J
Jesse Barnes 已提交
3068 3069
static void valleyview_irq_preinstall(struct drm_device *dev)
{
3070
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081
	int pipe;

	/* VLV magic */
	I915_WRITE(VLV_IMR, 0);
	I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
	I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
	I915_WRITE(RING_IMR(BLT_RING_BASE), 0);

	/* and GT */
	I915_WRITE(GTIIR, I915_READ(GTIIR));
	I915_WRITE(GTIIR, I915_READ(GTIIR));
3082

3083
	gen5_gt_irq_reset(dev);
J
Jesse Barnes 已提交
3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096

	I915_WRITE(DPINVGTT, 0xff);

	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe), 0xffff);
	I915_WRITE(VLV_IIR, 0xffffffff);
	I915_WRITE(VLV_IMR, 0xffffffff);
	I915_WRITE(VLV_IER, 0x0);
	POSTING_READ(VLV_IER);
}

P
Paulo Zanoni 已提交
3097
static void gen8_irq_reset(struct drm_device *dev)
3098 3099 3100 3101 3102 3103 3104
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

3105 3106 3107 3108
	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);
3109

P
Paulo Zanoni 已提交
3110
	for_each_pipe(pipe)
3111
		GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
3112

3113 3114 3115
	GEN5_IRQ_RESET(GEN8_DE_PORT_);
	GEN5_IRQ_RESET(GEN8_DE_MISC_);
	GEN5_IRQ_RESET(GEN8_PCU_);
3116

3117
	ibx_irq_reset(dev);
3118
}
3119

P
Paulo Zanoni 已提交
3120 3121 3122
static void gen8_irq_preinstall(struct drm_device *dev)
{
	gen8_irq_reset(dev);
3123 3124
}

3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155
static void cherryview_irq_preinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int pipe;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

	GEN8_IRQ_RESET_NDX(GT, 0);
	GEN8_IRQ_RESET_NDX(GT, 1);
	GEN8_IRQ_RESET_NDX(GT, 2);
	GEN8_IRQ_RESET_NDX(GT, 3);

	GEN5_IRQ_RESET(GEN8_PCU_);

	POSTING_READ(GEN8_PCU_IIR);

	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);

	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe), 0xffff);

	I915_WRITE(VLV_IMR, 0xffffffff);
	I915_WRITE(VLV_IER, 0x0);
	I915_WRITE(VLV_IIR, 0xffffffff);
	POSTING_READ(VLV_IIR);
}

3156
static void ibx_hpd_irq_setup(struct drm_device *dev)
3157
{
3158
	struct drm_i915_private *dev_priv = dev->dev_private;
3159 3160
	struct drm_mode_config *mode_config = &dev->mode_config;
	struct intel_encoder *intel_encoder;
3161
	u32 hotplug_irqs, hotplug, enabled_irqs = 0;
3162 3163

	if (HAS_PCH_IBX(dev)) {
3164
		hotplug_irqs = SDE_HOTPLUG_MASK;
3165
		list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
3166
			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
3167
				enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
3168
	} else {
3169
		hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
3170
		list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
3171
			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
3172
				enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
3173
	}
3174

3175
	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
3176 3177 3178 3179 3180 3181 3182

	/*
	 * Enable digital hotplug on the PCH, and configure the DP short pulse
	 * duration to 2ms (which is the minimum in the Display Port spec)
	 *
	 * This register is the same on all known PCH chips.
	 */
3183 3184 3185 3186 3187 3188 3189 3190
	hotplug = I915_READ(PCH_PORT_HOTPLUG);
	hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
	hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
	hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
	hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
	I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
}

P
Paulo Zanoni 已提交
3191 3192
static void ibx_irq_postinstall(struct drm_device *dev)
{
3193
	struct drm_i915_private *dev_priv = dev->dev_private;
3194
	u32 mask;
3195

D
Daniel Vetter 已提交
3196 3197 3198
	if (HAS_PCH_NOP(dev))
		return;

3199
	if (HAS_PCH_IBX(dev))
3200
		mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
3201
	else
3202
		mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
3203

3204
	GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
P
Paulo Zanoni 已提交
3205 3206 3207
	I915_WRITE(SDEIMR, ~mask);
}

3208 3209 3210 3211 3212 3213 3214 3215
static void gen5_gt_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 pm_irqs, gt_irqs;

	pm_irqs = gt_irqs = 0;

	dev_priv->gt_irq_mask = ~0;
3216
	if (HAS_L3_DPF(dev)) {
3217
		/* L3 parity interrupt is always unmasked. */
3218 3219
		dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
		gt_irqs |= GT_PARITY_ERROR(dev);
3220 3221 3222 3223 3224 3225 3226 3227 3228 3229
	}

	gt_irqs |= GT_RENDER_USER_INTERRUPT;
	if (IS_GEN5(dev)) {
		gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
			   ILK_BSD_USER_INTERRUPT;
	} else {
		gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
	}

P
Paulo Zanoni 已提交
3230
	GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
3231 3232

	if (INTEL_INFO(dev)->gen >= 6) {
3233
		pm_irqs |= dev_priv->pm_rps_events;
3234 3235 3236 3237

		if (HAS_VEBOX(dev))
			pm_irqs |= PM_VEBOX_USER_INTERRUPT;

3238
		dev_priv->pm_irq_mask = 0xffffffff;
P
Paulo Zanoni 已提交
3239
		GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
3240 3241 3242
	}
}

3243
static int ironlake_irq_postinstall(struct drm_device *dev)
3244
{
3245
	unsigned long irqflags;
3246
	struct drm_i915_private *dev_priv = dev->dev_private;
3247 3248 3249 3250 3251 3252
	u32 display_mask, extra_mask;

	if (INTEL_INFO(dev)->gen >= 7) {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
				DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
				DE_PLANEB_FLIP_DONE_IVB |
3253
				DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
3254
		extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
3255
			      DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB);
3256 3257 3258
	} else {
		display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
				DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
3259 3260 3261
				DE_AUX_CHANNEL_A |
				DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
				DE_POISON);
3262 3263
		extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
				DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN;
3264
	}
3265

3266
	dev_priv->irq_mask = ~display_mask;
3267

3268 3269
	I915_WRITE(HWSTAM, 0xeffe);

P
Paulo Zanoni 已提交
3270 3271
	ibx_irq_pre_postinstall(dev);

P
Paulo Zanoni 已提交
3272
	GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
3273

3274
	gen5_gt_irq_postinstall(dev);
3275

P
Paulo Zanoni 已提交
3276
	ibx_irq_postinstall(dev);
3277

3278
	if (IS_IRONLAKE_M(dev)) {
3279 3280 3281
		/* Enable PCU event interrupts
		 *
		 * spinlocking not required here for correctness since interrupt
3282 3283 3284
		 * setup is guaranteed to run in single-threaded context. But we
		 * need it to make the assert_spin_locked happy. */
		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3285
		ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
3286
		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3287 3288
	}

3289 3290 3291
	return 0;
}

3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329
static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;

	I915_WRITE(PIPESTAT(PIPE_A), pipestat_mask);
	I915_WRITE(PIPESTAT(PIPE_B), pipestat_mask);
	POSTING_READ(PIPESTAT(PIPE_A));

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

	i915_enable_pipestat(dev_priv, PIPE_A, pipestat_mask |
					       PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, pipestat_mask);

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
	dev_priv->irq_mask &= ~iir_mask;

	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
	POSTING_READ(VLV_IER);
}

static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
{
	u32 pipestat_mask;
	u32 iir_mask;

	iir_mask = I915_DISPLAY_PORT_INTERRUPT |
		   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3330
		   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378

	dev_priv->irq_mask |= iir_mask;
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	I915_WRITE(VLV_IIR, iir_mask);
	POSTING_READ(VLV_IIR);

	pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
			PIPE_CRC_DONE_INTERRUPT_STATUS;

	i915_disable_pipestat(dev_priv, PIPE_A, pipestat_mask |
					        PIPE_GMBUS_INTERRUPT_STATUS);
	i915_disable_pipestat(dev_priv, PIPE_B, pipestat_mask);

	pipestat_mask = PIPESTAT_INT_STATUS_MASK |
			PIPE_FIFO_UNDERRUN_STATUS;
	I915_WRITE(PIPESTAT(PIPE_A), pipestat_mask);
	I915_WRITE(PIPESTAT(PIPE_B), pipestat_mask);
	POSTING_READ(PIPESTAT(PIPE_A));
}

void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = true;

	if (dev_priv->dev->irq_enabled)
		valleyview_display_irqs_install(dev_priv);
}

void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
{
	assert_spin_locked(&dev_priv->irq_lock);

	if (!dev_priv->display_irqs_enabled)
		return;

	dev_priv->display_irqs_enabled = false;

	if (dev_priv->dev->irq_enabled)
		valleyview_display_irqs_uninstall(dev_priv);
}

J
Jesse Barnes 已提交
3379 3380
static int valleyview_irq_postinstall(struct drm_device *dev)
{
3381
	struct drm_i915_private *dev_priv = dev->dev_private;
3382
	unsigned long irqflags;
J
Jesse Barnes 已提交
3383

3384
	dev_priv->irq_mask = ~0;
J
Jesse Barnes 已提交
3385

3386 3387 3388
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	POSTING_READ(PORT_HOTPLUG_EN);

J
Jesse Barnes 已提交
3389
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3390
	I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
J
Jesse Barnes 已提交
3391 3392 3393
	I915_WRITE(VLV_IIR, 0xffffffff);
	POSTING_READ(VLV_IER);

3394 3395 3396
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3397 3398
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_install(dev_priv);
3399
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
3400

J
Jesse Barnes 已提交
3401 3402 3403
	I915_WRITE(VLV_IIR, 0xffffffff);
	I915_WRITE(VLV_IIR, 0xffffffff);

3404
	gen5_gt_irq_postinstall(dev);
J
Jesse Barnes 已提交
3405 3406 3407 3408 3409 3410 3411 3412

	/* ack & enable invalid PTE error interrupts */
#if 0 /* FIXME: add support to irq handler for checking these bits */
	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
#endif

	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
3413 3414 3415 3416

	return 0;
}

3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431
static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
{
	int i;

	/* These are interrupts we'll toggle with the ring mask register */
	uint32_t gt_interrupts[] = {
		GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
			GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
			GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
		GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
			GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
		0,
		GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT
		};

3432
	for (i = 0; i < ARRAY_SIZE(gt_interrupts); i++)
P
Paulo Zanoni 已提交
3433
		GEN8_IRQ_INIT_NDX(GT, i, ~gt_interrupts[i], gt_interrupts[i]);
3434 3435

	dev_priv->pm_irq_mask = 0xffffffff;
3436 3437 3438 3439 3440
}

static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
{
	struct drm_device *dev = dev_priv->dev;
3441
	uint32_t de_pipe_masked = GEN8_PIPE_PRIMARY_FLIP_DONE |
3442 3443
		GEN8_PIPE_CDCLK_CRC_DONE |
		GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3444 3445
	uint32_t de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
		GEN8_PIPE_FIFO_UNDERRUN;
3446
	int pipe;
3447 3448 3449
	dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
	dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
3450

3451
	for_each_pipe(pipe)
P
Paulo Zanoni 已提交
3452 3453
		GEN8_IRQ_INIT_NDX(DE_PIPE, pipe, dev_priv->de_irq_mask[pipe],
				  de_pipe_enables);
3454

P
Paulo Zanoni 已提交
3455
	GEN5_IRQ_INIT(GEN8_DE_PORT_, ~GEN8_AUX_CHANNEL_A, GEN8_AUX_CHANNEL_A);
3456 3457 3458 3459 3460 3461
}

static int gen8_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

P
Paulo Zanoni 已提交
3462 3463
	ibx_irq_pre_postinstall(dev);

3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474
	gen8_gt_irq_postinstall(dev_priv);
	gen8_de_irq_postinstall(dev_priv);

	ibx_irq_postinstall(dev);

	I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3475 3476 3477 3478 3479 3480
static int cherryview_irq_postinstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 enable_mask = I915_DISPLAY_PORT_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3481 3482 3483
		I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
	u32 pipestat_enable = PLANE_FLIP_DONE_INT_STATUS_VLV |
		PIPE_CRC_DONE_INTERRUPT_STATUS;
3484 3485 3486 3487 3488 3489 3490
	unsigned long irqflags;
	int pipe;

	/*
	 * Leave vblank interrupts masked initially.  enable/disable will
	 * toggle them based on usage.
	 */
3491
	dev_priv->irq_mask = ~enable_mask;
3492 3493 3494 3495 3496

	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe), 0xffff);

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3497
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513
	for_each_pipe(pipe)
		i915_enable_pipestat(dev_priv, pipe, pipestat_enable);
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	I915_WRITE(VLV_IIR, 0xffffffff);
	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
	I915_WRITE(VLV_IER, enable_mask);

	gen8_gt_irq_postinstall(dev_priv);

	I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
	POSTING_READ(GEN8_MASTER_IRQ);

	return 0;
}

3514 3515 3516 3517 3518 3519 3520
static void gen8_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (!dev_priv)
		return;

3521
	intel_hpd_irq_uninstall(dev_priv);
3522

P
Paulo Zanoni 已提交
3523
	gen8_irq_reset(dev);
3524 3525
}

J
Jesse Barnes 已提交
3526 3527
static void valleyview_irq_uninstall(struct drm_device *dev)
{
3528
	struct drm_i915_private *dev_priv = dev->dev_private;
3529
	unsigned long irqflags;
J
Jesse Barnes 已提交
3530 3531 3532 3533 3534
	int pipe;

	if (!dev_priv)
		return;

3535 3536
	I915_WRITE(VLV_MASTER_IER, 0);

3537
	intel_hpd_irq_uninstall(dev_priv);
3538

J
Jesse Barnes 已提交
3539 3540 3541 3542 3543 3544
	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe), 0xffff);

	I915_WRITE(HWSTAM, 0xffffffff);
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3545 3546 3547 3548 3549 3550 3551 3552

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
	if (dev_priv->display_irqs_enabled)
		valleyview_display_irqs_uninstall(dev_priv);
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	dev_priv->irq_mask = 0;

J
Jesse Barnes 已提交
3553 3554 3555 3556 3557 3558
	I915_WRITE(VLV_IIR, 0xffffffff);
	I915_WRITE(VLV_IMR, 0xffffffff);
	I915_WRITE(VLV_IER, 0x0);
	POSTING_READ(VLV_IER);
}

3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609
static void cherryview_irq_uninstall(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int pipe;

	if (!dev_priv)
		return;

	I915_WRITE(GEN8_MASTER_IRQ, 0);
	POSTING_READ(GEN8_MASTER_IRQ);

#define GEN8_IRQ_FINI_NDX(type, which)				\
do {								\
	I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff);	\
	I915_WRITE(GEN8_##type##_IER(which), 0);		\
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff);	\
	POSTING_READ(GEN8_##type##_IIR(which));			\
	I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff);	\
} while (0)

#define GEN8_IRQ_FINI(type)				\
do {							\
	I915_WRITE(GEN8_##type##_IMR, 0xffffffff);	\
	I915_WRITE(GEN8_##type##_IER, 0);		\
	I915_WRITE(GEN8_##type##_IIR, 0xffffffff);	\
	POSTING_READ(GEN8_##type##_IIR);		\
	I915_WRITE(GEN8_##type##_IIR, 0xffffffff);	\
} while (0)

	GEN8_IRQ_FINI_NDX(GT, 0);
	GEN8_IRQ_FINI_NDX(GT, 1);
	GEN8_IRQ_FINI_NDX(GT, 2);
	GEN8_IRQ_FINI_NDX(GT, 3);

	GEN8_IRQ_FINI(PCU);

#undef GEN8_IRQ_FINI
#undef GEN8_IRQ_FINI_NDX

	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));

	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe), 0xffff);

	I915_WRITE(VLV_IMR, 0xffffffff);
	I915_WRITE(VLV_IER, 0x0);
	I915_WRITE(VLV_IIR, 0xffffffff);
	POSTING_READ(VLV_IIR);
}

3610
static void ironlake_irq_uninstall(struct drm_device *dev)
3611
{
3612
	struct drm_i915_private *dev_priv = dev->dev_private;
3613 3614 3615 3616

	if (!dev_priv)
		return;

3617
	intel_hpd_irq_uninstall(dev_priv);
3618

P
Paulo Zanoni 已提交
3619
	ironlake_irq_reset(dev);
3620 3621
}

3622
static void i8xx_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
3623
{
3624
	struct drm_i915_private *dev_priv = dev->dev_private;
3625
	int pipe;
3626

3627 3628
	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe), 0);
3629 3630 3631
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	POSTING_READ16(IER);
C
Chris Wilson 已提交
3632 3633 3634 3635
}

static int i8xx_irq_postinstall(struct drm_device *dev)
{
3636
	struct drm_i915_private *dev_priv = dev->dev_private;
3637
	unsigned long irqflags;
C
Chris Wilson 已提交
3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657

	I915_WRITE16(EMR,
		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	I915_WRITE16(IMR, dev_priv->irq_mask);

	I915_WRITE16(IER,
		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		     I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
		     I915_USER_INTERRUPT);
	POSTING_READ16(IER);

3658 3659 3660
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3661 3662
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3663 3664
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

C
Chris Wilson 已提交
3665 3666 3667
	return 0;
}

3668 3669 3670 3671
/*
 * Returns true when a page flip has completed.
 */
static bool i8xx_handle_vblank(struct drm_device *dev,
3672
			       int plane, int pipe, u32 iir)
3673
{
3674
	struct drm_i915_private *dev_priv = dev->dev_private;
3675
	u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3676

3677
	if (!intel_pipe_handle_vblank(dev, pipe))
3678 3679 3680 3681 3682
		return false;

	if ((iir & flip_pending) == 0)
		return false;

3683
	intel_prepare_page_flip(dev, plane);
3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ16(ISR) & flip_pending)
		return false;

	intel_finish_page_flip(dev, pipe);

	return true;
}

3699
static irqreturn_t i8xx_irq_handler(int irq, void *arg)
C
Chris Wilson 已提交
3700
{
3701
	struct drm_device *dev = arg;
3702
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722
	u16 iir, new_iir;
	u32 pipe_stats[2];
	unsigned long irqflags;
	int pipe;
	u16 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;

	iir = I915_READ16(IIR);
	if (iir == 0)
		return IRQ_NONE;

	while (iir & ~flip_mask) {
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3723 3724 3725
			i915_handle_error(dev, false,
					  "Command parser error, iir 0x%08x",
					  iir);
C
Chris Wilson 已提交
3726 3727 3728 3729 3730 3731 3732 3733

		for_each_pipe(pipe) {
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
3734
			if (pipe_stats[pipe] & 0x8000ffff)
C
Chris Wilson 已提交
3735 3736 3737 3738 3739 3740 3741
				I915_WRITE(reg, pipe_stats[pipe]);
		}
		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

		I915_WRITE16(IIR, iir & ~flip_mask);
		new_iir = I915_READ16(IIR); /* Flush posted writes */

3742
		i915_update_dri1_breadcrumb(dev);
C
Chris Wilson 已提交
3743 3744 3745 3746

		if (iir & I915_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[RCS]);

3747
		for_each_pipe(pipe) {
3748
			int plane = pipe;
3749
			if (HAS_FBC(dev))
3750 3751
				plane = !plane;

3752
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3753 3754
			    i8xx_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
C
Chris Wilson 已提交
3755

3756
			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3757
				i9xx_pipe_crc_irq_handler(dev, pipe);
3758 3759 3760

			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
			    intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
3761
				DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
3762
		}
C
Chris Wilson 已提交
3763 3764 3765 3766 3767 3768 3769 3770 3771

		iir = new_iir;
	}

	return IRQ_HANDLED;
}

static void i8xx_irq_uninstall(struct drm_device * dev)
{
3772
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784
	int pipe;

	for_each_pipe(pipe) {
		/* Clear enable bits; then clear status bits */
		I915_WRITE(PIPESTAT(pipe), 0);
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
	I915_WRITE16(IMR, 0xffff);
	I915_WRITE16(IER, 0x0);
	I915_WRITE16(IIR, I915_READ16(IIR));
}

3785 3786
static void i915_irq_preinstall(struct drm_device * dev)
{
3787
	struct drm_i915_private *dev_priv = dev->dev_private;
3788 3789 3790 3791 3792 3793 3794
	int pipe;

	if (I915_HAS_HOTPLUG(dev)) {
		I915_WRITE(PORT_HOTPLUG_EN, 0);
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3795
	I915_WRITE16(HWSTAM, 0xeffe);
3796 3797 3798 3799 3800 3801 3802 3803 3804
	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i915_irq_postinstall(struct drm_device *dev)
{
3805
	struct drm_i915_private *dev_priv = dev->dev_private;
3806
	u32 enable_mask;
3807
	unsigned long irqflags;
3808

3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826
	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));

	/* Unmask the interrupts that we always want on. */
	dev_priv->irq_mask =
		~(I915_ASLE_INTERRUPT |
		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask =
		I915_ASLE_INTERRUPT |
		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
		I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
		I915_USER_INTERRUPT;

3827
	if (I915_HAS_HOTPLUG(dev)) {
3828 3829 3830
		I915_WRITE(PORT_HOTPLUG_EN, 0);
		POSTING_READ(PORT_HOTPLUG_EN);

3831 3832 3833 3834 3835 3836 3837 3838 3839 3840
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
	}

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

3841
	i915_enable_asle_pipestat(dev);
3842

3843 3844 3845
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
3846 3847
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
3848 3849
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

3850 3851 3852
	return 0;
}

3853 3854 3855 3856 3857 3858
/*
 * Returns true when a page flip has completed.
 */
static bool i915_handle_vblank(struct drm_device *dev,
			       int plane, int pipe, u32 iir)
{
3859
	struct drm_i915_private *dev_priv = dev->dev_private;
3860 3861
	u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);

3862
	if (!intel_pipe_handle_vblank(dev, pipe))
3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883
		return false;

	if ((iir & flip_pending) == 0)
		return false;

	intel_prepare_page_flip(dev, plane);

	/* We detect FlipDone by looking for the change in PendingFlip from '1'
	 * to '0' on the following vblank, i.e. IIR has the Pendingflip
	 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
	 * the flip is completed (no longer pending). Since this doesn't raise
	 * an interrupt per se, we watch for the change at vblank.
	 */
	if (I915_READ(ISR) & flip_pending)
		return false;

	intel_finish_page_flip(dev, pipe);

	return true;
}

3884
static irqreturn_t i915_irq_handler(int irq, void *arg)
3885
{
3886
	struct drm_device *dev = arg;
3887
	struct drm_i915_private *dev_priv = dev->dev_private;
3888
	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
3889
	unsigned long irqflags;
3890 3891 3892 3893
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
	int pipe, ret = IRQ_NONE;
3894 3895

	iir = I915_READ(IIR);
3896 3897
	do {
		bool irq_received = (iir & ~flip_mask) != 0;
3898
		bool blc_event = false;
3899 3900 3901 3902 3903 3904 3905 3906

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
3907 3908 3909
			i915_handle_error(dev, false,
					  "Command parser error, iir 0x%08x",
					  iir);
3910 3911 3912 3913 3914

		for_each_pipe(pipe) {
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

3915
			/* Clear the PIPE*STAT regs before the IIR */
3916 3917
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
3918
				irq_received = true;
3919 3920 3921 3922 3923 3924 3925 3926
			}
		}
		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

		if (!irq_received)
			break;

		/* Consume port.  Then clear IIR or we'll miss events */
3927 3928 3929
		if (I915_HAS_HOTPLUG(dev) &&
		    iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
3930

3931
		I915_WRITE(IIR, iir & ~flip_mask);
3932 3933 3934 3935 3936 3937
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[RCS]);

		for_each_pipe(pipe) {
3938
			int plane = pipe;
3939
			if (HAS_FBC(dev))
3940
				plane = !plane;
3941

3942
			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3943 3944
			    i915_handle_vblank(dev, plane, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
3945 3946 3947

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
3948 3949

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
3950
				i9xx_pipe_crc_irq_handler(dev, pipe);
3951 3952 3953

			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
			    intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
3954
				DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974
		}

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
3975
		ret = IRQ_HANDLED;
3976
		iir = new_iir;
3977
	} while (iir & ~flip_mask);
3978

3979
	i915_update_dri1_breadcrumb(dev);
3980

3981 3982 3983 3984 3985
	return ret;
}

static void i915_irq_uninstall(struct drm_device * dev)
{
3986
	struct drm_i915_private *dev_priv = dev->dev_private;
3987 3988
	int pipe;

3989
	intel_hpd_irq_uninstall(dev_priv);
3990

3991 3992 3993 3994 3995
	if (I915_HAS_HOTPLUG(dev)) {
		I915_WRITE(PORT_HOTPLUG_EN, 0);
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

3996
	I915_WRITE16(HWSTAM, 0xffff);
3997 3998
	for_each_pipe(pipe) {
		/* Clear enable bits; then clear status bits */
3999
		I915_WRITE(PIPESTAT(pipe), 0);
4000 4001
		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
	}
4002 4003 4004 4005 4006 4007 4008 4009
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	I915_WRITE(IIR, I915_READ(IIR));
}

static void i965_irq_preinstall(struct drm_device * dev)
{
4010
	struct drm_i915_private *dev_priv = dev->dev_private;
4011 4012
	int pipe;

4013 4014
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025

	I915_WRITE(HWSTAM, 0xeffe);
	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);
	POSTING_READ(IER);
}

static int i965_irq_postinstall(struct drm_device *dev)
{
4026
	struct drm_i915_private *dev_priv = dev->dev_private;
4027
	u32 enable_mask;
4028
	u32 error_mask;
4029
	unsigned long irqflags;
4030 4031

	/* Unmask the interrupts that we always want on. */
4032
	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
4033
			       I915_DISPLAY_PORT_INTERRUPT |
4034 4035 4036 4037 4038 4039 4040
			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);

	enable_mask = ~dev_priv->irq_mask;
4041 4042
	enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
			 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
4043 4044 4045 4046
	enable_mask |= I915_USER_INTERRUPT;

	if (IS_G4X(dev))
		enable_mask |= I915_BSD_USER_INTERRUPT;
4047

4048 4049 4050
	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked check happy. */
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4051 4052 4053
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
	i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
4054
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074

	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
	if (IS_G4X(dev)) {
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

	I915_WRITE(IMR, dev_priv->irq_mask);
	I915_WRITE(IER, enable_mask);
	POSTING_READ(IER);

4075 4076 4077
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	POSTING_READ(PORT_HOTPLUG_EN);

4078
	i915_enable_asle_pipestat(dev);
4079 4080 4081 4082

	return 0;
}

4083
static void i915_hpd_irq_setup(struct drm_device *dev)
4084
{
4085
	struct drm_i915_private *dev_priv = dev->dev_private;
4086
	struct drm_mode_config *mode_config = &dev->mode_config;
4087
	struct intel_encoder *intel_encoder;
4088 4089
	u32 hotplug_en;

4090 4091
	assert_spin_locked(&dev_priv->irq_lock);

4092 4093 4094 4095
	if (I915_HAS_HOTPLUG(dev)) {
		hotplug_en = I915_READ(PORT_HOTPLUG_EN);
		hotplug_en &= ~HOTPLUG_INT_EN_MASK;
		/* Note HDMI and DP share hotplug bits */
4096
		/* enable bits are the same for all generations */
4097 4098 4099
		list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
			if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
				hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
4100 4101 4102 4103 4104 4105
		/* Programming the CRT detection parameters tends
		   to generate a spurious hotplug event about three
		   seconds later.  So just do it once.
		*/
		if (IS_G4X(dev))
			hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
4106
		hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
4107
		hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
4108

4109 4110 4111
		/* Ignore TV since it's buggy */
		I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
	}
4112 4113
}

4114
static irqreturn_t i965_irq_handler(int irq, void *arg)
4115
{
4116
	struct drm_device *dev = arg;
4117
	struct drm_i915_private *dev_priv = dev->dev_private;
4118 4119 4120 4121
	u32 iir, new_iir;
	u32 pipe_stats[I915_MAX_PIPES];
	unsigned long irqflags;
	int ret = IRQ_NONE, pipe;
4122 4123 4124
	u32 flip_mask =
		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
4125 4126 4127 4128

	iir = I915_READ(IIR);

	for (;;) {
4129
		bool irq_received = (iir & ~flip_mask) != 0;
4130 4131
		bool blc_event = false;

4132 4133 4134 4135 4136 4137 4138
		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
4139 4140 4141
			i915_handle_error(dev, false,
					  "Command parser error, iir 0x%08x",
					  iir);
4142 4143 4144 4145 4146 4147 4148 4149 4150 4151

		for_each_pipe(pipe) {
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				I915_WRITE(reg, pipe_stats[pipe]);
4152
				irq_received = true;
4153 4154 4155 4156 4157 4158 4159 4160 4161 4162
			}
		}
		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;

		/* Consume port.  Then clear IIR or we'll miss events */
4163 4164
		if (iir & I915_DISPLAY_PORT_INTERRUPT)
			i9xx_hpd_irq_handler(dev);
4165

4166
		I915_WRITE(IIR, iir & ~flip_mask);
4167 4168 4169 4170 4171 4172 4173 4174
		new_iir = I915_READ(IIR); /* Flush posted writes */

		if (iir & I915_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[RCS]);
		if (iir & I915_BSD_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[VCS]);

		for_each_pipe(pipe) {
4175
			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
4176 4177
			    i915_handle_vblank(dev, pipe, pipe, iir))
				flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
4178 4179 4180

			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
4181 4182

			if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
4183
				i9xx_pipe_crc_irq_handler(dev, pipe);
4184

4185 4186
			if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
			    intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
4187
				DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
4188
		}
4189 4190 4191 4192

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
			intel_opregion_asle_intr(dev);

4193 4194 4195
		if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
			gmbus_irq_handler(dev);

4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213
		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
	}

4214
	i915_update_dri1_breadcrumb(dev);
4215

4216 4217 4218 4219 4220
	return ret;
}

static void i965_irq_uninstall(struct drm_device * dev)
{
4221
	struct drm_i915_private *dev_priv = dev->dev_private;
4222 4223 4224 4225 4226
	int pipe;

	if (!dev_priv)
		return;

4227
	intel_hpd_irq_uninstall(dev_priv);
4228

4229 4230
	I915_WRITE(PORT_HOTPLUG_EN, 0);
	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243

	I915_WRITE(HWSTAM, 0xffffffff);
	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe), 0);
	I915_WRITE(IMR, 0xffffffff);
	I915_WRITE(IER, 0x0);

	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
	I915_WRITE(IIR, I915_READ(IIR));
}

4244
static void intel_hpd_irq_reenable(unsigned long data)
4245
{
4246
	struct drm_i915_private *dev_priv = (struct drm_i915_private *)data;
4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278
	struct drm_device *dev = dev_priv->dev;
	struct drm_mode_config *mode_config = &dev->mode_config;
	unsigned long irqflags;
	int i;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
	for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
		struct drm_connector *connector;

		if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
			continue;

		dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;

		list_for_each_entry(connector, &mode_config->connector_list, head) {
			struct intel_connector *intel_connector = to_intel_connector(connector);

			if (intel_connector->encoder->hpd_pin == i) {
				if (connector->polled != intel_connector->polled)
					DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
							 drm_get_connector_name(connector));
				connector->polled = intel_connector->polled;
				if (!connector->polled)
					connector->polled = DRM_CONNECTOR_POLL_HPD;
			}
		}
	}
	if (dev_priv->display.hpd_irq_setup)
		dev_priv->display.hpd_irq_setup(dev);
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

4279 4280
void intel_irq_init(struct drm_device *dev)
{
4281 4282 4283
	struct drm_i915_private *dev_priv = dev->dev_private;

	INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
4284
	INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
4285
	INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
4286
	INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
4287

4288 4289 4290
	/* Let's track the enabled rps events */
	dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;

4291 4292
	setup_timer(&dev_priv->gpu_error.hangcheck_timer,
		    i915_hangcheck_elapsed,
4293
		    (unsigned long) dev);
4294
	setup_timer(&dev_priv->hotplug_reenable_timer, intel_hpd_irq_reenable,
4295
		    (unsigned long) dev_priv);
4296

4297
	pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
4298

4299 4300 4301 4302
	if (IS_GEN2(dev)) {
		dev->max_vblank_count = 0;
		dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
	} else if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
4303 4304
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
		dev->driver->get_vblank_counter = gm45_get_vblank_counter;
4305 4306 4307
	} else {
		dev->driver->get_vblank_counter = i915_get_vblank_counter;
		dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
4308 4309
	}

4310
	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
4311
		dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
4312 4313
		dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
	}
4314

4315 4316 4317 4318 4319 4320 4321 4322 4323
	if (IS_CHERRYVIEW(dev)) {
		dev->driver->irq_handler = cherryview_irq_handler;
		dev->driver->irq_preinstall = cherryview_irq_preinstall;
		dev->driver->irq_postinstall = cherryview_irq_postinstall;
		dev->driver->irq_uninstall = cherryview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
	} else if (IS_VALLEYVIEW(dev)) {
J
Jesse Barnes 已提交
4324 4325 4326 4327 4328 4329
		dev->driver->irq_handler = valleyview_irq_handler;
		dev->driver->irq_preinstall = valleyview_irq_preinstall;
		dev->driver->irq_postinstall = valleyview_irq_postinstall;
		dev->driver->irq_uninstall = valleyview_irq_uninstall;
		dev->driver->enable_vblank = valleyview_enable_vblank;
		dev->driver->disable_vblank = valleyview_disable_vblank;
4330
		dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
4331 4332 4333 4334 4335 4336 4337 4338
	} else if (IS_GEN8(dev)) {
		dev->driver->irq_handler = gen8_irq_handler;
		dev->driver->irq_preinstall = gen8_irq_preinstall;
		dev->driver->irq_postinstall = gen8_irq_postinstall;
		dev->driver->irq_uninstall = gen8_irq_uninstall;
		dev->driver->enable_vblank = gen8_enable_vblank;
		dev->driver->disable_vblank = gen8_disable_vblank;
		dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4339 4340 4341 4342 4343 4344 4345
	} else if (HAS_PCH_SPLIT(dev)) {
		dev->driver->irq_handler = ironlake_irq_handler;
		dev->driver->irq_preinstall = ironlake_irq_preinstall;
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
4346
		dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
4347
	} else {
C
Chris Wilson 已提交
4348 4349 4350 4351 4352
		if (INTEL_INFO(dev)->gen == 2) {
			dev->driver->irq_preinstall = i8xx_irq_preinstall;
			dev->driver->irq_postinstall = i8xx_irq_postinstall;
			dev->driver->irq_handler = i8xx_irq_handler;
			dev->driver->irq_uninstall = i8xx_irq_uninstall;
4353 4354 4355 4356 4357
		} else if (INTEL_INFO(dev)->gen == 3) {
			dev->driver->irq_preinstall = i915_irq_preinstall;
			dev->driver->irq_postinstall = i915_irq_postinstall;
			dev->driver->irq_uninstall = i915_irq_uninstall;
			dev->driver->irq_handler = i915_irq_handler;
4358
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
C
Chris Wilson 已提交
4359
		} else {
4360 4361 4362 4363
			dev->driver->irq_preinstall = i965_irq_preinstall;
			dev->driver->irq_postinstall = i965_irq_postinstall;
			dev->driver->irq_uninstall = i965_irq_uninstall;
			dev->driver->irq_handler = i965_irq_handler;
4364
			dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
C
Chris Wilson 已提交
4365
		}
4366 4367 4368 4369
		dev->driver->enable_vblank = i915_enable_vblank;
		dev->driver->disable_vblank = i915_disable_vblank;
	}
}
4370 4371 4372 4373

void intel_hpd_init(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
4374 4375
	struct drm_mode_config *mode_config = &dev->mode_config;
	struct drm_connector *connector;
4376
	unsigned long irqflags;
4377
	int i;
4378

4379 4380 4381 4382 4383 4384 4385 4386 4387 4388
	for (i = 1; i < HPD_NUM_PINS; i++) {
		dev_priv->hpd_stats[i].hpd_cnt = 0;
		dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
	}
	list_for_each_entry(connector, &mode_config->connector_list, head) {
		struct intel_connector *intel_connector = to_intel_connector(connector);
		connector->polled = intel_connector->polled;
		if (!connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
			connector->polled = DRM_CONNECTOR_POLL_HPD;
	}
4389 4390 4391 4392

	/* Interrupt setup is already guaranteed to be single-threaded, this is
	 * just to make the assert_spin_locked checks happy. */
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
4393 4394
	if (dev_priv->display.hpd_irq_setup)
		dev_priv->display.hpd_irq_setup(dev);
4395
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
4396
}
4397

4398
/* Disable interrupts so we can allow runtime PM. */
4399
void intel_runtime_pm_disable_interrupts(struct drm_device *dev)
4400 4401 4402
{
	struct drm_i915_private *dev_priv = dev->dev_private;

4403
	dev->driver->irq_uninstall(dev);
4404
	dev_priv->pm.irqs_disabled = true;
4405 4406
}

4407
/* Restore interrupts so we can recover from runtime PM. */
4408
void intel_runtime_pm_restore_interrupts(struct drm_device *dev)
4409 4410 4411
{
	struct drm_i915_private *dev_priv = dev->dev_private;

4412
	dev_priv->pm.irqs_disabled = false;
4413 4414
	dev->driver->irq_preinstall(dev);
	dev->driver->irq_postinstall(dev);
4415
}