i915_irq.c 57.1 KB
Newer Older
D
Dave Airlie 已提交
1
/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
L
Linus Torvalds 已提交
2
 */
D
Dave Airlie 已提交
3
/*
L
Linus Torvalds 已提交
4 5
 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
 * All Rights Reserved.
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
D
Dave Airlie 已提交
27
 */
L
Linus Torvalds 已提交
28

29
#include <linux/sysrq.h>
30
#include <linux/slab.h>
L
Linus Torvalds 已提交
31 32 33 34
#include "drmP.h"
#include "drm.h"
#include "i915_drm.h"
#include "i915_drv.h"
C
Chris Wilson 已提交
35
#include "i915_trace.h"
J
Jesse Barnes 已提交
36
#include "intel_drv.h"
L
Linus Torvalds 已提交
37 38 39

#define MAX_NOPID ((u32)~0)

40 41 42 43 44 45 46
/**
 * Interrupts that are always left unmasked.
 *
 * Since pipe events are edge-triggered from the PIPESTAT register to IIR,
 * we leave them always unmasked in IMR and then control enabling them through
 * PIPESTAT alone.
 */
47 48 49 50 51 52 53
#define I915_INTERRUPT_ENABLE_FIX			\
	(I915_ASLE_INTERRUPT |				\
	 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |		\
	 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |		\
	 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |	\
	 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |	\
	 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
54 55

/** Interrupts that we mask and unmask at runtime. */
56
#define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT | I915_BSD_USER_INTERRUPT)
57

J
Jesse Barnes 已提交
58 59 60 61 62 63 64 65 66
#define I915_PIPE_VBLANK_STATUS	(PIPE_START_VBLANK_INTERRUPT_STATUS |\
				 PIPE_VBLANK_INTERRUPT_STATUS)

#define I915_PIPE_VBLANK_ENABLE	(PIPE_START_VBLANK_INTERRUPT_ENABLE |\
				 PIPE_VBLANK_INTERRUPT_ENABLE)

#define DRM_I915_VBLANK_PIPE_ALL	(DRM_I915_VBLANK_PIPE_A | \
					 DRM_I915_VBLANK_PIPE_B)

67
/* For display hotplug interrupt */
68
static void
69
ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
70
{
71 72 73
	if ((dev_priv->irq_mask & mask) != 0) {
		dev_priv->irq_mask &= ~mask;
		I915_WRITE(DEIMR, dev_priv->irq_mask);
74
		POSTING_READ(DEIMR);
75 76 77 78
	}
}

static inline void
79
ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
80
{
81 82 83
	if ((dev_priv->irq_mask & mask) != mask) {
		dev_priv->irq_mask |= mask;
		I915_WRITE(DEIMR, dev_priv->irq_mask);
84
		POSTING_READ(DEIMR);
85 86 87
	}
}

88 89 90 91
void
i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
{
	if ((dev_priv->pipestat[pipe] & mask) != mask) {
92
		u32 reg = PIPESTAT(pipe);
93 94 95 96

		dev_priv->pipestat[pipe] |= mask;
		/* Enable the interrupt, clear any pending status */
		I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
97
		POSTING_READ(reg);
98 99 100 101 102 103 104
	}
}

void
i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
{
	if ((dev_priv->pipestat[pipe] & mask) != 0) {
105
		u32 reg = PIPESTAT(pipe);
106 107 108

		dev_priv->pipestat[pipe] &= ~mask;
		I915_WRITE(reg, dev_priv->pipestat[pipe]);
109
		POSTING_READ(reg);
110 111 112
	}
}

113 114 115
/**
 * intel_enable_asle - enable ASLE interrupt for OpRegion
 */
116
void intel_enable_asle(struct drm_device *dev)
117
{
118 119 120 121
	drm_i915_private_t *dev_priv = dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
122

123
	if (HAS_PCH_SPLIT(dev))
124
		ironlake_enable_display_irq(dev_priv, DE_GSE);
125
	else {
126
		i915_enable_pipestat(dev_priv, 1,
127
				     PIPE_LEGACY_BLC_EVENT_ENABLE);
128
		if (INTEL_INFO(dev)->gen >= 4)
129
			i915_enable_pipestat(dev_priv, 0,
130
					     PIPE_LEGACY_BLC_EVENT_ENABLE);
131
	}
132 133

	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
134 135
}

136 137 138 139 140 141 142 143 144 145 146 147 148
/**
 * i915_pipe_enabled - check if a pipe is enabled
 * @dev: DRM device
 * @pipe: pipe to check
 *
 * Reading certain registers when the pipe is disabled can hang the chip.
 * Use this routine to make sure the PLL is running and the pipe is active
 * before reading such registers if unsure.
 */
static int
i915_pipe_enabled(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
149
	return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
150 151
}

152 153 154
/* Called from drm generic code, passed a 'crtc', which
 * we use as a pipe index
 */
155
static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
156 157 158 159
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	unsigned long high_frame;
	unsigned long low_frame;
160
	u32 high1, high2, low;
161 162

	if (!i915_pipe_enabled(dev, pipe)) {
163
		DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
164
				"pipe %c\n", pipe_name(pipe));
165 166 167
		return 0;
	}

168 169
	high_frame = PIPEFRAME(pipe);
	low_frame = PIPEFRAMEPIXEL(pipe);
170

171 172 173 174 175 176
	/*
	 * High & low register fields aren't synchronized, so make sure
	 * we get a low value that's stable across two reads of the high
	 * register.
	 */
	do {
177 178 179
		high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
		low   = I915_READ(low_frame)  & PIPE_FRAME_LOW_MASK;
		high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
180 181
	} while (high1 != high2);

182 183 184
	high1 >>= PIPE_FRAME_HIGH_SHIFT;
	low >>= PIPE_FRAME_LOW_SHIFT;
	return (high1 << 8) | low;
185 186
}

187
static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
188 189
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
190
	int reg = PIPE_FRMCOUNT_GM45(pipe);
191 192

	if (!i915_pipe_enabled(dev, pipe)) {
193
		DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
194
				 "pipe %c\n", pipe_name(pipe));
195 196 197 198 199 200
		return 0;
	}

	return I915_READ(reg);
}

201
static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
202 203 204 205 206 207 208 209 210 211
			     int *vpos, int *hpos)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	u32 vbl = 0, position = 0;
	int vbl_start, vbl_end, htotal, vtotal;
	bool in_vbl = true;
	int ret = 0;

	if (!i915_pipe_enabled(dev, pipe)) {
		DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
212
				 "pipe %c\n", pipe_name(pipe));
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
		return 0;
	}

	/* Get vtotal. */
	vtotal = 1 + ((I915_READ(VTOTAL(pipe)) >> 16) & 0x1fff);

	if (INTEL_INFO(dev)->gen >= 4) {
		/* No obvious pixelcount register. Only query vertical
		 * scanout position from Display scan line register.
		 */
		position = I915_READ(PIPEDSL(pipe));

		/* Decode into vertical scanout position. Don't have
		 * horizontal scanout position.
		 */
		*vpos = position & 0x1fff;
		*hpos = 0;
	} else {
		/* Have access to pixelcount since start of frame.
		 * We can split this into vertical and horizontal
		 * scanout position.
		 */
		position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;

		htotal = 1 + ((I915_READ(HTOTAL(pipe)) >> 16) & 0x1fff);
		*vpos = position / htotal;
		*hpos = position - (*vpos * htotal);
	}

	/* Query vblank area. */
	vbl = I915_READ(VBLANK(pipe));

	/* Test position against vblank region. */
	vbl_start = vbl & 0x1fff;
	vbl_end = (vbl >> 16) & 0x1fff;

	if ((*vpos < vbl_start) || (*vpos > vbl_end))
		in_vbl = false;

	/* Inside "upper part" of vblank area? Apply corrective offset: */
	if (in_vbl && (*vpos >= vbl_start))
		*vpos = *vpos - vtotal;

	/* Readouts valid? */
	if (vbl > 0)
		ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;

	/* In vblank? */
	if (in_vbl)
		ret |= DRM_SCANOUTPOS_INVBL;

	return ret;
}

267
static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
268 269 270 271
			      int *max_error,
			      struct timeval *vblank_time,
			      unsigned flags)
{
272 273
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_crtc *crtc;
274

275 276
	if (pipe < 0 || pipe >= dev_priv->num_pipe) {
		DRM_ERROR("Invalid crtc %d\n", pipe);
277 278 279 280
		return -EINVAL;
	}

	/* Get drm_crtc to timestamp: */
281 282 283 284 285 286 287 288 289 290
	crtc = intel_get_crtc_for_pipe(dev, pipe);
	if (crtc == NULL) {
		DRM_ERROR("Invalid crtc %d\n", pipe);
		return -EINVAL;
	}

	if (!crtc->enabled) {
		DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
		return -EBUSY;
	}
291 292

	/* Helper routine in DRM core does all the work: */
293 294 295
	return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
						     vblank_time, flags,
						     crtc);
296 297
}

298 299 300 301 302 303 304 305
/*
 * Handle hotplug events outside the interrupt handler proper.
 */
static void i915_hotplug_work_func(struct work_struct *work)
{
	drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
						    hotplug_work);
	struct drm_device *dev = dev_priv->dev;
306
	struct drm_mode_config *mode_config = &dev->mode_config;
307 308
	struct intel_encoder *encoder;

309 310
	DRM_DEBUG_KMS("running encoder hotplug functions\n");

311 312 313 314
	list_for_each_entry(encoder, &mode_config->encoder_list, base.head)
		if (encoder->hot_plug)
			encoder->hot_plug(encoder);

315
	/* Just fire off a uevent and let userspace tell us what to do */
316
	drm_helper_hpd_irq_event(dev);
317 318
}

319 320 321
static void i915_handle_rps_change(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
322
	u32 busy_up, busy_down, max_avg, min_avg;
323 324
	u8 new_delay = dev_priv->cur_delay;

325
	I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
326 327
	busy_up = I915_READ(RCPREVBSYTUPAVG);
	busy_down = I915_READ(RCPREVBSYTDNAVG);
328 329 330 331
	max_avg = I915_READ(RCBMAXAVG);
	min_avg = I915_READ(RCBMINAVG);

	/* Handle RCS change request from hw */
332
	if (busy_up > max_avg) {
333 334 335 336
		if (dev_priv->cur_delay != dev_priv->max_delay)
			new_delay = dev_priv->cur_delay - 1;
		if (new_delay < dev_priv->max_delay)
			new_delay = dev_priv->max_delay;
337
	} else if (busy_down < min_avg) {
338 339 340 341 342 343
		if (dev_priv->cur_delay != dev_priv->min_delay)
			new_delay = dev_priv->cur_delay + 1;
		if (new_delay > dev_priv->min_delay)
			new_delay = dev_priv->min_delay;
	}

344 345
	if (ironlake_set_drps(dev, new_delay))
		dev_priv->cur_delay = new_delay;
346 347 348 349

	return;
}

350 351 352 353
static void notify_ring(struct drm_device *dev,
			struct intel_ring_buffer *ring)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
354
	u32 seqno;
355

356 357 358 359
	if (ring->obj == NULL)
		return;

	seqno = ring->get_seqno(ring);
C
Chris Wilson 已提交
360
	trace_i915_gem_request_complete(ring, seqno);
361 362

	ring->irq_seqno = seqno;
363
	wake_up_all(&ring->irq_queue);
364

365 366 367 368 369
	dev_priv->hangcheck_count = 0;
	mod_timer(&dev_priv->hangcheck_timer,
		  jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
}

370
static void gen6_pm_rps_work(struct work_struct *work)
371
{
372 373
	drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
						    rps_work);
374
	u8 new_delay = dev_priv->cur_delay;
375 376 377 378 379 380 381
	u32 pm_iir, pm_imr;

	spin_lock_irq(&dev_priv->rps_lock);
	pm_iir = dev_priv->pm_iir;
	dev_priv->pm_iir = 0;
	pm_imr = I915_READ(GEN6_PMIMR);
	spin_unlock_irq(&dev_priv->rps_lock);
382 383 384 385

	if (!pm_iir)
		return;

386
	mutex_lock(&dev_priv->dev->struct_mutex);
387 388 389 390 391 392
	if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
		if (dev_priv->cur_delay != dev_priv->max_delay)
			new_delay = dev_priv->cur_delay + 1;
		if (new_delay > dev_priv->max_delay)
			new_delay = dev_priv->max_delay;
	} else if (pm_iir & (GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT)) {
393
		gen6_gt_force_wake_get(dev_priv);
394 395 396 397 398 399 400 401 402 403 404 405 406
		if (dev_priv->cur_delay != dev_priv->min_delay)
			new_delay = dev_priv->cur_delay - 1;
		if (new_delay < dev_priv->min_delay) {
			new_delay = dev_priv->min_delay;
			I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
				   I915_READ(GEN6_RP_INTERRUPT_LIMITS) |
				   ((new_delay << 16) & 0x3f0000));
		} else {
			/* Make sure we continue to get down interrupts
			 * until we hit the minimum frequency */
			I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
				   I915_READ(GEN6_RP_INTERRUPT_LIMITS) & ~0x3f0000);
		}
407
		gen6_gt_force_wake_put(dev_priv);
408 409
	}

410
	gen6_set_rps(dev_priv->dev, new_delay);
411 412
	dev_priv->cur_delay = new_delay;

413 414 415 416 417 418 419
	/*
	 * rps_lock not held here because clearing is non-destructive. There is
	 * an *extremely* unlikely race with gen6_rps_enable() that is prevented
	 * by holding struct_mutex for the duration of the write.
	 */
	I915_WRITE(GEN6_PMIMR, pm_imr & ~pm_iir);
	mutex_unlock(&dev_priv->dev->struct_mutex);
420 421
}

422 423 424 425
static void pch_irq_handler(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	u32 pch_iir;
426
	int pipe;
427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446

	pch_iir = I915_READ(SDEIIR);

	if (pch_iir & SDE_AUDIO_POWER_MASK)
		DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
				 (pch_iir & SDE_AUDIO_POWER_MASK) >>
				 SDE_AUDIO_POWER_SHIFT);

	if (pch_iir & SDE_GMBUS)
		DRM_DEBUG_DRIVER("PCH GMBUS interrupt\n");

	if (pch_iir & SDE_AUDIO_HDCP_MASK)
		DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");

	if (pch_iir & SDE_AUDIO_TRANS_MASK)
		DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");

	if (pch_iir & SDE_POISON)
		DRM_ERROR("PCH poison interrupt\n");

447 448 449 450 451
	if (pch_iir & SDE_FDI_MASK)
		for_each_pipe(pipe)
			DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
					 pipe_name(pipe),
					 I915_READ(FDI_RX_IIR(pipe)));
452 453 454 455 456 457 458 459 460 461 462 463 464

	if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
		DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");

	if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
		DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");

	if (pch_iir & SDE_TRANSB_FIFO_UNDER)
		DRM_DEBUG_DRIVER("PCH transcoder B underrun interrupt\n");
	if (pch_iir & SDE_TRANSA_FIFO_UNDER)
		DRM_DEBUG_DRIVER("PCH transcoder A underrun interrupt\n");
}

465
static irqreturn_t ivybridge_irq_handler(DRM_IRQ_ARGS)
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
{
	struct drm_device *dev = (struct drm_device *) arg;
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	int ret = IRQ_NONE;
	u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir;
	struct drm_i915_master_private *master_priv;

	atomic_inc(&dev_priv->irq_received);

	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
	POSTING_READ(DEIER);

	de_iir = I915_READ(DEIIR);
	gt_iir = I915_READ(GTIIR);
	pch_iir = I915_READ(SDEIIR);
	pm_iir = I915_READ(GEN6_PMIIR);

	if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 && pm_iir == 0)
		goto done;

	ret = IRQ_HANDLED;

	if (dev->primary->master) {
		master_priv = dev->primary->master->driver_priv;
		if (master_priv->sarea_priv)
			master_priv->sarea_priv->last_dispatch =
				READ_BREADCRUMB(dev_priv);
	}

	if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
		notify_ring(dev, &dev_priv->ring[RCS]);
	if (gt_iir & GT_GEN6_BSD_USER_INTERRUPT)
		notify_ring(dev, &dev_priv->ring[VCS]);
	if (gt_iir & GT_BLT_USER_INTERRUPT)
		notify_ring(dev, &dev_priv->ring[BCS]);

	if (de_iir & DE_GSE_IVB)
		intel_opregion_gse_intr(dev);

	if (de_iir & DE_PLANEA_FLIP_DONE_IVB) {
		intel_prepare_page_flip(dev, 0);
		intel_finish_page_flip_plane(dev, 0);
	}

	if (de_iir & DE_PLANEB_FLIP_DONE_IVB) {
		intel_prepare_page_flip(dev, 1);
		intel_finish_page_flip_plane(dev, 1);
	}

	if (de_iir & DE_PIPEA_VBLANK_IVB)
		drm_handle_vblank(dev, 0);

520
	if (de_iir & DE_PIPEB_VBLANK_IVB)
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552
		drm_handle_vblank(dev, 1);

	/* check event from PCH */
	if (de_iir & DE_PCH_EVENT_IVB) {
		if (pch_iir & SDE_HOTPLUG_MASK_CPT)
			queue_work(dev_priv->wq, &dev_priv->hotplug_work);
		pch_irq_handler(dev);
	}

	if (pm_iir & GEN6_PM_DEFERRED_EVENTS) {
		unsigned long flags;
		spin_lock_irqsave(&dev_priv->rps_lock, flags);
		WARN(dev_priv->pm_iir & pm_iir, "Missed a PM interrupt\n");
		I915_WRITE(GEN6_PMIMR, pm_iir);
		dev_priv->pm_iir |= pm_iir;
		spin_unlock_irqrestore(&dev_priv->rps_lock, flags);
		queue_work(dev_priv->wq, &dev_priv->rps_work);
	}

	/* should clear PCH hotplug event before clear CPU irq */
	I915_WRITE(SDEIIR, pch_iir);
	I915_WRITE(GTIIR, gt_iir);
	I915_WRITE(DEIIR, de_iir);
	I915_WRITE(GEN6_PMIIR, pm_iir);

done:
	I915_WRITE(DEIER, de_ier);
	POSTING_READ(DEIER);

	return ret;
}

553
static irqreturn_t ironlake_irq_handler(DRM_IRQ_ARGS)
554
{
555
	struct drm_device *dev = (struct drm_device *) arg;
556 557
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	int ret = IRQ_NONE;
558
	u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir;
559
	u32 hotplug_mask;
560
	struct drm_i915_master_private *master_priv;
561 562
	u32 bsd_usr_interrupt = GT_BSD_USER_INTERRUPT;

563 564
	atomic_inc(&dev_priv->irq_received);

565 566
	if (IS_GEN6(dev))
		bsd_usr_interrupt = GT_GEN6_BSD_USER_INTERRUPT;
567

568 569 570
	/* disable master interrupt before clearing iir  */
	de_ier = I915_READ(DEIER);
	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
571
	POSTING_READ(DEIER);
572

573 574
	de_iir = I915_READ(DEIIR);
	gt_iir = I915_READ(GTIIR);
575
	pch_iir = I915_READ(SDEIIR);
576
	pm_iir = I915_READ(GEN6_PMIIR);
577

578 579
	if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 &&
	    (!IS_GEN6(dev) || pm_iir == 0))
580
		goto done;
581

582 583 584 585 586
	if (HAS_PCH_CPT(dev))
		hotplug_mask = SDE_HOTPLUG_MASK_CPT;
	else
		hotplug_mask = SDE_HOTPLUG_MASK;

587
	ret = IRQ_HANDLED;
588

589 590 591 592 593 594
	if (dev->primary->master) {
		master_priv = dev->primary->master->driver_priv;
		if (master_priv->sarea_priv)
			master_priv->sarea_priv->last_dispatch =
				READ_BREADCRUMB(dev_priv);
	}
595

596
	if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
597
		notify_ring(dev, &dev_priv->ring[RCS]);
598
	if (gt_iir & bsd_usr_interrupt)
599 600 601
		notify_ring(dev, &dev_priv->ring[VCS]);
	if (gt_iir & GT_BLT_USER_INTERRUPT)
		notify_ring(dev, &dev_priv->ring[BCS]);
602

603
	if (de_iir & DE_GSE)
604
		intel_opregion_gse_intr(dev);
605

606
	if (de_iir & DE_PLANEA_FLIP_DONE) {
607
		intel_prepare_page_flip(dev, 0);
608
		intel_finish_page_flip_plane(dev, 0);
609
	}
610

611
	if (de_iir & DE_PLANEB_FLIP_DONE) {
612
		intel_prepare_page_flip(dev, 1);
613
		intel_finish_page_flip_plane(dev, 1);
614
	}
615

616
	if (de_iir & DE_PIPEA_VBLANK)
617 618
		drm_handle_vblank(dev, 0);

619
	if (de_iir & DE_PIPEB_VBLANK)
620 621
		drm_handle_vblank(dev, 1);

622
	/* check event from PCH */
623 624 625 626 627
	if (de_iir & DE_PCH_EVENT) {
		if (pch_iir & hotplug_mask)
			queue_work(dev_priv->wq, &dev_priv->hotplug_work);
		pch_irq_handler(dev);
	}
628

629
	if (de_iir & DE_PCU_EVENT) {
630
		I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
631 632 633
		i915_handle_rps_change(dev);
	}

634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
	if (IS_GEN6(dev) && pm_iir & GEN6_PM_DEFERRED_EVENTS) {
		/*
		 * IIR bits should never already be set because IMR should
		 * prevent an interrupt from being shown in IIR. The warning
		 * displays a case where we've unsafely cleared
		 * dev_priv->pm_iir. Although missing an interrupt of the same
		 * type is not a problem, it displays a problem in the logic.
		 *
		 * The mask bit in IMR is cleared by rps_work.
		 */
		unsigned long flags;
		spin_lock_irqsave(&dev_priv->rps_lock, flags);
		WARN(dev_priv->pm_iir & pm_iir, "Missed a PM interrupt\n");
		I915_WRITE(GEN6_PMIMR, pm_iir);
		dev_priv->pm_iir |= pm_iir;
		spin_unlock_irqrestore(&dev_priv->rps_lock, flags);
		queue_work(dev_priv->wq, &dev_priv->rps_work);
	}
652

653 654 655 656
	/* should clear PCH hotplug event before clear CPU irq */
	I915_WRITE(SDEIIR, pch_iir);
	I915_WRITE(GTIIR, gt_iir);
	I915_WRITE(DEIIR, de_iir);
657
	I915_WRITE(GEN6_PMIIR, pm_iir);
658 659

done:
660
	I915_WRITE(DEIER, de_ier);
661
	POSTING_READ(DEIER);
662

663 664 665
	return ret;
}

666 667 668 669 670 671 672 673 674 675 676 677
/**
 * i915_error_work_func - do process context error handling work
 * @work: work struct
 *
 * Fire an error uevent so userspace can see that a hang or error
 * was detected.
 */
static void i915_error_work_func(struct work_struct *work)
{
	drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
						    error_work);
	struct drm_device *dev = dev_priv->dev;
678 679 680
	char *error_event[] = { "ERROR=1", NULL };
	char *reset_event[] = { "RESET=1", NULL };
	char *reset_done_event[] = { "ERROR=0", NULL };
681

682 683
	kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);

684
	if (atomic_read(&dev_priv->mm.wedged)) {
685 686 687 688 689
		DRM_DEBUG_DRIVER("resetting chip\n");
		kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_event);
		if (!i915_reset(dev, GRDOM_RENDER)) {
			atomic_set(&dev_priv->mm.wedged, 0);
			kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, reset_done_event);
690
		}
691
		complete_all(&dev_priv->error_completion);
692
	}
693 694
}

695
#ifdef CONFIG_DEBUG_FS
696
static struct drm_i915_error_object *
697
i915_error_object_create(struct drm_i915_private *dev_priv,
698
			 struct drm_i915_gem_object *src)
699 700 701
{
	struct drm_i915_error_object *dst;
	int page, page_count;
702
	u32 reloc_offset;
703

704
	if (src == NULL || src->pages == NULL)
705 706
		return NULL;

707
	page_count = src->base.size / PAGE_SIZE;
708 709 710 711 712

	dst = kmalloc(sizeof(*dst) + page_count * sizeof (u32 *), GFP_ATOMIC);
	if (dst == NULL)
		return NULL;

713
	reloc_offset = src->gtt_offset;
714
	for (page = 0; page < page_count; page++) {
715
		unsigned long flags;
716 717
		void __iomem *s;
		void *d;
718

719
		d = kmalloc(PAGE_SIZE, GFP_ATOMIC);
720 721
		if (d == NULL)
			goto unwind;
722

723
		local_irq_save(flags);
724
		s = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
P
Peter Zijlstra 已提交
725
					     reloc_offset);
726
		memcpy_fromio(d, s, PAGE_SIZE);
P
Peter Zijlstra 已提交
727
		io_mapping_unmap_atomic(s);
728
		local_irq_restore(flags);
729

730
		dst->pages[page] = d;
731 732

		reloc_offset += PAGE_SIZE;
733 734
	}
	dst->page_count = page_count;
735
	dst->gtt_offset = src->gtt_offset;
736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763

	return dst;

unwind:
	while (page--)
		kfree(dst->pages[page]);
	kfree(dst);
	return NULL;
}

static void
i915_error_object_free(struct drm_i915_error_object *obj)
{
	int page;

	if (obj == NULL)
		return;

	for (page = 0; page < obj->page_count; page++)
		kfree(obj->pages[page]);

	kfree(obj);
}

static void
i915_error_state_free(struct drm_device *dev,
		      struct drm_i915_error_state *error)
{
764 765 766 767 768 769 770 771
	int i;

	for (i = 0; i < ARRAY_SIZE(error->batchbuffer); i++)
		i915_error_object_free(error->batchbuffer[i]);

	for (i = 0; i < ARRAY_SIZE(error->ringbuffer); i++)
		i915_error_object_free(error->ringbuffer[i]);

772
	kfree(error->active_bo);
773
	kfree(error->overlay);
774 775 776
	kfree(error);
}

777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
static u32 capture_bo_list(struct drm_i915_error_buffer *err,
			   int count,
			   struct list_head *head)
{
	struct drm_i915_gem_object *obj;
	int i = 0;

	list_for_each_entry(obj, head, mm_list) {
		err->size = obj->base.size;
		err->name = obj->base.name;
		err->seqno = obj->last_rendering_seqno;
		err->gtt_offset = obj->gtt_offset;
		err->read_domains = obj->base.read_domains;
		err->write_domain = obj->base.write_domain;
		err->fence_reg = obj->fence_reg;
		err->pinned = 0;
		if (obj->pin_count > 0)
			err->pinned = 1;
		if (obj->user_pin_count > 0)
			err->pinned = -1;
		err->tiling = obj->tiling_mode;
		err->dirty = obj->dirty;
		err->purgeable = obj->madv != I915_MADV_WILLNEED;
800
		err->ring = obj->ring ? obj->ring->id : 0;
801
		err->cache_level = obj->cache_level;
802 803 804 805 806 807 808 809 810 811

		if (++i == count)
			break;

		err++;
	}

	return i;
}

812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
static void i915_gem_record_fences(struct drm_device *dev,
				   struct drm_i915_error_state *error)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int i;

	/* Fences */
	switch (INTEL_INFO(dev)->gen) {
	case 6:
		for (i = 0; i < 16; i++)
			error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8));
		break;
	case 5:
	case 4:
		for (i = 0; i < 16; i++)
			error->fence[i] = I915_READ64(FENCE_REG_965_0 + (i * 8));
		break;
	case 3:
		if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
			for (i = 0; i < 8; i++)
				error->fence[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4));
	case 2:
		for (i = 0; i < 8; i++)
			error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4));
		break;

	}
}

841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
static struct drm_i915_error_object *
i915_error_first_batchbuffer(struct drm_i915_private *dev_priv,
			     struct intel_ring_buffer *ring)
{
	struct drm_i915_gem_object *obj;
	u32 seqno;

	if (!ring->get_seqno)
		return NULL;

	seqno = ring->get_seqno(ring);
	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
		if (obj->ring != ring)
			continue;

856
		if (i915_seqno_passed(seqno, obj->last_rendering_seqno))
857 858 859 860 861 862 863 864 865 866 867 868 869 870
			continue;

		if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0)
			continue;

		/* We need to copy these to an anonymous buffer as the simplest
		 * method to avoid being overwritten by userspace.
		 */
		return i915_error_object_create(dev_priv, obj);
	}

	return NULL;
}

871 872 873 874 875 876 877 878 879
/**
 * i915_capture_error_state - capture an error record for later analysis
 * @dev: drm device
 *
 * Should be called when an error is detected (either a hang or an error
 * interrupt) to capture error state from the time of the error.  Fills
 * out a structure which becomes available in debugfs for user level tools
 * to pick up.
 */
880 881 882
static void i915_capture_error_state(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
883
	struct drm_i915_gem_object *obj;
884 885
	struct drm_i915_error_state *error;
	unsigned long flags;
886
	int i, pipe;
887 888

	spin_lock_irqsave(&dev_priv->error_lock, flags);
889 890 891 892
	error = dev_priv->first_error;
	spin_unlock_irqrestore(&dev_priv->error_lock, flags);
	if (error)
		return;
893

894
	/* Account for pipe specific data like PIPE*STAT */
895 896
	error = kmalloc(sizeof(*error), GFP_ATOMIC);
	if (!error) {
897 898
		DRM_DEBUG_DRIVER("out of memory, not capturing error state\n");
		return;
899 900
	}

901 902
	DRM_INFO("capturing error event; look for more information in /debug/dri/%d/i915_error_state\n",
		 dev->primary->index);
903

904
	error->seqno = dev_priv->ring[RCS].get_seqno(&dev_priv->ring[RCS]);
905 906
	error->eir = I915_READ(EIR);
	error->pgtbl_er = I915_READ(PGTBL_ER);
907 908
	for_each_pipe(pipe)
		error->pipestat[pipe] = I915_READ(PIPESTAT(pipe));
909
	error->instpm = I915_READ(INSTPM);
910 911 912
	error->error = 0;
	if (INTEL_INFO(dev)->gen >= 6) {
		error->error = I915_READ(ERROR_GEN6);
913

914 915 916 917 918
		error->bcs_acthd = I915_READ(BCS_ACTHD);
		error->bcs_ipehr = I915_READ(BCS_IPEHR);
		error->bcs_ipeir = I915_READ(BCS_IPEIR);
		error->bcs_instdone = I915_READ(BCS_INSTDONE);
		error->bcs_seqno = 0;
919 920
		if (dev_priv->ring[BCS].get_seqno)
			error->bcs_seqno = dev_priv->ring[BCS].get_seqno(&dev_priv->ring[BCS]);
921 922 923 924 925 926

		error->vcs_acthd = I915_READ(VCS_ACTHD);
		error->vcs_ipehr = I915_READ(VCS_IPEHR);
		error->vcs_ipeir = I915_READ(VCS_IPEIR);
		error->vcs_instdone = I915_READ(VCS_INSTDONE);
		error->vcs_seqno = 0;
927 928
		if (dev_priv->ring[VCS].get_seqno)
			error->vcs_seqno = dev_priv->ring[VCS].get_seqno(&dev_priv->ring[VCS]);
929 930
	}
	if (INTEL_INFO(dev)->gen >= 4) {
931 932 933 934 935 936
		error->ipeir = I915_READ(IPEIR_I965);
		error->ipehr = I915_READ(IPEHR_I965);
		error->instdone = I915_READ(INSTDONE_I965);
		error->instps = I915_READ(INSTPS);
		error->instdone1 = I915_READ(INSTDONE1);
		error->acthd = I915_READ(ACTHD_I965);
937
		error->bbaddr = I915_READ64(BB_ADDR);
938 939 940 941 942 943
	} else {
		error->ipeir = I915_READ(IPEIR);
		error->ipehr = I915_READ(IPEHR);
		error->instdone = I915_READ(INSTDONE);
		error->acthd = I915_READ(ACTHD);
		error->bbaddr = 0;
944
	}
945
	i915_gem_record_fences(dev, error);
946

947 948
	/* Record the active batch and ring buffers */
	for (i = 0; i < I915_NUM_RINGS; i++) {
949 950 951
		error->batchbuffer[i] =
			i915_error_first_batchbuffer(dev_priv,
						     &dev_priv->ring[i]);
952

953 954 955 956
		error->ringbuffer[i] =
			i915_error_object_create(dev_priv,
						 dev_priv->ring[i].obj);
	}
957

958
	/* Record buffers on the active and pinned lists. */
959
	error->active_bo = NULL;
960
	error->pinned_bo = NULL;
961

962 963 964 965
	i = 0;
	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list)
		i++;
	error->active_bo_count = i;
966
	list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
967 968
		i++;
	error->pinned_bo_count = i - error->active_bo_count;
969

970 971
	error->active_bo = NULL;
	error->pinned_bo = NULL;
972 973
	if (i) {
		error->active_bo = kmalloc(sizeof(*error->active_bo)*i,
974
					   GFP_ATOMIC);
975 976 977
		if (error->active_bo)
			error->pinned_bo =
				error->active_bo + error->active_bo_count;
978 979
	}

980 981 982 983 984 985 986 987 988 989 990 991
	if (error->active_bo)
		error->active_bo_count =
			capture_bo_list(error->active_bo,
					error->active_bo_count,
					&dev_priv->mm.active_list);

	if (error->pinned_bo)
		error->pinned_bo_count =
			capture_bo_list(error->pinned_bo,
					error->pinned_bo_count,
					&dev_priv->mm.pinned_list);

992 993
	do_gettimeofday(&error->time);

994
	error->overlay = intel_overlay_capture_error_state(dev);
995
	error->display = intel_display_capture_error_state(dev);
996

997 998 999 1000 1001
	spin_lock_irqsave(&dev_priv->error_lock, flags);
	if (dev_priv->first_error == NULL) {
		dev_priv->first_error = error;
		error = NULL;
	}
1002
	spin_unlock_irqrestore(&dev_priv->error_lock, flags);
1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019

	if (error)
		i915_error_state_free(dev, error);
}

void i915_destroy_error_state(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_error_state *error;

	spin_lock(&dev_priv->error_lock);
	error = dev_priv->first_error;
	dev_priv->first_error = NULL;
	spin_unlock(&dev_priv->error_lock);

	if (error)
		i915_error_state_free(dev, error);
1020
}
1021 1022 1023
#else
#define i915_capture_error_state(x)
#endif
1024

1025
static void i915_report_and_clear_eir(struct drm_device *dev)
1026 1027 1028
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 eir = I915_READ(EIR);
1029
	int pipe;
1030

1031 1032
	if (!eir)
		return;
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053

	printk(KERN_ERR "render error detected, EIR: 0x%08x\n",
	       eir);

	if (IS_G4X(dev)) {
		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
			u32 ipeir = I915_READ(IPEIR_I965);

			printk(KERN_ERR "  IPEIR: 0x%08x\n",
			       I915_READ(IPEIR_I965));
			printk(KERN_ERR "  IPEHR: 0x%08x\n",
			       I915_READ(IPEHR_I965));
			printk(KERN_ERR "  INSTDONE: 0x%08x\n",
			       I915_READ(INSTDONE_I965));
			printk(KERN_ERR "  INSTPS: 0x%08x\n",
			       I915_READ(INSTPS));
			printk(KERN_ERR "  INSTDONE1: 0x%08x\n",
			       I915_READ(INSTDONE1));
			printk(KERN_ERR "  ACTHD: 0x%08x\n",
			       I915_READ(ACTHD_I965));
			I915_WRITE(IPEIR_I965, ipeir);
1054
			POSTING_READ(IPEIR_I965);
1055 1056 1057 1058 1059 1060 1061
		}
		if (eir & GM45_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
			printk(KERN_ERR "page table error\n");
			printk(KERN_ERR "  PGTBL_ER: 0x%08x\n",
			       pgtbl_err);
			I915_WRITE(PGTBL_ER, pgtbl_err);
1062
			POSTING_READ(PGTBL_ER);
1063 1064 1065
		}
	}

1066
	if (!IS_GEN2(dev)) {
1067 1068 1069 1070 1071 1072
		if (eir & I915_ERROR_PAGE_TABLE) {
			u32 pgtbl_err = I915_READ(PGTBL_ER);
			printk(KERN_ERR "page table error\n");
			printk(KERN_ERR "  PGTBL_ER: 0x%08x\n",
			       pgtbl_err);
			I915_WRITE(PGTBL_ER, pgtbl_err);
1073
			POSTING_READ(PGTBL_ER);
1074 1075 1076 1077
		}
	}

	if (eir & I915_ERROR_MEMORY_REFRESH) {
1078 1079 1080 1081
		printk(KERN_ERR "memory refresh error:\n");
		for_each_pipe(pipe)
			printk(KERN_ERR "pipe %c stat: 0x%08x\n",
			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
1082 1083 1084 1085 1086 1087
		/* pipestat has already been acked */
	}
	if (eir & I915_ERROR_INSTRUCTION) {
		printk(KERN_ERR "instruction error\n");
		printk(KERN_ERR "  INSTPM: 0x%08x\n",
		       I915_READ(INSTPM));
1088
		if (INTEL_INFO(dev)->gen < 4) {
1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099
			u32 ipeir = I915_READ(IPEIR);

			printk(KERN_ERR "  IPEIR: 0x%08x\n",
			       I915_READ(IPEIR));
			printk(KERN_ERR "  IPEHR: 0x%08x\n",
			       I915_READ(IPEHR));
			printk(KERN_ERR "  INSTDONE: 0x%08x\n",
			       I915_READ(INSTDONE));
			printk(KERN_ERR "  ACTHD: 0x%08x\n",
			       I915_READ(ACTHD));
			I915_WRITE(IPEIR, ipeir);
1100
			POSTING_READ(IPEIR);
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
		} else {
			u32 ipeir = I915_READ(IPEIR_I965);

			printk(KERN_ERR "  IPEIR: 0x%08x\n",
			       I915_READ(IPEIR_I965));
			printk(KERN_ERR "  IPEHR: 0x%08x\n",
			       I915_READ(IPEHR_I965));
			printk(KERN_ERR "  INSTDONE: 0x%08x\n",
			       I915_READ(INSTDONE_I965));
			printk(KERN_ERR "  INSTPS: 0x%08x\n",
			       I915_READ(INSTPS));
			printk(KERN_ERR "  INSTDONE1: 0x%08x\n",
			       I915_READ(INSTDONE1));
			printk(KERN_ERR "  ACTHD: 0x%08x\n",
			       I915_READ(ACTHD_I965));
			I915_WRITE(IPEIR_I965, ipeir);
1117
			POSTING_READ(IPEIR_I965);
1118 1119 1120 1121
		}
	}

	I915_WRITE(EIR, eir);
1122
	POSTING_READ(EIR);
1123 1124 1125 1126 1127 1128 1129 1130 1131 1132
	eir = I915_READ(EIR);
	if (eir) {
		/*
		 * some errors might have become stuck,
		 * mask them.
		 */
		DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
		I915_WRITE(EMR, I915_READ(EMR) | eir);
		I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
	}
1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
}

/**
 * i915_handle_error - handle an error interrupt
 * @dev: drm device
 *
 * Do some basic checking of regsiter state at error interrupt time and
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
1145
void i915_handle_error(struct drm_device *dev, bool wedged)
1146 1147 1148 1149 1150
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	i915_capture_error_state(dev);
	i915_report_and_clear_eir(dev);
1151

1152
	if (wedged) {
1153
		INIT_COMPLETION(dev_priv->error_completion);
1154 1155
		atomic_set(&dev_priv->mm.wedged, 1);

1156 1157 1158
		/*
		 * Wakeup waiting processes so they don't hang
		 */
1159
		wake_up_all(&dev_priv->ring[RCS].irq_queue);
1160
		if (HAS_BSD(dev))
1161
			wake_up_all(&dev_priv->ring[VCS].irq_queue);
1162
		if (HAS_BLT(dev))
1163
			wake_up_all(&dev_priv->ring[BCS].irq_queue);
1164 1165
	}

1166
	queue_work(dev_priv->wq, &dev_priv->error_work);
1167 1168
}

1169 1170 1171 1172 1173
static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1174
	struct drm_i915_gem_object *obj;
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192
	struct intel_unpin_work *work;
	unsigned long flags;
	bool stall_detected;

	/* Ignore early vblank irqs */
	if (intel_crtc == NULL)
		return;

	spin_lock_irqsave(&dev->event_lock, flags);
	work = intel_crtc->unpin_work;

	if (work == NULL || work->pending || !work->enable_stall_check) {
		/* Either the pending flip IRQ arrived, or we're too early. Don't check */
		spin_unlock_irqrestore(&dev->event_lock, flags);
		return;
	}

	/* Potential stall - if we see that the flip has happened, assume a missed interrupt */
1193
	obj = work->pending_flip_obj;
1194
	if (INTEL_INFO(dev)->gen >= 4) {
1195
		int dspsurf = DSPSURF(intel_crtc->plane);
1196
		stall_detected = I915_READ(dspsurf) == obj->gtt_offset;
1197
	} else {
1198
		int dspaddr = DSPADDR(intel_crtc->plane);
1199
		stall_detected = I915_READ(dspaddr) == (obj->gtt_offset +
1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
							crtc->y * crtc->fb->pitch +
							crtc->x * crtc->fb->bits_per_pixel/8);
	}

	spin_unlock_irqrestore(&dev->event_lock, flags);

	if (stall_detected) {
		DRM_DEBUG_DRIVER("Pageflip stall detected\n");
		intel_prepare_page_flip(dev, intel_crtc->plane);
	}
}

1212
static irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
L
Linus Torvalds 已提交
1213
{
1214
	struct drm_device *dev = (struct drm_device *) arg;
L
Linus Torvalds 已提交
1215
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1216
	struct drm_i915_master_private *master_priv;
1217
	u32 iir, new_iir;
1218
	u32 pipe_stats[I915_MAX_PIPES];
1219
	u32 vblank_status;
1220
	int vblank = 0;
1221
	unsigned long irqflags;
1222
	int irq_received;
1223 1224
	int ret = IRQ_NONE, pipe;
	bool blc_event = false;
1225

1226 1227
	atomic_inc(&dev_priv->irq_received);

1228
	iir = I915_READ(IIR);
1229

1230
	if (INTEL_INFO(dev)->gen >= 4)
1231
		vblank_status = PIPE_START_VBLANK_INTERRUPT_STATUS;
1232
	else
1233
		vblank_status = PIPE_VBLANK_INTERRUPT_STATUS;
1234

1235 1236 1237 1238 1239 1240 1241 1242
	for (;;) {
		irq_received = iir != 0;

		/* Can't rely on pipestat interrupt bit in iir as it might
		 * have been cleared after the pipestat interrupt was received.
		 * It doesn't set the bit in iir again, but it still produces
		 * interrupts (for non-MSI).
		 */
1243
		spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1244
		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
1245
			i915_handle_error(dev, false);
1246

1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
		for_each_pipe(pipe) {
			int reg = PIPESTAT(pipe);
			pipe_stats[pipe] = I915_READ(reg);

			/*
			 * Clear the PIPE*STAT regs before the IIR
			 */
			if (pipe_stats[pipe] & 0x8000ffff) {
				if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
					DRM_DEBUG_DRIVER("pipe %c underrun\n",
							 pipe_name(pipe));
				I915_WRITE(reg, pipe_stats[pipe]);
				irq_received = 1;
			}
1261
		}
1262
		spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1263 1264 1265 1266 1267

		if (!irq_received)
			break;

		ret = IRQ_HANDLED;
1268

1269 1270 1271 1272 1273
		/* Consume port.  Then clear IIR or we'll miss events */
		if ((I915_HAS_HOTPLUG(dev)) &&
		    (iir & I915_DISPLAY_PORT_INTERRUPT)) {
			u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);

1274
			DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
1275 1276
				  hotplug_status);
			if (hotplug_status & dev_priv->hotplug_supported_mask)
1277 1278
				queue_work(dev_priv->wq,
					   &dev_priv->hotplug_work);
1279 1280 1281 1282 1283

			I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
			I915_READ(PORT_HOTPLUG_STAT);
		}

1284 1285
		I915_WRITE(IIR, iir);
		new_iir = I915_READ(IIR); /* Flush posted writes */
1286

1287 1288 1289 1290 1291 1292
		if (dev->primary->master) {
			master_priv = dev->primary->master->driver_priv;
			if (master_priv->sarea_priv)
				master_priv->sarea_priv->last_dispatch =
					READ_BREADCRUMB(dev_priv);
		}
1293

1294
		if (iir & I915_USER_INTERRUPT)
1295 1296 1297
			notify_ring(dev, &dev_priv->ring[RCS]);
		if (iir & I915_BSD_USER_INTERRUPT)
			notify_ring(dev, &dev_priv->ring[VCS]);
1298

1299
		if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) {
1300
			intel_prepare_page_flip(dev, 0);
1301 1302 1303
			if (dev_priv->flip_pending_is_done)
				intel_finish_page_flip_plane(dev, 0);
		}
1304

1305
		if (iir & I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT) {
1306
			intel_prepare_page_flip(dev, 1);
1307 1308 1309
			if (dev_priv->flip_pending_is_done)
				intel_finish_page_flip_plane(dev, 1);
		}
1310

1311 1312 1313 1314 1315 1316 1317 1318
		for_each_pipe(pipe) {
			if (pipe_stats[pipe] & vblank_status &&
			    drm_handle_vblank(dev, pipe)) {
				vblank++;
				if (!dev_priv->flip_pending_is_done) {
					i915_pageflip_stall_check(dev, pipe);
					intel_finish_page_flip(dev, pipe);
				}
1319
			}
1320

1321 1322
			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
				blc_event = true;
1323
		}
1324

1325 1326

		if (blc_event || (iir & I915_ASLE_INTERRUPT))
1327
			intel_opregion_asle_intr(dev);
1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344

		/* With MSI, interrupts are only generated when iir
		 * transitions from zero to nonzero.  If another bit got
		 * set while we were handling the existing iir bits, then
		 * we would never get another interrupt.
		 *
		 * This is fine on non-MSI as well, as if we hit this path
		 * we avoid exiting the interrupt handler only to generate
		 * another one.
		 *
		 * Note that for MSI this could cause a stray interrupt report
		 * if an interrupt landed in the time between writing IIR and
		 * the posting read.  This should be rare enough to never
		 * trigger the 99% of 100,000 interrupts test for disabling
		 * stray interrupts.
		 */
		iir = new_iir;
1345
	}
1346

1347
	return ret;
L
Linus Torvalds 已提交
1348 1349
}

1350
static int i915_emit_irq(struct drm_device * dev)
L
Linus Torvalds 已提交
1351 1352
{
	drm_i915_private_t *dev_priv = dev->dev_private;
1353
	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
L
Linus Torvalds 已提交
1354 1355 1356

	i915_kernel_lost_context(dev);

1357
	DRM_DEBUG_DRIVER("\n");
L
Linus Torvalds 已提交
1358

1359
	dev_priv->counter++;
1360
	if (dev_priv->counter > 0x7FFFFFFFUL)
1361
		dev_priv->counter = 1;
1362 1363
	if (master_priv->sarea_priv)
		master_priv->sarea_priv->last_enqueue = dev_priv->counter;
1364

1365 1366 1367 1368 1369 1370 1371
	if (BEGIN_LP_RING(4) == 0) {
		OUT_RING(MI_STORE_DWORD_INDEX);
		OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
		OUT_RING(dev_priv->counter);
		OUT_RING(MI_USER_INTERRUPT);
		ADVANCE_LP_RING();
	}
D
Dave Airlie 已提交
1372

1373
	return dev_priv->counter;
L
Linus Torvalds 已提交
1374 1375
}

1376
static int i915_wait_irq(struct drm_device * dev, int irq_nr)
L
Linus Torvalds 已提交
1377 1378
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1379
	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
L
Linus Torvalds 已提交
1380
	int ret = 0;
1381
	struct intel_ring_buffer *ring = LP_RING(dev_priv);
L
Linus Torvalds 已提交
1382

1383
	DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr,
L
Linus Torvalds 已提交
1384 1385
		  READ_BREADCRUMB(dev_priv));

1386
	if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
1387 1388
		if (master_priv->sarea_priv)
			master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
L
Linus Torvalds 已提交
1389
		return 0;
1390
	}
L
Linus Torvalds 已提交
1391

1392 1393
	if (master_priv->sarea_priv)
		master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
L
Linus Torvalds 已提交
1394

1395 1396 1397 1398
	if (ring->irq_get(ring)) {
		DRM_WAIT_ON(ret, ring->irq_queue, 3 * DRM_HZ,
			    READ_BREADCRUMB(dev_priv) >= irq_nr);
		ring->irq_put(ring);
1399 1400
	} else if (wait_for(READ_BREADCRUMB(dev_priv) >= irq_nr, 3000))
		ret = -EBUSY;
L
Linus Torvalds 已提交
1401

E
Eric Anholt 已提交
1402
	if (ret == -EBUSY) {
1403
		DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
L
Linus Torvalds 已提交
1404 1405 1406
			  READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
	}

1407 1408 1409
	return ret;
}

L
Linus Torvalds 已提交
1410 1411
/* Needs the lock as it touches the ring.
 */
1412 1413
int i915_irq_emit(struct drm_device *dev, void *data,
			 struct drm_file *file_priv)
L
Linus Torvalds 已提交
1414 1415
{
	drm_i915_private_t *dev_priv = dev->dev_private;
1416
	drm_i915_irq_emit_t *emit = data;
L
Linus Torvalds 已提交
1417 1418
	int result;

1419
	if (!dev_priv || !LP_RING(dev_priv)->virtual_start) {
1420
		DRM_ERROR("called with no initialization\n");
E
Eric Anholt 已提交
1421
		return -EINVAL;
L
Linus Torvalds 已提交
1422
	}
1423 1424 1425

	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);

1426
	mutex_lock(&dev->struct_mutex);
L
Linus Torvalds 已提交
1427
	result = i915_emit_irq(dev);
1428
	mutex_unlock(&dev->struct_mutex);
L
Linus Torvalds 已提交
1429

1430
	if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) {
L
Linus Torvalds 已提交
1431
		DRM_ERROR("copy_to_user\n");
E
Eric Anholt 已提交
1432
		return -EFAULT;
L
Linus Torvalds 已提交
1433 1434 1435 1436 1437 1438 1439
	}

	return 0;
}

/* Doesn't need the hardware lock.
 */
1440 1441
int i915_irq_wait(struct drm_device *dev, void *data,
			 struct drm_file *file_priv)
L
Linus Torvalds 已提交
1442 1443
{
	drm_i915_private_t *dev_priv = dev->dev_private;
1444
	drm_i915_irq_wait_t *irqwait = data;
L
Linus Torvalds 已提交
1445 1446

	if (!dev_priv) {
1447
		DRM_ERROR("called with no initialization\n");
E
Eric Anholt 已提交
1448
		return -EINVAL;
L
Linus Torvalds 已提交
1449 1450
	}

1451
	return i915_wait_irq(dev, irqwait->irq_seq);
L
Linus Torvalds 已提交
1452 1453
}

1454 1455 1456
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
1457
static int i915_enable_vblank(struct drm_device *dev, int pipe)
1458 1459
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1460
	unsigned long irqflags;
1461

1462
	if (!i915_pipe_enabled(dev, pipe))
1463
		return -EINVAL;
1464

1465
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1466
	if (INTEL_INFO(dev)->gen >= 4)
1467 1468
		i915_enable_pipestat(dev_priv, pipe,
				     PIPE_START_VBLANK_INTERRUPT_ENABLE);
1469
	else
1470 1471
		i915_enable_pipestat(dev_priv, pipe,
				     PIPE_VBLANK_INTERRUPT_ENABLE);
1472 1473 1474 1475

	/* maintain vblank delivery even in deep C-states */
	if (dev_priv->info->gen == 3)
		I915_WRITE(INSTPM, INSTPM_AGPBUSY_DIS << 16);
1476
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1477

1478 1479 1480
	return 0;
}

1481
static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	unsigned long irqflags;

	if (!i915_pipe_enabled(dev, pipe))
		return -EINVAL;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
	ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
				    DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

1497
static int ivybridge_enable_vblank(struct drm_device *dev, int pipe)
1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	unsigned long irqflags;

	if (!i915_pipe_enabled(dev, pipe))
		return -EINVAL;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
	ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
				    DE_PIPEA_VBLANK_IVB : DE_PIPEB_VBLANK_IVB);
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);

	return 0;
}

1513 1514 1515
/* Called from drm generic code, passed 'crtc' which
 * we use as a pipe index
 */
1516
static void i915_disable_vblank(struct drm_device *dev, int pipe)
1517 1518
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1519
	unsigned long irqflags;
1520

1521
	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
1522 1523 1524 1525
	if (dev_priv->info->gen == 3)
		I915_WRITE(INSTPM,
			   INSTPM_AGPBUSY_DIS << 16 | INSTPM_AGPBUSY_DIS);

1526 1527 1528 1529 1530 1531
	i915_disable_pipestat(dev_priv, pipe,
			      PIPE_VBLANK_INTERRUPT_ENABLE |
			      PIPE_START_VBLANK_INTERRUPT_ENABLE);
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

1532
static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
1533 1534 1535 1536 1537 1538 1539
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
	ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
				     DE_PIPEA_VBLANK: DE_PIPEB_VBLANK);
1540
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
1541 1542
}

1543
static void ivybridge_disable_vblank(struct drm_device *dev, int pipe)
1544 1545 1546 1547 1548 1549 1550 1551 1552 1553
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	unsigned long irqflags;

	spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
	ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
				     DE_PIPEA_VBLANK_IVB : DE_PIPEB_VBLANK_IVB);
	spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
}

1554 1555
/* Set the vblank monitor pipe
 */
1556 1557
int i915_vblank_pipe_set(struct drm_device *dev, void *data,
			 struct drm_file *file_priv)
1558 1559 1560 1561
{
	drm_i915_private_t *dev_priv = dev->dev_private;

	if (!dev_priv) {
1562
		DRM_ERROR("called with no initialization\n");
E
Eric Anholt 已提交
1563
		return -EINVAL;
1564 1565
	}

1566
	return 0;
1567 1568
}

1569 1570
int i915_vblank_pipe_get(struct drm_device *dev, void *data,
			 struct drm_file *file_priv)
1571 1572
{
	drm_i915_private_t *dev_priv = dev->dev_private;
1573
	drm_i915_vblank_pipe_t *pipe = data;
1574 1575

	if (!dev_priv) {
1576
		DRM_ERROR("called with no initialization\n");
E
Eric Anholt 已提交
1577
		return -EINVAL;
1578 1579
	}

1580
	pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
1581

1582 1583 1584
	return 0;
}

1585 1586 1587
/**
 * Schedule buffer swap at given vertical blank.
 */
1588 1589
int i915_vblank_swap(struct drm_device *dev, void *data,
		     struct drm_file *file_priv)
1590
{
1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
	/* The delayed swap mechanism was fundamentally racy, and has been
	 * removed.  The model was that the client requested a delayed flip/swap
	 * from the kernel, then waited for vblank before continuing to perform
	 * rendering.  The problem was that the kernel might wake the client
	 * up before it dispatched the vblank swap (since the lock has to be
	 * held while touching the ringbuffer), in which case the client would
	 * clear and start the next frame before the swap occurred, and
	 * flicker would occur in addition to likely missing the vblank.
	 *
	 * In the absence of this ioctl, userland falls back to a correct path
	 * of waiting for a vblank, then dispatching the swap on its own.
	 * Context switching to userland and back is plenty fast enough for
	 * meeting the requirements of vblank swapping.
1604
	 */
1605
	return -EINVAL;
1606 1607
}

1608 1609
static u32
ring_last_seqno(struct intel_ring_buffer *ring)
1610
{
1611 1612 1613 1614 1615 1616 1617 1618 1619
	return list_entry(ring->request_list.prev,
			  struct drm_i915_gem_request, list)->seqno;
}

static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err)
{
	if (list_empty(&ring->request_list) ||
	    i915_seqno_passed(ring->get_seqno(ring), ring_last_seqno(ring))) {
		/* Issue a wake-up to catch stuck h/w. */
1620
		if (ring->waiting_seqno && waitqueue_active(&ring->irq_queue)) {
1621 1622
			DRM_ERROR("Hangcheck timer elapsed... %s idle [waiting on %d, at %d], missed IRQ?\n",
				  ring->name,
1623
				  ring->waiting_seqno,
1624 1625 1626 1627 1628 1629 1630
				  ring->get_seqno(ring));
			wake_up_all(&ring->irq_queue);
			*err = true;
		}
		return true;
	}
	return false;
B
Ben Gamari 已提交
1631 1632
}

1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653
static bool kick_ring(struct intel_ring_buffer *ring)
{
	struct drm_device *dev = ring->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 tmp = I915_READ_CTL(ring);
	if (tmp & RING_WAIT) {
		DRM_ERROR("Kicking stuck wait on %s\n",
			  ring->name);
		I915_WRITE_CTL(ring, tmp);
		return true;
	}
	if (IS_GEN6(dev) &&
	    (tmp & RING_WAIT_SEMAPHORE)) {
		DRM_ERROR("Kicking stuck semaphore on %s\n",
			  ring->name);
		I915_WRITE_CTL(ring, tmp);
		return true;
	}
	return false;
}

B
Ben Gamari 已提交
1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
/**
 * This is called when the chip hasn't reported back with completed
 * batchbuffers in a long time. The first time this is called we simply record
 * ACTHD. If ACTHD hasn't changed by the time the hangcheck timer elapses
 * again, we assume the chip is wedged and try to fix it.
 */
void i915_hangcheck_elapsed(unsigned long data)
{
	struct drm_device *dev = (struct drm_device *)data;
	drm_i915_private_t *dev_priv = dev->dev_private;
1664
	uint32_t acthd, instdone, instdone1;
1665 1666 1667
	bool err = false;

	/* If all work is done then ACTHD clearly hasn't advanced. */
1668 1669 1670
	if (i915_hangcheck_ring_idle(&dev_priv->ring[RCS], &err) &&
	    i915_hangcheck_ring_idle(&dev_priv->ring[VCS], &err) &&
	    i915_hangcheck_ring_idle(&dev_priv->ring[BCS], &err)) {
1671 1672 1673 1674 1675
		dev_priv->hangcheck_count = 0;
		if (err)
			goto repeat;
		return;
	}
1676

1677
	if (INTEL_INFO(dev)->gen < 4) {
B
Ben Gamari 已提交
1678
		acthd = I915_READ(ACTHD);
1679 1680 1681
		instdone = I915_READ(INSTDONE);
		instdone1 = 0;
	} else {
B
Ben Gamari 已提交
1682
		acthd = I915_READ(ACTHD_I965);
1683 1684 1685
		instdone = I915_READ(INSTDONE_I965);
		instdone1 = I915_READ(INSTDONE1);
	}
B
Ben Gamari 已提交
1686

1687 1688 1689 1690 1691
	if (dev_priv->last_acthd == acthd &&
	    dev_priv->last_instdone == instdone &&
	    dev_priv->last_instdone1 == instdone1) {
		if (dev_priv->hangcheck_count++ > 1) {
			DRM_ERROR("Hangcheck timer elapsed... GPU hung\n");
1692 1693 1694 1695 1696 1697 1698

			if (!IS_GEN2(dev)) {
				/* Is the chip hanging on a WAIT_FOR_EVENT?
				 * If so we can simply poke the RB_WAIT bit
				 * and break the hang. This should work on
				 * all but the second generation chipsets.
				 */
1699 1700 1701 1702 1703 1704 1705 1706 1707 1708

				if (kick_ring(&dev_priv->ring[RCS]))
					goto repeat;

				if (HAS_BSD(dev) &&
				    kick_ring(&dev_priv->ring[VCS]))
					goto repeat;

				if (HAS_BLT(dev) &&
				    kick_ring(&dev_priv->ring[BCS]))
1709
					goto repeat;
1710 1711
			}

1712 1713 1714 1715 1716 1717 1718 1719 1720 1721
			i915_handle_error(dev, true);
			return;
		}
	} else {
		dev_priv->hangcheck_count = 0;

		dev_priv->last_acthd = acthd;
		dev_priv->last_instdone = instdone;
		dev_priv->last_instdone1 = instdone1;
	}
B
Ben Gamari 已提交
1722

1723
repeat:
B
Ben Gamari 已提交
1724
	/* Reset timer case chip hangs without another request being added */
1725 1726
	mod_timer(&dev_priv->hangcheck_timer,
		  jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
B
Ben Gamari 已提交
1727 1728
}

L
Linus Torvalds 已提交
1729 1730
/* drm_dma.h hooks
*/
1731
static void ironlake_irq_preinstall(struct drm_device *dev)
1732 1733 1734
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;

1735 1736 1737 1738
	atomic_set(&dev_priv->irq_received, 0);

	INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
	INIT_WORK(&dev_priv->error_work, i915_error_work_func);
1739 1740
	if (IS_GEN6(dev) || IS_IVYBRIDGE(dev))
		INIT_WORK(&dev_priv->rps_work, gen6_pm_rps_work);
1741

1742
	I915_WRITE(HWSTAM, 0xeffe);
1743
	if (IS_GEN6(dev) || IS_GEN7(dev)) {
1744 1745 1746 1747 1748 1749 1750 1751
		/* Workaround stalls observed on Sandy Bridge GPUs by
		 * making the blitter command streamer generate a
		 * write to the Hardware Status Page for
		 * MI_USER_INTERRUPT.  This appears to serialize the
		 * previous seqno write out before the interrupt
		 * happens.
		 */
		I915_WRITE(GEN6_BLITTER_HWSTAM, ~GEN6_BLITTER_USER_INTERRUPT);
1752
		I915_WRITE(GEN6_BSD_HWSTAM, ~GEN6_BSD_USER_INTERRUPT);
1753
	}
1754 1755 1756 1757 1758

	/* XXX hotplug from PCH */

	I915_WRITE(DEIMR, 0xffffffff);
	I915_WRITE(DEIER, 0x0);
1759
	POSTING_READ(DEIER);
1760 1761 1762 1763

	/* and GT */
	I915_WRITE(GTIMR, 0xffffffff);
	I915_WRITE(GTIER, 0x0);
1764
	POSTING_READ(GTIER);
1765 1766 1767 1768

	/* south display irq */
	I915_WRITE(SDEIMR, 0xffffffff);
	I915_WRITE(SDEIER, 0x0);
1769
	POSTING_READ(SDEIER);
1770 1771
}

1772
static int ironlake_irq_postinstall(struct drm_device *dev)
1773 1774 1775
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	/* enable kind of interrupts always enabled */
1776 1777
	u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
			   DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE;
1778
	u32 render_irqs;
1779
	u32 hotplug_mask;
1780

1781 1782 1783 1784 1785 1786 1787
	DRM_INIT_WAITQUEUE(&dev_priv->ring[RCS].irq_queue);
	if (HAS_BSD(dev))
		DRM_INIT_WAITQUEUE(&dev_priv->ring[VCS].irq_queue);
	if (HAS_BLT(dev))
		DRM_INIT_WAITQUEUE(&dev_priv->ring[BCS].irq_queue);

	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
1788
	dev_priv->irq_mask = ~display_mask;
1789 1790 1791

	/* should always can generate irq */
	I915_WRITE(DEIIR, I915_READ(DEIIR));
1792 1793
	I915_WRITE(DEIMR, dev_priv->irq_mask);
	I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK);
1794
	POSTING_READ(DEIER);
1795

1796
	dev_priv->gt_irq_mask = ~0;
1797 1798

	I915_WRITE(GTIIR, I915_READ(GTIIR));
1799
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
1800

1801 1802 1803 1804 1805 1806 1807
	if (IS_GEN6(dev))
		render_irqs =
			GT_USER_INTERRUPT |
			GT_GEN6_BSD_USER_INTERRUPT |
			GT_BLT_USER_INTERRUPT;
	else
		render_irqs =
1808
			GT_USER_INTERRUPT |
1809
			GT_PIPE_NOTIFY |
1810 1811
			GT_BSD_USER_INTERRUPT;
	I915_WRITE(GTIER, render_irqs);
1812
	POSTING_READ(GTIER);
1813

1814
	if (HAS_PCH_CPT(dev)) {
1815 1816 1817 1818
		hotplug_mask = (SDE_CRT_HOTPLUG_CPT |
				SDE_PORTB_HOTPLUG_CPT |
				SDE_PORTC_HOTPLUG_CPT |
				SDE_PORTD_HOTPLUG_CPT);
1819
	} else {
1820 1821 1822 1823 1824
		hotplug_mask = (SDE_CRT_HOTPLUG |
				SDE_PORTB_HOTPLUG |
				SDE_PORTC_HOTPLUG |
				SDE_PORTD_HOTPLUG |
				SDE_AUX_MASK);
1825 1826
	}

1827
	dev_priv->pch_irq_mask = ~hotplug_mask;
1828 1829

	I915_WRITE(SDEIIR, I915_READ(SDEIIR));
1830 1831
	I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
	I915_WRITE(SDEIER, hotplug_mask);
1832
	POSTING_READ(SDEIER);
1833

1834 1835 1836 1837 1838 1839 1840
	if (IS_IRONLAKE_M(dev)) {
		/* Clear & enable PCU event interrupts */
		I915_WRITE(DEIIR, DE_PCU_EVENT);
		I915_WRITE(DEIER, I915_READ(DEIER) | DE_PCU_EVENT);
		ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
	}

1841 1842 1843
	return 0;
}

1844
static int ivybridge_irq_postinstall(struct drm_device *dev)
1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
	/* enable kind of interrupts always enabled */
	u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
		DE_PCH_EVENT_IVB | DE_PLANEA_FLIP_DONE_IVB |
		DE_PLANEB_FLIP_DONE_IVB;
	u32 render_irqs;
	u32 hotplug_mask;

	DRM_INIT_WAITQUEUE(&dev_priv->ring[RCS].irq_queue);
	if (HAS_BSD(dev))
		DRM_INIT_WAITQUEUE(&dev_priv->ring[VCS].irq_queue);
	if (HAS_BLT(dev))
		DRM_INIT_WAITQUEUE(&dev_priv->ring[BCS].irq_queue);

	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
	dev_priv->irq_mask = ~display_mask;

	/* should always can generate irq */
	I915_WRITE(DEIIR, I915_READ(DEIIR));
	I915_WRITE(DEIMR, dev_priv->irq_mask);
	I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK_IVB |
		   DE_PIPEB_VBLANK_IVB);
	POSTING_READ(DEIER);

	dev_priv->gt_irq_mask = ~0;

	I915_WRITE(GTIIR, I915_READ(GTIIR));
	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);

	render_irqs = GT_USER_INTERRUPT | GT_GEN6_BSD_USER_INTERRUPT |
		GT_BLT_USER_INTERRUPT;
	I915_WRITE(GTIER, render_irqs);
	POSTING_READ(GTIER);

	hotplug_mask = (SDE_CRT_HOTPLUG_CPT |
			SDE_PORTB_HOTPLUG_CPT |
			SDE_PORTC_HOTPLUG_CPT |
			SDE_PORTD_HOTPLUG_CPT);
	dev_priv->pch_irq_mask = ~hotplug_mask;

	I915_WRITE(SDEIIR, I915_READ(SDEIIR));
	I915_WRITE(SDEIMR, dev_priv->pch_irq_mask);
	I915_WRITE(SDEIER, hotplug_mask);
	POSTING_READ(SDEIER);

	return 0;
}

1894
static void i915_driver_irq_preinstall(struct drm_device * dev)
L
Linus Torvalds 已提交
1895 1896
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1897
	int pipe;
L
Linus Torvalds 已提交
1898

J
Jesse Barnes 已提交
1899 1900
	atomic_set(&dev_priv->irq_received, 0);

1901
	INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
1902
	INIT_WORK(&dev_priv->error_work, i915_error_work_func);
1903

1904 1905 1906 1907 1908
	if (I915_HAS_HOTPLUG(dev)) {
		I915_WRITE(PORT_HOTPLUG_EN, 0);
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

1909
	I915_WRITE(HWSTAM, 0xeffe);
1910 1911
	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe), 0);
1912
	I915_WRITE(IMR, 0xffffffff);
1913
	I915_WRITE(IER, 0x0);
1914
	POSTING_READ(IER);
L
Linus Torvalds 已提交
1915 1916
}

1917 1918 1919 1920
/*
 * Must be called after intel_modeset_init or hotplug interrupts won't be
 * enabled correctly.
 */
1921
static int i915_driver_irq_postinstall(struct drm_device *dev)
L
Linus Torvalds 已提交
1922 1923
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1924
	u32 enable_mask = I915_INTERRUPT_ENABLE_FIX | I915_INTERRUPT_ENABLE_VAR;
1925
	u32 error_mask;
1926 1927 1928

	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;

1929
	/* Unmask the interrupts that we always want on. */
1930
	dev_priv->irq_mask = ~I915_INTERRUPT_ENABLE_FIX;
1931 1932 1933 1934

	dev_priv->pipestat[0] = 0;
	dev_priv->pipestat[1] = 0;

1935 1936 1937 1938
	if (I915_HAS_HOTPLUG(dev)) {
		/* Enable in IER... */
		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
		/* and unmask in IMR */
1939
		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
1940 1941
	}

1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956
	/*
	 * Enable some error detection, note the instruction error mask
	 * bit is reserved, so we leave it masked.
	 */
	if (IS_G4X(dev)) {
		error_mask = ~(GM45_ERROR_PAGE_TABLE |
			       GM45_ERROR_MEM_PRIV |
			       GM45_ERROR_CP_PRIV |
			       I915_ERROR_MEMORY_REFRESH);
	} else {
		error_mask = ~(I915_ERROR_PAGE_TABLE |
			       I915_ERROR_MEMORY_REFRESH);
	}
	I915_WRITE(EMR, error_mask);

1957
	I915_WRITE(IMR, dev_priv->irq_mask);
1958
	I915_WRITE(IER, enable_mask);
1959
	POSTING_READ(IER);
1960

1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974
	if (I915_HAS_HOTPLUG(dev)) {
		u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);

		/* Note HDMI and DP share bits */
		if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS)
			hotplug_en |= HDMIB_HOTPLUG_INT_EN;
		if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS)
			hotplug_en |= HDMIC_HOTPLUG_INT_EN;
		if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS)
			hotplug_en |= HDMID_HOTPLUG_INT_EN;
		if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS)
			hotplug_en |= SDVOC_HOTPLUG_INT_EN;
		if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS)
			hotplug_en |= SDVOB_HOTPLUG_INT_EN;
1975
		if (dev_priv->hotplug_supported_mask & CRT_HOTPLUG_INT_STATUS) {
1976
			hotplug_en |= CRT_HOTPLUG_INT_EN;
1977 1978 1979 1980 1981 1982 1983 1984 1985 1986

			/* Programming the CRT detection parameters tends
			   to generate a spurious hotplug event about three
			   seconds later.  So just do it once.
			*/
			if (IS_G4X(dev))
				hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
			hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
		}

1987 1988 1989 1990 1991
		/* Ignore TV since it's buggy */

		I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
	}

1992
	intel_opregion_enable_asle(dev);
1993 1994

	return 0;
L
Linus Torvalds 已提交
1995 1996
}

1997
static void ironlake_irq_uninstall(struct drm_device *dev)
1998 1999
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2000 2001 2002 2003 2004 2005

	if (!dev_priv)
		return;

	dev_priv->vblank_pipe = 0;

2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016
	I915_WRITE(HWSTAM, 0xffffffff);

	I915_WRITE(DEIMR, 0xffffffff);
	I915_WRITE(DEIER, 0x0);
	I915_WRITE(DEIIR, I915_READ(DEIIR));

	I915_WRITE(GTIMR, 0xffffffff);
	I915_WRITE(GTIER, 0x0);
	I915_WRITE(GTIIR, I915_READ(GTIIR));
}

2017
static void i915_driver_irq_uninstall(struct drm_device * dev)
L
Linus Torvalds 已提交
2018 2019
{
	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
2020
	int pipe;
2021

L
Linus Torvalds 已提交
2022 2023 2024
	if (!dev_priv)
		return;

2025 2026
	dev_priv->vblank_pipe = 0;

2027 2028 2029 2030 2031
	if (I915_HAS_HOTPLUG(dev)) {
		I915_WRITE(PORT_HOTPLUG_EN, 0);
		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
	}

2032
	I915_WRITE(HWSTAM, 0xffffffff);
2033 2034
	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe), 0);
2035
	I915_WRITE(IMR, 0xffffffff);
2036
	I915_WRITE(IER, 0x0);
2037

2038 2039 2040
	for_each_pipe(pipe)
		I915_WRITE(PIPESTAT(pipe),
			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
2041
	I915_WRITE(IIR, I915_READ(IIR));
L
Linus Torvalds 已提交
2042
}
2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080

void intel_irq_init(struct drm_device *dev)
{
	dev->driver->get_vblank_counter = i915_get_vblank_counter;
	dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
	if (IS_G4X(dev) || IS_GEN5(dev) || IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
		dev->driver->get_vblank_counter = gm45_get_vblank_counter;
	}


	dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;

	if (IS_IVYBRIDGE(dev)) {
		/* Share pre & uninstall handlers with ILK/SNB */
		dev->driver->irq_handler = ivybridge_irq_handler;
		dev->driver->irq_preinstall = ironlake_irq_preinstall;
		dev->driver->irq_postinstall = ivybridge_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ivybridge_enable_vblank;
		dev->driver->disable_vblank = ivybridge_disable_vblank;
	} else if (HAS_PCH_SPLIT(dev)) {
		dev->driver->irq_handler = ironlake_irq_handler;
		dev->driver->irq_preinstall = ironlake_irq_preinstall;
		dev->driver->irq_postinstall = ironlake_irq_postinstall;
		dev->driver->irq_uninstall = ironlake_irq_uninstall;
		dev->driver->enable_vblank = ironlake_enable_vblank;
		dev->driver->disable_vblank = ironlake_disable_vblank;
	} else {
		dev->driver->irq_preinstall = i915_driver_irq_preinstall;
		dev->driver->irq_postinstall = i915_driver_irq_postinstall;
		dev->driver->irq_uninstall = i915_driver_irq_uninstall;
		dev->driver->irq_handler = i915_driver_irq_handler;
		dev->driver->enable_vblank = i915_enable_vblank;
		dev->driver->disable_vblank = i915_disable_vblank;
	}
}