i915_gpu_error.c 38.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39
/*
 * Copyright (c) 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *    Keith Packard <keithp@keithp.com>
 *    Mika Kuoppala <mika.kuoppala@intel.com>
 *
 */

#include <generated/utsrelease.h>
#include "i915_drv.h"

static const char *ring_str(int ring)
{
	switch (ring) {
	case RCS: return "render";
	case VCS: return "bsd";
	case BCS: return "blt";
	case VECS: return "vebox";
40
	case VCS2: return "bsd2";
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
	default: return "";
	}
}

static const char *pin_flag(int pinned)
{
	if (pinned > 0)
		return " P";
	else if (pinned < 0)
		return " p";
	else
		return "";
}

static const char *tiling_flag(int tiling)
{
	switch (tiling) {
	default:
	case I915_TILING_NONE: return "";
	case I915_TILING_X: return " X";
	case I915_TILING_Y: return " Y";
	}
}

static const char *dirty_flag(int dirty)
{
	return dirty ? " dirty" : "";
}

static const char *purgeable_flag(int purgeable)
{
	return purgeable ? " purgeable" : "";
}

static bool __i915_error_ok(struct drm_i915_error_state_buf *e)
{

	if (!e->err && WARN(e->bytes > (e->size - 1), "overflow")) {
		e->err = -ENOSPC;
		return false;
	}

	if (e->bytes == e->size - 1 || e->err)
		return false;

	return true;
}

static bool __i915_error_seek(struct drm_i915_error_state_buf *e,
			      unsigned len)
{
	if (e->pos + len <= e->start) {
		e->pos += len;
		return false;
	}

	/* First vsnprintf needs to fit in its entirety for memmove */
	if (len >= e->size) {
		e->err = -EIO;
		return false;
	}

	return true;
}

static void __i915_error_advance(struct drm_i915_error_state_buf *e,
				 unsigned len)
{
	/* If this is first printf in this window, adjust it so that
	 * start position matches start of the buffer
	 */

	if (e->pos < e->start) {
		const size_t off = e->start - e->pos;

		/* Should not happen but be paranoid */
		if (off > len || e->bytes) {
			e->err = -EIO;
			return;
		}

		memmove(e->buf, e->buf + off, len - off);
		e->bytes = len - off;
		e->pos = e->start;
		return;
	}

	e->bytes += len;
	e->pos += len;
}

static void i915_error_vprintf(struct drm_i915_error_state_buf *e,
			       const char *f, va_list args)
{
	unsigned len;

	if (!__i915_error_ok(e))
		return;

	/* Seek the first printf which is hits start position */
	if (e->pos < e->start) {
142 143 144
		va_list tmp;

		va_copy(tmp, args);
145 146 147 148
		len = vsnprintf(NULL, 0, f, tmp);
		va_end(tmp);

		if (!__i915_error_seek(e, len))
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
			return;
	}

	len = vsnprintf(e->buf + e->bytes, e->size - e->bytes, f, args);
	if (len >= e->size - e->bytes)
		len = e->size - e->bytes - 1;

	__i915_error_advance(e, len);
}

static void i915_error_puts(struct drm_i915_error_state_buf *e,
			    const char *str)
{
	unsigned len;

	if (!__i915_error_ok(e))
		return;

	len = strlen(str);

	/* Seek the first printf which is hits start position */
	if (e->pos < e->start) {
		if (!__i915_error_seek(e, len))
			return;
	}

	if (len >= e->size - e->bytes)
		len = e->size - e->bytes - 1;
	memcpy(e->buf + e->bytes, str, len);

	__i915_error_advance(e, len);
}

#define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
#define err_puts(e, s) i915_error_puts(e, s)

static void print_error_buffers(struct drm_i915_error_state_buf *m,
				const char *name,
				struct drm_i915_error_buffer *err,
				int count)
{
190 191
	int i;

192
	err_printf(m, "  %s [%d]:\n", name, count);
193 194

	while (count--) {
195 196 197
		err_printf(m, "    %08x_%08x %8u %02x %02x [ ",
			   upper_32_bits(err->gtt_offset),
			   lower_32_bits(err->gtt_offset),
198 199
			   err->size,
			   err->read_domains,
200 201 202 203 204
			   err->write_domain);
		for (i = 0; i < I915_NUM_RINGS; i++)
			err_printf(m, "%02x ", err->rseqno[i]);

		err_printf(m, "] %02x", err->wseqno);
205 206 207 208
		err_puts(m, pin_flag(err->pinned));
		err_puts(m, tiling_flag(err->tiling));
		err_puts(m, dirty_flag(err->dirty));
		err_puts(m, purgeable_flag(err->purgeable));
209
		err_puts(m, err->userptr ? " userptr" : "");
210 211
		err_puts(m, err->ring != -1 ? " " : "");
		err_puts(m, ring_str(err->ring));
212
		err_puts(m, i915_cache_level_str(m->i915, err->cache_level));
213 214 215 216 217 218 219 220 221 222 223

		if (err->name)
			err_printf(m, " (name: %d)", err->name);
		if (err->fence_reg != I915_FENCE_REG_NONE)
			err_printf(m, " (fence: %d)", err->fence_reg);

		err_puts(m, "\n");
		err++;
	}
}

224 225 226 227 228 229 230 231 232
static const char *hangcheck_action_to_str(enum intel_ring_hangcheck_action a)
{
	switch (a) {
	case HANGCHECK_IDLE:
		return "idle";
	case HANGCHECK_WAIT:
		return "wait";
	case HANGCHECK_ACTIVE:
		return "active";
233 234
	case HANGCHECK_ACTIVE_LOOP:
		return "active (loop)";
235 236 237 238 239 240 241 242 243
	case HANGCHECK_KICK:
		return "kick";
	case HANGCHECK_HUNG:
		return "hung";
	}

	return "unknown";
}

244 245
static void i915_ring_error_state(struct drm_i915_error_state_buf *m,
				  struct drm_device *dev,
246 247
				  struct drm_i915_error_state *error,
				  int ring_idx)
248
{
249 250
	struct drm_i915_error_ring *ring = &error->ring[ring_idx];

251
	if (!ring->valid)
252 253
		return;

254
	err_printf(m, "%s command stream:\n", ring_str(ring_idx));
255 256 257 258 259
	err_printf(m, "  START: 0x%08x\n", ring->start);
	err_printf(m, "  HEAD:  0x%08x\n", ring->head);
	err_printf(m, "  TAIL:  0x%08x\n", ring->tail);
	err_printf(m, "  CTL:   0x%08x\n", ring->ctl);
	err_printf(m, "  HWS:   0x%08x\n", ring->hws);
260
	err_printf(m, "  ACTHD: 0x%08x %08x\n", (u32)(ring->acthd>>32), (u32)ring->acthd);
261 262 263
	err_printf(m, "  IPEIR: 0x%08x\n", ring->ipeir);
	err_printf(m, "  IPEHR: 0x%08x\n", ring->ipehr);
	err_printf(m, "  INSTDONE: 0x%08x\n", ring->instdone);
264
	if (INTEL_INFO(dev)->gen >= 4) {
265
		err_printf(m, "  BBADDR: 0x%08x %08x\n", (u32)(ring->bbaddr>>32), (u32)ring->bbaddr);
266 267
		err_printf(m, "  BB_STATE: 0x%08x\n", ring->bbstate);
		err_printf(m, "  INSTPS: 0x%08x\n", ring->instps);
268
	}
269
	err_printf(m, "  INSTPM: 0x%08x\n", ring->instpm);
270 271
	err_printf(m, "  FADDR: 0x%08x %08x\n", upper_32_bits(ring->faddr),
		   lower_32_bits(ring->faddr));
272
	if (INTEL_INFO(dev)->gen >= 6) {
273 274
		err_printf(m, "  RC PSMI: 0x%08x\n", ring->rc_psmi);
		err_printf(m, "  FAULT_REG: 0x%08x\n", ring->fault_reg);
275
		err_printf(m, "  SYNC_0: 0x%08x [last synced 0x%08x]\n",
276 277
			   ring->semaphore_mboxes[0],
			   ring->semaphore_seqno[0]);
278
		err_printf(m, "  SYNC_1: 0x%08x [last synced 0x%08x]\n",
279 280
			   ring->semaphore_mboxes[1],
			   ring->semaphore_seqno[1]);
281 282
		if (HAS_VEBOX(dev)) {
			err_printf(m, "  SYNC_2: 0x%08x [last synced 0x%08x]\n",
283 284
				   ring->semaphore_mboxes[2],
				   ring->semaphore_seqno[2]);
285
		}
286
	}
287 288 289 290 291 292 293 294 295 296 297 298 299
	if (USES_PPGTT(dev)) {
		err_printf(m, "  GFX_MODE: 0x%08x\n", ring->vm_info.gfx_mode);

		if (INTEL_INFO(dev)->gen >= 8) {
			int i;
			for (i = 0; i < 4; i++)
				err_printf(m, "  PDP%d: 0x%016llx\n",
					   i, ring->vm_info.pdp[i]);
		} else {
			err_printf(m, "  PP_DIR_BASE: 0x%08x\n",
				   ring->vm_info.pp_dir_base);
		}
	}
300 301 302 303
	err_printf(m, "  seqno: 0x%08x\n", ring->seqno);
	err_printf(m, "  waiting: %s\n", yesno(ring->waiting));
	err_printf(m, "  ring->head: 0x%08x\n", ring->cpu_ring_head);
	err_printf(m, "  ring->tail: 0x%08x\n", ring->cpu_ring_tail);
304
	err_printf(m, "  hangcheck: %s [%d]\n",
305 306
		   hangcheck_action_to_str(ring->hangcheck_action),
		   ring->hangcheck_score);
307 308 309 310 311 312 313 314 315 316 317
}

void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
{
	va_list args;

	va_start(args, f);
	i915_error_vprintf(e, f, args);
	va_end(args);
}

318 319 320 321 322 323 324 325 326 327 328 329 330 331
static void print_error_obj(struct drm_i915_error_state_buf *m,
			    struct drm_i915_error_object *obj)
{
	int page, offset, elt;

	for (page = offset = 0; page < obj->page_count; page++) {
		for (elt = 0; elt < PAGE_SIZE/4; elt++) {
			err_printf(m, "%08x :  %08x\n", offset,
				   obj->pages[page][elt]);
			offset += 4;
		}
	}
}

332 333 334 335
int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
			    const struct i915_error_state_file_priv *error_priv)
{
	struct drm_device *dev = error_priv->dev;
336
	struct drm_i915_private *dev_priv = dev->dev_private;
337
	struct drm_i915_error_state *error = error_priv->error;
338
	struct drm_i915_error_object *obj;
339 340
	int i, j, offset, elt;
	int max_hangcheck_score;
341 342 343 344 345 346

	if (!error) {
		err_printf(m, "no error state collected\n");
		goto out;
	}

347
	err_printf(m, "%s\n", error->error_msg);
348 349 350
	err_printf(m, "Time: %ld s %ld us\n", error->time.tv_sec,
		   error->time.tv_usec);
	err_printf(m, "Kernel: " UTS_RELEASE "\n");
351 352 353 354 355 356 357 358 359 360 361 362 363 364
	max_hangcheck_score = 0;
	for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
		if (error->ring[i].hangcheck_score > max_hangcheck_score)
			max_hangcheck_score = error->ring[i].hangcheck_score;
	}
	for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
		if (error->ring[i].hangcheck_score == max_hangcheck_score &&
		    error->ring[i].pid != -1) {
			err_printf(m, "Active process (on ring %s): %s [%d]\n",
				   ring_str(i),
				   error->ring[i].comm,
				   error->ring[i].pid);
		}
	}
365
	err_printf(m, "Reset count: %u\n", error->reset_count);
366
	err_printf(m, "Suspend count: %u\n", error->suspend_count);
367
	err_printf(m, "PCI ID: 0x%04x\n", dev->pdev->device);
368
	err_printf(m, "IOMMU enabled?: %d\n", error->iommu);
369 370
	err_printf(m, "EIR: 0x%08x\n", error->eir);
	err_printf(m, "IER: 0x%08x\n", error->ier);
371 372 373 374 375 376
	if (INTEL_INFO(dev)->gen >= 8) {
		for (i = 0; i < 4; i++)
			err_printf(m, "GTIER gt %d: 0x%08x\n", i,
				   error->gtier[i]);
	} else if (HAS_PCH_SPLIT(dev) || IS_VALLEYVIEW(dev))
		err_printf(m, "GTIER: 0x%08x\n", error->gtier[0]);
377 378 379 380
	err_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
	err_printf(m, "FORCEWAKE: 0x%08x\n", error->forcewake);
	err_printf(m, "DERRMR: 0x%08x\n", error->derrmr);
	err_printf(m, "CCID: 0x%08x\n", error->ccid);
381
	err_printf(m, "Missed interrupts: 0x%08lx\n", dev_priv->gpu_error.missed_irq_rings);
382 383 384 385 386 387 388 389 390 391

	for (i = 0; i < dev_priv->num_fence_regs; i++)
		err_printf(m, "  fence[%d] = %08llx\n", i, error->fence[i]);

	for (i = 0; i < ARRAY_SIZE(error->extra_instdone); i++)
		err_printf(m, "  INSTDONE_%d: 0x%08x\n", i,
			   error->extra_instdone[i]);

	if (INTEL_INFO(dev)->gen >= 6) {
		err_printf(m, "ERROR: 0x%08x\n", error->error);
392 393 394 395 396

		if (INTEL_INFO(dev)->gen >= 8)
			err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n",
				   error->fault_data1, error->fault_data0);

397 398 399 400 401 402
		err_printf(m, "DONE_REG: 0x%08x\n", error->done_reg);
	}

	if (INTEL_INFO(dev)->gen == 7)
		err_printf(m, "ERR_INT: 0x%08x\n", error->err_int);

403 404
	for (i = 0; i < ARRAY_SIZE(error->ring); i++)
		i915_ring_error_state(m, dev, error, i);
405

406 407 408
	for (i = 0; i < error->vm_count; i++) {
		err_printf(m, "vm[%d]\n", i);

409
		print_error_buffers(m, "Active",
410 411
				    error->active_bo[i],
				    error->active_bo_count[i]);
412 413

		print_error_buffers(m, "Pinned",
414 415 416
				    error->pinned_bo[i],
				    error->pinned_bo_count[i]);
	}
417 418

	for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
419 420 421 422 423 424 425
		obj = error->ring[i].batchbuffer;
		if (obj) {
			err_puts(m, dev_priv->ring[i].name);
			if (error->ring[i].pid != -1)
				err_printf(m, " (submitted by %s [%d])",
					   error->ring[i].comm,
					   error->ring[i].pid);
426 427 428
			err_printf(m, " --- gtt_offset = 0x%08x %08x\n",
				   upper_32_bits(obj->gtt_offset),
				   lower_32_bits(obj->gtt_offset));
429 430 431 432 433 434
			print_error_obj(m, obj);
		}

		obj = error->ring[i].wa_batchbuffer;
		if (obj) {
			err_printf(m, "%s (w/a) --- gtt_offset = 0x%08x\n",
435 436
				   dev_priv->ring[i].name,
				   lower_32_bits(obj->gtt_offset));
437
			print_error_obj(m, obj);
438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
		}

		if (error->ring[i].num_requests) {
			err_printf(m, "%s --- %d requests\n",
				   dev_priv->ring[i].name,
				   error->ring[i].num_requests);
			for (j = 0; j < error->ring[i].num_requests; j++) {
				err_printf(m, "  seqno 0x%08x, emitted %ld, tail 0x%08x\n",
					   error->ring[i].requests[j].seqno,
					   error->ring[i].requests[j].jiffies,
					   error->ring[i].requests[j].tail);
			}
		}

		if ((obj = error->ring[i].ringbuffer)) {
			err_printf(m, "%s --- ringbuffer = 0x%08x\n",
				   dev_priv->ring[i].name,
455
				   lower_32_bits(obj->gtt_offset));
456
			print_error_obj(m, obj);
457 458
		}

459
		if ((obj = error->ring[i].hws_page)) {
460 461 462 463 464 465 466
			u64 hws_offset = obj->gtt_offset;
			u32 *hws_page = &obj->pages[0][0];

			if (i915.enable_execlists) {
				hws_offset += LRC_PPHWSP_PN * PAGE_SIZE;
				hws_page = &obj->pages[LRC_PPHWSP_PN][0];
			}
467
			err_printf(m, "%s --- HW Status = 0x%08llx\n",
468
				   dev_priv->ring[i].name, hws_offset);
469 470 471 472
			offset = 0;
			for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
				err_printf(m, "[%04x] %08x %08x %08x %08x\n",
					   offset,
473 474 475 476
					   hws_page[elt],
					   hws_page[elt+1],
					   hws_page[elt+2],
					   hws_page[elt+3]);
477 478 479 480
					offset += 16;
			}
		}

481
		if ((obj = error->ring[i].ctx)) {
482 483
			err_printf(m, "%s --- HW Context = 0x%08x\n",
				   dev_priv->ring[i].name,
484
				   lower_32_bits(obj->gtt_offset));
485
			print_error_obj(m, obj);
486 487 488
		}
	}

489
	if ((obj = error->semaphore_obj)) {
490 491
		err_printf(m, "Semaphore page = 0x%08x\n",
			   lower_32_bits(obj->gtt_offset));
492 493 494 495 496 497 498 499 500 501
		for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
			err_printf(m, "[%04x] %08x %08x %08x %08x\n",
				   elt * 4,
				   obj->pages[0][elt],
				   obj->pages[0][elt+1],
				   obj->pages[0][elt+2],
				   obj->pages[0][elt+3]);
		}
	}

502 503 504 505 506 507 508 509 510 511 512 513 514 515
	if (error->overlay)
		intel_overlay_print_error_state(m, error->overlay);

	if (error->display)
		intel_display_print_error_state(m, dev, error->display);

out:
	if (m->bytes == 0 && m->err)
		return m->err;

	return 0;
}

int i915_error_state_buf_init(struct drm_i915_error_state_buf *ebuf,
516
			      struct drm_i915_private *i915,
517 518 519
			      size_t count, loff_t pos)
{
	memset(ebuf, 0, sizeof(*ebuf));
520
	ebuf->i915 = i915;
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567

	/* We need to have enough room to store any i915_error_state printf
	 * so that we can move it to start position.
	 */
	ebuf->size = count + 1 > PAGE_SIZE ? count + 1 : PAGE_SIZE;
	ebuf->buf = kmalloc(ebuf->size,
				GFP_TEMPORARY | __GFP_NORETRY | __GFP_NOWARN);

	if (ebuf->buf == NULL) {
		ebuf->size = PAGE_SIZE;
		ebuf->buf = kmalloc(ebuf->size, GFP_TEMPORARY);
	}

	if (ebuf->buf == NULL) {
		ebuf->size = 128;
		ebuf->buf = kmalloc(ebuf->size, GFP_TEMPORARY);
	}

	if (ebuf->buf == NULL)
		return -ENOMEM;

	ebuf->start = pos;

	return 0;
}

static void i915_error_object_free(struct drm_i915_error_object *obj)
{
	int page;

	if (obj == NULL)
		return;

	for (page = 0; page < obj->page_count; page++)
		kfree(obj->pages[page]);

	kfree(obj);
}

static void i915_error_state_free(struct kref *error_ref)
{
	struct drm_i915_error_state *error = container_of(error_ref,
							  typeof(*error), ref);
	int i;

	for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
		i915_error_object_free(error->ring[i].batchbuffer);
568
		i915_error_object_free(error->ring[i].wa_batchbuffer);
569
		i915_error_object_free(error->ring[i].ringbuffer);
570
		i915_error_object_free(error->ring[i].hws_page);
571 572 573 574
		i915_error_object_free(error->ring[i].ctx);
		kfree(error->ring[i].requests);
	}

575
	i915_error_object_free(error->semaphore_obj);
576 577 578 579

	for (i = 0; i < error->vm_count; i++)
		kfree(error->active_bo[i]);

580
	kfree(error->active_bo);
581 582 583
	kfree(error->active_bo_count);
	kfree(error->pinned_bo);
	kfree(error->pinned_bo_count);
584 585 586 587 588 589
	kfree(error->overlay);
	kfree(error->display);
	kfree(error);
}

static struct drm_i915_error_object *
590 591 592
i915_error_object_create(struct drm_i915_private *dev_priv,
			 struct drm_i915_gem_object *src,
			 struct i915_address_space *vm)
593 594
{
	struct drm_i915_error_object *dst;
595
	struct i915_vma *vma = NULL;
596
	int num_pages;
597 598
	bool use_ggtt;
	int i = 0;
599
	u64 reloc_offset;
600 601 602 603

	if (src == NULL || src->pages == NULL)
		return NULL;

604 605
	num_pages = src->base.size >> PAGE_SHIFT;

606 607 608 609
	dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *), GFP_ATOMIC);
	if (dst == NULL)
		return NULL;

610 611 612 613
	if (i915_gem_obj_bound(src, vm))
		dst->gtt_offset = i915_gem_obj_offset(src, vm);
	else
		dst->gtt_offset = -1;
614 615

	reloc_offset = dst->gtt_offset;
616 617
	if (i915_is_ggtt(vm))
		vma = i915_gem_obj_to_ggtt(src);
618
	use_ggtt = (src->cache_level == I915_CACHE_NONE &&
619 620
		   vma && (vma->bound & GLOBAL_BIND) &&
		   reloc_offset + num_pages * PAGE_SIZE <= dev_priv->gtt.mappable_end);
621 622 623 624 625

	/* Cannot access stolen address directly, try to use the aperture */
	if (src->stolen) {
		use_ggtt = true;

626
		if (!(vma && vma->bound & GLOBAL_BIND))
627 628 629 630 631 632 633 634 635 636 637 638 639
			goto unwind;

		reloc_offset = i915_gem_obj_ggtt_offset(src);
		if (reloc_offset + num_pages * PAGE_SIZE > dev_priv->gtt.mappable_end)
			goto unwind;
	}

	/* Cannot access snooped pages through the aperture */
	if (use_ggtt && src->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv->dev))
		goto unwind;

	dst->page_count = num_pages;
	while (num_pages--) {
640 641 642 643 644 645 646 647
		unsigned long flags;
		void *d;

		d = kmalloc(PAGE_SIZE, GFP_ATOMIC);
		if (d == NULL)
			goto unwind;

		local_irq_save(flags);
648
		if (use_ggtt) {
649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675
			void __iomem *s;

			/* Simply ignore tiling or any overlapping fence.
			 * It's part of the error state, and this hopefully
			 * captures what the GPU read.
			 */

			s = io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
						     reloc_offset);
			memcpy_fromio(d, s, PAGE_SIZE);
			io_mapping_unmap_atomic(s);
		} else {
			struct page *page;
			void *s;

			page = i915_gem_object_get_page(src, i);

			drm_clflush_pages(&page, 1);

			s = kmap_atomic(page);
			memcpy(d, s, PAGE_SIZE);
			kunmap_atomic(s);

			drm_clflush_pages(&page, 1);
		}
		local_irq_restore(flags);

676
		dst->pages[i++] = d;
677 678 679 680 681 682 683 684 685 686 687
		reloc_offset += PAGE_SIZE;
	}

	return dst;

unwind:
	while (i--)
		kfree(dst->pages[i]);
	kfree(dst);
	return NULL;
}
688
#define i915_error_ggtt_object_create(dev_priv, src) \
689
	i915_error_object_create((dev_priv), (src), &(dev_priv)->gtt.base)
690 691

static void capture_bo(struct drm_i915_error_buffer *err,
692
		       struct i915_vma *vma)
693
{
694
	struct drm_i915_gem_object *obj = vma->obj;
695
	int i;
696

697 698
	err->size = obj->base.size;
	err->name = obj->base.name;
699 700
	for (i = 0; i < I915_NUM_RINGS; i++)
		err->rseqno[i] = i915_gem_request_get_seqno(obj->last_read_req[i]);
701
	err->wseqno = i915_gem_request_get_seqno(obj->last_write_req);
702
	err->gtt_offset = vma->node.start;
703 704 705 706
	err->read_domains = obj->base.read_domains;
	err->write_domain = obj->base.write_domain;
	err->fence_reg = obj->fence_reg;
	err->pinned = 0;
B
Ben Widawsky 已提交
707
	if (i915_gem_obj_is_pinned(obj))
708 709 710 711
		err->pinned = 1;
	err->tiling = obj->tiling_mode;
	err->dirty = obj->dirty;
	err->purgeable = obj->madv != I915_MADV_WILLNEED;
712
	err->userptr = obj->userptr.mm != NULL;
713 714
	err->ring = obj->last_write_req ?
			i915_gem_request_get_ring(obj->last_write_req)->id : -1;
715 716 717 718 719 720
	err->cache_level = obj->cache_level;
}

static u32 capture_active_bo(struct drm_i915_error_buffer *err,
			     int count, struct list_head *head)
{
B
Ben Widawsky 已提交
721
	struct i915_vma *vma;
722 723
	int i = 0;

B
Ben Widawsky 已提交
724
	list_for_each_entry(vma, head, mm_list) {
725
		capture_bo(err++, vma);
726 727 728 729 730 731 732 733
		if (++i == count)
			break;
	}

	return i;
}

static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
734 735
			     int count, struct list_head *head,
			     struct i915_address_space *vm)
736 737
{
	struct drm_i915_gem_object *obj;
738 739
	struct drm_i915_error_buffer * const first = err;
	struct drm_i915_error_buffer * const last = err + count;
740 741

	list_for_each_entry(obj, head, global_list) {
742
		struct i915_vma *vma;
743

744
		if (err == last)
745
			break;
746 747

		list_for_each_entry(vma, &obj->vma_list, vma_link)
748
			if (vma->vm == vm && vma->pin_count > 0)
749
				capture_bo(err++, vma);
750 751
	}

752
	return err - first;
753 754
}

755 756 757 758 759 760 761 762 763 764
/* Generate a semi-unique error code. The code is not meant to have meaning, The
 * code's only purpose is to try to prevent false duplicated bug reports by
 * grossly estimating a GPU error state.
 *
 * TODO Ideally, hashing the batchbuffer would be a very nice way to determine
 * the hang if we could strip the GTT offset information from it.
 *
 * It's only a small step better than a random number in its current form.
 */
static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv,
765 766
					 struct drm_i915_error_state *error,
					 int *ring_id)
767 768 769 770 771 772 773 774 775
{
	uint32_t error_code = 0;
	int i;

	/* IPEHR would be an ideal way to detect errors, as it's the gross
	 * measure of "the command that hung." However, has some very common
	 * synchronization commands which almost always appear in the case
	 * strictly a client bug. Use instdone to differentiate those some.
	 */
776 777 778 779 780
	for (i = 0; i < I915_NUM_RINGS; i++) {
		if (error->ring[i].hangcheck_action == HANGCHECK_HUNG) {
			if (ring_id)
				*ring_id = i;

781
			return error->ring[i].ipehr ^ error->ring[i].instdone;
782 783
		}
	}
784 785 786 787

	return error_code;
}

788 789 790 791 792 793
static void i915_gem_record_fences(struct drm_device *dev,
				   struct drm_i915_error_state *error)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int i;

794 795
	if (IS_GEN3(dev) || IS_GEN2(dev)) {
		for (i = 0; i < dev_priv->num_fence_regs; i++)
796 797 798 799 800 801 802 803
			error->fence[i] = I915_READ(FENCE_REG(i));
	} else if (IS_GEN5(dev) || IS_GEN4(dev)) {
		for (i = 0; i < dev_priv->num_fence_regs; i++)
			error->fence[i] = I915_READ64(FENCE_REG_965_LO(i));
	} else if (INTEL_INFO(dev)->gen >= 6) {
		for (i = 0; i < dev_priv->num_fence_regs; i++)
			error->fence[i] = I915_READ64(FENCE_REG_GEN6_LO(i));
	}
804 805
}

806

807 808 809 810 811
static void gen8_record_semaphore_state(struct drm_i915_private *dev_priv,
					struct drm_i915_error_state *error,
					struct intel_engine_cs *ring,
					struct drm_i915_error_ring *ering)
{
812
	struct intel_engine_cs *to;
813 814 815 816 817 818 819
	int i;

	if (!i915_semaphore_is_enabled(dev_priv->dev))
		return;

	if (!error->semaphore_obj)
		error->semaphore_obj =
820 821
			i915_error_ggtt_object_create(dev_priv,
						      dev_priv->semaphore_obj);
822

823 824 825 826
	for_each_ring(to, dev_priv, i) {
		int idx;
		u16 signal_offset;
		u32 *tmp;
827

828 829 830
		if (ring == to)
			continue;

831 832
		signal_offset = (GEN8_SIGNAL_OFFSET(ring, i) & (PAGE_SIZE - 1))
				/ 4;
833 834 835 836 837
		tmp = error->semaphore_obj->pages[0];
		idx = intel_ring_sync_index(ring, to);

		ering->semaphore_mboxes[idx] = tmp[signal_offset];
		ering->semaphore_seqno[idx] = ring->semaphore.sync_seqno[idx];
838 839 840
	}
}

841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856
static void gen6_record_semaphore_state(struct drm_i915_private *dev_priv,
					struct intel_engine_cs *ring,
					struct drm_i915_error_ring *ering)
{
	ering->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(ring->mmio_base));
	ering->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(ring->mmio_base));
	ering->semaphore_seqno[0] = ring->semaphore.sync_seqno[0];
	ering->semaphore_seqno[1] = ring->semaphore.sync_seqno[1];

	if (HAS_VEBOX(dev_priv->dev)) {
		ering->semaphore_mboxes[2] =
			I915_READ(RING_SYNC_2(ring->mmio_base));
		ering->semaphore_seqno[2] = ring->semaphore.sync_seqno[2];
	}
}

857
static void i915_record_ring_state(struct drm_device *dev,
858
				   struct drm_i915_error_state *error,
859
				   struct intel_engine_cs *ring,
860
				   struct drm_i915_error_ring *ering)
861 862 863 864
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	if (INTEL_INFO(dev)->gen >= 6) {
865 866
		ering->rc_psmi = I915_READ(ring->mmio_base + 0x50);
		ering->fault_reg = I915_READ(RING_FAULT_REG(ring));
867 868 869 870
		if (INTEL_INFO(dev)->gen >= 8)
			gen8_record_semaphore_state(dev_priv, error, ring, ering);
		else
			gen6_record_semaphore_state(dev_priv, ring, ering);
871 872
	}

873
	if (INTEL_INFO(dev)->gen >= 4) {
874 875 876 877 878 879
		ering->faddr = I915_READ(RING_DMA_FADD(ring->mmio_base));
		ering->ipeir = I915_READ(RING_IPEIR(ring->mmio_base));
		ering->ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
		ering->instdone = I915_READ(RING_INSTDONE(ring->mmio_base));
		ering->instps = I915_READ(RING_INSTPS(ring->mmio_base));
		ering->bbaddr = I915_READ(RING_BBADDR(ring->mmio_base));
880 881
		if (INTEL_INFO(dev)->gen >= 8) {
			ering->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(ring->mmio_base)) << 32;
882
			ering->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(ring->mmio_base)) << 32;
883
		}
884
		ering->bbstate = I915_READ(RING_BBSTATE(ring->mmio_base));
885
	} else {
886 887 888
		ering->faddr = I915_READ(DMA_FADD_I8XX);
		ering->ipeir = I915_READ(IPEIR);
		ering->ipehr = I915_READ(IPEHR);
889
		ering->instdone = I915_READ(GEN2_INSTDONE);
890 891
	}

892 893 894 895
	ering->waiting = waitqueue_active(&ring->irq_queue);
	ering->instpm = I915_READ(RING_INSTPM(ring->mmio_base));
	ering->seqno = ring->get_seqno(ring, false);
	ering->acthd = intel_ring_get_active_head(ring);
896
	ering->start = I915_READ_START(ring);
897 898 899
	ering->head = I915_READ_HEAD(ring);
	ering->tail = I915_READ_TAIL(ring);
	ering->ctl = I915_READ_CTL(ring);
900

901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
	if (I915_NEED_GFX_HWS(dev)) {
		int mmio;

		if (IS_GEN7(dev)) {
			switch (ring->id) {
			default:
			case RCS:
				mmio = RENDER_HWS_PGA_GEN7;
				break;
			case BCS:
				mmio = BLT_HWS_PGA_GEN7;
				break;
			case VCS:
				mmio = BSD_HWS_PGA_GEN7;
				break;
			case VECS:
				mmio = VEBOX_HWS_PGA_GEN7;
				break;
			}
		} else if (IS_GEN6(ring->dev)) {
			mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
		} else {
			/* XXX: gen8 returns to sanity */
			mmio = RING_HWS_PGA(ring->mmio_base);
		}

927
		ering->hws = I915_READ(mmio);
928 929
	}

930 931
	ering->hangcheck_score = ring->hangcheck.score;
	ering->hangcheck_action = ring->hangcheck.action;
932 933 934 935 936 937

	if (USES_PPGTT(dev)) {
		int i;

		ering->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(ring));

938 939 940 941 942 943 944
		if (IS_GEN6(dev))
			ering->vm_info.pp_dir_base =
				I915_READ(RING_PP_DIR_BASE_READ(ring));
		else if (IS_GEN7(dev))
			ering->vm_info.pp_dir_base =
				I915_READ(RING_PP_DIR_BASE(ring));
		else if (INTEL_INFO(dev)->gen >= 8)
945 946 947 948 949 950 951 952
			for (i = 0; i < 4; i++) {
				ering->vm_info.pdp[i] =
					I915_READ(GEN8_RING_PDP_UDW(ring, i));
				ering->vm_info.pdp[i] <<= 32;
				ering->vm_info.pdp[i] |=
					I915_READ(GEN8_RING_PDP_LDW(ring, i));
			}
	}
953 954 955
}


956
static void i915_gem_record_active_context(struct intel_engine_cs *ring,
957 958 959 960 961 962 963 964 965 966 967
					   struct drm_i915_error_state *error,
					   struct drm_i915_error_ring *ering)
{
	struct drm_i915_private *dev_priv = ring->dev->dev_private;
	struct drm_i915_gem_object *obj;

	/* Currently render ring is the only HW context user */
	if (ring->id != RCS || !error->ccid)
		return;

	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
968 969 970
		if (!i915_gem_obj_ggtt_bound(obj))
			continue;

971
		if ((error->ccid & PAGE_MASK) == i915_gem_obj_ggtt_offset(obj)) {
972
			ering->ctx = i915_error_ggtt_object_create(dev_priv, obj);
973 974 975 976 977 978 979 980 981 982 983 984
			break;
		}
	}
}

static void i915_gem_record_rings(struct drm_device *dev,
				  struct drm_i915_error_state *error)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_gem_request *request;
	int i, count;

985
	for (i = 0; i < I915_NUM_RINGS; i++) {
986
		struct intel_engine_cs *ring = &dev_priv->ring[i];
987
		struct intel_ringbuffer *rbuf;
988

989 990
		error->ring[i].pid = -1;

991 992 993 994 995
		if (ring->dev == NULL)
			continue;

		error->ring[i].valid = true;

996
		i915_record_ring_state(dev, error, ring, &error->ring[i]);
997

998 999
		request = i915_gem_find_active_request(ring);
		if (request) {
1000 1001 1002 1003 1004 1005
			struct i915_address_space *vm;

			vm = request->ctx && request->ctx->ppgtt ?
				&request->ctx->ppgtt->base :
				&dev_priv->gtt.base;

1006 1007 1008 1009 1010 1011 1012
			/* We need to copy these to an anonymous buffer
			 * as the simplest method to avoid being overwritten
			 * by userspace.
			 */
			error->ring[i].batchbuffer =
				i915_error_object_create(dev_priv,
							 request->batch_obj,
1013
							 vm);
1014

1015
			if (HAS_BROKEN_CS_TLB(dev_priv->dev))
1016 1017 1018 1019
				error->ring[i].wa_batchbuffer =
					i915_error_ggtt_object_create(dev_priv,
							     ring->scratch.obj);

1020
			if (request->pid) {
1021 1022 1023
				struct task_struct *task;

				rcu_read_lock();
1024
				task = pid_task(request->pid, PIDTYPE_PID);
1025 1026 1027 1028 1029 1030 1031
				if (task) {
					strcpy(error->ring[i].comm, task->comm);
					error->ring[i].pid = task->pid;
				}
				rcu_read_unlock();
			}
		}
1032

1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048
		if (i915.enable_execlists) {
			/* TODO: This is only a small fix to keep basic error
			 * capture working, but we need to add more information
			 * for it to be useful (e.g. dump the context being
			 * executed).
			 */
			if (request)
				rbuf = request->ctx->engine[ring->id].ringbuf;
			else
				rbuf = ring->default_context->engine[ring->id].ringbuf;
		} else
			rbuf = ring->buffer;

		error->ring[i].cpu_ring_head = rbuf->head;
		error->ring[i].cpu_ring_tail = rbuf->tail;

1049
		error->ring[i].ringbuffer =
1050
			i915_error_ggtt_object_create(dev_priv, rbuf->obj);
1051

1052 1053
		error->ring[i].hws_page =
			i915_error_ggtt_object_create(dev_priv, ring->status_page.obj);
1054 1055 1056 1057 1058 1059 1060 1061 1062

		i915_gem_record_active_context(ring, error, &error->ring[i]);

		count = 0;
		list_for_each_entry(request, &ring->request_list, list)
			count++;

		error->ring[i].num_requests = count;
		error->ring[i].requests =
D
Daniel Vetter 已提交
1063
			kcalloc(count, sizeof(*error->ring[i].requests),
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
				GFP_ATOMIC);
		if (error->ring[i].requests == NULL) {
			error->ring[i].num_requests = 0;
			continue;
		}

		count = 0;
		list_for_each_entry(request, &ring->request_list, list) {
			struct drm_i915_error_request *erq;

1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
			if (count >= error->ring[i].num_requests) {
				/*
				 * If the ring request list was changed in
				 * between the point where the error request
				 * list was created and dimensioned and this
				 * point then just exit early to avoid crashes.
				 *
				 * We don't need to communicate that the
				 * request list changed state during error
				 * state capture and that the error state is
				 * slightly incorrect as a consequence since we
				 * are typically only interested in the request
				 * list state at the point of error state
				 * capture, not in any changes happening during
				 * the capture.
				 */
				break;
			}

1093 1094 1095
			erq = &error->ring[i].requests[count++];
			erq->seqno = request->seqno;
			erq->jiffies = request->emitted_jiffies;
1096
			erq->tail = request->postfix;
1097 1098 1099 1100
		}
	}
}

1101 1102 1103 1104 1105 1106 1107
/* FIXME: Since pin count/bound list is global, we duplicate what we capture per
 * VM.
 */
static void i915_gem_capture_vm(struct drm_i915_private *dev_priv,
				struct drm_i915_error_state *error,
				struct i915_address_space *vm,
				const int ndx)
1108
{
1109
	struct drm_i915_error_buffer *active_bo = NULL, *pinned_bo = NULL;
1110
	struct drm_i915_gem_object *obj;
1111
	struct i915_vma *vma;
1112 1113 1114
	int i;

	i = 0;
B
Ben Widawsky 已提交
1115
	list_for_each_entry(vma, &vm->active_list, mm_list)
1116
		i++;
1117
	error->active_bo_count[ndx] = i;
1118 1119 1120

	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
		list_for_each_entry(vma, &obj->vma_list, vma_link)
1121
			if (vma->vm == vm && vma->pin_count > 0)
1122 1123
				i++;
	}
1124
	error->pinned_bo_count[ndx] = i - error->active_bo_count[ndx];
1125 1126

	if (i) {
D
Daniel Vetter 已提交
1127
		active_bo = kcalloc(i, sizeof(*active_bo), GFP_ATOMIC);
1128 1129
		if (active_bo)
			pinned_bo = active_bo + error->active_bo_count[ndx];
1130 1131
	}

1132 1133 1134 1135
	if (active_bo)
		error->active_bo_count[ndx] =
			capture_active_bo(active_bo,
					  error->active_bo_count[ndx],
1136
					  &vm->active_list);
1137

1138 1139 1140 1141
	if (pinned_bo)
		error->pinned_bo_count[ndx] =
			capture_pinned_bo(pinned_bo,
					  error->pinned_bo_count[ndx],
1142
					  &dev_priv->mm.bound_list, vm);
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
	error->active_bo[ndx] = active_bo;
	error->pinned_bo[ndx] = pinned_bo;
}

static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv,
				     struct drm_i915_error_state *error)
{
	struct i915_address_space *vm;
	int cnt = 0, i = 0;

	list_for_each_entry(vm, &dev_priv->vm_list, global_link)
		cnt++;

	error->active_bo = kcalloc(cnt, sizeof(*error->active_bo), GFP_ATOMIC);
	error->pinned_bo = kcalloc(cnt, sizeof(*error->pinned_bo), GFP_ATOMIC);
	error->active_bo_count = kcalloc(cnt, sizeof(*error->active_bo_count),
					 GFP_ATOMIC);
	error->pinned_bo_count = kcalloc(cnt, sizeof(*error->pinned_bo_count),
					 GFP_ATOMIC);

1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
	if (error->active_bo == NULL ||
	    error->pinned_bo == NULL ||
	    error->active_bo_count == NULL ||
	    error->pinned_bo_count == NULL) {
		kfree(error->active_bo);
		kfree(error->active_bo_count);
		kfree(error->pinned_bo);
		kfree(error->pinned_bo_count);

		error->active_bo = NULL;
		error->active_bo_count = NULL;
		error->pinned_bo = NULL;
		error->pinned_bo_count = NULL;
	} else {
		list_for_each_entry(vm, &dev_priv->vm_list, global_link)
			i915_gem_capture_vm(dev_priv, error, vm, i++);

		error->vm_count = cnt;
	}
1182 1183
}

1184 1185 1186
/* Capture all registers which don't fit into another category. */
static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
				   struct drm_i915_error_state *error)
1187
{
1188
	struct drm_device *dev = dev_priv->dev;
1189
	int i;
1190

1191 1192 1193 1194 1195 1196 1197
	/* General organization
	 * 1. Registers specific to a single generation
	 * 2. Registers which belong to multiple generations
	 * 3. Feature specific registers.
	 * 4. Everything else
	 * Please try to follow the order.
	 */
1198

1199 1200
	/* 1: Registers specific to a single generation */
	if (IS_VALLEYVIEW(dev)) {
1201
		error->gtier[0] = I915_READ(GTIER);
1202
		error->ier = I915_READ(VLV_IER);
1203 1204
		error->forcewake = I915_READ(FORCEWAKE_VLV);
	}
1205

1206 1207
	if (IS_GEN7(dev))
		error->err_int = I915_READ(GEN7_ERR_INT);
1208

1209 1210 1211 1212 1213
	if (INTEL_INFO(dev)->gen >= 8) {
		error->fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0);
		error->fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1);
	}

1214
	if (IS_GEN6(dev)) {
1215
		error->forcewake = I915_READ(FORCEWAKE);
1216 1217 1218
		error->gab_ctl = I915_READ(GAB_CTL);
		error->gfx_mode = I915_READ(GFX_MODE);
	}
1219

1220 1221 1222
	/* 2: Registers which belong to multiple generations */
	if (INTEL_INFO(dev)->gen >= 7)
		error->forcewake = I915_READ(FORCEWAKE_MT);
1223 1224

	if (INTEL_INFO(dev)->gen >= 6) {
1225
		error->derrmr = I915_READ(DERRMR);
1226 1227 1228 1229
		error->error = I915_READ(ERROR_GEN6);
		error->done_reg = I915_READ(DONE_REG);
	}

1230
	/* 3: Feature specific registers */
1231 1232 1233 1234 1235 1236
	if (IS_GEN6(dev) || IS_GEN7(dev)) {
		error->gam_ecochk = I915_READ(GAM_ECOCHK);
		error->gac_eco = I915_READ(GAC_ECO_BITS);
	}

	/* 4: Everything else */
1237 1238 1239
	if (HAS_HW_CONTEXTS(dev))
		error->ccid = I915_READ(CCID);

1240 1241 1242 1243 1244
	if (INTEL_INFO(dev)->gen >= 8) {
		error->ier = I915_READ(GEN8_DE_MISC_IER);
		for (i = 0; i < 4; i++)
			error->gtier[i] = I915_READ(GEN8_GT_IER(i));
	} else if (HAS_PCH_SPLIT(dev)) {
1245
		error->ier = I915_READ(DEIER);
1246
		error->gtier[0] = I915_READ(GTIER);
1247 1248 1249 1250
	} else if (IS_GEN2(dev)) {
		error->ier = I915_READ16(IER);
	} else if (!IS_VALLEYVIEW(dev)) {
		error->ier = I915_READ(IER);
1251 1252 1253
	}
	error->eir = I915_READ(EIR);
	error->pgtbl_er = I915_READ(PGTBL_ER);
1254 1255

	i915_get_extra_instdone(dev, error->extra_instdone);
1256 1257
}

1258
static void i915_error_capture_msg(struct drm_device *dev,
1259 1260 1261
				   struct drm_i915_error_state *error,
				   bool wedged,
				   const char *error_msg)
1262 1263 1264
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	u32 ecode;
1265
	int ring_id = -1, len;
1266 1267 1268

	ecode = i915_error_generate_code(dev_priv, error, &ring_id);

1269
	len = scnprintf(error->error_msg, sizeof(error->error_msg),
1270 1271
			"GPU HANG: ecode %d:%d:0x%08x",
			INTEL_INFO(dev)->gen, ring_id, ecode);
1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283

	if (ring_id != -1 && error->ring[ring_id].pid != -1)
		len += scnprintf(error->error_msg + len,
				 sizeof(error->error_msg) - len,
				 ", in %s [%d]",
				 error->ring[ring_id].comm,
				 error->ring[ring_id].pid);

	scnprintf(error->error_msg + len, sizeof(error->error_msg) - len,
		  ", reason: %s, action: %s",
		  error_msg,
		  wedged ? "reset" : "continue");
1284 1285
}

1286 1287 1288
static void i915_capture_gen_state(struct drm_i915_private *dev_priv,
				   struct drm_i915_error_state *error)
{
1289 1290 1291 1292
	error->iommu = -1;
#ifdef CONFIG_INTEL_IOMMU
	error->iommu = intel_iommu_gfx_mapped;
#endif
1293
	error->reset_count = i915_reset_count(&dev_priv->gpu_error);
1294
	error->suspend_count = dev_priv->suspend_count;
1295 1296
}

1297 1298 1299 1300 1301 1302 1303 1304 1305
/**
 * i915_capture_error_state - capture an error record for later analysis
 * @dev: drm device
 *
 * Should be called when an error is detected (either a hang or an error
 * interrupt) to capture error state from the time of the error.  Fills
 * out a structure which becomes available in debugfs for user level tools
 * to pick up.
 */
1306 1307
void i915_capture_error_state(struct drm_device *dev, bool wedged,
			      const char *error_msg)
1308
{
1309
	static bool warned;
1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_error_state *error;
	unsigned long flags;

	/* Account for pipe specific data like PIPE*STAT */
	error = kzalloc(sizeof(*error), GFP_ATOMIC);
	if (!error) {
		DRM_DEBUG_DRIVER("out of memory, not capturing error state\n");
		return;
	}

1321 1322
	kref_init(&error->ref);

1323
	i915_capture_gen_state(dev_priv, error);
1324 1325 1326 1327
	i915_capture_reg_state(dev_priv, error);
	i915_gem_capture_buffers(dev_priv, error);
	i915_gem_record_fences(dev, error);
	i915_gem_record_rings(dev, error);
1328

1329 1330 1331 1332 1333
	do_gettimeofday(&error->time);

	error->overlay = intel_overlay_capture_error_state(dev);
	error->display = intel_display_capture_error_state(dev);

1334
	i915_error_capture_msg(dev, error, wedged, error_msg);
1335 1336
	DRM_INFO("%s\n", error->error_msg);

1337 1338 1339 1340 1341 1342 1343
	spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
	if (dev_priv->gpu_error.first_error == NULL) {
		dev_priv->gpu_error.first_error = error;
		error = NULL;
	}
	spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);

1344
	if (error) {
1345
		i915_error_state_free(&error->ref);
1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356
		return;
	}

	if (!warned) {
		DRM_INFO("GPU hangs can indicate a bug anywhere in the entire gfx stack, including userspace.\n");
		DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n");
		DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n");
		DRM_INFO("The gpu crash dump is required to analyze gpu hangs, so please always attach it.\n");
		DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n", dev->primary->index);
		warned = true;
	}
1357 1358 1359 1360 1361 1362 1363
}

void i915_error_state_get(struct drm_device *dev,
			  struct i915_error_state_file_priv *error_priv)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

1364
	spin_lock_irq(&dev_priv->gpu_error.lock);
1365 1366 1367
	error_priv->error = dev_priv->gpu_error.first_error;
	if (error_priv->error)
		kref_get(&error_priv->error->ref);
1368
	spin_unlock_irq(&dev_priv->gpu_error.lock);
1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382

}

void i915_error_state_put(struct i915_error_state_file_priv *error_priv)
{
	if (error_priv->error)
		kref_put(&error_priv->error->ref, i915_error_state_free);
}

void i915_destroy_error_state(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_error_state *error;

1383
	spin_lock_irq(&dev_priv->gpu_error.lock);
1384 1385
	error = dev_priv->gpu_error.first_error;
	dev_priv->gpu_error.first_error = NULL;
1386
	spin_unlock_irq(&dev_priv->gpu_error.lock);
1387 1388 1389 1390 1391

	if (error)
		kref_put(&error->ref, i915_error_state_free);
}

1392
const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
1393 1394 1395
{
	switch (type) {
	case I915_CACHE_NONE: return " uncached";
1396
	case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
1397
	case I915_CACHE_L3_LLC: return " L3+LLC";
1398
	case I915_CACHE_WT: return " WT";
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408
	default: return "";
	}
}

/* NB: please notice the memset */
void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	memset(instdone, 0, sizeof(*instdone) * I915_NUM_INSTDONE_REG);

1409
	if (IS_GEN2(dev) || IS_GEN3(dev))
1410
		instdone[0] = I915_READ(GEN2_INSTDONE);
1411
	else if (IS_GEN4(dev) || IS_GEN5(dev) || IS_GEN6(dev)) {
1412
		instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE));
1413
		instdone[1] = I915_READ(GEN4_INSTDONE1);
1414
	} else if (INTEL_INFO(dev)->gen >= 7) {
1415
		instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE));
1416 1417 1418 1419 1420
		instdone[1] = I915_READ(GEN7_SC_INSTDONE);
		instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE);
		instdone[3] = I915_READ(GEN7_ROW_INSTDONE);
	}
}