i915_gpu_error.c 48.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 * Copyright (c) 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *    Keith Packard <keithp@keithp.com>
 *    Mika Kuoppala <mika.kuoppala@intel.com>
 *
 */

#include <generated/utsrelease.h>
31
#include <linux/stop_machine.h>
32
#include <linux/zlib.h>
33 34
#include <drm/drm_print.h>

35 36
#include "i915_drv.h"

37
static const char *engine_str(int engine)
38
{
39
	switch (engine) {
40 41 42 43
	case RCS: return "render";
	case VCS: return "bsd";
	case BCS: return "blt";
	case VECS: return "vebox";
44
	case VCS2: return "bsd2";
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
	default: return "";
	}
}

static const char *tiling_flag(int tiling)
{
	switch (tiling) {
	default:
	case I915_TILING_NONE: return "";
	case I915_TILING_X: return " X";
	case I915_TILING_Y: return " Y";
	}
}

static const char *dirty_flag(int dirty)
{
	return dirty ? " dirty" : "";
}

static const char *purgeable_flag(int purgeable)
{
	return purgeable ? " purgeable" : "";
}

static bool __i915_error_ok(struct drm_i915_error_state_buf *e)
{

	if (!e->err && WARN(e->bytes > (e->size - 1), "overflow")) {
		e->err = -ENOSPC;
		return false;
	}

	if (e->bytes == e->size - 1 || e->err)
		return false;

	return true;
}

static bool __i915_error_seek(struct drm_i915_error_state_buf *e,
			      unsigned len)
{
	if (e->pos + len <= e->start) {
		e->pos += len;
		return false;
	}

	/* First vsnprintf needs to fit in its entirety for memmove */
	if (len >= e->size) {
		e->err = -EIO;
		return false;
	}

	return true;
}

static void __i915_error_advance(struct drm_i915_error_state_buf *e,
				 unsigned len)
{
	/* If this is first printf in this window, adjust it so that
	 * start position matches start of the buffer
	 */

	if (e->pos < e->start) {
		const size_t off = e->start - e->pos;

		/* Should not happen but be paranoid */
		if (off > len || e->bytes) {
			e->err = -EIO;
			return;
		}

		memmove(e->buf, e->buf + off, len - off);
		e->bytes = len - off;
		e->pos = e->start;
		return;
	}

	e->bytes += len;
	e->pos += len;
}

126
__printf(2, 0)
127 128 129 130 131 132 133 134 135 136
static void i915_error_vprintf(struct drm_i915_error_state_buf *e,
			       const char *f, va_list args)
{
	unsigned len;

	if (!__i915_error_ok(e))
		return;

	/* Seek the first printf which is hits start position */
	if (e->pos < e->start) {
137 138 139
		va_list tmp;

		va_copy(tmp, args);
140 141 142 143
		len = vsnprintf(NULL, 0, f, tmp);
		va_end(tmp);

		if (!__i915_error_seek(e, len))
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
			return;
	}

	len = vsnprintf(e->buf + e->bytes, e->size - e->bytes, f, args);
	if (len >= e->size - e->bytes)
		len = e->size - e->bytes - 1;

	__i915_error_advance(e, len);
}

static void i915_error_puts(struct drm_i915_error_state_buf *e,
			    const char *str)
{
	unsigned len;

	if (!__i915_error_ok(e))
		return;

	len = strlen(str);

	/* Seek the first printf which is hits start position */
	if (e->pos < e->start) {
		if (!__i915_error_seek(e, len))
			return;
	}

	if (len >= e->size - e->bytes)
		len = e->size - e->bytes - 1;
	memcpy(e->buf + e->bytes, str, len);

	__i915_error_advance(e, len);
}

#define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
#define err_puts(e, s) i915_error_puts(e, s)

180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
static void __i915_printfn_error(struct drm_printer *p, struct va_format *vaf)
{
	i915_error_vprintf(p->arg, vaf->fmt, *vaf->va);
}

static inline struct drm_printer
i915_error_printer(struct drm_i915_error_state_buf *e)
{
	struct drm_printer p = {
		.printfn = __i915_printfn_error,
		.arg = e,
	};
	return p;
}

195 196
#ifdef CONFIG_DRM_I915_COMPRESS_ERROR

197 198 199 200 201 202
struct compress {
	struct z_stream_s zstream;
	void *tmp;
};

static bool compress_init(struct compress *c)
203
{
204
	struct z_stream_s *zstream = memset(&c->zstream, 0, sizeof(c->zstream));
205 206 207 208 209 210 211 212 213 214 215 216

	zstream->workspace =
		kmalloc(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL),
			GFP_ATOMIC | __GFP_NOWARN);
	if (!zstream->workspace)
		return false;

	if (zlib_deflateInit(zstream, Z_DEFAULT_COMPRESSION) != Z_OK) {
		kfree(zstream->workspace);
		return false;
	}

217
	c->tmp = NULL;
218
	if (i915_has_memcpy_from_wc())
219 220
		c->tmp = (void *)__get_free_page(GFP_ATOMIC | __GFP_NOWARN);

221 222 223
	return true;
}

224
static int compress_page(struct compress *c,
225 226 227
			 void *src,
			 struct drm_i915_error_object *dst)
{
228 229
	struct z_stream_s *zstream = &c->zstream;

230
	zstream->next_in = src;
231 232
	if (c->tmp && i915_memcpy_from_wc(c->tmp, src, PAGE_SIZE))
		zstream->next_in = c->tmp;
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
	zstream->avail_in = PAGE_SIZE;

	do {
		if (zstream->avail_out == 0) {
			unsigned long page;

			page = __get_free_page(GFP_ATOMIC | __GFP_NOWARN);
			if (!page)
				return -ENOMEM;

			dst->pages[dst->page_count++] = (void *)page;

			zstream->next_out = (void *)page;
			zstream->avail_out = PAGE_SIZE;
		}

		if (zlib_deflate(zstream, Z_SYNC_FLUSH) != Z_OK)
			return -EIO;
	} while (zstream->avail_in);

	/* Fallback to uncompressed if we increase size? */
	if (0 && zstream->total_out > zstream->total_in)
		return -E2BIG;

	return 0;
}

260
static void compress_fini(struct compress *c,
261 262
			  struct drm_i915_error_object *dst)
{
263 264
	struct z_stream_s *zstream = &c->zstream;

265 266 267 268 269 270 271
	if (dst) {
		zlib_deflate(zstream, Z_FINISH);
		dst->unused = zstream->avail_out;
	}

	zlib_deflateEnd(zstream);
	kfree(zstream->workspace);
272 273 274

	if (c->tmp)
		free_page((unsigned long)c->tmp);
275 276 277 278 279 280 281 282 283
}

static void err_compression_marker(struct drm_i915_error_state_buf *m)
{
	err_puts(m, ":");
}

#else

284 285 286 287
struct compress {
};

static bool compress_init(struct compress *c)
288 289 290 291
{
	return true;
}

292
static int compress_page(struct compress *c,
293 294 295 296
			 void *src,
			 struct drm_i915_error_object *dst)
{
	unsigned long page;
297
	void *ptr;
298 299 300 301 302

	page = __get_free_page(GFP_ATOMIC | __GFP_NOWARN);
	if (!page)
		return -ENOMEM;

303 304 305 306
	ptr = (void *)page;
	if (!i915_memcpy_from_wc(ptr, src, PAGE_SIZE))
		memcpy(ptr, src, PAGE_SIZE);
	dst->pages[dst->page_count++] = ptr;
307 308 309 310

	return 0;
}

311
static void compress_fini(struct compress *c,
312 313 314 315 316 317 318 319 320 321 322
			  struct drm_i915_error_object *dst)
{
}

static void err_compression_marker(struct drm_i915_error_state_buf *m)
{
	err_puts(m, "~");
}

#endif

323 324 325 326 327
static void print_error_buffers(struct drm_i915_error_state_buf *m,
				const char *name,
				struct drm_i915_error_buffer *err,
				int count)
{
328 329
	int i;

330
	err_printf(m, "%s [%d]:\n", name, count);
331 332

	while (count--) {
333 334 335
		err_printf(m, "    %08x_%08x %8u %02x %02x [ ",
			   upper_32_bits(err->gtt_offset),
			   lower_32_bits(err->gtt_offset),
336 337
			   err->size,
			   err->read_domains,
338
			   err->write_domain);
339
		for (i = 0; i < I915_NUM_ENGINES; i++)
340 341 342
			err_printf(m, "%02x ", err->rseqno[i]);

		err_printf(m, "] %02x", err->wseqno);
343 344 345
		err_puts(m, tiling_flag(err->tiling));
		err_puts(m, dirty_flag(err->dirty));
		err_puts(m, purgeable_flag(err->purgeable));
346
		err_puts(m, err->userptr ? " userptr" : "");
347 348
		err_puts(m, err->engine != -1 ? " " : "");
		err_puts(m, engine_str(err->engine));
349
		err_puts(m, i915_cache_level_str(m->i915, err->cache_level));
350 351 352 353 354 355 356 357 358 359 360

		if (err->name)
			err_printf(m, " (name: %d)", err->name);
		if (err->fence_reg != I915_FENCE_REG_NONE)
			err_printf(m, " (fence: %d)", err->fence_reg);

		err_puts(m, "\n");
		err++;
	}
}

361
static void error_print_instdone(struct drm_i915_error_state_buf *m,
362
				 const struct drm_i915_error_engine *ee)
363
{
364 365 366
	int slice;
	int subslice;

367 368 369 370 371 372 373 374 375 376 377 378
	err_printf(m, "  INSTDONE: 0x%08x\n",
		   ee->instdone.instdone);

	if (ee->engine_id != RCS || INTEL_GEN(m->i915) <= 3)
		return;

	err_printf(m, "  SC_INSTDONE: 0x%08x\n",
		   ee->instdone.slice_common);

	if (INTEL_GEN(m->i915) <= 6)
		return;

379 380 381 382 383 384 385 386 387
	for_each_instdone_slice_subslice(m->i915, slice, subslice)
		err_printf(m, "  SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
			   slice, subslice,
			   ee->instdone.sampler[slice][subslice]);

	for_each_instdone_slice_subslice(m->i915, slice, subslice)
		err_printf(m, "  ROW_INSTDONE[%d][%d]: 0x%08x\n",
			   slice, subslice,
			   ee->instdone.row[slice][subslice]);
388 389
}

390 391
static void error_print_request(struct drm_i915_error_state_buf *m,
				const char *prefix,
392
				const struct drm_i915_error_request *erq)
393 394 395 396
{
	if (!erq->seqno)
		return;

397
	err_printf(m, "%s pid %d, ban score %d, seqno %8x:%08x, prio %d, emitted %dms ago, head %08x, tail %08x\n",
398
		   prefix, erq->pid, erq->ban_score,
399
		   erq->context, erq->seqno, erq->priority,
400 401 402 403
		   jiffies_to_msecs(jiffies - erq->jiffies),
		   erq->head, erq->tail);
}

404 405
static void error_print_context(struct drm_i915_error_state_buf *m,
				const char *header,
406
				const struct drm_i915_error_context *ctx)
407
{
408
	err_printf(m, "%s%s[%d] user_handle %d hw_id %d, prio %d, ban score %d guilty %d active %d\n",
409
		   header, ctx->comm, ctx->pid, ctx->handle, ctx->hw_id,
410
		   ctx->priority, ctx->ban_score, ctx->guilty, ctx->active);
411 412
}

413
static void error_print_engine(struct drm_i915_error_state_buf *m,
414
			       const struct drm_i915_error_engine *ee)
415
{
416 417
	int n;

418 419
	err_printf(m, "%s command stream:\n", engine_str(ee->engine_id));
	err_printf(m, "  START: 0x%08x\n", ee->start);
420
	err_printf(m, "  HEAD:  0x%08x [0x%08x]\n", ee->head, ee->rq_head);
421 422
	err_printf(m, "  TAIL:  0x%08x [0x%08x, 0x%08x]\n",
		   ee->tail, ee->rq_post, ee->rq_tail);
423
	err_printf(m, "  CTL:   0x%08x\n", ee->ctl);
424
	err_printf(m, "  MODE:  0x%08x\n", ee->mode);
425 426 427 428 429
	err_printf(m, "  HWS:   0x%08x\n", ee->hws);
	err_printf(m, "  ACTHD: 0x%08x %08x\n",
		   (u32)(ee->acthd>>32), (u32)ee->acthd);
	err_printf(m, "  IPEIR: 0x%08x\n", ee->ipeir);
	err_printf(m, "  IPEHR: 0x%08x\n", ee->ipehr);
430 431 432

	error_print_instdone(m, ee);

433 434 435 436 437 438 439 440
	if (ee->batchbuffer) {
		u64 start = ee->batchbuffer->gtt_offset;
		u64 end = start + ee->batchbuffer->gtt_size;

		err_printf(m, "  batch: [0x%08x_%08x, 0x%08x_%08x]\n",
			   upper_32_bits(start), lower_32_bits(start),
			   upper_32_bits(end), lower_32_bits(end));
	}
441
	if (INTEL_GEN(m->i915) >= 4) {
442
		err_printf(m, "  BBADDR: 0x%08x_%08x\n",
443 444 445
			   (u32)(ee->bbaddr>>32), (u32)ee->bbaddr);
		err_printf(m, "  BB_STATE: 0x%08x\n", ee->bbstate);
		err_printf(m, "  INSTPS: 0x%08x\n", ee->instps);
446
	}
447 448 449 450 451 452
	err_printf(m, "  INSTPM: 0x%08x\n", ee->instpm);
	err_printf(m, "  FADDR: 0x%08x %08x\n", upper_32_bits(ee->faddr),
		   lower_32_bits(ee->faddr));
	if (INTEL_GEN(m->i915) >= 6) {
		err_printf(m, "  RC PSMI: 0x%08x\n", ee->rc_psmi);
		err_printf(m, "  FAULT_REG: 0x%08x\n", ee->fault_reg);
453 454 455 456 457 458 459
		err_printf(m, "  SYNC_0: 0x%08x\n",
			   ee->semaphore_mboxes[0]);
		err_printf(m, "  SYNC_1: 0x%08x\n",
			   ee->semaphore_mboxes[1]);
		if (HAS_VEBOX(m->i915))
			err_printf(m, "  SYNC_2: 0x%08x\n",
				   ee->semaphore_mboxes[2]);
460
	}
461 462
	if (USES_PPGTT(m->i915)) {
		err_printf(m, "  GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode);
463

464
		if (INTEL_GEN(m->i915) >= 8) {
465 466 467
			int i;
			for (i = 0; i < 4; i++)
				err_printf(m, "  PDP%d: 0x%016llx\n",
468
					   i, ee->vm_info.pdp[i]);
469 470
		} else {
			err_printf(m, "  PP_DIR_BASE: 0x%08x\n",
471
				   ee->vm_info.pp_dir_base);
472 473
		}
	}
474 475 476 477 478
	err_printf(m, "  seqno: 0x%08x\n", ee->seqno);
	err_printf(m, "  last_seqno: 0x%08x\n", ee->last_seqno);
	err_printf(m, "  waiting: %s\n", yesno(ee->waiting));
	err_printf(m, "  ring->head: 0x%08x\n", ee->cpu_ring_head);
	err_printf(m, "  ring->tail: 0x%08x\n", ee->cpu_ring_tail);
479 480 481 482 483 484
	err_printf(m, "  hangcheck stall: %s\n", yesno(ee->hangcheck_stalled));
	err_printf(m, "  hangcheck action: %s\n",
		   hangcheck_action_to_str(ee->hangcheck_action));
	err_printf(m, "  hangcheck action timestamp: %lu, %u ms ago\n",
		   ee->hangcheck_timestamp,
		   jiffies_to_msecs(jiffies - ee->hangcheck_timestamp));
485
	err_printf(m, "  engine reset count: %u\n", ee->reset_count);
486

487 488 489 490 491
	for (n = 0; n < ee->num_ports; n++) {
		err_printf(m, "  ELSP[%d]:", n);
		error_print_request(m, " ", &ee->execlist[n]);
	}

492
	error_print_context(m, "  Active context: ", &ee->context);
493 494 495 496 497 498 499 500 501 502 503
}

void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
{
	va_list args;

	va_start(args, f);
	i915_error_vprintf(e, f, args);
	va_end(args);
}

504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
static int
ascii85_encode_len(int len)
{
	return DIV_ROUND_UP(len, 4);
}

static bool
ascii85_encode(u32 in, char *out)
{
	int i;

	if (in == 0)
		return false;

	out[5] = '\0';
	for (i = 5; i--; ) {
		out[i] = '!' + in % 85;
		in /= 85;
	}

	return true;
}

527
static void print_error_obj(struct drm_i915_error_state_buf *m,
528 529
			    struct intel_engine_cs *engine,
			    const char *name,
530 531
			    struct drm_i915_error_object *obj)
{
532 533
	char out[6];
	int page;
534

535 536 537 538 539 540 541 542 543 544
	if (!obj)
		return;

	if (name) {
		err_printf(m, "%s --- %s = 0x%08x %08x\n",
			   engine ? engine->name : "global", name,
			   upper_32_bits(obj->gtt_offset),
			   lower_32_bits(obj->gtt_offset));
	}

545 546 547 548 549 550 551 552 553 554 555 556 557 558
	err_compression_marker(m);
	for (page = 0; page < obj->page_count; page++) {
		int i, len;

		len = PAGE_SIZE;
		if (page == obj->page_count - 1)
			len -= obj->unused;
		len = ascii85_encode_len(len);

		for (i = 0; i < len; i++) {
			if (ascii85_encode(obj->pages[page][i], out))
				err_puts(m, out);
			else
				err_puts(m, "z");
559 560
		}
	}
561
	err_puts(m, "\n");
562 563
}

564 565 566 567
static void err_print_capabilities(struct drm_i915_error_state_buf *m,
				   const struct intel_device_info *info)
{
#define PRINT_FLAG(x)  err_printf(m, #x ": %s\n", yesno(info->x))
568
	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
569 570 571
#undef PRINT_FLAG
}

572 573 574 575 576 577 578 579 580 581 582
static __always_inline void err_print_param(struct drm_i915_error_state_buf *m,
					    const char *name,
					    const char *type,
					    const void *x)
{
	if (!__builtin_strcmp(type, "bool"))
		err_printf(m, "i915.%s=%s\n", name, yesno(*(const bool *)x));
	else if (!__builtin_strcmp(type, "int"))
		err_printf(m, "i915.%s=%d\n", name, *(const int *)x);
	else if (!__builtin_strcmp(type, "unsigned int"))
		err_printf(m, "i915.%s=%u\n", name, *(const unsigned int *)x);
583 584
	else if (!__builtin_strcmp(type, "char *"))
		err_printf(m, "i915.%s=%s\n", name, *(const char **)x);
585 586 587 588 589 590 591
	else
		BUILD_BUG();
}

static void err_print_params(struct drm_i915_error_state_buf *m,
			     const struct i915_params *p)
{
592
#define PRINT(T, x, ...) err_print_param(m, #x, #T, &p->x);
593 594 595 596
	I915_PARAMS_FOR_EACH(PRINT);
#undef PRINT
}

597 598 599 600 601 602 603 604 605 606 607 608
static void err_print_pciid(struct drm_i915_error_state_buf *m,
			    struct drm_i915_private *i915)
{
	struct pci_dev *pdev = i915->drm.pdev;

	err_printf(m, "PCI ID: 0x%04x\n", pdev->device);
	err_printf(m, "PCI Revision: 0x%02x\n", pdev->revision);
	err_printf(m, "PCI Subsystem: %04x:%04x\n",
		   pdev->subsystem_vendor,
		   pdev->subsystem_device);
}

609 610 611 612 613 614 615 616 617 618 619 620
static void err_print_uc(struct drm_i915_error_state_buf *m,
			 const struct i915_error_uc *error_uc)
{
	struct drm_printer p = i915_error_printer(m);
	const struct i915_gpu_state *error =
		container_of(error_uc, typeof(*error), uc);

	if (!error->device_info.has_guc)
		return;

	intel_uc_fw_dump(&error_uc->guc_fw, &p);
	intel_uc_fw_dump(&error_uc->huc_fw, &p);
621
	print_error_obj(m, NULL, "GuC log buffer", error_uc->guc_log);
622 623
}

624
int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
625
			    const struct i915_gpu_state *error)
626
{
627
	struct drm_i915_private *dev_priv = m->i915;
628
	struct drm_i915_error_object *obj;
629
	int i, j;
630 631

	if (!error) {
632 633
		err_printf(m, "No error state collected\n");
		return 0;
634 635
	}

636 637
	if (*error->error_msg)
		err_printf(m, "%s\n", error->error_msg);
638
	err_printf(m, "Kernel: " UTS_RELEASE "\n");
639 640 641 642 643 644
	err_printf(m, "Time: %ld s %ld us\n",
		   error->time.tv_sec, error->time.tv_usec);
	err_printf(m, "Boottime: %ld s %ld us\n",
		   error->boottime.tv_sec, error->boottime.tv_usec);
	err_printf(m, "Uptime: %ld s %ld us\n",
		   error->uptime.tv_sec, error->uptime.tv_usec);
645

646
	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
647
		if (error->engine[i].hangcheck_stalled &&
648 649
		    error->engine[i].context.pid) {
			err_printf(m, "Active process (on ring %s): %s [%d], score %d\n",
650
				   engine_str(i),
651 652 653
				   error->engine[i].context.comm,
				   error->engine[i].context.pid,
				   error->engine[i].context.ban_score);
654 655
		}
	}
656
	err_printf(m, "Reset count: %u\n", error->reset_count);
657
	err_printf(m, "Suspend count: %u\n", error->suspend_count);
658
	err_printf(m, "Platform: %s\n", intel_platform_name(error->device_info.platform));
659
	err_print_pciid(m, error->i915);
660

661
	err_printf(m, "IOMMU enabled?: %d\n", error->iommu);
662

663
	if (HAS_CSR(dev_priv)) {
664 665 666 667 668 669 670 671 672
		struct intel_csr *csr = &dev_priv->csr;

		err_printf(m, "DMC loaded: %s\n",
			   yesno(csr->dmc_payload != NULL));
		err_printf(m, "DMC fw version: %d.%d\n",
			   CSR_VERSION_MAJOR(csr->version),
			   CSR_VERSION_MINOR(csr->version));
	}

673
	err_printf(m, "GT awake: %s\n", yesno(error->awake));
674 675
	err_printf(m, "RPM wakelock: %s\n", yesno(error->wakelock));
	err_printf(m, "PM suspended: %s\n", yesno(error->suspended));
676 677
	err_printf(m, "EIR: 0x%08x\n", error->eir);
	err_printf(m, "IER: 0x%08x\n", error->ier);
678 679
	for (i = 0; i < error->ngtier; i++)
		err_printf(m, "GTIER[%d]: 0x%08x\n", i, error->gtier[i]);
680 681 682 683
	err_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
	err_printf(m, "FORCEWAKE: 0x%08x\n", error->forcewake);
	err_printf(m, "DERRMR: 0x%08x\n", error->derrmr);
	err_printf(m, "CCID: 0x%08x\n", error->ccid);
684
	err_printf(m, "Missed interrupts: 0x%08lx\n", dev_priv->gpu_error.missed_irq_rings);
685

686
	for (i = 0; i < error->nfence; i++)
687 688
		err_printf(m, "  fence[%d] = %08llx\n", i, error->fence[i]);

689
	if (INTEL_GEN(dev_priv) >= 6) {
690
		err_printf(m, "ERROR: 0x%08x\n", error->error);
691

692
		if (INTEL_GEN(dev_priv) >= 8)
693 694 695
			err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n",
				   error->fault_data1, error->fault_data0);

696 697 698
		err_printf(m, "DONE_REG: 0x%08x\n", error->done_reg);
	}

699
	if (IS_GEN7(dev_priv))
700 701
		err_printf(m, "ERR_INT: 0x%08x\n", error->err_int);

702 703 704 705
	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
		if (error->engine[i].engine_id != -1)
			error_print_engine(m, &error->engine[i]);
	}
706

707 708 709
	for (i = 0; i < ARRAY_SIZE(error->active_vm); i++) {
		char buf[128];
		int len, first = 1;
710

711 712 713 714 715 716 717 718 719 720
		if (!error->active_vm[i])
			break;

		len = scnprintf(buf, sizeof(buf), "Active (");
		for (j = 0; j < ARRAY_SIZE(error->engine); j++) {
			if (error->engine[j].vm != error->active_vm[i])
				continue;

			len += scnprintf(buf + len, sizeof(buf), "%s%s",
					 first ? "" : ", ",
721
					 dev_priv->engine[j]->name);
722 723 724 725
			first = 0;
		}
		scnprintf(buf + len, sizeof(buf), ")");
		print_error_buffers(m, buf,
726 727 728
				    error->active_bo[i],
				    error->active_bo_count[i]);
	}
729

730 731 732 733
	print_error_buffers(m, "Pinned (global)",
			    error->pinned_bo,
			    error->pinned_bo_count);

734
	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
735
		const struct drm_i915_error_engine *ee = &error->engine[i];
736 737

		obj = ee->batchbuffer;
738
		if (obj) {
739
			err_puts(m, dev_priv->engine[i]->name);
740 741 742 743 744 745 746
			if (ee->context.pid)
				err_printf(m, " (submitted by %s [%d], ctx %d [%d], score %d)",
					   ee->context.comm,
					   ee->context.pid,
					   ee->context.handle,
					   ee->context.hw_id,
					   ee->context.ban_score);
747 748 749
			err_printf(m, " --- gtt_offset = 0x%08x %08x\n",
				   upper_32_bits(obj->gtt_offset),
				   lower_32_bits(obj->gtt_offset));
750
			print_error_obj(m, dev_priv->engine[i], NULL, obj);
751 752
		}

753 754 755 756
		for (j = 0; j < ee->user_bo_count; j++)
			print_error_obj(m, dev_priv->engine[i],
					"user", ee->user_bo[j]);

757
		if (ee->num_requests) {
758
			err_printf(m, "%s --- %d requests\n",
759
				   dev_priv->engine[i]->name,
760
				   ee->num_requests);
761 762
			for (j = 0; j < ee->num_requests; j++)
				error_print_request(m, " ", &ee->requests[j]);
763 764
		}

765 766
		if (IS_ERR(ee->waiters)) {
			err_printf(m, "%s --- ? waiters [unable to acquire spinlock]\n",
767
				   dev_priv->engine[i]->name);
768
		} else if (ee->num_waiters) {
769
			err_printf(m, "%s --- %d waiters\n",
770
				   dev_priv->engine[i]->name,
771 772
				   ee->num_waiters);
			for (j = 0; j < ee->num_waiters; j++) {
773
				err_printf(m, " seqno 0x%08x for %s [%d]\n",
774 775 776
					   ee->waiters[j].seqno,
					   ee->waiters[j].comm,
					   ee->waiters[j].pid);
777 778 779
			}
		}

780
		print_error_obj(m, dev_priv->engine[i],
781
				"ringbuffer", ee->ringbuffer);
782

783
		print_error_obj(m, dev_priv->engine[i],
784
				"HW Status", ee->hws_page);
785

786
		print_error_obj(m, dev_priv->engine[i],
787
				"HW context", ee->ctx);
788

789
		print_error_obj(m, dev_priv->engine[i],
790
				"WA context", ee->wa_ctx);
791

792
		print_error_obj(m, dev_priv->engine[i],
793
				"WA batchbuffer", ee->wa_batchbuffer);
794 795
	}

796
	print_error_obj(m, NULL, "Semaphores", error->semaphore);
797

798 799 800 801
	if (error->overlay)
		intel_overlay_print_error_state(m, error->overlay);

	if (error->display)
802
		intel_display_print_error_state(m, error->display);
803

804 805
	err_print_capabilities(m, &error->device_info);
	err_print_params(m, &error->params);
806
	err_print_uc(m, &error->uc);
807

808 809 810 811 812 813 814
	if (m->bytes == 0 && m->err)
		return m->err;

	return 0;
}

int i915_error_state_buf_init(struct drm_i915_error_state_buf *ebuf,
815
			      struct drm_i915_private *i915,
816 817 818
			      size_t count, loff_t pos)
{
	memset(ebuf, 0, sizeof(*ebuf));
819
	ebuf->i915 = i915;
820 821 822 823 824 825

	/* We need to have enough room to store any i915_error_state printf
	 * so that we can move it to start position.
	 */
	ebuf->size = count + 1 > PAGE_SIZE ? count + 1 : PAGE_SIZE;
	ebuf->buf = kmalloc(ebuf->size,
826
				GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN);
827 828 829

	if (ebuf->buf == NULL) {
		ebuf->size = PAGE_SIZE;
830
		ebuf->buf = kmalloc(ebuf->size, GFP_KERNEL);
831 832 833 834
	}

	if (ebuf->buf == NULL) {
		ebuf->size = 128;
835
		ebuf->buf = kmalloc(ebuf->size, GFP_KERNEL);
836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853
	}

	if (ebuf->buf == NULL)
		return -ENOMEM;

	ebuf->start = pos;

	return 0;
}

static void i915_error_object_free(struct drm_i915_error_object *obj)
{
	int page;

	if (obj == NULL)
		return;

	for (page = 0; page < obj->page_count; page++)
854
		free_page((unsigned long)obj->pages[page]);
855 856 857 858

	kfree(obj);
}

859 860 861 862 863 864
static __always_inline void free_param(const char *type, void *x)
{
	if (!__builtin_strcmp(type, "char *"))
		kfree(*(void **)x);
}

865 866 867 868 869 870
static void cleanup_uc_state(struct i915_gpu_state *error)
{
	struct i915_error_uc *error_uc = &error->uc;

	kfree(error_uc->guc_fw.path);
	kfree(error_uc->huc_fw.path);
871
	i915_error_object_free(error_uc->guc_log);
872 873
}

874
void __i915_gpu_state_free(struct kref *error_ref)
875
{
876 877
	struct i915_gpu_state *error =
		container_of(error_ref, typeof(*error), ref);
878
	long i, j;
879

880 881 882
	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
		struct drm_i915_error_engine *ee = &error->engine[i];

883 884 885 886
		for (j = 0; j < ee->user_bo_count; j++)
			i915_error_object_free(ee->user_bo[j]);
		kfree(ee->user_bo);

887 888 889 890 891 892 893 894
		i915_error_object_free(ee->batchbuffer);
		i915_error_object_free(ee->wa_batchbuffer);
		i915_error_object_free(ee->ringbuffer);
		i915_error_object_free(ee->hws_page);
		i915_error_object_free(ee->ctx);
		i915_error_object_free(ee->wa_ctx);

		kfree(ee->requests);
895 896
		if (!IS_ERR_OR_NULL(ee->waiters))
			kfree(ee->waiters);
897 898
	}

899
	i915_error_object_free(error->semaphore);
900

901
	for (i = 0; i < ARRAY_SIZE(error->active_bo); i++)
902 903
		kfree(error->active_bo[i]);
	kfree(error->pinned_bo);
904

905 906
	kfree(error->overlay);
	kfree(error->display);
907

908
#define FREE(T, x, ...) free_param(#T, &error->params.x);
909 910 911
	I915_PARAMS_FOR_EACH(FREE);
#undef FREE

912 913
	cleanup_uc_state(error);

914 915 916 917
	kfree(error);
}

static struct drm_i915_error_object *
918
i915_error_object_create(struct drm_i915_private *i915,
C
Chris Wilson 已提交
919
			 struct i915_vma *vma)
920
{
921 922
	struct i915_ggtt *ggtt = &i915->ggtt;
	const u64 slot = ggtt->error_capture.start;
923
	struct drm_i915_error_object *dst;
924
	struct compress compress;
925 926 927
	unsigned long num_pages;
	struct sgt_iter iter;
	dma_addr_t dma;
928

C
Chris Wilson 已提交
929 930 931
	if (!vma)
		return NULL;

932
	num_pages = min_t(u64, vma->size, vma->obj->base.size) >> PAGE_SHIFT;
933
	num_pages = DIV_ROUND_UP(10 * num_pages, 8); /* worstcase zlib growth */
934 935
	dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *),
		      GFP_ATOMIC | __GFP_NOWARN);
C
Chris Wilson 已提交
936
	if (!dst)
937 938
		return NULL;

939 940
	dst->gtt_offset = vma->node.start;
	dst->gtt_size = vma->node.size;
941
	dst->page_count = 0;
942 943
	dst->unused = 0;

944
	if (!compress_init(&compress)) {
945 946 947
		kfree(dst);
		return NULL;
	}
948

949 950 951
	for_each_sgt_dma(dma, iter, vma->pages) {
		void __iomem *s;
		int ret;
952

953 954
		ggtt->base.insert_page(&ggtt->base, dma, slot,
				       I915_CACHE_NONE, 0);
955

956
		s = io_mapping_map_atomic_wc(&ggtt->mappable, slot);
957
		ret = compress_page(&compress, (void  __force *)s, dst);
958
		io_mapping_unmap_atomic(s);
959

960
		if (ret)
961 962
			goto unwind;
	}
963
	goto out;
964 965

unwind:
966 967
	while (dst->page_count--)
		free_page((unsigned long)dst->pages[dst->page_count]);
968
	kfree(dst);
969 970 971
	dst = NULL;

out:
972
	compress_fini(&compress, dst);
973
	ggtt->base.clear_range(&ggtt->base, slot, PAGE_SIZE);
974
	return dst;
975 976
}

977 978 979 980 981 982
/* The error capture is special as tries to run underneath the normal
 * locking rules - so we use the raw version of the i915_gem_active lookup.
 */
static inline uint32_t
__active_get_seqno(struct i915_gem_active *active)
{
983 984 985 986
	struct drm_i915_gem_request *request;

	request = __i915_gem_active_peek(active);
	return request ? request->global_seqno : 0;
987 988 989 990 991
}

static inline int
__active_get_engine_id(struct i915_gem_active *active)
{
992
	struct drm_i915_gem_request *request;
993

994 995
	request = __i915_gem_active_peek(active);
	return request ? request->engine->id : -1;
996 997
}

998
static void capture_bo(struct drm_i915_error_buffer *err,
999
		       struct i915_vma *vma)
1000
{
1001
	struct drm_i915_gem_object *obj = vma->obj;
1002
	int i;
1003

1004 1005
	err->size = obj->base.size;
	err->name = obj->base.name;
1006

1007
	for (i = 0; i < I915_NUM_ENGINES; i++)
1008
		err->rseqno[i] = __active_get_seqno(&vma->last_read[i]);
1009 1010
	err->wseqno = __active_get_seqno(&obj->frontbuffer_write);
	err->engine = __active_get_engine_id(&obj->frontbuffer_write);
1011

1012
	err->gtt_offset = vma->node.start;
1013 1014
	err->read_domains = obj->base.read_domains;
	err->write_domain = obj->base.write_domain;
1015
	err->fence_reg = vma->fence ? vma->fence->id : -1;
1016
	err->tiling = i915_gem_object_get_tiling(obj);
C
Chris Wilson 已提交
1017 1018
	err->dirty = obj->mm.dirty;
	err->purgeable = obj->mm.madv != I915_MADV_WILLNEED;
1019
	err->userptr = obj->userptr.mm != NULL;
1020 1021 1022
	err->cache_level = obj->cache_level;
}

1023 1024 1025
static u32 capture_error_bo(struct drm_i915_error_buffer *err,
			    int count, struct list_head *head,
			    bool pinned_only)
1026
{
B
Ben Widawsky 已提交
1027
	struct i915_vma *vma;
1028 1029
	int i = 0;

1030
	list_for_each_entry(vma, head, vm_link) {
1031 1032 1033
		if (pinned_only && !i915_vma_is_pinned(vma))
			continue;

1034
		capture_bo(err++, vma);
1035 1036 1037 1038 1039 1040 1041
		if (++i == count)
			break;
	}

	return i;
}

1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
/* Generate a semi-unique error code. The code is not meant to have meaning, The
 * code's only purpose is to try to prevent false duplicated bug reports by
 * grossly estimating a GPU error state.
 *
 * TODO Ideally, hashing the batchbuffer would be a very nice way to determine
 * the hang if we could strip the GTT offset information from it.
 *
 * It's only a small step better than a random number in its current form.
 */
static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv,
1052
					 struct i915_gpu_state *error,
1053
					 int *engine_id)
1054 1055 1056 1057 1058 1059 1060 1061 1062
{
	uint32_t error_code = 0;
	int i;

	/* IPEHR would be an ideal way to detect errors, as it's the gross
	 * measure of "the command that hung." However, has some very common
	 * synchronization commands which almost always appear in the case
	 * strictly a client bug. Use instdone to differentiate those some.
	 */
1063
	for (i = 0; i < I915_NUM_ENGINES; i++) {
1064
		if (error->engine[i].hangcheck_stalled) {
1065 1066
			if (engine_id)
				*engine_id = i;
1067

1068 1069
			return error->engine[i].ipehr ^
			       error->engine[i].instdone.instdone;
1070 1071
		}
	}
1072 1073 1074 1075

	return error_code;
}

1076
static void i915_gem_record_fences(struct drm_i915_private *dev_priv,
1077
				   struct i915_gpu_state *error)
1078 1079 1080
{
	int i;

1081
	if (INTEL_GEN(dev_priv) >= 6) {
1082
		for (i = 0; i < dev_priv->num_fence_regs; i++)
1083 1084
			error->fence[i] = I915_READ64(FENCE_REG_GEN6_LO(i));
	} else if (INTEL_GEN(dev_priv) >= 4) {
1085 1086
		for (i = 0; i < dev_priv->num_fence_regs; i++)
			error->fence[i] = I915_READ64(FENCE_REG_965_LO(i));
1087
	} else {
1088
		for (i = 0; i < dev_priv->num_fence_regs; i++)
1089
			error->fence[i] = I915_READ(FENCE_REG(i));
1090
	}
1091
	error->nfence = i;
1092 1093
}

1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
static inline u32
gen8_engine_sync_index(struct intel_engine_cs *engine,
		       struct intel_engine_cs *other)
{
	int idx;

	/*
	 * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
	 * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
	 * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
	 * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
	 * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
	 */

	idx = (other - engine) - 1;
	if (idx < 0)
		idx += I915_NUM_ENGINES;

	return idx;
}
1114

1115
static void gen8_record_semaphore_state(struct i915_gpu_state *error,
1116
					struct intel_engine_cs *engine,
1117
					struct drm_i915_error_engine *ee)
1118
{
1119
	struct drm_i915_private *dev_priv = engine->i915;
1120
	struct intel_engine_cs *to;
1121
	enum intel_engine_id id;
1122

1123
	if (!error->semaphore)
1124
		return;
1125

1126
	for_each_engine(to, dev_priv, id) {
1127 1128 1129
		int idx;
		u16 signal_offset;
		u32 *tmp;
1130

1131
		if (engine == to)
1132 1133
			continue;

1134 1135
		signal_offset =
			(GEN8_SIGNAL_OFFSET(engine, id) & (PAGE_SIZE - 1)) / 4;
1136
		tmp = error->semaphore->pages[0];
1137
		idx = gen8_engine_sync_index(engine, to);
1138

1139
		ee->semaphore_mboxes[idx] = tmp[signal_offset];
1140 1141 1142
	}
}

1143 1144
static void gen6_record_semaphore_state(struct intel_engine_cs *engine,
					struct drm_i915_error_engine *ee)
1145
{
1146 1147 1148 1149
	struct drm_i915_private *dev_priv = engine->i915;

	ee->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base));
	ee->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base));
1150
	if (HAS_VEBOX(dev_priv))
1151
		ee->semaphore_mboxes[2] =
1152
			I915_READ(RING_SYNC_2(engine->mmio_base));
1153 1154
}

1155 1156
static void error_record_engine_waiters(struct intel_engine_cs *engine,
					struct drm_i915_error_engine *ee)
1157 1158 1159 1160 1161 1162
{
	struct intel_breadcrumbs *b = &engine->breadcrumbs;
	struct drm_i915_error_waiter *waiter;
	struct rb_node *rb;
	int count;

1163 1164
	ee->num_waiters = 0;
	ee->waiters = NULL;
1165

1166 1167 1168
	if (RB_EMPTY_ROOT(&b->waiters))
		return;

1169
	if (!spin_trylock_irq(&b->rb_lock)) {
1170 1171 1172 1173
		ee->waiters = ERR_PTR(-EDEADLK);
		return;
	}

1174 1175 1176
	count = 0;
	for (rb = rb_first(&b->waiters); rb != NULL; rb = rb_next(rb))
		count++;
1177
	spin_unlock_irq(&b->rb_lock);
1178 1179 1180 1181 1182 1183 1184 1185 1186

	waiter = NULL;
	if (count)
		waiter = kmalloc_array(count,
				       sizeof(struct drm_i915_error_waiter),
				       GFP_ATOMIC);
	if (!waiter)
		return;

1187
	if (!spin_trylock_irq(&b->rb_lock)) {
1188 1189 1190 1191
		kfree(waiter);
		ee->waiters = ERR_PTR(-EDEADLK);
		return;
	}
1192

1193
	ee->waiters = waiter;
1194
	for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
G
Geliang Tang 已提交
1195
		struct intel_wait *w = rb_entry(rb, typeof(*w), node);
1196 1197 1198 1199 1200 1201

		strcpy(waiter->comm, w->tsk->comm);
		waiter->pid = w->tsk->pid;
		waiter->seqno = w->seqno;
		waiter++;

1202
		if (++ee->num_waiters == count)
1203 1204
			break;
	}
1205
	spin_unlock_irq(&b->rb_lock);
1206 1207
}

1208
static void error_record_engine_registers(struct i915_gpu_state *error,
1209 1210
					  struct intel_engine_cs *engine,
					  struct drm_i915_error_engine *ee)
1211
{
1212 1213
	struct drm_i915_private *dev_priv = engine->i915;

1214
	if (INTEL_GEN(dev_priv) >= 6) {
1215 1216
		ee->rc_psmi = I915_READ(RING_PSMI_CTL(engine->mmio_base));
		ee->fault_reg = I915_READ(RING_FAULT_REG(engine));
1217
		if (INTEL_GEN(dev_priv) >= 8)
1218
			gen8_record_semaphore_state(error, engine, ee);
1219
		else
1220
			gen6_record_semaphore_state(engine, ee);
1221 1222
	}

1223
	if (INTEL_GEN(dev_priv) >= 4) {
1224 1225 1226 1227 1228
		ee->faddr = I915_READ(RING_DMA_FADD(engine->mmio_base));
		ee->ipeir = I915_READ(RING_IPEIR(engine->mmio_base));
		ee->ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
		ee->instps = I915_READ(RING_INSTPS(engine->mmio_base));
		ee->bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
1229
		if (INTEL_GEN(dev_priv) >= 8) {
1230 1231
			ee->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(engine->mmio_base)) << 32;
			ee->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(engine->mmio_base)) << 32;
1232
		}
1233
		ee->bbstate = I915_READ(RING_BBSTATE(engine->mmio_base));
1234
	} else {
1235 1236 1237
		ee->faddr = I915_READ(DMA_FADD_I8XX);
		ee->ipeir = I915_READ(IPEIR);
		ee->ipehr = I915_READ(IPEHR);
1238 1239
	}

1240
	intel_engine_get_instdone(engine, &ee->instdone);
1241

1242 1243
	ee->waiting = intel_engine_has_waiter(engine);
	ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base));
1244
	ee->acthd = intel_engine_get_active_head(engine);
1245
	ee->seqno = intel_engine_get_seqno(engine);
1246
	ee->last_seqno = intel_engine_last_submit(engine);
1247 1248 1249 1250
	ee->start = I915_READ_START(engine);
	ee->head = I915_READ_HEAD(engine);
	ee->tail = I915_READ_TAIL(engine);
	ee->ctl = I915_READ_CTL(engine);
1251 1252
	if (INTEL_GEN(dev_priv) > 2)
		ee->mode = I915_READ_MODE(engine);
1253

1254
	if (!HWS_NEEDS_PHYSICAL(dev_priv)) {
1255
		i915_reg_t mmio;
1256

1257
		if (IS_GEN7(dev_priv)) {
1258
			switch (engine->id) {
1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
			default:
			case RCS:
				mmio = RENDER_HWS_PGA_GEN7;
				break;
			case BCS:
				mmio = BLT_HWS_PGA_GEN7;
				break;
			case VCS:
				mmio = BSD_HWS_PGA_GEN7;
				break;
			case VECS:
				mmio = VEBOX_HWS_PGA_GEN7;
				break;
			}
1273
		} else if (IS_GEN6(engine->i915)) {
1274
			mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
1275 1276
		} else {
			/* XXX: gen8 returns to sanity */
1277
			mmio = RING_HWS_PGA(engine->mmio_base);
1278 1279
		}

1280
		ee->hws = I915_READ(mmio);
1281 1282
	}

1283
	ee->hangcheck_timestamp = engine->hangcheck.action_timestamp;
1284
	ee->hangcheck_action = engine->hangcheck.action;
1285
	ee->hangcheck_stalled = engine->hangcheck.stalled;
1286 1287
	ee->reset_count = i915_reset_engine_count(&dev_priv->gpu_error,
						  engine);
1288

1289
	if (USES_PPGTT(dev_priv)) {
1290 1291
		int i;

1292
		ee->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(engine));
1293

1294
		if (IS_GEN6(dev_priv))
1295
			ee->vm_info.pp_dir_base =
1296
				I915_READ(RING_PP_DIR_BASE_READ(engine));
1297
		else if (IS_GEN7(dev_priv))
1298
			ee->vm_info.pp_dir_base =
1299
				I915_READ(RING_PP_DIR_BASE(engine));
1300
		else if (INTEL_GEN(dev_priv) >= 8)
1301
			for (i = 0; i < 4; i++) {
1302
				ee->vm_info.pdp[i] =
1303
					I915_READ(GEN8_RING_PDP_UDW(engine, i));
1304 1305
				ee->vm_info.pdp[i] <<= 32;
				ee->vm_info.pdp[i] |=
1306
					I915_READ(GEN8_RING_PDP_LDW(engine, i));
1307 1308
			}
	}
1309 1310
}

1311 1312 1313 1314
static void record_request(struct drm_i915_gem_request *request,
			   struct drm_i915_error_request *erq)
{
	erq->context = request->ctx->hw_id;
1315
	erq->priority = request->priotree.priority;
1316
	erq->ban_score = atomic_read(&request->ctx->ban_score);
1317
	erq->seqno = request->global_seqno;
1318 1319 1320 1321 1322 1323 1324 1325 1326
	erq->jiffies = request->emitted_jiffies;
	erq->head = request->head;
	erq->tail = request->tail;

	rcu_read_lock();
	erq->pid = request->ctx->pid ? pid_nr(request->ctx->pid) : 0;
	rcu_read_unlock();
}

1327 1328 1329 1330 1331 1332 1333 1334 1335
static void engine_record_requests(struct intel_engine_cs *engine,
				   struct drm_i915_gem_request *first,
				   struct drm_i915_error_engine *ee)
{
	struct drm_i915_gem_request *request;
	int count;

	count = 0;
	request = first;
1336
	list_for_each_entry_from(request, &engine->timeline->requests, link)
1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
		count++;
	if (!count)
		return;

	ee->requests = kcalloc(count, sizeof(*ee->requests), GFP_ATOMIC);
	if (!ee->requests)
		return;

	ee->num_requests = count;

	count = 0;
	request = first;
1349
	list_for_each_entry_from(request, &engine->timeline->requests, link) {
1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
		if (count >= ee->num_requests) {
			/*
			 * If the ring request list was changed in
			 * between the point where the error request
			 * list was created and dimensioned and this
			 * point then just exit early to avoid crashes.
			 *
			 * We don't need to communicate that the
			 * request list changed state during error
			 * state capture and that the error state is
			 * slightly incorrect as a consequence since we
			 * are typically only interested in the request
			 * list state at the point of error state
			 * capture, not in any changes happening during
			 * the capture.
			 */
			break;
		}

1369
		record_request(request, &ee->requests[count++]);
1370 1371 1372 1373
	}
	ee->num_requests = count;
}

1374 1375 1376
static void error_record_engine_execlists(struct intel_engine_cs *engine,
					  struct drm_i915_error_engine *ee)
{
1377
	const struct intel_engine_execlists * const execlists = &engine->execlists;
1378 1379
	unsigned int n;

1380 1381
	for (n = 0; n < execlists_num_ports(execlists); n++) {
		struct drm_i915_gem_request *rq = port_request(&execlists->port[n]);
1382 1383 1384 1385 1386 1387

		if (!rq)
			break;

		record_request(rq, &ee->execlist[n]);
	}
1388 1389

	ee->num_ports = n;
1390 1391
}

1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408
static void record_context(struct drm_i915_error_context *e,
			   struct i915_gem_context *ctx)
{
	if (ctx->pid) {
		struct task_struct *task;

		rcu_read_lock();
		task = pid_task(ctx->pid, PIDTYPE_PID);
		if (task) {
			strcpy(e->comm, task->comm);
			e->pid = task->pid;
		}
		rcu_read_unlock();
	}

	e->handle = ctx->user_handle;
	e->hw_id = ctx->hw_id;
1409
	e->priority = ctx->priority;
1410 1411 1412
	e->ban_score = atomic_read(&ctx->ban_score);
	e->guilty = atomic_read(&ctx->guilty_count);
	e->active = atomic_read(&ctx->active_count);
1413 1414
}

1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443
static void request_record_user_bo(struct drm_i915_gem_request *request,
				   struct drm_i915_error_engine *ee)
{
	struct i915_gem_capture_list *c;
	struct drm_i915_error_object **bo;
	long count;

	count = 0;
	for (c = request->capture_list; c; c = c->next)
		count++;

	bo = NULL;
	if (count)
		bo = kcalloc(count, sizeof(*bo), GFP_ATOMIC);
	if (!bo)
		return;

	count = 0;
	for (c = request->capture_list; c; c = c->next) {
		bo[count] = i915_error_object_create(request->i915, c->vma);
		if (!bo[count])
			break;
		count++;
	}

	ee->user_bo = bo;
	ee->user_bo_count = count;
}

1444
static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
1445
				  struct i915_gpu_state *error)
1446
{
1447
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
1448
	int i;
1449

1450
	error->semaphore =
C
Chris Wilson 已提交
1451
		i915_error_object_create(dev_priv, dev_priv->semaphore);
1452

1453
	for (i = 0; i < I915_NUM_ENGINES; i++) {
1454
		struct intel_engine_cs *engine = dev_priv->engine[i];
1455
		struct drm_i915_error_engine *ee = &error->engine[i];
1456
		struct drm_i915_gem_request *request;
1457

1458
		ee->engine_id = -1;
1459

1460
		if (!engine)
1461 1462
			continue;

1463
		ee->engine_id = i;
1464

1465 1466
		error_record_engine_registers(error, engine, ee);
		error_record_engine_waiters(engine, ee);
1467
		error_record_engine_execlists(engine, ee);
1468

1469
		request = i915_gem_find_active_request(engine);
1470
		if (request) {
1471
			struct intel_ring *ring;
1472

1473
			ee->vm = request->ctx->ppgtt ?
1474
				&request->ctx->ppgtt->base : &ggtt->base;
1475

1476 1477
			record_context(&ee->context, request->ctx);

1478 1479 1480 1481
			/* We need to copy these to an anonymous buffer
			 * as the simplest method to avoid being overwritten
			 * by userspace.
			 */
1482
			ee->batchbuffer =
1483
				i915_error_object_create(dev_priv,
C
Chris Wilson 已提交
1484
							 request->batch);
1485

1486
			if (HAS_BROKEN_CS_TLB(dev_priv))
1487
				ee->wa_batchbuffer =
C
Chris Wilson 已提交
1488 1489
					i915_error_object_create(dev_priv,
								 engine->scratch);
1490
			request_record_user_bo(request, ee);
1491

C
Chris Wilson 已提交
1492 1493 1494
			ee->ctx =
				i915_error_object_create(dev_priv,
							 request->ctx->engine[i].state);
1495

1496
			error->simulated |=
1497
				i915_gem_context_no_error_capture(request->ctx);
1498

1499 1500 1501 1502
			ee->rq_head = request->head;
			ee->rq_post = request->postfix;
			ee->rq_tail = request->tail;

1503 1504 1505
			ring = request->ring;
			ee->cpu_ring_head = ring->head;
			ee->cpu_ring_tail = ring->tail;
1506
			ee->ringbuffer =
C
Chris Wilson 已提交
1507
				i915_error_object_create(dev_priv, ring->vma);
1508 1509

			engine_record_requests(engine, request, ee);
1510
		}
1511

1512
		ee->hws_page =
C
Chris Wilson 已提交
1513 1514
			i915_error_object_create(dev_priv,
						 engine->status_page.vma);
1515

C
Chris Wilson 已提交
1516 1517
		ee->wa_ctx =
			i915_error_object_create(dev_priv, engine->wa_ctx.vma);
1518 1519 1520
	}
}

1521
static void i915_gem_capture_vm(struct drm_i915_private *dev_priv,
1522
				struct i915_gpu_state *error,
1523
				struct i915_address_space *vm,
1524
				int idx)
1525
{
1526
	struct drm_i915_error_buffer *active_bo;
1527
	struct i915_vma *vma;
1528
	int count;
1529

1530
	count = 0;
1531
	list_for_each_entry(vma, &vm->active_list, vm_link)
1532
		count++;
1533

1534 1535 1536
	active_bo = NULL;
	if (count)
		active_bo = kcalloc(count, sizeof(*active_bo), GFP_ATOMIC);
1537
	if (active_bo)
1538 1539 1540 1541 1542 1543 1544
		count = capture_error_bo(active_bo, count, &vm->active_list, false);
	else
		count = 0;

	error->active_vm[idx] = vm;
	error->active_bo[idx] = active_bo;
	error->active_bo_count[idx] = count;
1545 1546
}

1547
static void i915_capture_active_buffers(struct drm_i915_private *dev_priv,
1548
					struct i915_gpu_state *error)
1549
{
1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562
	int cnt = 0, i, j;

	BUILD_BUG_ON(ARRAY_SIZE(error->engine) > ARRAY_SIZE(error->active_bo));
	BUILD_BUG_ON(ARRAY_SIZE(error->active_bo) != ARRAY_SIZE(error->active_vm));
	BUILD_BUG_ON(ARRAY_SIZE(error->active_bo) != ARRAY_SIZE(error->active_bo_count));

	/* Scan each engine looking for unique active contexts/vm */
	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
		struct drm_i915_error_engine *ee = &error->engine[i];
		bool found;

		if (!ee->vm)
			continue;
1563

1564 1565 1566 1567 1568
		found = false;
		for (j = 0; j < i && !found; j++)
			found = error->engine[j].vm == ee->vm;
		if (!found)
			i915_gem_capture_vm(dev_priv, error, ee->vm, cnt++);
1569
	}
1570 1571
}

1572
static void i915_capture_pinned_buffers(struct drm_i915_private *dev_priv,
1573
					struct i915_gpu_state *error)
1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
{
	struct i915_address_space *vm = &dev_priv->ggtt.base;
	struct drm_i915_error_buffer *bo;
	struct i915_vma *vma;
	int count_inactive, count_active;

	count_inactive = 0;
	list_for_each_entry(vma, &vm->active_list, vm_link)
		count_inactive++;

	count_active = 0;
	list_for_each_entry(vma, &vm->inactive_list, vm_link)
		count_active++;

	bo = NULL;
	if (count_inactive + count_active)
		bo = kcalloc(count_inactive + count_active,
			     sizeof(*bo), GFP_ATOMIC);
	if (!bo)
		return;

	count_inactive = capture_error_bo(bo, count_inactive,
					  &vm->active_list, true);
	count_active = capture_error_bo(bo + count_inactive, count_active,
					&vm->inactive_list, true);
	error->pinned_bo_count = count_inactive + count_active;
	error->pinned_bo = bo;
}

1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620
static void capture_uc_state(struct i915_gpu_state *error)
{
	struct drm_i915_private *i915 = error->i915;
	struct i915_error_uc *error_uc = &error->uc;

	/* Capturing uC state won't be useful if there is no GuC */
	if (!error->device_info.has_guc)
		return;

	error_uc->guc_fw = i915->guc.fw;
	error_uc->huc_fw = i915->huc.fw;

	/* Non-default firmware paths will be specified by the modparam.
	 * As modparams are generally accesible from the userspace make
	 * explicit copies of the firmware paths.
	 */
	error_uc->guc_fw.path = kstrdup(i915->guc.fw.path, GFP_ATOMIC);
	error_uc->huc_fw.path = kstrdup(i915->huc.fw.path, GFP_ATOMIC);
1621
	error_uc->guc_log = i915_error_object_create(i915, i915->guc.log.vma);
1622 1623
}

1624 1625
/* Capture all registers which don't fit into another category. */
static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
1626
				   struct i915_gpu_state *error)
1627
{
1628
	int i;
1629

1630 1631 1632 1633 1634 1635 1636
	/* General organization
	 * 1. Registers specific to a single generation
	 * 2. Registers which belong to multiple generations
	 * 3. Feature specific registers.
	 * 4. Everything else
	 * Please try to follow the order.
	 */
1637

1638
	/* 1: Registers specific to a single generation */
1639
	if (IS_VALLEYVIEW(dev_priv)) {
1640
		error->gtier[0] = I915_READ(GTIER);
1641
		error->ier = I915_READ(VLV_IER);
1642
		error->forcewake = I915_READ_FW(FORCEWAKE_VLV);
1643
	}
1644

1645
	if (IS_GEN7(dev_priv))
1646
		error->err_int = I915_READ(GEN7_ERR_INT);
1647

1648
	if (INTEL_GEN(dev_priv) >= 8) {
1649 1650 1651 1652
		error->fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0);
		error->fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1);
	}

1653
	if (IS_GEN6(dev_priv)) {
1654
		error->forcewake = I915_READ_FW(FORCEWAKE);
1655 1656 1657
		error->gab_ctl = I915_READ(GAB_CTL);
		error->gfx_mode = I915_READ(GFX_MODE);
	}
1658

1659
	/* 2: Registers which belong to multiple generations */
1660
	if (INTEL_GEN(dev_priv) >= 7)
1661
		error->forcewake = I915_READ_FW(FORCEWAKE_MT);
1662

1663
	if (INTEL_GEN(dev_priv) >= 6) {
1664
		error->derrmr = I915_READ(DERRMR);
1665 1666 1667 1668
		error->error = I915_READ(ERROR_GEN6);
		error->done_reg = I915_READ(DONE_REG);
	}

J
Joonas Lahtinen 已提交
1669
	if (INTEL_GEN(dev_priv) >= 5)
1670 1671
		error->ccid = I915_READ(CCID);

1672
	/* 3: Feature specific registers */
1673
	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
1674 1675 1676 1677 1678
		error->gam_ecochk = I915_READ(GAM_ECOCHK);
		error->gac_eco = I915_READ(GAC_ECO_BITS);
	}

	/* 4: Everything else */
1679
	if (INTEL_GEN(dev_priv) >= 8) {
1680 1681 1682
		error->ier = I915_READ(GEN8_DE_MISC_IER);
		for (i = 0; i < 4; i++)
			error->gtier[i] = I915_READ(GEN8_GT_IER(i));
1683
		error->ngtier = 4;
1684
	} else if (HAS_PCH_SPLIT(dev_priv)) {
1685
		error->ier = I915_READ(DEIER);
1686
		error->gtier[0] = I915_READ(GTIER);
1687
		error->ngtier = 1;
1688
	} else if (IS_GEN2(dev_priv)) {
1689
		error->ier = I915_READ16(IER);
1690
	} else if (!IS_VALLEYVIEW(dev_priv)) {
1691
		error->ier = I915_READ(IER);
1692 1693 1694
	}
	error->eir = I915_READ(EIR);
	error->pgtbl_er = I915_READ(PGTBL_ER);
1695 1696
}

1697
static void i915_error_capture_msg(struct drm_i915_private *dev_priv,
1698
				   struct i915_gpu_state *error,
1699
				   u32 engine_mask,
1700
				   const char *error_msg)
1701 1702
{
	u32 ecode;
1703
	int engine_id = -1, len;
1704

1705
	ecode = i915_error_generate_code(dev_priv, error, &engine_id);
1706

1707
	len = scnprintf(error->error_msg, sizeof(error->error_msg),
1708
			"GPU HANG: ecode %d:%d:0x%08x",
1709
			INTEL_GEN(dev_priv), engine_id, ecode);
1710

1711
	if (engine_id != -1 && error->engine[engine_id].context.pid)
1712 1713 1714
		len += scnprintf(error->error_msg + len,
				 sizeof(error->error_msg) - len,
				 ", in %s [%d]",
1715 1716
				 error->engine[engine_id].context.comm,
				 error->engine[engine_id].context.pid);
1717 1718 1719 1720

	scnprintf(error->error_msg + len, sizeof(error->error_msg) - len,
		  ", reason: %s, action: %s",
		  error_msg,
1721
		  engine_mask ? "reset" : "continue");
1722 1723
}

1724
static void i915_capture_gen_state(struct drm_i915_private *dev_priv,
1725
				   struct i915_gpu_state *error)
1726
{
1727
	error->awake = dev_priv->gt.awake;
1728 1729
	error->wakelock = atomic_read(&dev_priv->runtime_pm.wakeref_count);
	error->suspended = dev_priv->runtime_pm.suspended;
1730

1731 1732 1733 1734
	error->iommu = -1;
#ifdef CONFIG_INTEL_IOMMU
	error->iommu = intel_iommu_gfx_mapped;
#endif
1735
	error->reset_count = i915_reset_count(&dev_priv->gpu_error);
1736
	error->suspend_count = dev_priv->suspend_count;
1737 1738 1739 1740

	memcpy(&error->device_info,
	       INTEL_INFO(dev_priv),
	       sizeof(error->device_info));
1741 1742
}

1743 1744 1745 1746 1747 1748
static __always_inline void dup_param(const char *type, void *x)
{
	if (!__builtin_strcmp(type, "char *"))
		*(void **)x = kstrdup(*(void **)x, GFP_ATOMIC);
}

1749 1750
static int capture(void *data)
{
1751
	struct i915_gpu_state *error = data;
1752

1753 1754 1755 1756 1757 1758
	do_gettimeofday(&error->time);
	error->boottime = ktime_to_timeval(ktime_get_boottime());
	error->uptime =
		ktime_to_timeval(ktime_sub(ktime_get(),
					   error->i915->gt.last_init_time));

1759
	error->params = i915_modparams;
1760
#define DUP(T, x, ...) dup_param(#T, &error->params.x);
1761 1762
	I915_PARAMS_FOR_EACH(DUP);
#undef DUP
1763

1764 1765
	capture_uc_state(error);

1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778
	i915_capture_gen_state(error->i915, error);
	i915_capture_reg_state(error->i915, error);
	i915_gem_record_fences(error->i915, error);
	i915_gem_record_rings(error->i915, error);
	i915_capture_active_buffers(error->i915, error);
	i915_capture_pinned_buffers(error->i915, error);

	error->overlay = intel_overlay_capture_error_state(error->i915);
	error->display = intel_display_capture_error_state(error->i915);

	return 0;
}

1779 1780
#define DAY_AS_SECONDS(x) (24 * 60 * 60 * (x))

1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797
struct i915_gpu_state *
i915_capture_gpu_state(struct drm_i915_private *i915)
{
	struct i915_gpu_state *error;

	error = kzalloc(sizeof(*error), GFP_ATOMIC);
	if (!error)
		return NULL;

	kref_init(&error->ref);
	error->i915 = i915;

	stop_machine(capture, error, NULL);

	return error;
}

1798 1799 1800 1801 1802 1803 1804 1805 1806
/**
 * i915_capture_error_state - capture an error record for later analysis
 * @dev: drm device
 *
 * Should be called when an error is detected (either a hang or an error
 * interrupt) to capture error state from the time of the error.  Fills
 * out a structure which becomes available in debugfs for user level tools
 * to pick up.
 */
1807 1808
void i915_capture_error_state(struct drm_i915_private *dev_priv,
			      u32 engine_mask,
1809
			      const char *error_msg)
1810
{
1811
	static bool warned;
1812
	struct i915_gpu_state *error;
1813 1814
	unsigned long flags;

1815
	if (!i915_modparams.error_capture)
1816 1817
		return;

1818 1819 1820
	if (READ_ONCE(dev_priv->gpu_error.first_error))
		return;

1821
	error = i915_capture_gpu_state(dev_priv);
1822 1823 1824 1825 1826
	if (!error) {
		DRM_DEBUG_DRIVER("out of memory, not capturing error state\n");
		return;
	}

1827
	i915_error_capture_msg(dev_priv, error, engine_mask, error_msg);
1828 1829
	DRM_INFO("%s\n", error->error_msg);

1830 1831 1832 1833 1834 1835 1836
	if (!error->simulated) {
		spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
		if (!dev_priv->gpu_error.first_error) {
			dev_priv->gpu_error.first_error = error;
			error = NULL;
		}
		spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
1837 1838
	}

1839
	if (error) {
1840
		__i915_gpu_state_free(&error->ref);
1841 1842 1843
		return;
	}

1844 1845
	if (!warned &&
	    ktime_get_real_seconds() - DRIVER_TIMESTAMP < DAY_AS_SECONDS(180)) {
1846 1847 1848 1849
		DRM_INFO("GPU hangs can indicate a bug anywhere in the entire gfx stack, including userspace.\n");
		DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n");
		DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n");
		DRM_INFO("The gpu crash dump is required to analyze gpu hangs, so please always attach it.\n");
1850 1851
		DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n",
			 dev_priv->drm.primary->index);
1852 1853
		warned = true;
	}
1854 1855
}

1856 1857
struct i915_gpu_state *
i915_first_error_state(struct drm_i915_private *i915)
1858
{
1859
	struct i915_gpu_state *error;
1860

1861 1862 1863 1864 1865
	spin_lock_irq(&i915->gpu_error.lock);
	error = i915->gpu_error.first_error;
	if (error)
		i915_gpu_state_get(error);
	spin_unlock_irq(&i915->gpu_error.lock);
1866

1867
	return error;
1868 1869
}

1870
void i915_reset_error_state(struct drm_i915_private *i915)
1871
{
1872
	struct i915_gpu_state *error;
1873

1874 1875 1876 1877
	spin_lock_irq(&i915->gpu_error.lock);
	error = i915->gpu_error.first_error;
	i915->gpu_error.first_error = NULL;
	spin_unlock_irq(&i915->gpu_error.lock);
1878

1879
	i915_gpu_state_put(error);
1880
}