i915_gpu_error.c 43.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 * Copyright (c) 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *    Keith Packard <keithp@keithp.com>
 *    Mika Kuoppala <mika.kuoppala@intel.com>
 *
 */

#include <generated/utsrelease.h>
31
#include <linux/stop_machine.h>
32
#include <linux/zlib.h>
33 34
#include "i915_drv.h"

35
static const char *engine_str(int engine)
36
{
37
	switch (engine) {
38 39 40 41
	case RCS: return "render";
	case VCS: return "bsd";
	case BCS: return "blt";
	case VECS: return "vebox";
42
	case VCS2: return "bsd2";
43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
	default: return "";
	}
}

static const char *tiling_flag(int tiling)
{
	switch (tiling) {
	default:
	case I915_TILING_NONE: return "";
	case I915_TILING_X: return " X";
	case I915_TILING_Y: return " Y";
	}
}

static const char *dirty_flag(int dirty)
{
	return dirty ? " dirty" : "";
}

static const char *purgeable_flag(int purgeable)
{
	return purgeable ? " purgeable" : "";
}

static bool __i915_error_ok(struct drm_i915_error_state_buf *e)
{

	if (!e->err && WARN(e->bytes > (e->size - 1), "overflow")) {
		e->err = -ENOSPC;
		return false;
	}

	if (e->bytes == e->size - 1 || e->err)
		return false;

	return true;
}

static bool __i915_error_seek(struct drm_i915_error_state_buf *e,
			      unsigned len)
{
	if (e->pos + len <= e->start) {
		e->pos += len;
		return false;
	}

	/* First vsnprintf needs to fit in its entirety for memmove */
	if (len >= e->size) {
		e->err = -EIO;
		return false;
	}

	return true;
}

static void __i915_error_advance(struct drm_i915_error_state_buf *e,
				 unsigned len)
{
	/* If this is first printf in this window, adjust it so that
	 * start position matches start of the buffer
	 */

	if (e->pos < e->start) {
		const size_t off = e->start - e->pos;

		/* Should not happen but be paranoid */
		if (off > len || e->bytes) {
			e->err = -EIO;
			return;
		}

		memmove(e->buf, e->buf + off, len - off);
		e->bytes = len - off;
		e->pos = e->start;
		return;
	}

	e->bytes += len;
	e->pos += len;
}

124
__printf(2, 0)
125 126 127 128 129 130 131 132 133 134
static void i915_error_vprintf(struct drm_i915_error_state_buf *e,
			       const char *f, va_list args)
{
	unsigned len;

	if (!__i915_error_ok(e))
		return;

	/* Seek the first printf which is hits start position */
	if (e->pos < e->start) {
135 136 137
		va_list tmp;

		va_copy(tmp, args);
138 139 140 141
		len = vsnprintf(NULL, 0, f, tmp);
		va_end(tmp);

		if (!__i915_error_seek(e, len))
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
			return;
	}

	len = vsnprintf(e->buf + e->bytes, e->size - e->bytes, f, args);
	if (len >= e->size - e->bytes)
		len = e->size - e->bytes - 1;

	__i915_error_advance(e, len);
}

static void i915_error_puts(struct drm_i915_error_state_buf *e,
			    const char *str)
{
	unsigned len;

	if (!__i915_error_ok(e))
		return;

	len = strlen(str);

	/* Seek the first printf which is hits start position */
	if (e->pos < e->start) {
		if (!__i915_error_seek(e, len))
			return;
	}

	if (len >= e->size - e->bytes)
		len = e->size - e->bytes - 1;
	memcpy(e->buf + e->bytes, str, len);

	__i915_error_advance(e, len);
}

#define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
#define err_puts(e, s) i915_error_puts(e, s)

178 179
#ifdef CONFIG_DRM_I915_COMPRESS_ERROR

180 181 182 183 184 185
struct compress {
	struct z_stream_s zstream;
	void *tmp;
};

static bool compress_init(struct compress *c)
186
{
187
	struct z_stream_s *zstream = memset(&c->zstream, 0, sizeof(c->zstream));
188 189 190 191 192 193 194 195 196 197 198 199

	zstream->workspace =
		kmalloc(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL),
			GFP_ATOMIC | __GFP_NOWARN);
	if (!zstream->workspace)
		return false;

	if (zlib_deflateInit(zstream, Z_DEFAULT_COMPRESSION) != Z_OK) {
		kfree(zstream->workspace);
		return false;
	}

200
	c->tmp = NULL;
201
	if (i915_has_memcpy_from_wc())
202 203
		c->tmp = (void *)__get_free_page(GFP_ATOMIC | __GFP_NOWARN);

204 205 206
	return true;
}

207
static int compress_page(struct compress *c,
208 209 210
			 void *src,
			 struct drm_i915_error_object *dst)
{
211 212
	struct z_stream_s *zstream = &c->zstream;

213
	zstream->next_in = src;
214 215
	if (c->tmp && i915_memcpy_from_wc(c->tmp, src, PAGE_SIZE))
		zstream->next_in = c->tmp;
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
	zstream->avail_in = PAGE_SIZE;

	do {
		if (zstream->avail_out == 0) {
			unsigned long page;

			page = __get_free_page(GFP_ATOMIC | __GFP_NOWARN);
			if (!page)
				return -ENOMEM;

			dst->pages[dst->page_count++] = (void *)page;

			zstream->next_out = (void *)page;
			zstream->avail_out = PAGE_SIZE;
		}

		if (zlib_deflate(zstream, Z_SYNC_FLUSH) != Z_OK)
			return -EIO;
	} while (zstream->avail_in);

	/* Fallback to uncompressed if we increase size? */
	if (0 && zstream->total_out > zstream->total_in)
		return -E2BIG;

	return 0;
}

243
static void compress_fini(struct compress *c,
244 245
			  struct drm_i915_error_object *dst)
{
246 247
	struct z_stream_s *zstream = &c->zstream;

248 249 250 251 252 253 254
	if (dst) {
		zlib_deflate(zstream, Z_FINISH);
		dst->unused = zstream->avail_out;
	}

	zlib_deflateEnd(zstream);
	kfree(zstream->workspace);
255 256 257

	if (c->tmp)
		free_page((unsigned long)c->tmp);
258 259 260 261 262 263 264 265 266
}

static void err_compression_marker(struct drm_i915_error_state_buf *m)
{
	err_puts(m, ":");
}

#else

267 268 269 270
struct compress {
};

static bool compress_init(struct compress *c)
271 272 273 274
{
	return true;
}

275
static int compress_page(struct compress *c,
276 277 278 279
			 void *src,
			 struct drm_i915_error_object *dst)
{
	unsigned long page;
280
	void *ptr;
281 282 283 284 285

	page = __get_free_page(GFP_ATOMIC | __GFP_NOWARN);
	if (!page)
		return -ENOMEM;

286 287 288 289
	ptr = (void *)page;
	if (!i915_memcpy_from_wc(ptr, src, PAGE_SIZE))
		memcpy(ptr, src, PAGE_SIZE);
	dst->pages[dst->page_count++] = ptr;
290 291 292 293

	return 0;
}

294
static void compress_fini(struct compress *c,
295 296 297 298 299 300 301 302 303 304 305
			  struct drm_i915_error_object *dst)
{
}

static void err_compression_marker(struct drm_i915_error_state_buf *m)
{
	err_puts(m, "~");
}

#endif

306 307 308 309 310
static void print_error_buffers(struct drm_i915_error_state_buf *m,
				const char *name,
				struct drm_i915_error_buffer *err,
				int count)
{
311 312
	int i;

313
	err_printf(m, "%s [%d]:\n", name, count);
314 315

	while (count--) {
316 317 318
		err_printf(m, "    %08x_%08x %8u %02x %02x [ ",
			   upper_32_bits(err->gtt_offset),
			   lower_32_bits(err->gtt_offset),
319 320
			   err->size,
			   err->read_domains,
321
			   err->write_domain);
322
		for (i = 0; i < I915_NUM_ENGINES; i++)
323 324 325
			err_printf(m, "%02x ", err->rseqno[i]);

		err_printf(m, "] %02x", err->wseqno);
326 327 328
		err_puts(m, tiling_flag(err->tiling));
		err_puts(m, dirty_flag(err->dirty));
		err_puts(m, purgeable_flag(err->purgeable));
329
		err_puts(m, err->userptr ? " userptr" : "");
330 331
		err_puts(m, err->engine != -1 ? " " : "");
		err_puts(m, engine_str(err->engine));
332
		err_puts(m, i915_cache_level_str(m->i915, err->cache_level));
333 334 335 336 337 338 339 340 341 342 343

		if (err->name)
			err_printf(m, " (name: %d)", err->name);
		if (err->fence_reg != I915_FENCE_REG_NONE)
			err_printf(m, " (fence: %d)", err->fence_reg);

		err_puts(m, "\n");
		err++;
	}
}

344 345 346
static void error_print_instdone(struct drm_i915_error_state_buf *m,
				 struct drm_i915_error_engine *ee)
{
347 348 349
	int slice;
	int subslice;

350 351 352 353 354 355 356 357 358 359 360 361
	err_printf(m, "  INSTDONE: 0x%08x\n",
		   ee->instdone.instdone);

	if (ee->engine_id != RCS || INTEL_GEN(m->i915) <= 3)
		return;

	err_printf(m, "  SC_INSTDONE: 0x%08x\n",
		   ee->instdone.slice_common);

	if (INTEL_GEN(m->i915) <= 6)
		return;

362 363 364 365 366 367 368 369 370
	for_each_instdone_slice_subslice(m->i915, slice, subslice)
		err_printf(m, "  SAMPLER_INSTDONE[%d][%d]: 0x%08x\n",
			   slice, subslice,
			   ee->instdone.sampler[slice][subslice]);

	for_each_instdone_slice_subslice(m->i915, slice, subslice)
		err_printf(m, "  ROW_INSTDONE[%d][%d]: 0x%08x\n",
			   slice, subslice,
			   ee->instdone.row[slice][subslice]);
371 372
}

373 374 375 376 377 378 379
static void error_print_request(struct drm_i915_error_state_buf *m,
				const char *prefix,
				struct drm_i915_error_request *erq)
{
	if (!erq->seqno)
		return;

380 381
	err_printf(m, "%s pid %d, ban score %d, seqno %8x:%08x, emitted %dms ago, head %08x, tail %08x\n",
		   prefix, erq->pid, erq->ban_score,
382 383 384 385 386
		   erq->context, erq->seqno,
		   jiffies_to_msecs(jiffies - erq->jiffies),
		   erq->head, erq->tail);
}

387 388
static void error_print_engine(struct drm_i915_error_state_buf *m,
			       struct drm_i915_error_engine *ee)
389
{
390 391
	err_printf(m, "%s command stream:\n", engine_str(ee->engine_id));
	err_printf(m, "  START: 0x%08x\n", ee->start);
392
	err_printf(m, "  HEAD:  0x%08x [0x%08x]\n", ee->head, ee->rq_head);
393 394
	err_printf(m, "  TAIL:  0x%08x [0x%08x, 0x%08x]\n",
		   ee->tail, ee->rq_post, ee->rq_tail);
395
	err_printf(m, "  CTL:   0x%08x\n", ee->ctl);
396
	err_printf(m, "  MODE:  0x%08x\n", ee->mode);
397 398 399 400 401
	err_printf(m, "  HWS:   0x%08x\n", ee->hws);
	err_printf(m, "  ACTHD: 0x%08x %08x\n",
		   (u32)(ee->acthd>>32), (u32)ee->acthd);
	err_printf(m, "  IPEIR: 0x%08x\n", ee->ipeir);
	err_printf(m, "  IPEHR: 0x%08x\n", ee->ipehr);
402 403 404

	error_print_instdone(m, ee);

405 406 407 408 409 410 411 412
	if (ee->batchbuffer) {
		u64 start = ee->batchbuffer->gtt_offset;
		u64 end = start + ee->batchbuffer->gtt_size;

		err_printf(m, "  batch: [0x%08x_%08x, 0x%08x_%08x]\n",
			   upper_32_bits(start), lower_32_bits(start),
			   upper_32_bits(end), lower_32_bits(end));
	}
413
	if (INTEL_GEN(m->i915) >= 4) {
414
		err_printf(m, "  BBADDR: 0x%08x_%08x\n",
415 416 417
			   (u32)(ee->bbaddr>>32), (u32)ee->bbaddr);
		err_printf(m, "  BB_STATE: 0x%08x\n", ee->bbstate);
		err_printf(m, "  INSTPS: 0x%08x\n", ee->instps);
418
	}
419 420 421 422 423 424
	err_printf(m, "  INSTPM: 0x%08x\n", ee->instpm);
	err_printf(m, "  FADDR: 0x%08x %08x\n", upper_32_bits(ee->faddr),
		   lower_32_bits(ee->faddr));
	if (INTEL_GEN(m->i915) >= 6) {
		err_printf(m, "  RC PSMI: 0x%08x\n", ee->rc_psmi);
		err_printf(m, "  FAULT_REG: 0x%08x\n", ee->fault_reg);
425 426 427 428 429 430 431
		err_printf(m, "  SYNC_0: 0x%08x\n",
			   ee->semaphore_mboxes[0]);
		err_printf(m, "  SYNC_1: 0x%08x\n",
			   ee->semaphore_mboxes[1]);
		if (HAS_VEBOX(m->i915))
			err_printf(m, "  SYNC_2: 0x%08x\n",
				   ee->semaphore_mboxes[2]);
432
	}
433 434
	if (USES_PPGTT(m->i915)) {
		err_printf(m, "  GFX_MODE: 0x%08x\n", ee->vm_info.gfx_mode);
435

436
		if (INTEL_GEN(m->i915) >= 8) {
437 438 439
			int i;
			for (i = 0; i < 4; i++)
				err_printf(m, "  PDP%d: 0x%016llx\n",
440
					   i, ee->vm_info.pdp[i]);
441 442
		} else {
			err_printf(m, "  PP_DIR_BASE: 0x%08x\n",
443
				   ee->vm_info.pp_dir_base);
444 445
		}
	}
446 447 448 449 450
	err_printf(m, "  seqno: 0x%08x\n", ee->seqno);
	err_printf(m, "  last_seqno: 0x%08x\n", ee->last_seqno);
	err_printf(m, "  waiting: %s\n", yesno(ee->waiting));
	err_printf(m, "  ring->head: 0x%08x\n", ee->cpu_ring_head);
	err_printf(m, "  ring->tail: 0x%08x\n", ee->cpu_ring_tail);
451 452 453 454 455 456 457
	err_printf(m, "  hangcheck stall: %s\n", yesno(ee->hangcheck_stalled));
	err_printf(m, "  hangcheck action: %s\n",
		   hangcheck_action_to_str(ee->hangcheck_action));
	err_printf(m, "  hangcheck action timestamp: %lu, %u ms ago\n",
		   ee->hangcheck_timestamp,
		   jiffies_to_msecs(jiffies - ee->hangcheck_timestamp));

458 459
	error_print_request(m, "  ELSP[0]: ", &ee->execlist[0]);
	error_print_request(m, "  ELSP[1]: ", &ee->execlist[1]);
460 461 462 463 464 465 466 467 468 469 470
}

void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
{
	va_list args;

	va_start(args, f);
	i915_error_vprintf(e, f, args);
	va_end(args);
}

471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493
static int
ascii85_encode_len(int len)
{
	return DIV_ROUND_UP(len, 4);
}

static bool
ascii85_encode(u32 in, char *out)
{
	int i;

	if (in == 0)
		return false;

	out[5] = '\0';
	for (i = 5; i--; ) {
		out[i] = '!' + in % 85;
		in /= 85;
	}

	return true;
}

494
static void print_error_obj(struct drm_i915_error_state_buf *m,
495 496
			    struct intel_engine_cs *engine,
			    const char *name,
497 498
			    struct drm_i915_error_object *obj)
{
499 500
	char out[6];
	int page;
501

502 503 504 505 506 507 508 509 510 511
	if (!obj)
		return;

	if (name) {
		err_printf(m, "%s --- %s = 0x%08x %08x\n",
			   engine ? engine->name : "global", name,
			   upper_32_bits(obj->gtt_offset),
			   lower_32_bits(obj->gtt_offset));
	}

512 513 514 515 516 517 518 519 520 521 522 523 524 525
	err_compression_marker(m);
	for (page = 0; page < obj->page_count; page++) {
		int i, len;

		len = PAGE_SIZE;
		if (page == obj->page_count - 1)
			len -= obj->unused;
		len = ascii85_encode_len(len);

		for (i = 0; i < len; i++) {
			if (ascii85_encode(obj->pages[page][i], out))
				err_puts(m, out);
			else
				err_puts(m, "z");
526 527
		}
	}
528
	err_puts(m, "\n");
529 530
}

531 532 533 534
static void err_print_capabilities(struct drm_i915_error_state_buf *m,
				   const struct intel_device_info *info)
{
#define PRINT_FLAG(x)  err_printf(m, #x ": %s\n", yesno(info->x))
535
	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
536 537 538
#undef PRINT_FLAG
}

539 540 541
int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
			    const struct i915_error_state_file_priv *error_priv)
{
542
	struct drm_i915_private *dev_priv = error_priv->i915;
D
David Weinehall 已提交
543
	struct pci_dev *pdev = dev_priv->drm.pdev;
544
	struct drm_i915_error_state *error = error_priv->error;
545
	struct drm_i915_error_object *obj;
546
	int i, j;
547 548 549 550 551 552

	if (!error) {
		err_printf(m, "no error state collected\n");
		goto out;
	}

553
	err_printf(m, "%s\n", error->error_msg);
554
	err_printf(m, "Kernel: " UTS_RELEASE "\n");
555 556 557 558 559 560
	err_printf(m, "Time: %ld s %ld us\n",
		   error->time.tv_sec, error->time.tv_usec);
	err_printf(m, "Boottime: %ld s %ld us\n",
		   error->boottime.tv_sec, error->boottime.tv_usec);
	err_printf(m, "Uptime: %ld s %ld us\n",
		   error->uptime.tv_sec, error->uptime.tv_usec);
561
	err_print_capabilities(m, &error->device_info);
562

563
	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
564
		if (error->engine[i].hangcheck_stalled &&
565
		    error->engine[i].pid != -1) {
566
			err_printf(m, "Active process (on ring %s): %s [%d], context bans %d\n",
567 568
				   engine_str(i),
				   error->engine[i].comm,
569 570
				   error->engine[i].pid,
				   error->engine[i].context_bans);
571 572
		}
	}
573
	err_printf(m, "Reset count: %u\n", error->reset_count);
574
	err_printf(m, "Suspend count: %u\n", error->suspend_count);
575
	err_printf(m, "Platform: %s\n", intel_platform_name(error->device_info.platform));
D
David Weinehall 已提交
576 577
	err_printf(m, "PCI ID: 0x%04x\n", pdev->device);
	err_printf(m, "PCI Revision: 0x%02x\n", pdev->revision);
578
	err_printf(m, "PCI Subsystem: %04x:%04x\n",
D
David Weinehall 已提交
579 580
		   pdev->subsystem_vendor,
		   pdev->subsystem_device);
581
	err_printf(m, "IOMMU enabled?: %d\n", error->iommu);
582

583
	if (HAS_CSR(dev_priv)) {
584 585 586 587 588 589 590 591 592
		struct intel_csr *csr = &dev_priv->csr;

		err_printf(m, "DMC loaded: %s\n",
			   yesno(csr->dmc_payload != NULL));
		err_printf(m, "DMC fw version: %d.%d\n",
			   CSR_VERSION_MAJOR(csr->version),
			   CSR_VERSION_MINOR(csr->version));
	}

593 594
	err_printf(m, "EIR: 0x%08x\n", error->eir);
	err_printf(m, "IER: 0x%08x\n", error->ier);
595
	if (INTEL_GEN(dev_priv) >= 8) {
596 597 598
		for (i = 0; i < 4; i++)
			err_printf(m, "GTIER gt %d: 0x%08x\n", i,
				   error->gtier[i]);
599
	} else if (HAS_PCH_SPLIT(dev_priv) || IS_VALLEYVIEW(dev_priv))
600
		err_printf(m, "GTIER: 0x%08x\n", error->gtier[0]);
601 602 603 604
	err_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
	err_printf(m, "FORCEWAKE: 0x%08x\n", error->forcewake);
	err_printf(m, "DERRMR: 0x%08x\n", error->derrmr);
	err_printf(m, "CCID: 0x%08x\n", error->ccid);
605
	err_printf(m, "Missed interrupts: 0x%08lx\n", dev_priv->gpu_error.missed_irq_rings);
606 607 608 609

	for (i = 0; i < dev_priv->num_fence_regs; i++)
		err_printf(m, "  fence[%d] = %08llx\n", i, error->fence[i]);

610
	if (INTEL_GEN(dev_priv) >= 6) {
611
		err_printf(m, "ERROR: 0x%08x\n", error->error);
612

613
		if (INTEL_GEN(dev_priv) >= 8)
614 615 616
			err_printf(m, "FAULT_TLB_DATA: 0x%08x 0x%08x\n",
				   error->fault_data1, error->fault_data0);

617 618 619
		err_printf(m, "DONE_REG: 0x%08x\n", error->done_reg);
	}

620
	if (IS_GEN7(dev_priv))
621 622
		err_printf(m, "ERR_INT: 0x%08x\n", error->err_int);

623 624 625 626
	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
		if (error->engine[i].engine_id != -1)
			error_print_engine(m, &error->engine[i]);
	}
627

628 629 630
	for (i = 0; i < ARRAY_SIZE(error->active_vm); i++) {
		char buf[128];
		int len, first = 1;
631

632 633 634 635 636 637 638 639 640 641
		if (!error->active_vm[i])
			break;

		len = scnprintf(buf, sizeof(buf), "Active (");
		for (j = 0; j < ARRAY_SIZE(error->engine); j++) {
			if (error->engine[j].vm != error->active_vm[i])
				continue;

			len += scnprintf(buf + len, sizeof(buf), "%s%s",
					 first ? "" : ", ",
642
					 dev_priv->engine[j]->name);
643 644 645 646
			first = 0;
		}
		scnprintf(buf + len, sizeof(buf), ")");
		print_error_buffers(m, buf,
647 648 649
				    error->active_bo[i],
				    error->active_bo_count[i]);
	}
650

651 652 653 654
	print_error_buffers(m, "Pinned (global)",
			    error->pinned_bo,
			    error->pinned_bo_count);

655 656 657 658
	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
		struct drm_i915_error_engine *ee = &error->engine[i];

		obj = ee->batchbuffer;
659
		if (obj) {
660
			err_puts(m, dev_priv->engine[i]->name);
661
			if (ee->pid != -1)
662
				err_printf(m, " (submitted by %s [%d], bans %d)",
663
					   ee->comm,
664 665
					   ee->pid,
					   ee->context_bans);
666 667 668
			err_printf(m, " --- gtt_offset = 0x%08x %08x\n",
				   upper_32_bits(obj->gtt_offset),
				   lower_32_bits(obj->gtt_offset));
669
			print_error_obj(m, dev_priv->engine[i], NULL, obj);
670 671
		}

672
		if (ee->num_requests) {
673
			err_printf(m, "%s --- %d requests\n",
674
				   dev_priv->engine[i]->name,
675
				   ee->num_requests);
676 677
			for (j = 0; j < ee->num_requests; j++)
				error_print_request(m, " ", &ee->requests[j]);
678 679
		}

680 681
		if (IS_ERR(ee->waiters)) {
			err_printf(m, "%s --- ? waiters [unable to acquire spinlock]\n",
682
				   dev_priv->engine[i]->name);
683
		} else if (ee->num_waiters) {
684
			err_printf(m, "%s --- %d waiters\n",
685
				   dev_priv->engine[i]->name,
686 687
				   ee->num_waiters);
			for (j = 0; j < ee->num_waiters; j++) {
688
				err_printf(m, " seqno 0x%08x for %s [%d]\n",
689 690 691
					   ee->waiters[j].seqno,
					   ee->waiters[j].comm,
					   ee->waiters[j].pid);
692 693 694
			}
		}

695
		print_error_obj(m, dev_priv->engine[i],
696
				"ringbuffer", ee->ringbuffer);
697

698
		print_error_obj(m, dev_priv->engine[i],
699
				"HW Status", ee->hws_page);
700

701
		print_error_obj(m, dev_priv->engine[i],
702
				"HW context", ee->ctx);
703

704
		print_error_obj(m, dev_priv->engine[i],
705
				"WA context", ee->wa_ctx);
706

707
		print_error_obj(m, dev_priv->engine[i],
708
				"WA batchbuffer", ee->wa_batchbuffer);
709 710
	}

711
	print_error_obj(m, NULL, "Semaphores", error->semaphore);
712

713 714
	print_error_obj(m, NULL, "GuC log buffer", error->guc_log);

715 716 717 718
	if (error->overlay)
		intel_overlay_print_error_state(m, error->overlay);

	if (error->display)
719
		intel_display_print_error_state(m, dev_priv, error->display);
720 721 722 723 724 725 726 727 728

out:
	if (m->bytes == 0 && m->err)
		return m->err;

	return 0;
}

int i915_error_state_buf_init(struct drm_i915_error_state_buf *ebuf,
729
			      struct drm_i915_private *i915,
730 731 732
			      size_t count, loff_t pos)
{
	memset(ebuf, 0, sizeof(*ebuf));
733
	ebuf->i915 = i915;
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767

	/* We need to have enough room to store any i915_error_state printf
	 * so that we can move it to start position.
	 */
	ebuf->size = count + 1 > PAGE_SIZE ? count + 1 : PAGE_SIZE;
	ebuf->buf = kmalloc(ebuf->size,
				GFP_TEMPORARY | __GFP_NORETRY | __GFP_NOWARN);

	if (ebuf->buf == NULL) {
		ebuf->size = PAGE_SIZE;
		ebuf->buf = kmalloc(ebuf->size, GFP_TEMPORARY);
	}

	if (ebuf->buf == NULL) {
		ebuf->size = 128;
		ebuf->buf = kmalloc(ebuf->size, GFP_TEMPORARY);
	}

	if (ebuf->buf == NULL)
		return -ENOMEM;

	ebuf->start = pos;

	return 0;
}

static void i915_error_object_free(struct drm_i915_error_object *obj)
{
	int page;

	if (obj == NULL)
		return;

	for (page = 0; page < obj->page_count; page++)
768
		free_page((unsigned long)obj->pages[page]);
769 770 771 772 773 774 775 776 777 778

	kfree(obj);
}

static void i915_error_state_free(struct kref *error_ref)
{
	struct drm_i915_error_state *error = container_of(error_ref,
							  typeof(*error), ref);
	int i;

779 780 781 782 783 784 785 786 787 788 789
	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
		struct drm_i915_error_engine *ee = &error->engine[i];

		i915_error_object_free(ee->batchbuffer);
		i915_error_object_free(ee->wa_batchbuffer);
		i915_error_object_free(ee->ringbuffer);
		i915_error_object_free(ee->hws_page);
		i915_error_object_free(ee->ctx);
		i915_error_object_free(ee->wa_ctx);

		kfree(ee->requests);
790 791
		if (!IS_ERR_OR_NULL(ee->waiters))
			kfree(ee->waiters);
792 793
	}

794
	i915_error_object_free(error->semaphore);
795
	i915_error_object_free(error->guc_log);
796

797
	for (i = 0; i < ARRAY_SIZE(error->active_bo); i++)
798 799
		kfree(error->active_bo[i]);
	kfree(error->pinned_bo);
800

801 802 803 804 805 806
	kfree(error->overlay);
	kfree(error->display);
	kfree(error);
}

static struct drm_i915_error_object *
807
i915_error_object_create(struct drm_i915_private *i915,
C
Chris Wilson 已提交
808
			 struct i915_vma *vma)
809
{
810 811
	struct i915_ggtt *ggtt = &i915->ggtt;
	const u64 slot = ggtt->error_capture.start;
812
	struct drm_i915_error_object *dst;
813
	struct compress compress;
814 815 816
	unsigned long num_pages;
	struct sgt_iter iter;
	dma_addr_t dma;
817

C
Chris Wilson 已提交
818 819 820
	if (!vma)
		return NULL;

821
	num_pages = min_t(u64, vma->size, vma->obj->base.size) >> PAGE_SHIFT;
822
	num_pages = DIV_ROUND_UP(10 * num_pages, 8); /* worstcase zlib growth */
823 824
	dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *),
		      GFP_ATOMIC | __GFP_NOWARN);
C
Chris Wilson 已提交
825
	if (!dst)
826 827
		return NULL;

828 829
	dst->gtt_offset = vma->node.start;
	dst->gtt_size = vma->node.size;
830
	dst->page_count = 0;
831 832
	dst->unused = 0;

833
	if (!compress_init(&compress)) {
834 835 836
		kfree(dst);
		return NULL;
	}
837

838 839 840
	for_each_sgt_dma(dma, iter, vma->pages) {
		void __iomem *s;
		int ret;
841

842 843
		ggtt->base.insert_page(&ggtt->base, dma, slot,
				       I915_CACHE_NONE, 0);
844

845
		s = io_mapping_map_atomic_wc(&ggtt->mappable, slot);
846
		ret = compress_page(&compress, (void  __force *)s, dst);
847
		io_mapping_unmap_atomic(s);
848

849
		if (ret)
850 851
			goto unwind;
	}
852
	goto out;
853 854

unwind:
855 856
	while (dst->page_count--)
		free_page((unsigned long)dst->pages[dst->page_count]);
857
	kfree(dst);
858 859 860
	dst = NULL;

out:
861
	compress_fini(&compress, dst);
862
	ggtt->base.clear_range(&ggtt->base, slot, PAGE_SIZE);
863
	return dst;
864 865
}

866 867 868 869 870 871
/* The error capture is special as tries to run underneath the normal
 * locking rules - so we use the raw version of the i915_gem_active lookup.
 */
static inline uint32_t
__active_get_seqno(struct i915_gem_active *active)
{
872 873 874 875
	struct drm_i915_gem_request *request;

	request = __i915_gem_active_peek(active);
	return request ? request->global_seqno : 0;
876 877 878 879 880
}

static inline int
__active_get_engine_id(struct i915_gem_active *active)
{
881
	struct drm_i915_gem_request *request;
882

883 884
	request = __i915_gem_active_peek(active);
	return request ? request->engine->id : -1;
885 886
}

887
static void capture_bo(struct drm_i915_error_buffer *err,
888
		       struct i915_vma *vma)
889
{
890
	struct drm_i915_gem_object *obj = vma->obj;
891
	int i;
892

893 894
	err->size = obj->base.size;
	err->name = obj->base.name;
895

896
	for (i = 0; i < I915_NUM_ENGINES; i++)
897
		err->rseqno[i] = __active_get_seqno(&vma->last_read[i]);
898 899
	err->wseqno = __active_get_seqno(&obj->frontbuffer_write);
	err->engine = __active_get_engine_id(&obj->frontbuffer_write);
900

901
	err->gtt_offset = vma->node.start;
902 903
	err->read_domains = obj->base.read_domains;
	err->write_domain = obj->base.write_domain;
904
	err->fence_reg = vma->fence ? vma->fence->id : -1;
905
	err->tiling = i915_gem_object_get_tiling(obj);
C
Chris Wilson 已提交
906 907
	err->dirty = obj->mm.dirty;
	err->purgeable = obj->mm.madv != I915_MADV_WILLNEED;
908
	err->userptr = obj->userptr.mm != NULL;
909 910 911
	err->cache_level = obj->cache_level;
}

912 913 914
static u32 capture_error_bo(struct drm_i915_error_buffer *err,
			    int count, struct list_head *head,
			    bool pinned_only)
915
{
B
Ben Widawsky 已提交
916
	struct i915_vma *vma;
917 918
	int i = 0;

919
	list_for_each_entry(vma, head, vm_link) {
920 921 922
		if (pinned_only && !i915_vma_is_pinned(vma))
			continue;

923
		capture_bo(err++, vma);
924 925 926 927 928 929 930
		if (++i == count)
			break;
	}

	return i;
}

931 932 933 934 935 936 937 938 939 940
/* Generate a semi-unique error code. The code is not meant to have meaning, The
 * code's only purpose is to try to prevent false duplicated bug reports by
 * grossly estimating a GPU error state.
 *
 * TODO Ideally, hashing the batchbuffer would be a very nice way to determine
 * the hang if we could strip the GTT offset information from it.
 *
 * It's only a small step better than a random number in its current form.
 */
static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv,
941
					 struct drm_i915_error_state *error,
942
					 int *engine_id)
943 944 945 946 947 948 949 950 951
{
	uint32_t error_code = 0;
	int i;

	/* IPEHR would be an ideal way to detect errors, as it's the gross
	 * measure of "the command that hung." However, has some very common
	 * synchronization commands which almost always appear in the case
	 * strictly a client bug. Use instdone to differentiate those some.
	 */
952
	for (i = 0; i < I915_NUM_ENGINES; i++) {
953
		if (error->engine[i].hangcheck_stalled) {
954 955
			if (engine_id)
				*engine_id = i;
956

957 958
			return error->engine[i].ipehr ^
			       error->engine[i].instdone.instdone;
959 960
		}
	}
961 962 963 964

	return error_code;
}

965
static void i915_gem_record_fences(struct drm_i915_private *dev_priv,
966 967 968 969
				   struct drm_i915_error_state *error)
{
	int i;

970
	if (IS_GEN3(dev_priv) || IS_GEN2(dev_priv)) {
971
		for (i = 0; i < dev_priv->num_fence_regs; i++)
972
			error->fence[i] = I915_READ(FENCE_REG(i));
973
	} else if (IS_GEN5(dev_priv) || IS_GEN4(dev_priv)) {
974 975
		for (i = 0; i < dev_priv->num_fence_regs; i++)
			error->fence[i] = I915_READ64(FENCE_REG_965_LO(i));
976
	} else if (INTEL_GEN(dev_priv) >= 6) {
977 978 979
		for (i = 0; i < dev_priv->num_fence_regs; i++)
			error->fence[i] = I915_READ64(FENCE_REG_GEN6_LO(i));
	}
980 981
}

982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001
static inline u32
gen8_engine_sync_index(struct intel_engine_cs *engine,
		       struct intel_engine_cs *other)
{
	int idx;

	/*
	 * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
	 * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
	 * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
	 * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
	 * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
	 */

	idx = (other - engine) - 1;
	if (idx < 0)
		idx += I915_NUM_ENGINES;

	return idx;
}
1002

1003
static void gen8_record_semaphore_state(struct drm_i915_error_state *error,
1004
					struct intel_engine_cs *engine,
1005
					struct drm_i915_error_engine *ee)
1006
{
1007
	struct drm_i915_private *dev_priv = engine->i915;
1008
	struct intel_engine_cs *to;
1009
	enum intel_engine_id id;
1010

1011
	if (!error->semaphore)
1012
		return;
1013

1014
	for_each_engine(to, dev_priv, id) {
1015 1016 1017
		int idx;
		u16 signal_offset;
		u32 *tmp;
1018

1019
		if (engine == to)
1020 1021
			continue;

1022 1023
		signal_offset =
			(GEN8_SIGNAL_OFFSET(engine, id) & (PAGE_SIZE - 1)) / 4;
1024
		tmp = error->semaphore->pages[0];
1025
		idx = gen8_engine_sync_index(engine, to);
1026

1027
		ee->semaphore_mboxes[idx] = tmp[signal_offset];
1028 1029 1030
	}
}

1031 1032
static void gen6_record_semaphore_state(struct intel_engine_cs *engine,
					struct drm_i915_error_engine *ee)
1033
{
1034 1035 1036 1037
	struct drm_i915_private *dev_priv = engine->i915;

	ee->semaphore_mboxes[0] = I915_READ(RING_SYNC_0(engine->mmio_base));
	ee->semaphore_mboxes[1] = I915_READ(RING_SYNC_1(engine->mmio_base));
1038
	if (HAS_VEBOX(dev_priv))
1039
		ee->semaphore_mboxes[2] =
1040
			I915_READ(RING_SYNC_2(engine->mmio_base));
1041 1042
}

1043 1044
static void error_record_engine_waiters(struct intel_engine_cs *engine,
					struct drm_i915_error_engine *ee)
1045 1046 1047 1048 1049 1050
{
	struct intel_breadcrumbs *b = &engine->breadcrumbs;
	struct drm_i915_error_waiter *waiter;
	struct rb_node *rb;
	int count;

1051 1052
	ee->num_waiters = 0;
	ee->waiters = NULL;
1053

1054 1055 1056
	if (RB_EMPTY_ROOT(&b->waiters))
		return;

1057
	if (!spin_trylock_irq(&b->lock)) {
1058 1059 1060 1061
		ee->waiters = ERR_PTR(-EDEADLK);
		return;
	}

1062 1063 1064
	count = 0;
	for (rb = rb_first(&b->waiters); rb != NULL; rb = rb_next(rb))
		count++;
1065
	spin_unlock_irq(&b->lock);
1066 1067 1068 1069 1070 1071 1072 1073 1074

	waiter = NULL;
	if (count)
		waiter = kmalloc_array(count,
				       sizeof(struct drm_i915_error_waiter),
				       GFP_ATOMIC);
	if (!waiter)
		return;

1075
	if (!spin_trylock_irq(&b->lock)) {
1076 1077 1078 1079
		kfree(waiter);
		ee->waiters = ERR_PTR(-EDEADLK);
		return;
	}
1080

1081
	ee->waiters = waiter;
1082
	for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
G
Geliang Tang 已提交
1083
		struct intel_wait *w = rb_entry(rb, typeof(*w), node);
1084 1085 1086 1087 1088 1089

		strcpy(waiter->comm, w->tsk->comm);
		waiter->pid = w->tsk->pid;
		waiter->seqno = w->seqno;
		waiter++;

1090
		if (++ee->num_waiters == count)
1091 1092
			break;
	}
1093
	spin_unlock_irq(&b->lock);
1094 1095
}

1096 1097 1098
static void error_record_engine_registers(struct drm_i915_error_state *error,
					  struct intel_engine_cs *engine,
					  struct drm_i915_error_engine *ee)
1099
{
1100 1101
	struct drm_i915_private *dev_priv = engine->i915;

1102
	if (INTEL_GEN(dev_priv) >= 6) {
1103 1104
		ee->rc_psmi = I915_READ(RING_PSMI_CTL(engine->mmio_base));
		ee->fault_reg = I915_READ(RING_FAULT_REG(engine));
1105
		if (INTEL_GEN(dev_priv) >= 8)
1106
			gen8_record_semaphore_state(error, engine, ee);
1107
		else
1108
			gen6_record_semaphore_state(engine, ee);
1109 1110
	}

1111
	if (INTEL_GEN(dev_priv) >= 4) {
1112 1113 1114 1115 1116
		ee->faddr = I915_READ(RING_DMA_FADD(engine->mmio_base));
		ee->ipeir = I915_READ(RING_IPEIR(engine->mmio_base));
		ee->ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
		ee->instps = I915_READ(RING_INSTPS(engine->mmio_base));
		ee->bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
1117
		if (INTEL_GEN(dev_priv) >= 8) {
1118 1119
			ee->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(engine->mmio_base)) << 32;
			ee->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(engine->mmio_base)) << 32;
1120
		}
1121
		ee->bbstate = I915_READ(RING_BBSTATE(engine->mmio_base));
1122
	} else {
1123 1124 1125
		ee->faddr = I915_READ(DMA_FADD_I8XX);
		ee->ipeir = I915_READ(IPEIR);
		ee->ipehr = I915_READ(IPEHR);
1126 1127
	}

1128
	intel_engine_get_instdone(engine, &ee->instdone);
1129

1130 1131
	ee->waiting = intel_engine_has_waiter(engine);
	ee->instpm = I915_READ(RING_INSTPM(engine->mmio_base));
1132
	ee->acthd = intel_engine_get_active_head(engine);
1133
	ee->seqno = intel_engine_get_seqno(engine);
1134
	ee->last_seqno = intel_engine_last_submit(engine);
1135 1136 1137 1138
	ee->start = I915_READ_START(engine);
	ee->head = I915_READ_HEAD(engine);
	ee->tail = I915_READ_TAIL(engine);
	ee->ctl = I915_READ_CTL(engine);
1139 1140
	if (INTEL_GEN(dev_priv) > 2)
		ee->mode = I915_READ_MODE(engine);
1141

1142
	if (!HWS_NEEDS_PHYSICAL(dev_priv)) {
1143
		i915_reg_t mmio;
1144

1145
		if (IS_GEN7(dev_priv)) {
1146
			switch (engine->id) {
1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160
			default:
			case RCS:
				mmio = RENDER_HWS_PGA_GEN7;
				break;
			case BCS:
				mmio = BLT_HWS_PGA_GEN7;
				break;
			case VCS:
				mmio = BSD_HWS_PGA_GEN7;
				break;
			case VECS:
				mmio = VEBOX_HWS_PGA_GEN7;
				break;
			}
1161
		} else if (IS_GEN6(engine->i915)) {
1162
			mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
1163 1164
		} else {
			/* XXX: gen8 returns to sanity */
1165
			mmio = RING_HWS_PGA(engine->mmio_base);
1166 1167
		}

1168
		ee->hws = I915_READ(mmio);
1169 1170
	}

1171
	ee->hangcheck_timestamp = engine->hangcheck.action_timestamp;
1172
	ee->hangcheck_action = engine->hangcheck.action;
1173
	ee->hangcheck_stalled = engine->hangcheck.stalled;
1174

1175
	if (USES_PPGTT(dev_priv)) {
1176 1177
		int i;

1178
		ee->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(engine));
1179

1180
		if (IS_GEN6(dev_priv))
1181
			ee->vm_info.pp_dir_base =
1182
				I915_READ(RING_PP_DIR_BASE_READ(engine));
1183
		else if (IS_GEN7(dev_priv))
1184
			ee->vm_info.pp_dir_base =
1185
				I915_READ(RING_PP_DIR_BASE(engine));
1186
		else if (INTEL_GEN(dev_priv) >= 8)
1187
			for (i = 0; i < 4; i++) {
1188
				ee->vm_info.pdp[i] =
1189
					I915_READ(GEN8_RING_PDP_UDW(engine, i));
1190 1191
				ee->vm_info.pdp[i] <<= 32;
				ee->vm_info.pdp[i] |=
1192
					I915_READ(GEN8_RING_PDP_LDW(engine, i));
1193 1194
			}
	}
1195 1196
}

1197 1198 1199 1200
static void record_request(struct drm_i915_gem_request *request,
			   struct drm_i915_error_request *erq)
{
	erq->context = request->ctx->hw_id;
1201
	erq->ban_score = request->ctx->ban_score;
1202
	erq->seqno = request->global_seqno;
1203 1204 1205 1206 1207 1208 1209 1210 1211
	erq->jiffies = request->emitted_jiffies;
	erq->head = request->head;
	erq->tail = request->tail;

	rcu_read_lock();
	erq->pid = request->ctx->pid ? pid_nr(request->ctx->pid) : 0;
	rcu_read_unlock();
}

1212 1213 1214 1215 1216 1217 1218 1219 1220
static void engine_record_requests(struct intel_engine_cs *engine,
				   struct drm_i915_gem_request *first,
				   struct drm_i915_error_engine *ee)
{
	struct drm_i915_gem_request *request;
	int count;

	count = 0;
	request = first;
1221
	list_for_each_entry_from(request, &engine->timeline->requests, link)
1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
		count++;
	if (!count)
		return;

	ee->requests = kcalloc(count, sizeof(*ee->requests), GFP_ATOMIC);
	if (!ee->requests)
		return;

	ee->num_requests = count;

	count = 0;
	request = first;
1234
	list_for_each_entry_from(request, &engine->timeline->requests, link) {
1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253
		if (count >= ee->num_requests) {
			/*
			 * If the ring request list was changed in
			 * between the point where the error request
			 * list was created and dimensioned and this
			 * point then just exit early to avoid crashes.
			 *
			 * We don't need to communicate that the
			 * request list changed state during error
			 * state capture and that the error state is
			 * slightly incorrect as a consequence since we
			 * are typically only interested in the request
			 * list state at the point of error state
			 * capture, not in any changes happening during
			 * the capture.
			 */
			break;
		}

1254
		record_request(request, &ee->requests[count++]);
1255 1256 1257 1258
	}
	ee->num_requests = count;
}

1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269
static void error_record_engine_execlists(struct intel_engine_cs *engine,
					  struct drm_i915_error_engine *ee)
{
	unsigned int n;

	for (n = 0; n < ARRAY_SIZE(engine->execlist_port); n++)
		if (engine->execlist_port[n].request)
			record_request(engine->execlist_port[n].request,
				       &ee->execlist[n]);
}

1270
static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
1271 1272
				  struct drm_i915_error_state *error)
{
1273
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
1274
	int i;
1275

1276
	error->semaphore =
C
Chris Wilson 已提交
1277
		i915_error_object_create(dev_priv, dev_priv->semaphore);
1278

1279
	for (i = 0; i < I915_NUM_ENGINES; i++) {
1280
		struct intel_engine_cs *engine = dev_priv->engine[i];
1281
		struct drm_i915_error_engine *ee = &error->engine[i];
1282
		struct drm_i915_gem_request *request;
1283

1284 1285
		ee->pid = -1;
		ee->engine_id = -1;
1286

1287
		if (!engine)
1288 1289
			continue;

1290
		ee->engine_id = i;
1291

1292 1293
		error_record_engine_registers(error, engine, ee);
		error_record_engine_waiters(engine, ee);
1294
		error_record_engine_execlists(engine, ee);
1295

1296
		request = i915_gem_find_active_request(engine);
1297
		if (request) {
1298
			struct intel_ring *ring;
1299
			struct pid *pid;
1300

1301
			ee->vm = request->ctx->ppgtt ?
1302
				&request->ctx->ppgtt->base : &ggtt->base;
1303

1304 1305 1306 1307
			/* We need to copy these to an anonymous buffer
			 * as the simplest method to avoid being overwritten
			 * by userspace.
			 */
1308
			ee->batchbuffer =
1309
				i915_error_object_create(dev_priv,
C
Chris Wilson 已提交
1310
							 request->batch);
1311

1312
			if (HAS_BROKEN_CS_TLB(dev_priv))
1313
				ee->wa_batchbuffer =
C
Chris Wilson 已提交
1314 1315
					i915_error_object_create(dev_priv,
								 engine->scratch);
1316

C
Chris Wilson 已提交
1317 1318 1319
			ee->ctx =
				i915_error_object_create(dev_priv,
							 request->ctx->engine[i].state);
1320

1321 1322
			pid = request->ctx->pid;
			if (pid) {
1323 1324 1325
				struct task_struct *task;

				rcu_read_lock();
1326
				task = pid_task(pid, PIDTYPE_PID);
1327
				if (task) {
1328 1329
					strcpy(ee->comm, task->comm);
					ee->pid = task->pid;
1330 1331 1332
				}
				rcu_read_unlock();
			}
1333

1334
			error->simulated |=
1335
				i915_gem_context_no_error_capture(request->ctx);
1336

1337 1338 1339 1340
			ee->rq_head = request->head;
			ee->rq_post = request->postfix;
			ee->rq_tail = request->tail;

1341 1342 1343
			ring = request->ring;
			ee->cpu_ring_head = ring->head;
			ee->cpu_ring_tail = ring->tail;
1344
			ee->ringbuffer =
C
Chris Wilson 已提交
1345
				i915_error_object_create(dev_priv, ring->vma);
1346 1347

			engine_record_requests(engine, request, ee);
1348
		}
1349

1350
		ee->hws_page =
C
Chris Wilson 已提交
1351 1352
			i915_error_object_create(dev_priv,
						 engine->status_page.vma);
1353

C
Chris Wilson 已提交
1354 1355
		ee->wa_ctx =
			i915_error_object_create(dev_priv, engine->wa_ctx.vma);
1356 1357 1358
	}
}

1359 1360 1361
static void i915_gem_capture_vm(struct drm_i915_private *dev_priv,
				struct drm_i915_error_state *error,
				struct i915_address_space *vm,
1362
				int idx)
1363
{
1364
	struct drm_i915_error_buffer *active_bo;
1365
	struct i915_vma *vma;
1366
	int count;
1367

1368
	count = 0;
1369
	list_for_each_entry(vma, &vm->active_list, vm_link)
1370
		count++;
1371

1372 1373 1374
	active_bo = NULL;
	if (count)
		active_bo = kcalloc(count, sizeof(*active_bo), GFP_ATOMIC);
1375
	if (active_bo)
1376 1377 1378 1379 1380 1381 1382
		count = capture_error_bo(active_bo, count, &vm->active_list, false);
	else
		count = 0;

	error->active_vm[idx] = vm;
	error->active_bo[idx] = active_bo;
	error->active_bo_count[idx] = count;
1383 1384
}

1385 1386
static void i915_capture_active_buffers(struct drm_i915_private *dev_priv,
					struct drm_i915_error_state *error)
1387
{
1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
	int cnt = 0, i, j;

	BUILD_BUG_ON(ARRAY_SIZE(error->engine) > ARRAY_SIZE(error->active_bo));
	BUILD_BUG_ON(ARRAY_SIZE(error->active_bo) != ARRAY_SIZE(error->active_vm));
	BUILD_BUG_ON(ARRAY_SIZE(error->active_bo) != ARRAY_SIZE(error->active_bo_count));

	/* Scan each engine looking for unique active contexts/vm */
	for (i = 0; i < ARRAY_SIZE(error->engine); i++) {
		struct drm_i915_error_engine *ee = &error->engine[i];
		bool found;

		if (!ee->vm)
			continue;
1401

1402 1403 1404 1405 1406
		found = false;
		for (j = 0; j < i && !found; j++)
			found = error->engine[j].vm == ee->vm;
		if (!found)
			i915_gem_capture_vm(dev_priv, error, ee->vm, cnt++);
1407
	}
1408 1409
}

1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440
static void i915_capture_pinned_buffers(struct drm_i915_private *dev_priv,
					struct drm_i915_error_state *error)
{
	struct i915_address_space *vm = &dev_priv->ggtt.base;
	struct drm_i915_error_buffer *bo;
	struct i915_vma *vma;
	int count_inactive, count_active;

	count_inactive = 0;
	list_for_each_entry(vma, &vm->active_list, vm_link)
		count_inactive++;

	count_active = 0;
	list_for_each_entry(vma, &vm->inactive_list, vm_link)
		count_active++;

	bo = NULL;
	if (count_inactive + count_active)
		bo = kcalloc(count_inactive + count_active,
			     sizeof(*bo), GFP_ATOMIC);
	if (!bo)
		return;

	count_inactive = capture_error_bo(bo, count_inactive,
					  &vm->active_list, true);
	count_active = capture_error_bo(bo + count_inactive, count_active,
					&vm->inactive_list, true);
	error->pinned_bo_count = count_inactive + count_active;
	error->pinned_bo = bo;
}

1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
static void i915_gem_capture_guc_log_buffer(struct drm_i915_private *dev_priv,
					    struct drm_i915_error_state *error)
{
	/* Capturing log buf contents won't be useful if logging was disabled */
	if (!dev_priv->guc.log.vma || (i915.guc_log_level < 0))
		return;

	error->guc_log = i915_error_object_create(dev_priv,
						  dev_priv->guc.log.vma);
}

1452 1453 1454
/* Capture all registers which don't fit into another category. */
static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
				   struct drm_i915_error_state *error)
1455
{
1456
	int i;
1457

1458 1459 1460 1461 1462 1463 1464
	/* General organization
	 * 1. Registers specific to a single generation
	 * 2. Registers which belong to multiple generations
	 * 3. Feature specific registers.
	 * 4. Everything else
	 * Please try to follow the order.
	 */
1465

1466
	/* 1: Registers specific to a single generation */
1467
	if (IS_VALLEYVIEW(dev_priv)) {
1468
		error->gtier[0] = I915_READ(GTIER);
1469
		error->ier = I915_READ(VLV_IER);
1470
		error->forcewake = I915_READ_FW(FORCEWAKE_VLV);
1471
	}
1472

1473
	if (IS_GEN7(dev_priv))
1474
		error->err_int = I915_READ(GEN7_ERR_INT);
1475

1476
	if (INTEL_GEN(dev_priv) >= 8) {
1477 1478 1479 1480
		error->fault_data0 = I915_READ(GEN8_FAULT_TLB_DATA0);
		error->fault_data1 = I915_READ(GEN8_FAULT_TLB_DATA1);
	}

1481
	if (IS_GEN6(dev_priv)) {
1482
		error->forcewake = I915_READ_FW(FORCEWAKE);
1483 1484 1485
		error->gab_ctl = I915_READ(GAB_CTL);
		error->gfx_mode = I915_READ(GFX_MODE);
	}
1486

1487
	/* 2: Registers which belong to multiple generations */
1488
	if (INTEL_GEN(dev_priv) >= 7)
1489
		error->forcewake = I915_READ_FW(FORCEWAKE_MT);
1490

1491
	if (INTEL_GEN(dev_priv) >= 6) {
1492
		error->derrmr = I915_READ(DERRMR);
1493 1494 1495 1496
		error->error = I915_READ(ERROR_GEN6);
		error->done_reg = I915_READ(DONE_REG);
	}

1497
	/* 3: Feature specific registers */
1498
	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
1499 1500 1501 1502 1503
		error->gam_ecochk = I915_READ(GAM_ECOCHK);
		error->gac_eco = I915_READ(GAC_ECO_BITS);
	}

	/* 4: Everything else */
1504
	if (HAS_HW_CONTEXTS(dev_priv))
1505 1506
		error->ccid = I915_READ(CCID);

1507
	if (INTEL_GEN(dev_priv) >= 8) {
1508 1509 1510
		error->ier = I915_READ(GEN8_DE_MISC_IER);
		for (i = 0; i < 4; i++)
			error->gtier[i] = I915_READ(GEN8_GT_IER(i));
1511
	} else if (HAS_PCH_SPLIT(dev_priv)) {
1512
		error->ier = I915_READ(DEIER);
1513
		error->gtier[0] = I915_READ(GTIER);
1514
	} else if (IS_GEN2(dev_priv)) {
1515
		error->ier = I915_READ16(IER);
1516
	} else if (!IS_VALLEYVIEW(dev_priv)) {
1517
		error->ier = I915_READ(IER);
1518 1519 1520
	}
	error->eir = I915_READ(EIR);
	error->pgtbl_er = I915_READ(PGTBL_ER);
1521 1522
}

1523
static void i915_error_capture_msg(struct drm_i915_private *dev_priv,
1524
				   struct drm_i915_error_state *error,
1525
				   u32 engine_mask,
1526
				   const char *error_msg)
1527 1528
{
	u32 ecode;
1529
	int engine_id = -1, len;
1530

1531
	ecode = i915_error_generate_code(dev_priv, error, &engine_id);
1532

1533
	len = scnprintf(error->error_msg, sizeof(error->error_msg),
1534
			"GPU HANG: ecode %d:%d:0x%08x",
1535
			INTEL_GEN(dev_priv), engine_id, ecode);
1536

1537
	if (engine_id != -1 && error->engine[engine_id].pid != -1)
1538 1539 1540
		len += scnprintf(error->error_msg + len,
				 sizeof(error->error_msg) - len,
				 ", in %s [%d]",
1541 1542
				 error->engine[engine_id].comm,
				 error->engine[engine_id].pid);
1543 1544 1545 1546

	scnprintf(error->error_msg + len, sizeof(error->error_msg) - len,
		  ", reason: %s, action: %s",
		  error_msg,
1547
		  engine_mask ? "reset" : "continue");
1548 1549
}

1550 1551 1552
static void i915_capture_gen_state(struct drm_i915_private *dev_priv,
				   struct drm_i915_error_state *error)
{
1553 1554 1555 1556
	error->iommu = -1;
#ifdef CONFIG_INTEL_IOMMU
	error->iommu = intel_iommu_gfx_mapped;
#endif
1557
	error->reset_count = i915_reset_count(&dev_priv->gpu_error);
1558
	error->suspend_count = dev_priv->suspend_count;
1559 1560 1561 1562

	memcpy(&error->device_info,
	       INTEL_INFO(dev_priv),
	       sizeof(error->device_info));
1563 1564
}

1565 1566 1567 1568 1569 1570 1571 1572 1573 1574
static int capture(void *data)
{
	struct drm_i915_error_state *error = data;

	i915_capture_gen_state(error->i915, error);
	i915_capture_reg_state(error->i915, error);
	i915_gem_record_fences(error->i915, error);
	i915_gem_record_rings(error->i915, error);
	i915_capture_active_buffers(error->i915, error);
	i915_capture_pinned_buffers(error->i915, error);
1575
	i915_gem_capture_guc_log_buffer(error->i915, error);
1576 1577

	do_gettimeofday(&error->time);
1578 1579 1580 1581
	error->boottime = ktime_to_timeval(ktime_get_boottime());
	error->uptime =
		ktime_to_timeval(ktime_sub(ktime_get(),
					   error->i915->gt.last_init_time));
1582 1583 1584 1585 1586 1587 1588

	error->overlay = intel_overlay_capture_error_state(error->i915);
	error->display = intel_display_capture_error_state(error->i915);

	return 0;
}

1589 1590
#define DAY_AS_SECONDS(x) (24 * 60 * 60 * (x))

1591 1592 1593 1594 1595 1596 1597 1598 1599
/**
 * i915_capture_error_state - capture an error record for later analysis
 * @dev: drm device
 *
 * Should be called when an error is detected (either a hang or an error
 * interrupt) to capture error state from the time of the error.  Fills
 * out a structure which becomes available in debugfs for user level tools
 * to pick up.
 */
1600 1601
void i915_capture_error_state(struct drm_i915_private *dev_priv,
			      u32 engine_mask,
1602
			      const char *error_msg)
1603
{
1604
	static bool warned;
1605 1606 1607
	struct drm_i915_error_state *error;
	unsigned long flags;

1608 1609 1610
	if (!i915.error_capture)
		return;

1611 1612 1613
	if (READ_ONCE(dev_priv->gpu_error.first_error))
		return;

1614 1615 1616 1617 1618 1619 1620
	/* Account for pipe specific data like PIPE*STAT */
	error = kzalloc(sizeof(*error), GFP_ATOMIC);
	if (!error) {
		DRM_DEBUG_DRIVER("out of memory, not capturing error state\n");
		return;
	}

1621
	kref_init(&error->ref);
1622
	error->i915 = dev_priv;
1623

1624
	stop_machine(capture, error, NULL);
1625

1626
	i915_error_capture_msg(dev_priv, error, engine_mask, error_msg);
1627 1628
	DRM_INFO("%s\n", error->error_msg);

1629 1630 1631 1632 1633 1634 1635
	if (!error->simulated) {
		spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
		if (!dev_priv->gpu_error.first_error) {
			dev_priv->gpu_error.first_error = error;
			error = NULL;
		}
		spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
1636 1637
	}

1638
	if (error) {
1639
		i915_error_state_free(&error->ref);
1640 1641 1642
		return;
	}

1643 1644
	if (!warned &&
	    ktime_get_real_seconds() - DRIVER_TIMESTAMP < DAY_AS_SECONDS(180)) {
1645 1646 1647 1648
		DRM_INFO("GPU hangs can indicate a bug anywhere in the entire gfx stack, including userspace.\n");
		DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n");
		DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n");
		DRM_INFO("The gpu crash dump is required to analyze gpu hangs, so please always attach it.\n");
1649 1650
		DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n",
			 dev_priv->drm.primary->index);
1651 1652
		warned = true;
	}
1653 1654 1655 1656 1657
}

void i915_error_state_get(struct drm_device *dev,
			  struct i915_error_state_file_priv *error_priv)
{
1658
	struct drm_i915_private *dev_priv = to_i915(dev);
1659

1660
	spin_lock_irq(&dev_priv->gpu_error.lock);
1661 1662 1663
	error_priv->error = dev_priv->gpu_error.first_error;
	if (error_priv->error)
		kref_get(&error_priv->error->ref);
1664
	spin_unlock_irq(&dev_priv->gpu_error.lock);
1665 1666 1667 1668 1669 1670 1671 1672
}

void i915_error_state_put(struct i915_error_state_file_priv *error_priv)
{
	if (error_priv->error)
		kref_put(&error_priv->error->ref, i915_error_state_free);
}

1673
void i915_destroy_error_state(struct drm_i915_private *dev_priv)
1674 1675 1676
{
	struct drm_i915_error_state *error;

1677
	spin_lock_irq(&dev_priv->gpu_error.lock);
1678 1679
	error = dev_priv->gpu_error.first_error;
	dev_priv->gpu_error.first_error = NULL;
1680
	spin_unlock_irq(&dev_priv->gpu_error.lock);
1681 1682 1683 1684

	if (error)
		kref_put(&error->ref, i915_error_state_free);
}