intel_reset.c 34.8 KB
Newer Older
1 2 3 4 5 6 7
/*
 * SPDX-License-Identifier: MIT
 *
 * Copyright © 2008-2018 Intel Corporation
 */

#include <linux/sched/mm.h>
8
#include <linux/stop_machine.h>
9

10
#include "display/intel_display_types.h"
11 12
#include "display/intel_overlay.h"

13 14
#include "gem/i915_gem_context.h"

15 16
#include "i915_drv.h"
#include "i915_gpu_error.h"
17
#include "i915_irq.h"
18
#include "intel_engine_pm.h"
19
#include "intel_gt.h"
20
#include "intel_gt_pm.h"
21
#include "intel_reset.h"
22

23
#include "uc/intel_guc.h"
24

25 26
#define RESET_MAX_RETRIES 3

27 28 29
/* XXX How to handle concurrent GGTT updates using tiling registers? */
#define RESET_UNDER_STOP_MACHINE 0

30 31 32 33 34 35 36 37 38 39
static void rmw_set_fw(struct intel_uncore *uncore, i915_reg_t reg, u32 set)
{
	intel_uncore_rmw_fw(uncore, reg, 0, set);
}

static void rmw_clear_fw(struct intel_uncore *uncore, i915_reg_t reg, u32 clr)
{
	intel_uncore_rmw_fw(uncore, reg, clr, 0);
}

40 41 42 43 44
static void engine_skip_context(struct i915_request *rq)
{
	struct intel_engine_cs *engine = rq->engine;
	struct i915_gem_context *hung_ctx = rq->gem_context;

45 46
	if (!i915_request_is_active(rq))
		return;
47

48
	lockdep_assert_held(&engine->active.lock);
49
	list_for_each_entry_continue(rq, &engine->active.requests, sched.link)
50 51
		if (rq->gem_context == hung_ctx)
			i915_request_skip(rq, -EIO);
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
}

static void client_mark_guilty(struct drm_i915_file_private *file_priv,
			       const struct i915_gem_context *ctx)
{
	unsigned int score;
	unsigned long prev_hang;

	if (i915_gem_context_is_banned(ctx))
		score = I915_CLIENT_SCORE_CONTEXT_BAN;
	else
		score = 0;

	prev_hang = xchg(&file_priv->hang_timestamp, jiffies);
	if (time_before(jiffies, prev_hang + I915_CLIENT_FAST_HANG_JIFFIES))
		score += I915_CLIENT_SCORE_HANG_FAST;

	if (score) {
		atomic_add(score, &file_priv->ban_score);

		DRM_DEBUG_DRIVER("client %s: gained %u ban score, now %u\n",
				 ctx->name, score,
				 atomic_read(&file_priv->ban_score));
	}
}

78
static bool context_mark_guilty(struct i915_gem_context *ctx)
79
{
80 81 82
	unsigned long prev_hang;
	bool banned;
	int i;
83 84 85

	atomic_inc(&ctx->guilty_count);

86 87
	/* Cool contexts are too cool to be banned! (Used for reset testing.) */
	if (!i915_gem_context_is_bannable(ctx))
88
		return false;
89

90 91 92 93 94 95 96 97 98 99
	/* Record the timestamp for the last N hangs */
	prev_hang = ctx->hang_timestamp[0];
	for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp) - 1; i++)
		ctx->hang_timestamp[i] = ctx->hang_timestamp[i + 1];
	ctx->hang_timestamp[i] = jiffies;

	/* If we have hung N+1 times in rapid succession, we ban the context! */
	banned = !i915_gem_context_is_recoverable(ctx);
	if (time_before(jiffies, prev_hang + CONTEXT_FAST_HANG_JIFFIES))
		banned = true;
100
	if (banned) {
101 102
		DRM_DEBUG_DRIVER("context %s: guilty %d, banned\n",
				 ctx->name, atomic_read(&ctx->guilty_count));
103 104 105 106 107
		i915_gem_context_set_banned(ctx);
	}

	if (!IS_ERR_OR_NULL(ctx->file_priv))
		client_mark_guilty(ctx->file_priv, ctx);
108 109

	return banned;
110 111 112 113 114 115 116
}

static void context_mark_innocent(struct i915_gem_context *ctx)
{
	atomic_inc(&ctx->active_count);
}

117
void __i915_request_reset(struct i915_request *rq, bool guilty)
118
{
119 120 121 122 123 124
	GEM_TRACE("%s rq=%llx:%lld, guilty? %s\n",
		  rq->engine->name,
		  rq->fence.context,
		  rq->fence.seqno,
		  yesno(guilty));

125 126 127 128 129 130 131 132 133 134 135 136
	GEM_BUG_ON(i915_request_completed(rq));

	if (guilty) {
		i915_request_skip(rq, -EIO);
		if (context_mark_guilty(rq->gem_context))
			engine_skip_context(rq);
	} else {
		dma_fence_set_error(&rq->fence, -EAGAIN);
		context_mark_innocent(rq->gem_context);
	}
}

137 138 139 140 141 142 143 144
static bool i915_in_reset(struct pci_dev *pdev)
{
	u8 gdrst;

	pci_read_config_byte(pdev, I915_GDRST, &gdrst);
	return gdrst & GRDOM_RESET_STATUS;
}

145
static int i915_do_reset(struct intel_gt *gt,
146
			 intel_engine_mask_t engine_mask,
147 148
			 unsigned int retry)
{
149
	struct pci_dev *pdev = gt->i915->drm.pdev;
150 151 152 153
	int err;

	/* Assert reset for at least 20 usec, and wait for acknowledgement. */
	pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
154 155
	udelay(50);
	err = wait_for_atomic(i915_in_reset(pdev), 50);
156 157 158

	/* Clear the reset request. */
	pci_write_config_byte(pdev, I915_GDRST, 0);
159
	udelay(50);
160
	if (!err)
161
		err = wait_for_atomic(!i915_in_reset(pdev), 50);
162 163 164 165 166 167 168 169 170 171 172 173

	return err;
}

static bool g4x_reset_complete(struct pci_dev *pdev)
{
	u8 gdrst;

	pci_read_config_byte(pdev, I915_GDRST, &gdrst);
	return (gdrst & GRDOM_RESET_ENABLE) == 0;
}

174
static int g33_do_reset(struct intel_gt *gt,
175
			intel_engine_mask_t engine_mask,
176 177
			unsigned int retry)
{
178
	struct pci_dev *pdev = gt->i915->drm.pdev;
179 180

	pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
181
	return wait_for_atomic(g4x_reset_complete(pdev), 50);
182 183
}

184
static int g4x_do_reset(struct intel_gt *gt,
185
			intel_engine_mask_t engine_mask,
186 187
			unsigned int retry)
{
188 189
	struct pci_dev *pdev = gt->i915->drm.pdev;
	struct intel_uncore *uncore = gt->uncore;
190 191 192
	int ret;

	/* WaVcpClkGateDisableForMediaReset:ctg,elk */
193 194
	rmw_set_fw(uncore, VDECCLK_GATE_D, VCP_UNIT_CLOCK_GATE_DISABLE);
	intel_uncore_posting_read_fw(uncore, VDECCLK_GATE_D);
195 196 197

	pci_write_config_byte(pdev, I915_GDRST,
			      GRDOM_MEDIA | GRDOM_RESET_ENABLE);
198
	ret =  wait_for_atomic(g4x_reset_complete(pdev), 50);
199 200 201 202 203 204 205
	if (ret) {
		DRM_DEBUG_DRIVER("Wait for media reset failed\n");
		goto out;
	}

	pci_write_config_byte(pdev, I915_GDRST,
			      GRDOM_RENDER | GRDOM_RESET_ENABLE);
206
	ret =  wait_for_atomic(g4x_reset_complete(pdev), 50);
207 208 209 210 211 212 213 214
	if (ret) {
		DRM_DEBUG_DRIVER("Wait for render reset failed\n");
		goto out;
	}

out:
	pci_write_config_byte(pdev, I915_GDRST, 0);

215 216
	rmw_clear_fw(uncore, VDECCLK_GATE_D, VCP_UNIT_CLOCK_GATE_DISABLE);
	intel_uncore_posting_read_fw(uncore, VDECCLK_GATE_D);
217 218 219 220

	return ret;
}

221
static int ironlake_do_reset(struct intel_gt *gt,
222
			     intel_engine_mask_t engine_mask,
223 224
			     unsigned int retry)
{
225
	struct intel_uncore *uncore = gt->uncore;
226 227
	int ret;

228 229 230
	intel_uncore_write_fw(uncore, ILK_GDSR,
			      ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
	ret = __intel_wait_for_register_fw(uncore, ILK_GDSR,
231 232 233
					   ILK_GRDOM_RESET_ENABLE, 0,
					   5000, 0,
					   NULL);
234 235 236 237 238
	if (ret) {
		DRM_DEBUG_DRIVER("Wait for render reset failed\n");
		goto out;
	}

239 240 241
	intel_uncore_write_fw(uncore, ILK_GDSR,
			      ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
	ret = __intel_wait_for_register_fw(uncore, ILK_GDSR,
242 243 244
					   ILK_GRDOM_RESET_ENABLE, 0,
					   5000, 0,
					   NULL);
245 246 247 248 249 250
	if (ret) {
		DRM_DEBUG_DRIVER("Wait for media reset failed\n");
		goto out;
	}

out:
251 252
	intel_uncore_write_fw(uncore, ILK_GDSR, 0);
	intel_uncore_posting_read_fw(uncore, ILK_GDSR);
253 254 255 256
	return ret;
}

/* Reset the hardware domains (GENX_GRDOM_*) specified by mask */
257
static int gen6_hw_domain_reset(struct intel_gt *gt, u32 hw_domain_mask)
258
{
259
	struct intel_uncore *uncore = gt->uncore;
260 261 262 263 264 265 266
	int err;

	/*
	 * GEN6_GDRST is not in the gt power well, no need to check
	 * for fifo space for the write or forcewake the chip for
	 * the read
	 */
267
	intel_uncore_write_fw(uncore, GEN6_GDRST, hw_domain_mask);
268 269

	/* Wait for the device to ack the reset requests */
270
	err = __intel_wait_for_register_fw(uncore,
271 272 273 274 275 276 277 278 279 280
					   GEN6_GDRST, hw_domain_mask, 0,
					   500, 0,
					   NULL);
	if (err)
		DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n",
				 hw_domain_mask);

	return err;
}

281
static int gen6_reset_engines(struct intel_gt *gt,
282
			      intel_engine_mask_t engine_mask,
283 284 285
			      unsigned int retry)
{
	struct intel_engine_cs *engine;
286 287 288 289 290 291
	const u32 hw_engine_mask[] = {
		[RCS0]  = GEN6_GRDOM_RENDER,
		[BCS0]  = GEN6_GRDOM_BLT,
		[VCS0]  = GEN6_GRDOM_MEDIA,
		[VCS1]  = GEN8_GRDOM_MEDIA2,
		[VECS0] = GEN6_GRDOM_VECS,
292 293 294 295 296 297
	};
	u32 hw_mask;

	if (engine_mask == ALL_ENGINES) {
		hw_mask = GEN6_GRDOM_FULL;
	} else {
298
		intel_engine_mask_t tmp;
299 300

		hw_mask = 0;
301
		for_each_engine_masked(engine, gt->i915, engine_mask, tmp) {
302
			GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask));
303
			hw_mask |= hw_engine_mask[engine->id];
304
		}
305 306
	}

307
	return gen6_hw_domain_reset(gt, hw_mask);
308 309
}

310
static int gen11_lock_sfc(struct intel_engine_cs *engine, u32 *hw_mask)
311
{
312 313
	struct intel_uncore *uncore = engine->uncore;
	u8 vdbox_sfc_access = RUNTIME_INFO(engine->i915)->vdbox_sfc_access;
314 315 316 317 318
	i915_reg_t sfc_forced_lock, sfc_forced_lock_ack;
	u32 sfc_forced_lock_bit, sfc_forced_lock_ack_bit;
	i915_reg_t sfc_usage;
	u32 sfc_usage_bit;
	u32 sfc_reset_bit;
319
	int ret;
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353

	switch (engine->class) {
	case VIDEO_DECODE_CLASS:
		if ((BIT(engine->instance) & vdbox_sfc_access) == 0)
			return 0;

		sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine);
		sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT;

		sfc_forced_lock_ack = GEN11_VCS_SFC_LOCK_STATUS(engine);
		sfc_forced_lock_ack_bit  = GEN11_VCS_SFC_LOCK_ACK_BIT;

		sfc_usage = GEN11_VCS_SFC_LOCK_STATUS(engine);
		sfc_usage_bit = GEN11_VCS_SFC_USAGE_BIT;
		sfc_reset_bit = GEN11_VCS_SFC_RESET_BIT(engine->instance);
		break;

	case VIDEO_ENHANCEMENT_CLASS:
		sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine);
		sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT;

		sfc_forced_lock_ack = GEN11_VECS_SFC_LOCK_ACK(engine);
		sfc_forced_lock_ack_bit  = GEN11_VECS_SFC_LOCK_ACK_BIT;

		sfc_usage = GEN11_VECS_SFC_USAGE(engine);
		sfc_usage_bit = GEN11_VECS_SFC_USAGE_BIT;
		sfc_reset_bit = GEN11_VECS_SFC_RESET_BIT(engine->instance);
		break;

	default:
		return 0;
	}

	/*
354 355 356 357 358
	 * If the engine is using a SFC, tell the engine that a software reset
	 * is going to happen. The engine will then try to force lock the SFC.
	 * If SFC ends up being locked to the engine we want to reset, we have
	 * to reset it as well (we will unlock it once the reset sequence is
	 * completed).
359
	 */
360 361 362
	if (!(intel_uncore_read_fw(uncore, sfc_usage) & sfc_usage_bit))
		return 0;

363
	rmw_set_fw(uncore, sfc_forced_lock, sfc_forced_lock_bit);
364

365 366 367 368 369 370 371 372
	ret = __intel_wait_for_register_fw(uncore,
					   sfc_forced_lock_ack,
					   sfc_forced_lock_ack_bit,
					   sfc_forced_lock_ack_bit,
					   1000, 0, NULL);

	/* Was the SFC released while we were trying to lock it? */
	if (!(intel_uncore_read_fw(uncore, sfc_usage) & sfc_usage_bit))
373 374
		return 0;

375 376 377 378
	if (ret) {
		DRM_DEBUG_DRIVER("Wait for SFC forced lock ack failed\n");
		return ret;
	}
379

380
	*hw_mask |= sfc_reset_bit;
381 382 383
	return 0;
}

384
static void gen11_unlock_sfc(struct intel_engine_cs *engine)
385
{
386 387
	struct intel_uncore *uncore = engine->uncore;
	u8 vdbox_sfc_access = RUNTIME_INFO(engine->i915)->vdbox_sfc_access;
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
	i915_reg_t sfc_forced_lock;
	u32 sfc_forced_lock_bit;

	switch (engine->class) {
	case VIDEO_DECODE_CLASS:
		if ((BIT(engine->instance) & vdbox_sfc_access) == 0)
			return;

		sfc_forced_lock = GEN11_VCS_SFC_FORCED_LOCK(engine);
		sfc_forced_lock_bit = GEN11_VCS_SFC_FORCED_LOCK_BIT;
		break;

	case VIDEO_ENHANCEMENT_CLASS:
		sfc_forced_lock = GEN11_VECS_SFC_FORCED_LOCK(engine);
		sfc_forced_lock_bit = GEN11_VECS_SFC_FORCED_LOCK_BIT;
		break;

	default:
		return;
	}

409
	rmw_clear_fw(uncore, sfc_forced_lock, sfc_forced_lock_bit);
410 411
}

412
static int gen11_reset_engines(struct intel_gt *gt,
413
			       intel_engine_mask_t engine_mask,
414 415
			       unsigned int retry)
{
416 417 418 419 420 421 422 423 424
	const u32 hw_engine_mask[] = {
		[RCS0]  = GEN11_GRDOM_RENDER,
		[BCS0]  = GEN11_GRDOM_BLT,
		[VCS0]  = GEN11_GRDOM_MEDIA,
		[VCS1]  = GEN11_GRDOM_MEDIA2,
		[VCS2]  = GEN11_GRDOM_MEDIA3,
		[VCS3]  = GEN11_GRDOM_MEDIA4,
		[VECS0] = GEN11_GRDOM_VECS,
		[VECS1] = GEN11_GRDOM_VECS2,
425 426
	};
	struct intel_engine_cs *engine;
427
	intel_engine_mask_t tmp;
428 429 430 431 432 433 434
	u32 hw_mask;
	int ret;

	if (engine_mask == ALL_ENGINES) {
		hw_mask = GEN11_GRDOM_FULL;
	} else {
		hw_mask = 0;
435
		for_each_engine_masked(engine, gt->i915, engine_mask, tmp) {
436
			GEM_BUG_ON(engine->id >= ARRAY_SIZE(hw_engine_mask));
437
			hw_mask |= hw_engine_mask[engine->id];
438 439 440
			ret = gen11_lock_sfc(engine, &hw_mask);
			if (ret)
				goto sfc_unlock;
441 442 443
		}
	}

444
	ret = gen6_hw_domain_reset(gt, hw_mask);
445

446 447 448 449 450 451 452
sfc_unlock:
	/*
	 * We unlock the SFC based on the lock status and not the result of
	 * gen11_lock_sfc to make sure that we clean properly if something
	 * wrong happened during the lock (e.g. lock acquired after timeout
	 * expiration).
	 */
453
	if (engine_mask != ALL_ENGINES)
454
		for_each_engine_masked(engine, gt->i915, engine_mask, tmp)
455
			gen11_unlock_sfc(engine);
456 457 458 459 460 461

	return ret;
}

static int gen8_engine_reset_prepare(struct intel_engine_cs *engine)
{
462
	struct intel_uncore *uncore = engine->uncore;
463 464
	const i915_reg_t reg = RING_RESET_CTL(engine->mmio_base);
	u32 request, mask, ack;
465 466
	int ret;

467
	ack = intel_uncore_read_fw(uncore, reg);
468 469 470 471 472 473 474 475 476 477 478
	if (ack & RESET_CTL_CAT_ERROR) {
		/*
		 * For catastrophic errors, ready-for-reset sequence
		 * needs to be bypassed: HAS#396813
		 */
		request = RESET_CTL_CAT_ERROR;
		mask = RESET_CTL_CAT_ERROR;

		/* Catastrophic errors need to be cleared by HW */
		ack = 0;
	} else if (!(ack & RESET_CTL_READY_TO_RESET)) {
479 480 481 482 483 484
		request = RESET_CTL_REQUEST_RESET;
		mask = RESET_CTL_READY_TO_RESET;
		ack = RESET_CTL_READY_TO_RESET;
	} else {
		return 0;
	}
485

486 487 488
	intel_uncore_write_fw(uncore, reg, _MASKED_BIT_ENABLE(request));
	ret = __intel_wait_for_register_fw(uncore, reg, mask, ack,
					   700, 0, NULL);
489
	if (ret)
490 491 492
		DRM_ERROR("%s reset request timed out: {request: %08x, RESET_CTL: %08x}\n",
			  engine->name, request,
			  intel_uncore_read_fw(uncore, reg));
493 494 495 496 497 498

	return ret;
}

static void gen8_engine_reset_cancel(struct intel_engine_cs *engine)
{
499 500 501
	intel_uncore_write_fw(engine->uncore,
			      RING_RESET_CTL(engine->mmio_base),
			      _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
502 503
}

504
static int gen8_reset_engines(struct intel_gt *gt,
505
			      intel_engine_mask_t engine_mask,
506 507 508 509
			      unsigned int retry)
{
	struct intel_engine_cs *engine;
	const bool reset_non_ready = retry >= 1;
510
	intel_engine_mask_t tmp;
511 512
	int ret;

513
	for_each_engine_masked(engine, gt->i915, engine_mask, tmp) {
514 515 516 517 518 519 520 521 522 523 524 525 526 527 528
		ret = gen8_engine_reset_prepare(engine);
		if (ret && !reset_non_ready)
			goto skip_reset;

		/*
		 * If this is not the first failed attempt to prepare,
		 * we decide to proceed anyway.
		 *
		 * By doing so we risk context corruption and with
		 * some gens (kbl), possible system hang if reset
		 * happens during active bb execution.
		 *
		 * We rather take context corruption instead of
		 * failed reset with a wedged driver/gpu. And
		 * active bb execution case should be covered by
529
		 * stop_engines() we have before the reset.
530 531 532
		 */
	}

533 534
	if (INTEL_GEN(gt->i915) >= 11)
		ret = gen11_reset_engines(gt, engine_mask, retry);
535
	else
536
		ret = gen6_reset_engines(gt, engine_mask, retry);
537 538

skip_reset:
539
	for_each_engine_masked(engine, gt->i915, engine_mask, tmp)
540 541 542 543 544
		gen8_engine_reset_cancel(engine);

	return ret;
}

545 546 547 548 549 550 551
static int mock_reset(struct intel_gt *gt,
		      intel_engine_mask_t mask,
		      unsigned int retry)
{
	return 0;
}

552
typedef int (*reset_func)(struct intel_gt *,
553
			  intel_engine_mask_t engine_mask,
554 555
			  unsigned int retry);

556
static reset_func intel_get_gpu_reset(const struct intel_gt *gt)
557
{
558 559
	struct drm_i915_private *i915 = gt->i915;

560 561 562
	if (is_mock_gt(gt))
		return mock_reset;
	else if (INTEL_GEN(i915) >= 8)
563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
		return gen8_reset_engines;
	else if (INTEL_GEN(i915) >= 6)
		return gen6_reset_engines;
	else if (INTEL_GEN(i915) >= 5)
		return ironlake_do_reset;
	else if (IS_G4X(i915))
		return g4x_do_reset;
	else if (IS_G33(i915) || IS_PINEVIEW(i915))
		return g33_do_reset;
	else if (INTEL_GEN(i915) >= 3)
		return i915_do_reset;
	else
		return NULL;
}

578
int __intel_gt_reset(struct intel_gt *gt, intel_engine_mask_t engine_mask)
579
{
580 581 582
	const int retries = engine_mask == ALL_ENGINES ? RESET_MAX_RETRIES : 1;
	reset_func reset;
	int ret = -ETIMEDOUT;
583 584
	int retry;

585
	reset = intel_get_gpu_reset(gt);
586 587
	if (!reset)
		return -ENODEV;
588 589 590 591 592

	/*
	 * If the power well sleeps during the reset, the reset
	 * request may be dropped and never completes (causing -EIO).
	 */
593
	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
594 595 596
	for (retry = 0; ret == -ETIMEDOUT && retry < retries; retry++) {
		GEM_TRACE("engine_mask=%x\n", engine_mask);
		preempt_disable();
597
		ret = reset(gt, engine_mask, retry);
598
		preempt_enable();
599
	}
600
	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
601 602 603 604

	return ret;
}

605
bool intel_has_gpu_reset(const struct intel_gt *gt)
606
{
607 608 609
	if (!i915_modparams.reset)
		return NULL;

610
	return intel_get_gpu_reset(gt);
611 612
}

613
bool intel_has_reset_engine(const struct intel_gt *gt)
614
{
615 616 617 618
	if (i915_modparams.reset < 2)
		return false;

	return INTEL_INFO(gt->i915)->has_reset_engine;
619 620
}

621
int intel_reset_guc(struct intel_gt *gt)
622 623
{
	u32 guc_domain =
624
		INTEL_GEN(gt->i915) >= 11 ? GEN11_GRDOM_GUC : GEN9_GRDOM_GUC;
625 626
	int ret;

627
	GEM_BUG_ON(!HAS_GT_UC(gt->i915));
628

629 630 631
	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
	ret = gen6_hw_domain_reset(gt, guc_domain);
	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
632 633 634 635 636 637 638 639

	return ret;
}

/*
 * Ensure irq handler finishes, and not run again.
 * Also return the active request so that we only search for it once.
 */
640
static void reset_prepare_engine(struct intel_engine_cs *engine)
641 642 643 644 645 646 647 648
{
	/*
	 * During the reset sequence, we must prevent the engine from
	 * entering RC6. As the context state is undefined until we restart
	 * the engine, if it does enter RC6 during the reset, the state
	 * written to the powercontext is undefined and so we may lose
	 * GPU state upon resume, i.e. fail to restart after a reset.
	 */
649
	intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL);
650
	engine->reset.prepare(engine);
651 652
}

653
static void revoke_mmaps(struct intel_gt *gt)
654 655 656
{
	int i;

657
	for (i = 0; i < gt->ggtt->num_fences; i++) {
658 659 660 661
		struct drm_vma_offset_node *node;
		struct i915_vma *vma;
		u64 vma_offset;

662
		vma = READ_ONCE(gt->ggtt->fence_regs[i].vma);
663 664 665 666 667 668
		if (!vma)
			continue;

		if (!i915_vma_has_userfault(vma))
			continue;

669
		GEM_BUG_ON(vma->fence != &gt->ggtt->fence_regs[i]);
670 671
		node = &vma->obj->base.vma_node;
		vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
672
		unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping,
673 674 675 676 677 678
				    drm_vma_node_offset_addr(node) + vma_offset,
				    vma->size,
				    1);
	}
}

679
static intel_engine_mask_t reset_prepare(struct intel_gt *gt)
680 681
{
	struct intel_engine_cs *engine;
682
	intel_engine_mask_t awake = 0;
683 684
	enum intel_engine_id id;

685
	for_each_engine(engine, gt->i915, id) {
686 687
		if (intel_engine_pm_get_if_awake(engine))
			awake |= engine->mask;
688
		reset_prepare_engine(engine);
689
	}
690

691
	intel_uc_reset_prepare(&gt->uc);
692 693

	return awake;
694 695
}

696
static void gt_revoke(struct intel_gt *gt)
697
{
698
	revoke_mmaps(gt);
699 700
}

701
static int gt_reset(struct intel_gt *gt, intel_engine_mask_t stalled_mask)
702
{
703 704 705 706
	struct intel_engine_cs *engine;
	enum intel_engine_id id;
	int err;

707
	/*
708 709
	 * Everything depends on having the GTT running, so we need to start
	 * there.
710
	 */
711
	err = i915_ggtt_enable_hw(gt->i915);
712 713
	if (err)
		return err;
714

715 716
	for_each_engine(engine, gt->i915, id)
		__intel_engine_reset(engine, stalled_mask & engine->mask);
717

718
	i915_gem_restore_fences(gt->i915);
719

720
	return err;
721 722
}

723
static void reset_finish_engine(struct intel_engine_cs *engine)
724
{
725
	engine->reset.finish(engine);
726
	intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL);
727

728
	intel_engine_breadcrumbs_irq(engine);
729 730
}

731
static void reset_finish(struct intel_gt *gt, intel_engine_mask_t awake)
732 733 734 735
{
	struct intel_engine_cs *engine;
	enum intel_engine_id id;

736
	for_each_engine(engine, gt->i915, id) {
737
		reset_finish_engine(engine);
738 739
		if (awake & engine->mask)
			intel_engine_pm_put(engine);
740
	}
741 742 743 744
}

static void nop_submit_request(struct i915_request *request)
{
745
	struct intel_engine_cs *engine = request->engine;
746 747 748
	unsigned long flags;

	GEM_TRACE("%s fence %llx:%lld -> -EIO\n",
749
		  engine->name, request->fence.context, request->fence.seqno);
750 751
	dma_fence_set_error(&request->fence, -EIO);

752
	spin_lock_irqsave(&engine->active.lock, flags);
753
	__i915_request_submit(request);
754
	i915_request_mark_complete(request);
755
	spin_unlock_irqrestore(&engine->active.lock, flags);
756 757

	intel_engine_queue_breadcrumbs(engine);
758 759
}

760
static void __intel_gt_set_wedged(struct intel_gt *gt)
761 762
{
	struct intel_engine_cs *engine;
763
	intel_engine_mask_t awake;
764 765
	enum intel_engine_id id;

766
	if (test_bit(I915_WEDGED, &gt->reset.flags))
767 768
		return;

769
	if (GEM_SHOW_DEBUG() && !intel_engines_are_idle(gt)) {
770 771
		struct drm_printer p = drm_debug_printer(__func__);

772
		for_each_engine(engine, gt->i915, id)
773 774 775 776 777 778 779 780 781 782
			intel_engine_dump(engine, &p, "%s\n", engine->name);
	}

	GEM_TRACE("start\n");

	/*
	 * First, stop submission to hw, but do not yet complete requests by
	 * rolling the global seqno forward (since this would complete requests
	 * for which we haven't set the fence error to EIO yet).
	 */
783
	awake = reset_prepare(gt);
784

785
	/* Even if the GPU reset fails, it should still stop the engines */
786 787
	if (!INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
		__intel_gt_reset(gt, ALL_ENGINES);
788

789
	for_each_engine(engine, gt->i915, id)
790 791 792 793 794 795 796
		engine->submit_request = nop_submit_request;

	/*
	 * Make sure no request can slip through without getting completed by
	 * either this call here to intel_engine_write_global_seqno, or the one
	 * in nop_submit_request.
	 */
C
Chris Wilson 已提交
797
	synchronize_rcu_expedited();
798
	set_bit(I915_WEDGED, &gt->reset.flags);
799 800

	/* Mark all executing requests as skipped */
801
	for_each_engine(engine, gt->i915, id)
802 803
		engine->cancel_requests(engine);

804
	reset_finish(gt, awake);
805 806

	GEM_TRACE("end\n");
807
}
808

809
void intel_gt_set_wedged(struct intel_gt *gt)
810
{
811
	intel_wakeref_t wakeref;
812

813 814 815 816
	mutex_lock(&gt->reset.mutex);
	with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)
		__intel_gt_set_wedged(gt);
	mutex_unlock(&gt->reset.mutex);
817 818
}

819
static bool __intel_gt_unset_wedged(struct intel_gt *gt)
820
{
821
	struct intel_gt_timelines *timelines = &gt->timelines;
822
	struct intel_timeline *tl;
823
	unsigned long flags;
824

825
	if (!test_bit(I915_WEDGED, &gt->reset.flags))
826 827
		return true;

828 829
	/* Never fully initialised, recovery impossible */
	if (test_bit(I915_WEDGED_ON_INIT, &gt->reset.flags))
830 831 832 833 834 835 836 837 838 839 840 841 842 843
		return false;

	GEM_TRACE("start\n");

	/*
	 * Before unwedging, make sure that all pending operations
	 * are flushed and errored out - we may have requests waiting upon
	 * third party fences. We marked all inflight requests as EIO, and
	 * every execbuf since returned EIO, for consistency we want all
	 * the currently pending requests to also be marked as EIO, which
	 * is done inside our nop_submit_request - and so we must wait.
	 *
	 * No more can be submitted until we reset the wedged bit.
	 */
844
	spin_lock_irqsave(&timelines->lock, flags);
845
	list_for_each_entry(tl, &timelines->active_list, link) {
846 847
		struct i915_request *rq;

848
		rq = i915_active_request_get_unlocked(&tl->last_request);
849 850 851
		if (!rq)
			continue;

852
		spin_unlock_irqrestore(&timelines->lock, flags);
853

854
		/*
855 856 857 858 859
		 * All internal dependencies (i915_requests) will have
		 * been flushed by the set-wedge, but we may be stuck waiting
		 * for external fences. These should all be capped to 10s
		 * (I915_FENCE_TIMEOUT) so this wait should not be unbounded
		 * in the worst case.
860
		 */
861
		dma_fence_default_wait(&rq->fence, false, MAX_SCHEDULE_TIMEOUT);
862
		i915_request_put(rq);
863 864

		/* Restart iteration after droping lock */
865
		spin_lock_irqsave(&timelines->lock, flags);
866
		tl = list_entry(&timelines->active_list, typeof(*tl), link);
867
	}
868
	spin_unlock_irqrestore(&timelines->lock, flags);
869

870
	intel_gt_sanitize(gt, false);
871 872 873 874 875 876 877 878 879 880

	/*
	 * Undo nop_submit_request. We prevent all new i915 requests from
	 * being queued (by disallowing execbuf whilst wedged) so having
	 * waited for all active requests above, we know the system is idle
	 * and do not have to worry about a thread being inside
	 * engine->submit_request() as we swap over. So unlike installing
	 * the nop_submit_request on reset, we can do this from normal
	 * context and do not require stop_machine().
	 */
881
	intel_engines_reset_default_submission(gt);
882 883 884 885

	GEM_TRACE("end\n");

	smp_mb__before_atomic(); /* complete takeover before enabling execbuf */
886
	clear_bit(I915_WEDGED, &gt->reset.flags);
887 888

	return true;
889 890
}

891
bool intel_gt_unset_wedged(struct intel_gt *gt)
892 893 894
{
	bool result;

895 896 897
	mutex_lock(&gt->reset.mutex);
	result = __intel_gt_unset_wedged(gt);
	mutex_unlock(&gt->reset.mutex);
898 899 900 901

	return result;
}

902
static int do_reset(struct intel_gt *gt, intel_engine_mask_t stalled_mask)
903 904 905
{
	int err, i;

906
	gt_revoke(gt);
907

908
	err = __intel_gt_reset(gt, ALL_ENGINES);
909
	for (i = 0; err && i < RESET_MAX_RETRIES; i++) {
910
		msleep(10 * (i + 1));
911
		err = __intel_gt_reset(gt, ALL_ENGINES);
912
	}
913 914
	if (err)
		return err;
915

916
	return gt_reset(gt, stalled_mask);
917 918
}

919
static int resume(struct intel_gt *gt)
920 921 922 923 924
{
	struct intel_engine_cs *engine;
	enum intel_engine_id id;
	int ret;

925
	for_each_engine(engine, gt->i915, id) {
926 927 928 929 930 931 932 933
		ret = engine->resume(engine);
		if (ret)
			return ret;
	}

	return 0;
}

934
/**
935 936
 * intel_gt_reset - reset chip after a hang
 * @gt: #intel_gt to reset
937 938 939 940 941 942 943 944 945 946 947 948 949 950
 * @stalled_mask: mask of the stalled engines with the guilty requests
 * @reason: user error message for why we are resetting
 *
 * Reset the chip.  Useful if a hang is detected. Marks the device as wedged
 * on failure.
 *
 * Procedure is fairly simple:
 *   - reset the chip using the reset reg
 *   - re-init context state
 *   - re-init hardware status page
 *   - re-init ring buffer
 *   - re-init interrupt state
 *   - re-init display
 */
951 952 953
void intel_gt_reset(struct intel_gt *gt,
		    intel_engine_mask_t stalled_mask,
		    const char *reason)
954
{
955
	intel_engine_mask_t awake;
956 957
	int ret;

958
	GEM_TRACE("flags=%lx\n", gt->reset.flags);
959 960

	might_sleep();
961 962
	GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &gt->reset.flags));
	mutex_lock(&gt->reset.mutex);
963 964

	/* Clear any previous failed attempts at recovery. Time to try again. */
965
	if (!__intel_gt_unset_wedged(gt))
966
		goto unlock;
967 968

	if (reason)
969 970 971
		dev_notice(gt->i915->drm.dev,
			   "Resetting chip for %s\n", reason);
	atomic_inc(&gt->i915->gpu_error.reset_count);
972

973
	awake = reset_prepare(gt);
974

975
	if (!intel_has_gpu_reset(gt)) {
976
		if (i915_modparams.reset)
977
			dev_err(gt->i915->drm.dev, "GPU reset not supported\n");
978 979 980 981 982
		else
			DRM_DEBUG_DRIVER("GPU reset disabled\n");
		goto error;
	}

983 984
	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
		intel_runtime_pm_disable_interrupts(gt->i915);
985

986 987
	if (do_reset(gt, stalled_mask)) {
		dev_err(gt->i915->drm.dev, "Failed to reset chip\n");
988 989 990
		goto taint;
	}

991 992
	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
		intel_runtime_pm_enable_interrupts(gt->i915);
993

994
	intel_overlay_reset(gt->i915);
995 996 997 998 999 1000 1001 1002 1003

	/*
	 * Next we need to restore the context, but we don't use those
	 * yet either...
	 *
	 * Ring buffer needs to be re-initialized in the KMS case, or if X
	 * was running at the time of the reset (i.e. we weren't VT
	 * switched away).
	 */
1004
	ret = intel_gt_init_hw(gt);
1005 1006 1007
	if (ret) {
		DRM_ERROR("Failed to initialise HW following reset (%d)\n",
			  ret);
1008
		goto taint;
1009 1010
	}

1011
	ret = resume(gt);
1012 1013 1014
	if (ret)
		goto taint;

1015
	intel_gt_queue_hangcheck(gt);
1016 1017

finish:
1018
	reset_finish(gt, awake);
1019
unlock:
1020
	mutex_unlock(&gt->reset.mutex);
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
	return;

taint:
	/*
	 * History tells us that if we cannot reset the GPU now, we
	 * never will. This then impacts everything that is run
	 * subsequently. On failing the reset, we mark the driver
	 * as wedged, preventing further execution on the GPU.
	 * We also want to go one step further and add a taint to the
	 * kernel so that any subsequent faults can be traced back to
	 * this failure. This is important for CI, where if the
	 * GPU/driver fails we would like to reboot and restart testing
	 * rather than continue on into oblivion. For everyone else,
	 * the system should still plod along, but they have been warned!
	 */
1036
	add_taint_for_CI(TAINT_WARN);
1037
error:
1038
	__intel_gt_set_wedged(gt);
1039 1040 1041
	goto finish;
}

1042
static inline int intel_gt_reset_engine(struct intel_engine_cs *engine)
1043
{
1044
	return __intel_gt_reset(engine->gt, engine->mask);
1045 1046 1047
}

/**
1048
 * intel_engine_reset - reset GPU engine to recover from a hang
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
 * @engine: engine to reset
 * @msg: reason for GPU reset; or NULL for no dev_notice()
 *
 * Reset a specific GPU engine. Useful if a hang is detected.
 * Returns zero on successful reset or otherwise an error code.
 *
 * Procedure is:
 *  - identifies the request that caused the hang and it is dropped
 *  - reset engine (which will force the engine to idle)
 *  - re-init/configure engine
 */
1060
int intel_engine_reset(struct intel_engine_cs *engine, const char *msg)
1061
{
1062
	struct intel_gt *gt = engine->gt;
1063 1064
	int ret;

1065 1066
	GEM_TRACE("%s flags=%lx\n", engine->name, gt->reset.flags);
	GEM_BUG_ON(!test_bit(I915_RESET_ENGINE + engine->id, &gt->reset.flags));
1067

1068
	if (!intel_engine_pm_get_if_awake(engine))
1069 1070
		return 0;

1071
	reset_prepare_engine(engine);
1072 1073 1074 1075

	if (msg)
		dev_notice(engine->i915->drm.dev,
			   "Resetting %s for %s\n", engine->name, msg);
1076
	atomic_inc(&engine->i915->gpu_error.reset_engine_count[engine->uabi_class]);
1077

1078
	if (!engine->gt->uc.guc.execbuf_client)
1079
		ret = intel_gt_reset_engine(engine);
1080
	else
1081
		ret = intel_guc_reset_engine(&engine->gt->uc.guc, engine);
1082 1083 1084
	if (ret) {
		/* If we fail here, we expect to fallback to a global reset */
		DRM_DEBUG_DRIVER("%sFailed to reset %s, ret=%d\n",
1085
				 engine->gt->uc.guc.execbuf_client ? "GuC " : "",
1086 1087 1088 1089 1090 1091 1092 1093 1094
				 engine->name, ret);
		goto out;
	}

	/*
	 * The request that caused the hang is stuck on elsp, we know the
	 * active request and can drop it, adjust head to skip the offending
	 * request to resume executing remaining requests in the queue.
	 */
1095
	__intel_engine_reset(engine, true);
1096 1097 1098 1099 1100 1101

	/*
	 * The engine and its registers (and workarounds in case of render)
	 * have been reset to their default values. Follow the init_ring
	 * process to program RING_MODE, HWSP and re-enable submission.
	 */
1102
	ret = engine->resume(engine);
1103 1104 1105 1106

out:
	intel_engine_cancel_stop_cs(engine);
	reset_finish_engine(engine);
1107
	intel_engine_pm_put(engine);
1108 1109 1110
	return ret;
}

1111 1112 1113
static void intel_gt_reset_global(struct intel_gt *gt,
				  u32 engine_mask,
				  const char *reason)
1114
{
1115
	struct kobject *kobj = &gt->i915->drm.primary->kdev->kobj;
1116 1117 1118
	char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
	char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
	char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
1119
	struct intel_wedge_me w;
1120 1121 1122 1123 1124 1125 1126

	kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);

	DRM_DEBUG_DRIVER("resetting chip\n");
	kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);

	/* Use a watchdog to ensure that our reset completes */
1127 1128
	intel_wedge_on_timeout(&w, gt, 5 * HZ) {
		intel_prepare_reset(gt->i915);
1129

1130
		/* Flush everyone using a resource about to be clobbered */
1131
		synchronize_srcu_expedited(&gt->reset.backoff_srcu);
1132

1133
		intel_gt_reset(gt, engine_mask, reason);
1134

1135
		intel_finish_reset(gt->i915);
1136 1137
	}

1138
	if (!test_bit(I915_WEDGED, &gt->reset.flags))
1139 1140 1141 1142
		kobject_uevent_env(kobj, KOBJ_CHANGE, reset_done_event);
}

/**
1143 1144
 * intel_gt_handle_error - handle a gpu error
 * @gt: the intel_gt
1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
 * @engine_mask: mask representing engines that are hung
 * @flags: control flags
 * @fmt: Error message format string
 *
 * Do some basic checking of register state at error time and
 * dump it to the syslog.  Also call i915_capture_error_state() to make
 * sure we get a record and make it available in debugfs.  Fire a uevent
 * so userspace knows something bad happened (should trigger collection
 * of a ring dump etc.).
 */
1155 1156 1157 1158
void intel_gt_handle_error(struct intel_gt *gt,
			   intel_engine_mask_t engine_mask,
			   unsigned long flags,
			   const char *fmt, ...)
1159 1160 1161
{
	struct intel_engine_cs *engine;
	intel_wakeref_t wakeref;
1162
	intel_engine_mask_t tmp;
1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
	char error_msg[80];
	char *msg = NULL;

	if (fmt) {
		va_list args;

		va_start(args, fmt);
		vscnprintf(error_msg, sizeof(error_msg), fmt, args);
		va_end(args);

		msg = error_msg;
	}

	/*
	 * In most cases it's guaranteed that we get here with an RPM
	 * reference held, for example because there is a pending GPU
	 * request that won't finish until the reset is done. This
	 * isn't the case at least when we get here by doing a
	 * simulated reset via debugfs, so get an RPM reference.
	 */
1183
	wakeref = intel_runtime_pm_get(&gt->i915->runtime_pm);
1184

1185
	engine_mask &= INTEL_INFO(gt->i915)->engine_mask;
1186 1187

	if (flags & I915_ERROR_CAPTURE) {
1188 1189
		i915_capture_error_state(gt->i915, engine_mask, msg);
		intel_gt_clear_error_registers(gt, engine_mask);
1190 1191 1192 1193 1194 1195
	}

	/*
	 * Try engine reset when available. We fall back to full reset if
	 * single reset fails.
	 */
1196
	if (intel_has_reset_engine(gt) && !intel_gt_is_wedged(gt)) {
1197
		for_each_engine_masked(engine, gt->i915, engine_mask, tmp) {
1198 1199
			BUILD_BUG_ON(I915_RESET_MODESET >= I915_RESET_ENGINE);
			if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
1200
					     &gt->reset.flags))
1201 1202
				continue;

1203
			if (intel_engine_reset(engine, msg) == 0)
1204
				engine_mask &= ~engine->mask;
1205

1206 1207
			clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id,
					      &gt->reset.flags);
1208 1209 1210 1211 1212 1213 1214
		}
	}

	if (!engine_mask)
		goto out;

	/* Full reset needs the mutex, stop any other user trying to do so. */
1215 1216 1217
	if (test_and_set_bit(I915_RESET_BACKOFF, &gt->reset.flags)) {
		wait_event(gt->reset.queue,
			   !test_bit(I915_RESET_BACKOFF, &gt->reset.flags));
1218
		goto out; /* piggy-back on the other reset */
1219 1220
	}

1221 1222 1223
	/* Make sure i915_reset_trylock() sees the I915_RESET_BACKOFF */
	synchronize_rcu_expedited();

1224
	/* Prevent any other reset-engine attempt. */
1225
	for_each_engine(engine, gt->i915, tmp) {
1226
		while (test_and_set_bit(I915_RESET_ENGINE + engine->id,
1227 1228
					&gt->reset.flags))
			wait_on_bit(&gt->reset.flags,
1229 1230 1231 1232
				    I915_RESET_ENGINE + engine->id,
				    TASK_UNINTERRUPTIBLE);
	}

1233
	intel_gt_reset_global(gt, engine_mask, msg);
1234

1235 1236 1237 1238 1239 1240
	for_each_engine(engine, gt->i915, tmp)
		clear_bit_unlock(I915_RESET_ENGINE + engine->id,
				 &gt->reset.flags);
	clear_bit_unlock(I915_RESET_BACKOFF, &gt->reset.flags);
	smp_mb__after_atomic();
	wake_up_all(&gt->reset.queue);
1241 1242

out:
1243
	intel_runtime_pm_put(&gt->i915->runtime_pm, wakeref);
1244 1245
}

1246
int intel_gt_reset_trylock(struct intel_gt *gt, int *srcu)
1247
{
1248
	might_lock(&gt->reset.backoff_srcu);
1249 1250
	might_sleep();

1251
	rcu_read_lock();
1252
	while (test_bit(I915_RESET_BACKOFF, &gt->reset.flags)) {
1253 1254
		rcu_read_unlock();

1255
		if (wait_event_interruptible(gt->reset.queue,
1256
					     !test_bit(I915_RESET_BACKOFF,
1257
						       &gt->reset.flags)))
1258 1259 1260 1261
			return -EINTR;

		rcu_read_lock();
	}
1262
	*srcu = srcu_read_lock(&gt->reset.backoff_srcu);
1263 1264
	rcu_read_unlock();

1265
	return 0;
1266 1267
}

1268 1269
void intel_gt_reset_unlock(struct intel_gt *gt, int tag)
__releases(&gt->reset.backoff_srcu)
1270
{
1271
	srcu_read_unlock(&gt->reset.backoff_srcu, tag);
1272 1273
}

1274
int intel_gt_terminally_wedged(struct intel_gt *gt)
1275 1276 1277
{
	might_sleep();

1278
	if (!intel_gt_is_wedged(gt))
1279 1280 1281
		return 0;

	/* Reset still in progress? Maybe we will recover? */
1282
	if (!test_bit(I915_RESET_BACKOFF, &gt->reset.flags))
1283 1284 1285
		return -EIO;

	/* XXX intel_reset_finish() still takes struct_mutex!!! */
1286
	if (mutex_is_locked(&gt->i915->drm.struct_mutex))
1287 1288
		return -EAGAIN;

1289
	if (wait_event_interruptible(gt->reset.queue,
1290
				     !test_bit(I915_RESET_BACKOFF,
1291
					       &gt->reset.flags)))
1292 1293
		return -EINTR;

1294 1295 1296
	return intel_gt_is_wedged(gt) ? -EIO : 0;
}

1297 1298 1299 1300 1301 1302 1303 1304
void intel_gt_set_wedged_on_init(struct intel_gt *gt)
{
	BUILD_BUG_ON(I915_RESET_ENGINE + I915_NUM_ENGINES >
		     I915_WEDGED_ON_INIT);
	intel_gt_set_wedged(gt);
	set_bit(I915_WEDGED_ON_INIT, &gt->reset.flags);
}

1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
void intel_gt_init_reset(struct intel_gt *gt)
{
	init_waitqueue_head(&gt->reset.queue);
	mutex_init(&gt->reset.mutex);
	init_srcu_struct(&gt->reset.backoff_srcu);
}

void intel_gt_fini_reset(struct intel_gt *gt)
{
	cleanup_srcu_struct(&gt->reset.backoff_srcu);
1315 1316
}

1317
static void intel_wedge_me(struct work_struct *work)
1318
{
1319
	struct intel_wedge_me *w = container_of(work, typeof(*w), work.work);
1320

1321
	dev_err(w->gt->i915->drm.dev,
1322 1323
		"%s timed out, cancelling all in-flight rendering.\n",
		w->name);
1324
	intel_gt_set_wedged(w->gt);
1325 1326
}

1327 1328 1329 1330
void __intel_init_wedge(struct intel_wedge_me *w,
			struct intel_gt *gt,
			long timeout,
			const char *name)
1331
{
1332
	w->gt = gt;
1333 1334
	w->name = name;

1335
	INIT_DELAYED_WORK_ONSTACK(&w->work, intel_wedge_me);
1336 1337 1338
	schedule_delayed_work(&w->work, timeout);
}

1339
void __intel_fini_wedge(struct intel_wedge_me *w)
1340 1341 1342
{
	cancel_delayed_work_sync(&w->work);
	destroy_delayed_work_on_stack(&w->work);
1343
	w->gt = NULL;
1344
}
1345 1346 1347 1348

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftest_reset.c"
#endif