intel_engine_cs.c 44.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
/*
 * Copyright © 2016 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

25 26
#include <drm/drm_print.h>

27 28
#include "gem/i915_gem_context.h"

29
#include "i915_drv.h"
30

31 32
#include "gt/intel_gt.h"

33
#include "intel_engine.h"
34
#include "intel_engine_pm.h"
35
#include "intel_context.h"
36
#include "intel_lrc.h"
37
#include "intel_reset.h"
38

39 40 41 42 43 44 45 46 47
/* Haswell does have the CXT_SIZE register however it does not appear to be
 * valid. Now, docs explain in dwords what is in the context object. The full
 * size is 70720 bytes, however, the power context and execlist context will
 * never be saved (power context is stored elsewhere, and execlists don't work
 * on HSW) - so the final size, including the extra state required for the
 * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
 */
#define HSW_CXT_TOTAL_SIZE		(17 * PAGE_SIZE)

48
#define DEFAULT_LR_CONTEXT_RENDER_SIZE	(22 * PAGE_SIZE)
49 50
#define GEN8_LR_CONTEXT_RENDER_SIZE	(20 * PAGE_SIZE)
#define GEN9_LR_CONTEXT_RENDER_SIZE	(22 * PAGE_SIZE)
51
#define GEN10_LR_CONTEXT_RENDER_SIZE	(18 * PAGE_SIZE)
52
#define GEN11_LR_CONTEXT_RENDER_SIZE	(14 * PAGE_SIZE)
53 54 55

#define GEN8_LR_CONTEXT_OTHER_SIZE	( 2 * PAGE_SIZE)

56
struct engine_class_info {
57
	const char *name;
58
	u8 uabi_class;
59 60 61 62 63
};

static const struct engine_class_info intel_engine_classes[] = {
	[RENDER_CLASS] = {
		.name = "rcs",
64
		.uabi_class = I915_ENGINE_CLASS_RENDER,
65 66 67
	},
	[COPY_ENGINE_CLASS] = {
		.name = "bcs",
68
		.uabi_class = I915_ENGINE_CLASS_COPY,
69 70 71
	},
	[VIDEO_DECODE_CLASS] = {
		.name = "vcs",
72
		.uabi_class = I915_ENGINE_CLASS_VIDEO,
73 74 75
	},
	[VIDEO_ENHANCEMENT_CLASS] = {
		.name = "vecs",
76
		.uabi_class = I915_ENGINE_CLASS_VIDEO_ENHANCE,
77 78 79
	},
};

80
#define MAX_MMIO_BASES 3
81
struct engine_info {
82
	unsigned int hw_id;
83 84
	u8 class;
	u8 instance;
85 86 87 88 89
	/* mmio bases table *must* be sorted in reverse gen order */
	struct engine_mmio_base {
		u32 gen : 8;
		u32 base : 24;
	} mmio_bases[MAX_MMIO_BASES];
90 91 92
};

static const struct engine_info intel_engines[] = {
93 94
	[RCS0] = {
		.hw_id = RCS0_HW,
95 96
		.class = RENDER_CLASS,
		.instance = 0,
97 98 99
		.mmio_bases = {
			{ .gen = 1, .base = RENDER_RING_BASE }
		},
100
	},
101 102
	[BCS0] = {
		.hw_id = BCS0_HW,
103 104
		.class = COPY_ENGINE_CLASS,
		.instance = 0,
105 106 107
		.mmio_bases = {
			{ .gen = 6, .base = BLT_RING_BASE }
		},
108
	},
109 110
	[VCS0] = {
		.hw_id = VCS0_HW,
111 112
		.class = VIDEO_DECODE_CLASS,
		.instance = 0,
113 114 115 116 117
		.mmio_bases = {
			{ .gen = 11, .base = GEN11_BSD_RING_BASE },
			{ .gen = 6, .base = GEN6_BSD_RING_BASE },
			{ .gen = 4, .base = BSD_RING_BASE }
		},
118
	},
119 120
	[VCS1] = {
		.hw_id = VCS1_HW,
121 122
		.class = VIDEO_DECODE_CLASS,
		.instance = 1,
123 124 125 126
		.mmio_bases = {
			{ .gen = 11, .base = GEN11_BSD2_RING_BASE },
			{ .gen = 8, .base = GEN8_BSD2_RING_BASE }
		},
127
	},
128 129
	[VCS2] = {
		.hw_id = VCS2_HW,
130 131
		.class = VIDEO_DECODE_CLASS,
		.instance = 2,
132 133 134
		.mmio_bases = {
			{ .gen = 11, .base = GEN11_BSD3_RING_BASE }
		},
135
	},
136 137
	[VCS3] = {
		.hw_id = VCS3_HW,
138 139
		.class = VIDEO_DECODE_CLASS,
		.instance = 3,
140 141 142
		.mmio_bases = {
			{ .gen = 11, .base = GEN11_BSD4_RING_BASE }
		},
143
	},
144 145
	[VECS0] = {
		.hw_id = VECS0_HW,
146 147
		.class = VIDEO_ENHANCEMENT_CLASS,
		.instance = 0,
148 149 150 151
		.mmio_bases = {
			{ .gen = 11, .base = GEN11_VEBOX_RING_BASE },
			{ .gen = 7, .base = VEBOX_RING_BASE }
		},
152
	},
153 154
	[VECS1] = {
		.hw_id = VECS1_HW,
155 156
		.class = VIDEO_ENHANCEMENT_CLASS,
		.instance = 1,
157 158 159
		.mmio_bases = {
			{ .gen = 11, .base = GEN11_VEBOX2_RING_BASE }
		},
160
	},
161 162
};

163
/**
164
 * intel_engine_context_size() - return the size of the context for an engine
165 166 167 168 169 170 171 172 173 174 175 176
 * @dev_priv: i915 device private
 * @class: engine class
 *
 * Each engine class may require a different amount of space for a context
 * image.
 *
 * Return: size (in bytes) of an engine class specific context image
 *
 * Note: this size includes the HWSP, which is part of the context image
 * in LRC mode, but does not include the "shared data page" used with
 * GuC submission. The caller should account for this if using the GuC.
 */
177
u32 intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
178 179 180 181 182 183 184 185 186 187
{
	u32 cxt_size;

	BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);

	switch (class) {
	case RENDER_CLASS:
		switch (INTEL_GEN(dev_priv)) {
		default:
			MISSING_CASE(INTEL_GEN(dev_priv));
188
			return DEFAULT_LR_CONTEXT_RENDER_SIZE;
189 190
		case 11:
			return GEN11_LR_CONTEXT_RENDER_SIZE;
191
		case 10:
O
Oscar Mateo 已提交
192
			return GEN10_LR_CONTEXT_RENDER_SIZE;
193 194 195
		case 9:
			return GEN9_LR_CONTEXT_RENDER_SIZE;
		case 8:
196
			return GEN8_LR_CONTEXT_RENDER_SIZE;
197 198 199 200 201 202 203 204 205 206 207 208
		case 7:
			if (IS_HASWELL(dev_priv))
				return HSW_CXT_TOTAL_SIZE;

			cxt_size = I915_READ(GEN7_CXT_SIZE);
			return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
					PAGE_SIZE);
		case 6:
			cxt_size = I915_READ(CXT_SIZE);
			return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
					PAGE_SIZE);
		case 5:
209
		case 4:
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
			/*
			 * There is a discrepancy here between the size reported
			 * by the register and the size of the context layout
			 * in the docs. Both are described as authorative!
			 *
			 * The discrepancy is on the order of a few cachelines,
			 * but the total is under one page (4k), which is our
			 * minimum allocation anyway so it should all come
			 * out in the wash.
			 */
			cxt_size = I915_READ(CXT_SIZE) + 1;
			DRM_DEBUG_DRIVER("gen%d CXT_SIZE = %d bytes [0x%08x]\n",
					 INTEL_GEN(dev_priv),
					 cxt_size * 64,
					 cxt_size - 1);
			return round_up(cxt_size * 64, PAGE_SIZE);
226 227 228 229 230 231 232 233 234
		case 3:
		case 2:
		/* For the special day when i810 gets merged. */
		case 1:
			return 0;
		}
		break;
	default:
		MISSING_CASE(class);
235
		/* fall through */
236 237 238 239 240 241 242 243 244
	case VIDEO_DECODE_CLASS:
	case VIDEO_ENHANCEMENT_CLASS:
	case COPY_ENGINE_CLASS:
		if (INTEL_GEN(dev_priv) < 8)
			return 0;
		return GEN8_LR_CONTEXT_OTHER_SIZE;
	}
}

245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
static u32 __engine_mmio_base(struct drm_i915_private *i915,
			      const struct engine_mmio_base *bases)
{
	int i;

	for (i = 0; i < MAX_MMIO_BASES; i++)
		if (INTEL_GEN(i915) >= bases[i].gen)
			break;

	GEM_BUG_ON(i == MAX_MMIO_BASES);
	GEM_BUG_ON(!bases[i].base);

	return bases[i].base;
}

260 261 262 263 264 265 266
static void __sprint_engine_name(char *name, const struct engine_info *info)
{
	WARN_ON(snprintf(name, INTEL_ENGINE_CS_MAX_NAME, "%s%u",
			 intel_engine_classes[info->class].name,
			 info->instance) >= INTEL_ENGINE_CS_MAX_NAME);
}

267 268 269 270 271 272
void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask)
{
	/*
	 * Though they added more rings on g4x/ilk, they did not add
	 * per-engine HWSTAM until gen6.
	 */
273
	if (INTEL_GEN(engine->i915) < 6 && engine->class != RENDER_CLASS)
274 275
		return;

276 277
	if (INTEL_GEN(engine->i915) >= 3)
		ENGINE_WRITE(engine, RING_HWSTAM, mask);
278
	else
279
		ENGINE_WRITE16(engine, RING_HWSTAM, mask);
280 281 282 283 284 285 286 287
}

static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine)
{
	/* Mask off all writes into the unknown HWSP */
	intel_engine_set_hwsp_writemask(engine, ~0u);
}

288
static int
289 290 291 292
intel_engine_setup(struct drm_i915_private *dev_priv,
		   enum intel_engine_id id)
{
	const struct engine_info *info = &intel_engines[id];
293 294
	struct intel_engine_cs *engine;

295 296
	GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));

297 298 299
	BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH));
	BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH));

300
	if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS))
301 302
		return -EINVAL;

303
	if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE))
304 305
		return -EINVAL;

306
	if (GEM_DEBUG_WARN_ON(dev_priv->engine_class[info->class][info->instance]))
307 308
		return -EINVAL;

309 310 311 312
	GEM_BUG_ON(dev_priv->engine[id]);
	engine = kzalloc(sizeof(*engine), GFP_KERNEL);
	if (!engine)
		return -ENOMEM;
313

314 315
	BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES);

316
	engine->id = id;
317
	engine->mask = BIT(id);
318
	engine->i915 = dev_priv;
319
	engine->gt = &dev_priv->gt;
320
	engine->uncore = &dev_priv->uncore;
321
	__sprint_engine_name(engine->name, info);
322
	engine->hw_id = engine->guc_id = info->hw_id;
323
	engine->mmio_base = __engine_mmio_base(dev_priv, info->mmio_bases);
324 325
	engine->class = info->class;
	engine->instance = info->instance;
326

327 328 329 330 331 332
	/*
	 * To be overridden by the backend on setup. However to facilitate
	 * cleanup on error during setup, we always provide the destroy vfunc.
	 */
	engine->destroy = (typeof(engine->destroy))kfree;

333
	engine->uabi_class = intel_engine_classes[info->class].uabi_class;
334

335 336
	engine->context_size = intel_engine_context_size(dev_priv,
							 engine->class);
337 338
	if (WARN_ON(engine->context_size > BIT(20)))
		engine->context_size = 0;
339 340
	if (engine->context_size)
		DRIVER_CAPS(dev_priv)->has_logical_contexts = true;
341

342 343 344
	/* Nothing to do here, execute in order of dependencies */
	engine->schedule = NULL;

345
	seqlock_init(&engine->stats.lock);
346

347 348
	ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);

349 350 351
	/* Scrub mmio state on takeover */
	intel_engine_sanitize_mmio(engine);

352
	dev_priv->engine_class[info->class][info->instance] = engine;
353 354
	dev_priv->engine[id] = engine;
	return 0;
355 356
}

357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395
static void __setup_engine_capabilities(struct intel_engine_cs *engine)
{
	struct drm_i915_private *i915 = engine->i915;

	if (engine->class == VIDEO_DECODE_CLASS) {
		/*
		 * HEVC support is present on first engine instance
		 * before Gen11 and on all instances afterwards.
		 */
		if (INTEL_GEN(i915) >= 11 ||
		    (INTEL_GEN(i915) >= 9 && engine->instance == 0))
			engine->uabi_capabilities |=
				I915_VIDEO_CLASS_CAPABILITY_HEVC;

		/*
		 * SFC block is present only on even logical engine
		 * instances.
		 */
		if ((INTEL_GEN(i915) >= 11 &&
		     RUNTIME_INFO(i915)->vdbox_sfc_access & engine->mask) ||
		    (INTEL_GEN(i915) >= 9 && engine->instance == 0))
			engine->uabi_capabilities |=
				I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
	} else if (engine->class == VIDEO_ENHANCEMENT_CLASS) {
		if (INTEL_GEN(i915) >= 9)
			engine->uabi_capabilities |=
				I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
	}
}

static void intel_setup_engine_capabilities(struct drm_i915_private *i915)
{
	struct intel_engine_cs *engine;
	enum intel_engine_id id;

	for_each_engine(engine, i915, id)
		__setup_engine_capabilities(engine);
}

396 397 398 399 400 401 402 403 404 405 406 407 408 409 410
/**
 * intel_engines_cleanup() - free the resources allocated for Command Streamers
 * @i915: the i915 devic
 */
void intel_engines_cleanup(struct drm_i915_private *i915)
{
	struct intel_engine_cs *engine;
	enum intel_engine_id id;

	for_each_engine(engine, i915, id) {
		engine->destroy(engine);
		i915->engine[id] = NULL;
	}
}

411
/**
412
 * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
413
 * @i915: the i915 device
414 415 416
 *
 * Return: non-zero if the initialization failed.
 */
417
int intel_engines_init_mmio(struct drm_i915_private *i915)
418
{
419 420
	struct intel_device_info *device_info = mkwrite_device_info(i915);
	const unsigned int engine_mask = INTEL_INFO(i915)->engine_mask;
421
	unsigned int mask = 0;
422
	unsigned int i;
423
	int err;
424

425 426
	WARN_ON(engine_mask == 0);
	WARN_ON(engine_mask &
427
		GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES));
428

429 430 431
	if (i915_inject_load_failure())
		return -ENODEV;

432
	for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
433
		if (!HAS_ENGINE(i915, i))
434 435
			continue;

436
		err = intel_engine_setup(i915, i);
437 438 439
		if (err)
			goto cleanup;

440
		mask |= BIT(i);
441 442 443 444 445 446 447
	}

	/*
	 * Catch failures to update intel_engines table when the new engines
	 * are added to the driver by a warning and disabling the forgotten
	 * engines.
	 */
448 449
	if (WARN_ON(mask != engine_mask))
		device_info->engine_mask = mask;
450

451
	/* We always presume we have at least RCS available for later probing */
452
	if (WARN_ON(!HAS_ENGINE(i915, RCS0))) {
453 454 455 456
		err = -ENODEV;
		goto cleanup;
	}

457
	RUNTIME_INFO(i915)->num_engines = hweight32(mask);
458

459
	intel_gt_check_and_clear_faults(&i915->gt);
460

461 462
	intel_setup_engine_capabilities(i915);

463 464 465
	return 0;

cleanup:
466
	intel_engines_cleanup(i915);
467 468 469 470
	return err;
}

/**
471
 * intel_engines_init() - init the Engine Command Streamers
472
 * @i915: i915 device private
473 474 475
 *
 * Return: non-zero if the initialization failed.
 */
476
int intel_engines_init(struct drm_i915_private *i915)
477
{
478
	int (*init)(struct intel_engine_cs *engine);
479
	struct intel_engine_cs *engine;
480
	enum intel_engine_id id;
481
	int err;
482

483 484 485 486
	if (HAS_EXECLISTS(i915))
		init = intel_execlists_submission_init;
	else
		init = intel_ring_submission_init;
487

488
	for_each_engine(engine, i915, id) {
489
		err = init(engine);
490
		if (err)
491 492 493 494 495 496
			goto cleanup;
	}

	return 0;

cleanup:
497
	intel_engines_cleanup(i915);
498
	return err;
499 500
}

501 502 503 504 505
static void intel_engine_init_batch_pool(struct intel_engine_cs *engine)
{
	i915_gem_batch_pool_init(&engine->batch_pool, engine);
}

506
void intel_engine_init_execlists(struct intel_engine_cs *engine)
507 508 509
{
	struct intel_engine_execlists * const execlists = &engine->execlists;

510
	execlists->port_mask = 1;
511
	GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists)));
512 513
	GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);

514 515 516 517
	memset(execlists->pending, 0, sizeof(execlists->pending));
	execlists->active =
		memset(execlists->inflight, 0, sizeof(execlists->inflight));

518
	execlists->queue_priority_hint = INT_MIN;
519
	execlists->queue = RB_ROOT_CACHED;
520 521
}

522
static void cleanup_status_page(struct intel_engine_cs *engine)
523
{
524 525
	struct i915_vma *vma;

526 527 528
	/* Prevent writes into HWSP after returning the page to the system */
	intel_engine_set_hwsp_writemask(engine, ~0u);

529 530 531
	vma = fetch_and_zero(&engine->status_page.vma);
	if (!vma)
		return;
532

533 534 535 536
	if (!HWS_NEEDS_PHYSICAL(engine->i915))
		i915_vma_unpin(vma);

	i915_gem_object_unpin_map(vma->obj);
537
	i915_gem_object_put(vma->obj);
538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
}

static int pin_ggtt_status_page(struct intel_engine_cs *engine,
				struct i915_vma *vma)
{
	unsigned int flags;

	flags = PIN_GLOBAL;
	if (!HAS_LLC(engine->i915))
		/*
		 * On g33, we cannot place HWS above 256MiB, so
		 * restrict its pinning to the low mappable arena.
		 * Though this restriction is not documented for
		 * gen4, gen5, or byt, they also behave similarly
		 * and hang if the HWS is placed at the top of the
		 * GTT. To generalise, it appears that all !llc
		 * platforms have issues with us placing the HWS
		 * above the mappable region (even though we never
		 * actually map it).
		 */
		flags |= PIN_MAPPABLE;
	else
		flags |= PIN_HIGH;
561

562
	return i915_vma_pin(vma, 0, 0, flags);
563 564 565 566 567 568 569 570 571
}

static int init_status_page(struct intel_engine_cs *engine)
{
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma;
	void *vaddr;
	int ret;

572 573 574 575 576 577 578
	/*
	 * Though the HWS register does support 36bit addresses, historically
	 * we have had hangs and corruption reported due to wild writes if
	 * the HWS is placed above 4G. We only allow objects to be allocated
	 * in GFP_DMA32 for i965, and no earlier physical address users had
	 * access to more than 4G.
	 */
579 580 581 582 583 584
	obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
	if (IS_ERR(obj)) {
		DRM_ERROR("Failed to allocate status page\n");
		return PTR_ERR(obj);
	}

585
	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
586

587
	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
588 589 590 591 592 593 594 595
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
		goto err;
	}

	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
	if (IS_ERR(vaddr)) {
		ret = PTR_ERR(vaddr);
596
		goto err;
597 598
	}

599
	engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE);
600
	engine->status_page.vma = vma;
601 602 603 604 605 606 607

	if (!HWS_NEEDS_PHYSICAL(engine->i915)) {
		ret = pin_ggtt_status_page(engine, vma);
		if (ret)
			goto err_unpin;
	}

608 609 610
	return 0;

err_unpin:
611
	i915_gem_object_unpin_map(obj);
612 613 614 615 616
err:
	i915_gem_object_put(obj);
	return ret;
}

617
static int intel_engine_setup_common(struct intel_engine_cs *engine)
618 619 620
{
	int err;

621 622
	init_llist_head(&engine->barrier_tasks);

623 624 625 626
	err = init_status_page(engine);
	if (err)
		return err;

627
	intel_engine_init_active(engine, ENGINE_PHYSICAL);
628
	intel_engine_init_breadcrumbs(engine);
629
	intel_engine_init_execlists(engine);
630 631 632
	intel_engine_init_hangcheck(engine);
	intel_engine_init_batch_pool(engine);
	intel_engine_init_cmd_parser(engine);
633
	intel_engine_init__pm(engine);
634

635 636 637 638
	/* Use the whole device by default */
	engine->sseu =
		intel_sseu_from_device_info(&RUNTIME_INFO(engine->i915)->sseu);

639 640 641
	return 0;
}

642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671
/**
 * intel_engines_setup- setup engine state not requiring hw access
 * @i915: Device to setup.
 *
 * Initializes engine structure members shared between legacy and execlists
 * submission modes which do not require hardware access.
 *
 * Typically done early in the submission mode specific engine setup stage.
 */
int intel_engines_setup(struct drm_i915_private *i915)
{
	int (*setup)(struct intel_engine_cs *engine);
	struct intel_engine_cs *engine;
	enum intel_engine_id id;
	int err;

	if (HAS_EXECLISTS(i915))
		setup = intel_execlists_submission_setup;
	else
		setup = intel_ring_submission_setup;

	for_each_engine(engine, i915, id) {
		err = intel_engine_setup_common(engine);
		if (err)
			goto cleanup;

		err = setup(engine);
		if (err)
			goto cleanup;

672 673 674
		/* We expect the backend to take control over its state */
		GEM_BUG_ON(engine->destroy == (typeof(engine->destroy))kfree);

675 676 677 678 679 680
		GEM_BUG_ON(!engine->cops);
	}

	return 0;

cleanup:
681
	intel_engines_cleanup(i915);
682 683 684
	return err;
}

685 686 687 688 689 690
void intel_engines_set_scheduler_caps(struct drm_i915_private *i915)
{
	static const struct {
		u8 engine;
		u8 sched;
	} map[] = {
691 692 693 694
#define MAP(x, y) { ilog2(I915_ENGINE_##x), ilog2(I915_SCHEDULER_CAP_##y) }
		MAP(HAS_PREEMPTION, PREEMPTION),
		MAP(HAS_SEMAPHORES, SEMAPHORES),
		MAP(SUPPORTS_STATS, ENGINE_BUSY_STATS),
695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
#undef MAP
	};
	struct intel_engine_cs *engine;
	enum intel_engine_id id;
	u32 enabled, disabled;

	enabled = 0;
	disabled = 0;
	for_each_engine(engine, i915, id) { /* all engines must agree! */
		int i;

		if (engine->schedule)
			enabled |= (I915_SCHEDULER_CAP_ENABLED |
				    I915_SCHEDULER_CAP_PRIORITY);
		else
			disabled |= (I915_SCHEDULER_CAP_ENABLED |
				     I915_SCHEDULER_CAP_PRIORITY);

		for (i = 0; i < ARRAY_SIZE(map); i++) {
			if (engine->flags & BIT(map[i].engine))
				enabled |= BIT(map[i].sched);
			else
				disabled |= BIT(map[i].sched);
		}
	}

	i915->caps.scheduler = enabled & ~disabled;
	if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_ENABLED))
		i915->caps.scheduler = 0;
}

726 727
struct measure_breadcrumb {
	struct i915_request rq;
728
	struct intel_timeline timeline;
729 730 731 732
	struct intel_ring ring;
	u32 cs[1024];
};

733
static int measure_breadcrumb_dw(struct intel_engine_cs *engine)
734 735
{
	struct measure_breadcrumb *frame;
736
	int dw = -ENOMEM;
737

738
	GEM_BUG_ON(!engine->gt->scratch);
739 740 741 742 743

	frame = kzalloc(sizeof(*frame), GFP_KERNEL);
	if (!frame)
		return -ENOMEM;

744 745 746
	if (intel_timeline_init(&frame->timeline,
				engine->gt,
				engine->status_page.vma))
747
		goto out_frame;
748 749 750 751 752 753 754 755 756 757 758 759 760

	INIT_LIST_HEAD(&frame->ring.request_list);
	frame->ring.timeline = &frame->timeline;
	frame->ring.vaddr = frame->cs;
	frame->ring.size = sizeof(frame->cs);
	frame->ring.effective_size = frame->ring.size;
	intel_ring_update_space(&frame->ring);

	frame->rq.i915 = engine->i915;
	frame->rq.engine = engine;
	frame->rq.ring = &frame->ring;
	frame->rq.timeline = &frame->timeline;

761
	dw = intel_timeline_pin(&frame->timeline);
762 763 764
	if (dw < 0)
		goto out_timeline;

765
	dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs;
766
	GEM_BUG_ON(dw & 1); /* RING_TAIL must be qword aligned */
767

768
	intel_timeline_unpin(&frame->timeline);
769

770
out_timeline:
771
	intel_timeline_fini(&frame->timeline);
772 773
out_frame:
	kfree(frame);
774 775 776
	return dw;
}

777 778 779 780 781
static int pin_context(struct i915_gem_context *ctx,
		       struct intel_engine_cs *engine,
		       struct intel_context **out)
{
	struct intel_context *ce;
782
	int err;
783

784
	ce = i915_gem_context_get_engine(ctx, engine->id);
785 786 787
	if (IS_ERR(ce))
		return PTR_ERR(ce);

788 789 790 791 792
	err = intel_context_pin(ce);
	intel_context_put(ce);
	if (err)
		return err;

793 794 795 796
	*out = ce;
	return 0;
}

797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
void
intel_engine_init_active(struct intel_engine_cs *engine, unsigned int subclass)
{
	INIT_LIST_HEAD(&engine->active.requests);

	spin_lock_init(&engine->active.lock);
	lockdep_set_subclass(&engine->active.lock, subclass);

	/*
	 * Due to an interesting quirk in lockdep's internal debug tracking,
	 * after setting a subclass we must ensure the lock is used. Otherwise,
	 * nr_unused_locks is incremented once too often.
	 */
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	local_irq_disable();
	lock_map_acquire(&engine->active.lock.dep_map);
	lock_map_release(&engine->active.lock.dep_map);
	local_irq_enable();
#endif
}

818 819 820 821 822 823 824 825 826 827 828 829 830
/**
 * intel_engines_init_common - initialize cengine state which might require hw access
 * @engine: Engine to initialize.
 *
 * Initializes @engine@ structure members shared between legacy and execlists
 * submission modes which do require hardware access.
 *
 * Typcally done at later stages of submission mode specific engine setup.
 *
 * Returns zero on success or an error code on failure.
 */
int intel_engine_init_common(struct intel_engine_cs *engine)
{
831
	struct drm_i915_private *i915 = engine->i915;
832 833
	int ret;

834 835 836 837 838 839 840
	/* We may need to do things with the shrinker which
	 * require us to immediately switch back to the default
	 * context. This can cause a problem as pinning the
	 * default context also requires GTT space which may not
	 * be available. To avoid this we always pin the default
	 * context.
	 */
841 842 843 844
	ret = pin_context(i915->kernel_context, engine,
			  &engine->kernel_context);
	if (ret)
		return ret;
845

846 847
	/*
	 * Similarly the preempt context must always be available so that
848 849
	 * we can interrupt the engine at any time. However, as preemption
	 * is optional, we allow it to fail.
850
	 */
851 852 853
	if (i915->preempt_context)
		pin_context(i915->preempt_context, engine,
			    &engine->preempt_context);
854

855
	ret = measure_breadcrumb_dw(engine);
856
	if (ret < 0)
857
		goto err_unpin;
858

859
	engine->emit_fini_breadcrumb_dw = ret;
860

861
	engine->set_default_submission(engine);
862

863
	return 0;
864

865 866 867 868
err_unpin:
	if (engine->preempt_context)
		intel_context_unpin(engine->preempt_context);
	intel_context_unpin(engine->kernel_context);
869
	return ret;
870
}
871 872 873 874 875 876 877 878 879 880

/**
 * intel_engines_cleanup_common - cleans up the engine state created by
 *                                the common initiailizers.
 * @engine: Engine to cleanup.
 *
 * This cleans up everything created by the common helpers.
 */
void intel_engine_cleanup_common(struct intel_engine_cs *engine)
{
881 882
	GEM_BUG_ON(!list_empty(&engine->active.requests));

883
	cleanup_status_page(engine);
884

885
	intel_engine_fini_breadcrumbs(engine);
886
	intel_engine_cleanup_cmd_parser(engine);
887
	i915_gem_batch_pool_fini(&engine->batch_pool);
888

889 890 891
	if (engine->default_state)
		i915_gem_object_put(engine->default_state);

892 893 894
	if (engine->preempt_context)
		intel_context_unpin(engine->preempt_context);
	intel_context_unpin(engine->kernel_context);
895
	GEM_BUG_ON(!llist_empty(&engine->barrier_tasks));
896

897
	intel_wa_list_free(&engine->ctx_wa_list);
898
	intel_wa_list_free(&engine->wa_list);
899
	intel_wa_list_free(&engine->whitelist);
900
}
901

902
u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
903
{
904 905
	struct drm_i915_private *i915 = engine->i915;

906 907
	u64 acthd;

908 909 910 911
	if (INTEL_GEN(i915) >= 8)
		acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW);
	else if (INTEL_GEN(i915) >= 4)
		acthd = ENGINE_READ(engine, RING_ACTHD);
912
	else
913
		acthd = ENGINE_READ(engine, ACTHD);
914 915 916 917

	return acthd;
}

918
u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
919 920 921
{
	u64 bbaddr;

922 923
	if (INTEL_GEN(engine->i915) >= 8)
		bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW);
924
	else
925
		bbaddr = ENGINE_READ(engine, RING_BBADDR);
926 927 928

	return bbaddr;
}
929

930 931
int intel_engine_stop_cs(struct intel_engine_cs *engine)
{
932
	struct intel_uncore *uncore = engine->uncore;
933 934 935 936
	const u32 base = engine->mmio_base;
	const i915_reg_t mode = RING_MI_MODE(base);
	int err;

937
	if (INTEL_GEN(engine->i915) < 3)
938 939 940 941
		return -ENODEV;

	GEM_TRACE("%s\n", engine->name);

942
	intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING));
943 944

	err = 0;
945
	if (__intel_wait_for_register_fw(uncore,
946 947 948 949 950 951 952 953
					 mode, MODE_IDLE, MODE_IDLE,
					 1000, 0,
					 NULL)) {
		GEM_TRACE("%s: timed out on STOP_RING -> IDLE\n", engine->name);
		err = -ETIMEDOUT;
	}

	/* A final mmio read to let GPU writes be hopefully flushed to memory */
954
	intel_uncore_posting_read_fw(uncore, mode);
955 956 957 958

	return err;
}

959 960 961 962
void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
{
	GEM_TRACE("%s\n", engine->name);

963
	ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
964 965
}

966 967 968 969 970 971 972 973 974 975 976
const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
{
	switch (type) {
	case I915_CACHE_NONE: return " uncached";
	case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
	case I915_CACHE_L3_LLC: return " L3+LLC";
	case I915_CACHE_WT: return " WT";
	default: return "";
	}
}

977 978
u32 intel_calculate_mcr_s_ss_select(struct drm_i915_private *dev_priv)
{
979
	const struct sseu_dev_info *sseu = &RUNTIME_INFO(dev_priv)->sseu;
980 981
	u32 mcr_s_ss_select;
	u32 slice = fls(sseu->slice_mask);
982
	u32 subslice = fls(sseu->subslice_mask[slice]);
983

984
	if (IS_GEN(dev_priv, 10))
985 986
		mcr_s_ss_select = GEN8_MCR_SLICE(slice) |
				  GEN8_MCR_SUBSLICE(subslice);
987 988 989
	else if (INTEL_GEN(dev_priv) >= 11)
		mcr_s_ss_select = GEN11_MCR_SLICE(slice) |
				  GEN11_MCR_SUBSLICE(subslice);
990 991 992 993 994 995
	else
		mcr_s_ss_select = 0;

	return mcr_s_ss_select;
}

996 997 998
static u32
read_subslice_reg(struct intel_engine_cs *engine, int slice, int subslice,
		  i915_reg_t reg)
999
{
1000 1001
	struct drm_i915_private *i915 = engine->i915;
	struct intel_uncore *uncore = engine->uncore;
1002 1003 1004 1005 1006
	u32 mcr_slice_subslice_mask;
	u32 mcr_slice_subslice_select;
	u32 default_mcr_s_ss_select;
	u32 mcr;
	u32 ret;
1007 1008
	enum forcewake_domains fw_domains;

1009
	if (INTEL_GEN(i915) >= 11) {
1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
		mcr_slice_subslice_mask = GEN11_MCR_SLICE_MASK |
					  GEN11_MCR_SUBSLICE_MASK;
		mcr_slice_subslice_select = GEN11_MCR_SLICE(slice) |
					    GEN11_MCR_SUBSLICE(subslice);
	} else {
		mcr_slice_subslice_mask = GEN8_MCR_SLICE_MASK |
					  GEN8_MCR_SUBSLICE_MASK;
		mcr_slice_subslice_select = GEN8_MCR_SLICE(slice) |
					    GEN8_MCR_SUBSLICE(subslice);
	}

1021
	default_mcr_s_ss_select = intel_calculate_mcr_s_ss_select(i915);
1022

1023
	fw_domains = intel_uncore_forcewake_for_reg(uncore, reg,
1024
						    FW_REG_READ);
1025
	fw_domains |= intel_uncore_forcewake_for_reg(uncore,
1026 1027 1028
						     GEN8_MCR_SELECTOR,
						     FW_REG_READ | FW_REG_WRITE);

1029 1030
	spin_lock_irq(&uncore->lock);
	intel_uncore_forcewake_get__locked(uncore, fw_domains);
1031

1032
	mcr = intel_uncore_read_fw(uncore, GEN8_MCR_SELECTOR);
1033 1034 1035 1036

	WARN_ON_ONCE((mcr & mcr_slice_subslice_mask) !=
		     default_mcr_s_ss_select);

1037 1038
	mcr &= ~mcr_slice_subslice_mask;
	mcr |= mcr_slice_subslice_select;
1039
	intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
1040

1041
	ret = intel_uncore_read_fw(uncore, reg);
1042

1043
	mcr &= ~mcr_slice_subslice_mask;
1044 1045
	mcr |= default_mcr_s_ss_select;

1046
	intel_uncore_write_fw(uncore, GEN8_MCR_SELECTOR, mcr);
1047

1048 1049
	intel_uncore_forcewake_put__locked(uncore, fw_domains);
	spin_unlock_irq(&uncore->lock);
1050 1051 1052 1053 1054 1055 1056 1057

	return ret;
}

/* NB: please notice the memset */
void intel_engine_get_instdone(struct intel_engine_cs *engine,
			       struct intel_instdone *instdone)
{
1058
	struct drm_i915_private *i915 = engine->i915;
1059
	struct intel_uncore *uncore = engine->uncore;
1060 1061 1062 1063 1064 1065
	u32 mmio_base = engine->mmio_base;
	int slice;
	int subslice;

	memset(instdone, 0, sizeof(*instdone));

1066
	switch (INTEL_GEN(i915)) {
1067
	default:
1068 1069
		instdone->instdone =
			intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1070

1071
		if (engine->id != RCS0)
1072 1073
			break;

1074 1075
		instdone->slice_common =
			intel_uncore_read(uncore, GEN7_SC_INSTDONE);
1076
		for_each_instdone_slice_subslice(i915, slice, subslice) {
1077
			instdone->sampler[slice][subslice] =
1078
				read_subslice_reg(engine, slice, subslice,
1079 1080
						  GEN7_SAMPLER_INSTDONE);
			instdone->row[slice][subslice] =
1081
				read_subslice_reg(engine, slice, subslice,
1082 1083 1084 1085
						  GEN7_ROW_INSTDONE);
		}
		break;
	case 7:
1086 1087
		instdone->instdone =
			intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1088

1089
		if (engine->id != RCS0)
1090 1091
			break;

1092 1093 1094 1095 1096 1097
		instdone->slice_common =
			intel_uncore_read(uncore, GEN7_SC_INSTDONE);
		instdone->sampler[0][0] =
			intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE);
		instdone->row[0][0] =
			intel_uncore_read(uncore, GEN7_ROW_INSTDONE);
1098 1099 1100 1101 1102

		break;
	case 6:
	case 5:
	case 4:
1103 1104
		instdone->instdone =
			intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1105
		if (engine->id == RCS0)
1106
			/* HACK: Using the wrong struct member */
1107 1108
			instdone->slice_common =
				intel_uncore_read(uncore, GEN4_INSTDONE1);
1109 1110 1111
		break;
	case 3:
	case 2:
1112
		instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE);
1113 1114 1115
		break;
	}
}
1116

1117 1118 1119
static bool ring_is_idle(struct intel_engine_cs *engine)
{
	struct drm_i915_private *dev_priv = engine->i915;
1120
	intel_wakeref_t wakeref;
1121 1122
	bool idle = true;

1123 1124 1125
	if (I915_SELFTEST_ONLY(!engine->mmio_base))
		return true;

1126
	/* If the whole device is asleep, the engine must be idle */
1127
	wakeref = intel_runtime_pm_get_if_in_use(&dev_priv->runtime_pm);
1128
	if (!wakeref)
1129
		return true;
1130

1131
	/* First check that no commands are left in the ring */
1132 1133
	if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) !=
	    (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR))
1134
		idle = false;
1135

1136
	/* No bit for gen2, so assume the CS parser is idle */
1137 1138
	if (INTEL_GEN(dev_priv) > 2 &&
	    !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE))
1139 1140
		idle = false;

1141
	intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
1142 1143 1144 1145

	return idle;
}

1146 1147 1148 1149 1150 1151 1152 1153 1154
/**
 * intel_engine_is_idle() - Report if the engine has finished process all work
 * @engine: the intel_engine_cs
 *
 * Return true if there are no requests pending, nothing left to be submitted
 * to hardware, and that the engine is idle.
 */
bool intel_engine_is_idle(struct intel_engine_cs *engine)
{
1155
	/* More white lies, if wedged, hw state is inconsistent */
1156
	if (i915_reset_failed(engine->i915))
1157 1158
		return true;

1159
	if (!intel_engine_pm_is_awake(engine))
1160 1161
		return true;

1162
	/* Waiting to drain ELSP? */
1163
	if (execlists_active(&engine->execlists)) {
1164
		struct tasklet_struct *t = &engine->execlists.tasklet;
1165

1166
		synchronize_hardirq(engine->i915->drm.pdev->irq);
1167

1168
		local_bh_disable();
1169 1170 1171 1172 1173
		if (tasklet_trylock(t)) {
			/* Must wait for any GPU reset in progress. */
			if (__tasklet_is_enabled(t))
				t->func(t->data);
			tasklet_unlock(t);
1174
		}
1175
		local_bh_enable();
1176

1177 1178 1179
		/* Otherwise flush the tasklet if it was on another cpu */
		tasklet_unlock_wait(t);

1180
		if (execlists_active(&engine->execlists))
1181 1182
			return false;
	}
1183

1184
	/* ELSP is empty, but there are ready requests? E.g. after reset */
1185
	if (!RB_EMPTY_ROOT(&engine->execlists.queue.rb_root))
1186 1187
		return false;

1188
	/* Ring stopped? */
1189
	return ring_is_idle(engine);
1190 1191
}

1192
bool intel_engines_are_idle(struct drm_i915_private *i915)
1193 1194 1195 1196
{
	struct intel_engine_cs *engine;
	enum intel_engine_id id;

1197 1198
	/*
	 * If the driver is wedged, HW state may be very inconsistent and
1199 1200
	 * report that it is still busy, even though we have stopped using it.
	 */
1201
	if (i915_reset_failed(i915))
1202 1203
		return true;

1204 1205 1206 1207 1208
	/* Already parked (and passed an idleness test); must still be idle */
	if (!READ_ONCE(i915->gt.awake))
		return true;

	for_each_engine(engine, i915, id) {
1209 1210 1211 1212 1213 1214 1215
		if (!intel_engine_is_idle(engine))
			return false;
	}

	return true;
}

1216 1217 1218 1219 1220 1221 1222 1223 1224
void intel_engines_reset_default_submission(struct drm_i915_private *i915)
{
	struct intel_engine_cs *engine;
	enum intel_engine_id id;

	for_each_engine(engine, i915, id)
		engine->set_default_submission(engine);
}

1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
{
	switch (INTEL_GEN(engine->i915)) {
	case 2:
		return false; /* uses physical not virtual addresses */
	case 3:
		/* maybe only uses physical not virtual addresses */
		return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
	case 6:
		return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
	default:
		return true;
	}
}

1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253
unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915)
{
	struct intel_engine_cs *engine;
	enum intel_engine_id id;
	unsigned int which;

	which = 0;
	for_each_engine(engine, i915, id)
		if (engine->default_state)
			which |= BIT(engine->uabi_class);

	return which;
}

1254 1255 1256
static int print_sched_attr(struct drm_i915_private *i915,
			    const struct i915_sched_attr *attr,
			    char *buf, int x, int len)
1257 1258
{
	if (attr->priority == I915_PRIORITY_INVALID)
1259 1260 1261 1262
		return x;

	x += snprintf(buf + x, len - x,
		      " prio=%d", attr->priority);
1263

1264
	return x;
1265 1266
}

1267
static void print_request(struct drm_printer *m,
1268
			  struct i915_request *rq,
1269 1270
			  const char *prefix)
{
1271
	const char *name = rq->fence.ops->get_timeline_name(&rq->fence);
1272
	char buf[80] = "";
1273 1274 1275
	int x = 0;

	x = print_sched_attr(rq->i915, &rq->sched.attr, buf, x, sizeof(buf));
1276

1277
	drm_printf(m, "%s %llx:%llx%s%s %s @ %dms: %s\n",
1278
		   prefix,
1279
		   rq->fence.context, rq->fence.seqno,
1280 1281 1282
		   i915_request_completed(rq) ? "!" :
		   i915_request_started(rq) ? "*" :
		   "",
1283 1284
		   test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
			    &rq->fence.flags) ? "+" :
1285
		   test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
1286 1287
			    &rq->fence.flags) ? "-" :
		   "",
1288
		   buf,
1289
		   jiffies_to_msecs(jiffies - rq->emitted_jiffies),
1290
		   name);
1291 1292
}

1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
static void hexdump(struct drm_printer *m, const void *buf, size_t len)
{
	const size_t rowsize = 8 * sizeof(u32);
	const void *prev = NULL;
	bool skip = false;
	size_t pos;

	for (pos = 0; pos < len; pos += rowsize) {
		char line[128];

		if (prev && !memcmp(prev, buf + pos, rowsize)) {
			if (!skip) {
				drm_printf(m, "*\n");
				skip = true;
			}
			continue;
		}

		WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
						rowsize, sizeof(u32),
						line, sizeof(line),
						false) >= sizeof(line));
1315
		drm_printf(m, "[%04zx] %s\n", pos, line);
1316 1317 1318 1319 1320 1321

		prev = buf + pos;
		skip = false;
	}
}

1322
static void intel_engine_print_registers(struct intel_engine_cs *engine,
1323
					 struct drm_printer *m)
1324 1325
{
	struct drm_i915_private *dev_priv = engine->i915;
1326 1327
	const struct intel_engine_execlists * const execlists =
		&engine->execlists;
1328
	unsigned long flags;
1329 1330
	u64 addr;

1331
	if (engine->id == RCS0 && IS_GEN_RANGE(dev_priv, 4, 7))
1332
		drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID));
1333
	drm_printf(m, "\tRING_START: 0x%08x\n",
1334
		   ENGINE_READ(engine, RING_START));
1335
	drm_printf(m, "\tRING_HEAD:  0x%08x\n",
1336
		   ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR);
1337
	drm_printf(m, "\tRING_TAIL:  0x%08x\n",
1338
		   ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR);
1339
	drm_printf(m, "\tRING_CTL:   0x%08x%s\n",
1340 1341
		   ENGINE_READ(engine, RING_CTL),
		   ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
1342 1343
	if (INTEL_GEN(engine->i915) > 2) {
		drm_printf(m, "\tRING_MODE:  0x%08x%s\n",
1344 1345
			   ENGINE_READ(engine, RING_MI_MODE),
			   ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : "");
1346
	}
1347 1348

	if (INTEL_GEN(dev_priv) >= 6) {
1349 1350
		drm_printf(m, "\tRING_IMR: %08x\n",
			   ENGINE_READ(engine, RING_IMR));
1351 1352
	}

1353 1354 1355 1356 1357 1358
	addr = intel_engine_get_active_head(engine);
	drm_printf(m, "\tACTHD:  0x%08x_%08x\n",
		   upper_32_bits(addr), lower_32_bits(addr));
	addr = intel_engine_get_last_batch_head(engine);
	drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
		   upper_32_bits(addr), lower_32_bits(addr));
1359
	if (INTEL_GEN(dev_priv) >= 8)
1360
		addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW);
1361
	else if (INTEL_GEN(dev_priv) >= 4)
1362
		addr = ENGINE_READ(engine, RING_DMA_FADD);
1363
	else
1364
		addr = ENGINE_READ(engine, DMA_FADD_I8XX);
1365 1366 1367 1368
	drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
		   upper_32_bits(addr), lower_32_bits(addr));
	if (INTEL_GEN(dev_priv) >= 4) {
		drm_printf(m, "\tIPEIR: 0x%08x\n",
1369
			   ENGINE_READ(engine, RING_IPEIR));
1370
		drm_printf(m, "\tIPEHR: 0x%08x\n",
1371
			   ENGINE_READ(engine, RING_IPEHR));
1372
	} else {
1373 1374
		drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR));
		drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR));
1375
	}
1376

1377
	if (HAS_EXECLISTS(dev_priv)) {
1378
		struct i915_request * const *port, *rq;
1379 1380
		const u32 *hws =
			&engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
1381
		const u8 num_entries = execlists->csb_size;
1382
		unsigned int idx;
1383
		u8 read, write;
1384

1385
		drm_printf(m, "\tExeclist status: 0x%08x %08x, entries %u\n",
1386
			   ENGINE_READ(engine, RING_EXECLIST_STATUS_LO),
1387 1388
			   ENGINE_READ(engine, RING_EXECLIST_STATUS_HI),
			   num_entries);
1389

1390 1391 1392
		read = execlists->csb_head;
		write = READ_ONCE(*execlists->csb_write);

1393
		drm_printf(m, "\tExeclist CSB read %d, write %d, tasklet queued? %s (%s)\n",
1394
			   read, write,
1395 1396 1397
			   yesno(test_bit(TASKLET_STATE_SCHED,
					  &engine->execlists.tasklet.state)),
			   enableddisabled(!atomic_read(&engine->execlists.tasklet.count)));
1398
		if (read >= num_entries)
1399
			read = 0;
1400
		if (write >= num_entries)
1401 1402
			write = 0;
		if (read > write)
1403
			write += num_entries;
1404
		while (read < write) {
1405 1406 1407
			idx = ++read % num_entries;
			drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
				   idx, hws[idx * 2], hws[idx * 2 + 1]);
1408 1409
		}

1410
		spin_lock_irqsave(&engine->active.lock, flags);
1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
		for (port = execlists->active; (rq = *port); port++) {
			char hdr[80];
			int len;

			len = snprintf(hdr, sizeof(hdr),
				       "\t\tActive[%d: ",
				       (int)(port - execlists->active));
			if (!i915_request_signaled(rq))
				len += snprintf(hdr + len, sizeof(hdr) - len,
						"ring:{start:%08x, hwsp:%08x, seqno:%08x}, ",
						i915_ggtt_offset(rq->ring->vma),
						rq->timeline->hwsp_offset,
						hwsp_seqno(rq));
			snprintf(hdr + len, sizeof(hdr) - len, "rq: ");
			print_request(m, rq, hdr);
		}
		for (port = execlists->pending; (rq = *port); port++) {
1428
			char hdr[80];
1429

1430 1431 1432 1433 1434 1435 1436
			snprintf(hdr, sizeof(hdr),
				 "\t\tPending[%d] ring:{start:%08x, hwsp:%08x, seqno:%08x}, rq: ",
				 (int)(port - execlists->pending),
				 i915_ggtt_offset(rq->ring->vma),
				 rq->timeline->hwsp_offset,
				 hwsp_seqno(rq));
			print_request(m, rq, hdr);
1437
		}
1438
		spin_unlock_irqrestore(&engine->active.lock, flags);
1439 1440
	} else if (INTEL_GEN(dev_priv) > 6) {
		drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1441
			   ENGINE_READ(engine, RING_PP_DIR_BASE));
1442
		drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1443
			   ENGINE_READ(engine, RING_PP_DIR_BASE_READ));
1444
		drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1445
			   ENGINE_READ(engine, RING_PP_DIR_DCLV));
1446
	}
1447 1448
}

1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481
static void print_request_ring(struct drm_printer *m, struct i915_request *rq)
{
	void *ring;
	int size;

	drm_printf(m,
		   "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n",
		   rq->head, rq->postfix, rq->tail,
		   rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
		   rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);

	size = rq->tail - rq->head;
	if (rq->tail < rq->head)
		size += rq->ring->size;

	ring = kmalloc(size, GFP_ATOMIC);
	if (ring) {
		const void *vaddr = rq->ring->vaddr;
		unsigned int head = rq->head;
		unsigned int len = 0;

		if (rq->tail < head) {
			len = rq->ring->size - head;
			memcpy(ring, vaddr + head, len);
			head = 0;
		}
		memcpy(ring + len, vaddr + head, size - len);

		hexdump(m, ring, size);
		kfree(ring);
	}
}

1482 1483 1484 1485 1486
void intel_engine_dump(struct intel_engine_cs *engine,
		       struct drm_printer *m,
		       const char *header, ...)
{
	struct i915_gpu_error * const error = &engine->i915->gpu_error;
1487
	struct i915_request *rq;
1488
	intel_wakeref_t wakeref;
1489 1490 1491 1492 1493 1494 1495 1496 1497

	if (header) {
		va_list ap;

		va_start(ap, header);
		drm_vprintf(m, header, &ap);
		va_end(ap);
	}

1498
	if (i915_reset_failed(engine->i915))
1499 1500
		drm_printf(m, "*** WEDGED ***\n");

1501
	drm_printf(m, "\tAwake? %d\n", atomic_read(&engine->wakeref.count));
1502
	drm_printf(m, "\tHangcheck: %d ms ago\n",
1503
		   jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp));
1504 1505 1506 1507 1508 1509 1510 1511
	drm_printf(m, "\tReset count: %d (global %d)\n",
		   i915_reset_engine_count(error, engine),
		   i915_reset_count(error));

	rcu_read_lock();

	drm_printf(m, "\tRequests:\n");

1512
	rq = intel_engine_find_active_request(engine);
1513 1514
	if (rq) {
		print_request(m, rq, "\t\tactive ");
1515

1516
		drm_printf(m, "\t\tring->start:  0x%08x\n",
1517
			   i915_ggtt_offset(rq->ring->vma));
1518
		drm_printf(m, "\t\tring->head:   0x%08x\n",
1519
			   rq->ring->head);
1520
		drm_printf(m, "\t\tring->tail:   0x%08x\n",
1521
			   rq->ring->tail);
1522 1523 1524 1525
		drm_printf(m, "\t\tring->emit:   0x%08x\n",
			   rq->ring->emit);
		drm_printf(m, "\t\tring->space:  0x%08x\n",
			   rq->ring->space);
1526 1527
		drm_printf(m, "\t\tring->hwsp:   0x%08x\n",
			   rq->timeline->hwsp_offset);
1528 1529

		print_request_ring(m, rq);
1530 1531 1532 1533
	}

	rcu_read_unlock();

1534
	wakeref = intel_runtime_pm_get_if_in_use(&engine->i915->runtime_pm);
1535
	if (wakeref) {
1536
		intel_engine_print_registers(engine, m);
1537
		intel_runtime_pm_put(&engine->i915->runtime_pm, wakeref);
1538 1539 1540
	} else {
		drm_printf(m, "\tDevice is asleep; skipping register dump\n");
	}
1541

1542
	intel_execlists_show_requests(engine, m, print_request, 8);
1543

1544
	drm_printf(m, "HWSP:\n");
1545
	hexdump(m, engine->status_page.addr, PAGE_SIZE);
1546

1547
	drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine)));
1548 1549

	intel_engine_print_breadcrumbs(engine, m);
1550 1551
}

1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574
static u8 user_class_map[] = {
	[I915_ENGINE_CLASS_RENDER] = RENDER_CLASS,
	[I915_ENGINE_CLASS_COPY] = COPY_ENGINE_CLASS,
	[I915_ENGINE_CLASS_VIDEO] = VIDEO_DECODE_CLASS,
	[I915_ENGINE_CLASS_VIDEO_ENHANCE] = VIDEO_ENHANCEMENT_CLASS,
};

struct intel_engine_cs *
intel_engine_lookup_user(struct drm_i915_private *i915, u8 class, u8 instance)
{
	if (class >= ARRAY_SIZE(user_class_map))
		return NULL;

	class = user_class_map[class];

	GEM_BUG_ON(class > MAX_ENGINE_CLASS);

	if (instance > MAX_ENGINE_INSTANCE)
		return NULL;

	return i915->engine_class[class][instance];
}

1575 1576 1577 1578 1579 1580 1581 1582 1583 1584
/**
 * intel_enable_engine_stats() - Enable engine busy tracking on engine
 * @engine: engine to enable stats collection
 *
 * Start collecting the engine busyness data for @engine.
 *
 * Returns 0 on success or a negative error code.
 */
int intel_enable_engine_stats(struct intel_engine_cs *engine)
{
1585
	struct intel_engine_execlists *execlists = &engine->execlists;
1586
	unsigned long flags;
1587
	int err = 0;
1588

1589
	if (!intel_engine_supports_stats(engine))
1590 1591
		return -ENODEV;

1592
	spin_lock_irqsave(&engine->active.lock, flags);
1593
	write_seqlock(&engine->stats.lock);
1594 1595 1596 1597 1598 1599

	if (unlikely(engine->stats.enabled == ~0)) {
		err = -EBUSY;
		goto unlock;
	}

1600
	if (engine->stats.enabled++ == 0) {
1601 1602
		struct i915_request * const *port;
		struct i915_request *rq;
1603

1604
		engine->stats.enabled_at = ktime_get();
1605 1606

		/* XXX submission method oblivious? */
1607
		for (port = execlists->active; (rq = *port); port++)
1608
			engine->stats.active++;
1609 1610 1611 1612 1613

		for (port = execlists->pending; (rq = *port); port++) {
			/* Exclude any contexts already counted in active */
			if (intel_context_inflight_count(rq->hw_context) == 1)
				engine->stats.active++;
1614 1615 1616 1617 1618
		}

		if (engine->stats.active)
			engine->stats.start = engine->stats.enabled_at;
	}
1619

1620
unlock:
1621
	write_sequnlock(&engine->stats.lock);
1622
	spin_unlock_irqrestore(&engine->active.lock, flags);
1623

1624
	return err;
1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649
}

static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine)
{
	ktime_t total = engine->stats.total;

	/*
	 * If the engine is executing something at the moment
	 * add it to the total.
	 */
	if (engine->stats.active)
		total = ktime_add(total,
				  ktime_sub(ktime_get(), engine->stats.start));

	return total;
}

/**
 * intel_engine_get_busy_time() - Return current accumulated engine busyness
 * @engine: engine to report on
 *
 * Returns accumulated time @engine was busy since engine stats were enabled.
 */
ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine)
{
1650
	unsigned int seq;
1651 1652
	ktime_t total;

1653 1654 1655 1656
	do {
		seq = read_seqbegin(&engine->stats.lock);
		total = __intel_engine_get_busy_time(engine);
	} while (read_seqretry(&engine->stats.lock, seq));
1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670

	return total;
}

/**
 * intel_disable_engine_stats() - Disable engine busy tracking on engine
 * @engine: engine to disable stats collection
 *
 * Stops collecting the engine busyness data for @engine.
 */
void intel_disable_engine_stats(struct intel_engine_cs *engine)
{
	unsigned long flags;

1671
	if (!intel_engine_supports_stats(engine))
1672 1673
		return;

1674
	write_seqlock_irqsave(&engine->stats.lock, flags);
1675 1676 1677 1678 1679
	WARN_ON_ONCE(engine->stats.enabled == 0);
	if (--engine->stats.enabled == 0) {
		engine->stats.total = __intel_engine_get_busy_time(engine);
		engine->stats.active = 0;
	}
1680
	write_sequnlock_irqrestore(&engine->stats.lock, flags);
1681 1682
}

1683 1684
static bool match_ring(struct i915_request *rq)
{
1685
	u32 ring = ENGINE_READ(rq->engine, RING_START);
1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706

	return ring == i915_ggtt_offset(rq->ring->vma);
}

struct i915_request *
intel_engine_find_active_request(struct intel_engine_cs *engine)
{
	struct i915_request *request, *active = NULL;
	unsigned long flags;

	/*
	 * We are called by the error capture, reset and to dump engine
	 * state at random points in time. In particular, note that neither is
	 * crucially ordered with an interrupt. After a hang, the GPU is dead
	 * and we assume that no more writes can happen (we waited long enough
	 * for all writes that were in transaction to be flushed) - adding an
	 * extra delay for a recent interrupt is pointless. Hence, we do
	 * not need an engine->irq_seqno_barrier() before the seqno reads.
	 * At all other times, we must assume the GPU is still running, but
	 * we only care about the snapshot of this moment.
	 */
1707 1708
	spin_lock_irqsave(&engine->active.lock, flags);
	list_for_each_entry(request, &engine->active.requests, sched.link) {
1709 1710 1711 1712
		if (i915_request_completed(request))
			continue;

		if (!i915_request_started(request))
1713
			continue;
1714 1715 1716

		/* More than one preemptible request may match! */
		if (!match_ring(request))
1717
			continue;
1718 1719 1720 1721

		active = request;
		break;
	}
1722
	spin_unlock_irqrestore(&engine->active.lock, flags);
1723 1724 1725 1726

	return active;
}

1727
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1728
#include "selftest_engine_cs.c"
1729
#endif