i915_guc_submission.c 32.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
/*
 * Copyright © 2014 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */
#include <linux/firmware.h>
#include <linux/circ_buf.h>
#include "i915_drv.h"
#include "intel_guc.h"

29
/**
A
Alex Dai 已提交
30
 * DOC: GuC-based command submission
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
 *
 * i915_guc_client:
 * We use the term client to avoid confusion with contexts. A i915_guc_client is
 * equivalent to GuC object guc_context_desc. This context descriptor is
 * allocated from a pool of 1024 entries. Kernel driver will allocate doorbell
 * and workqueue for it. Also the process descriptor (guc_process_desc), which
 * is mapped to client space. So the client can write Work Item then ring the
 * doorbell.
 *
 * To simplify the implementation, we allocate one gem object that contains all
 * pages for doorbell, process descriptor and workqueue.
 *
 * The Scratch registers:
 * There are 16 MMIO-based registers start from 0xC180. The kernel driver writes
 * a value to the action register (SOFT_SCRATCH_0) along with any data. It then
 * triggers an interrupt on the GuC via another register write (0xC4C8).
 * Firmware writes a success/fail code back to the action register after
 * processes the request. The kernel driver polls waiting for this update and
 * then proceeds.
 * See host2guc_action()
 *
 * Doorbells:
 * Doorbells are interrupts to uKernel. A doorbell is a single cache line (QW)
 * mapped into process space.
 *
 * Work Items:
 * There are several types of work items that the host may place into a
 * workqueue, each with its own requirements and limitations. Currently only
 * WQ_TYPE_INORDER is needed to support legacy submission via GuC, which
 * represents in-order queue. The kernel driver packs ring tail pointer and an
 * ELSP context descriptor dword into Work Item.
62
 * See guc_wq_item_append()
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
 *
 */

/*
 * Read GuC command/status register (SOFT_SCRATCH_0)
 * Return true if it contains a response rather than a command
 */
static inline bool host2guc_action_response(struct drm_i915_private *dev_priv,
					    u32 *status)
{
	u32 val = I915_READ(SOFT_SCRATCH(0));
	*status = val;
	return GUC2HOST_IS_RESPONSE(val);
}

static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len)
{
	struct drm_i915_private *dev_priv = guc_to_i915(guc);
	u32 status;
	int i;
	int ret;

	if (WARN_ON(len < 1 || len > 15))
		return -EINVAL;

	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);

	dev_priv->guc.action_count += 1;
	dev_priv->guc.action_cmd = data[0];

	for (i = 0; i < len; i++)
		I915_WRITE(SOFT_SCRATCH(i), data[i]);

	POSTING_READ(SOFT_SCRATCH(i - 1));

	I915_WRITE(HOST2GUC_INTERRUPT, HOST2GUC_TRIGGER);

100 101 102 103 104 105 106 107
	/*
	 * Fast commands should complete in less than 10us, so sample quickly
	 * up to that length of time, then switch to a slower sleep-wait loop.
	 * No HOST2GUC command should ever take longer than 10ms.
	 */
	ret = wait_for_us(host2guc_action_response(dev_priv, &status), 10);
	if (ret)
		ret = wait_for(host2guc_action_response(dev_priv, &status), 10);
108 109 110 111 112 113 114 115 116
	if (status != GUC2HOST_STATUS_SUCCESS) {
		/*
		 * Either the GuC explicitly returned an error (which
		 * we convert to -EIO here) or no response at all was
		 * received within the timeout limit (-ETIMEDOUT)
		 */
		if (ret != -ETIMEDOUT)
			ret = -EIO;

117 118
		DRM_WARN("Action 0x%X failed; ret=%d status=0x%08X response=0x%08X\n",
			 data[0], ret, status, I915_READ(SOFT_SCRATCH(15)));
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155

		dev_priv->guc.action_fail += 1;
		dev_priv->guc.action_err = ret;
	}
	dev_priv->guc.action_status = status;

	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);

	return ret;
}

/*
 * Tell the GuC to allocate or deallocate a specific doorbell
 */

static int host2guc_allocate_doorbell(struct intel_guc *guc,
				      struct i915_guc_client *client)
{
	u32 data[2];

	data[0] = HOST2GUC_ACTION_ALLOCATE_DOORBELL;
	data[1] = client->ctx_index;

	return host2guc_action(guc, data, 2);
}

static int host2guc_release_doorbell(struct intel_guc *guc,
				     struct i915_guc_client *client)
{
	u32 data[2];

	data[0] = HOST2GUC_ACTION_DEALLOCATE_DOORBELL;
	data[1] = client->ctx_index;

	return host2guc_action(guc, data, 2);
}

A
Alex Dai 已提交
156 157 158 159 160 161 162
static int host2guc_sample_forcewake(struct intel_guc *guc,
				     struct i915_guc_client *client)
{
	struct drm_i915_private *dev_priv = guc_to_i915(guc);
	u32 data[2];

	data[0] = HOST2GUC_ACTION_SAMPLE_FORCEWAKE;
163
	/* WaRsDisableCoarsePowerGating:skl,bxt */
164
	if (!intel_enable_rc6() || NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
165 166 167 168 169 170
		data[1] = 0;
	else
		/* bit 0 and 1 are for Render and Media domain separately */
		data[1] = GUC_FORCEWAKE_RENDER | GUC_FORCEWAKE_MEDIA;

	return host2guc_action(guc, data, ARRAY_SIZE(data));
A
Alex Dai 已提交
171 172
}

173 174 175 176 177 178 179
/*
 * Initialise, update, or clear doorbell data shared with the GuC
 *
 * These functions modify shared data and so need access to the mapped
 * client object which contains the page being used for the doorbell
 */

180 181 182
static int guc_update_doorbell_id(struct intel_guc *guc,
				  struct i915_guc_client *client,
				  u16 new_id)
183
{
184
	struct sg_table *sg = guc->ctx_pool_vma->pages;
185
	void *doorbell_bitmap = guc->doorbell_bitmap;
186
	struct guc_doorbell_info *doorbell;
187 188
	struct guc_context_desc desc;
	size_t len;
189

190
	doorbell = client->client_base + client->doorbell_offset;
191

192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
	if (client->doorbell_id != GUC_INVALID_DOORBELL_ID &&
	    test_bit(client->doorbell_id, doorbell_bitmap)) {
		/* Deactivate the old doorbell */
		doorbell->db_status = GUC_DOORBELL_DISABLED;
		(void)host2guc_release_doorbell(guc, client);
		__clear_bit(client->doorbell_id, doorbell_bitmap);
	}

	/* Update the GuC's idea of the doorbell ID */
	len = sg_pcopy_to_buffer(sg->sgl, sg->nents, &desc, sizeof(desc),
			     sizeof(desc) * client->ctx_index);
	if (len != sizeof(desc))
		return -EFAULT;
	desc.db_id = new_id;
	len = sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc),
			     sizeof(desc) * client->ctx_index);
	if (len != sizeof(desc))
		return -EFAULT;

	client->doorbell_id = new_id;
	if (new_id == GUC_INVALID_DOORBELL_ID)
		return 0;

	/* Activate the new doorbell */
	__set_bit(new_id, doorbell_bitmap);
217
	doorbell->cookie = 0;
218 219 220 221 222 223 224 225 226
	doorbell->db_status = GUC_DOORBELL_ENABLED;
	return host2guc_allocate_doorbell(guc, client);
}

static int guc_init_doorbell(struct intel_guc *guc,
			      struct i915_guc_client *client,
			      uint16_t db_id)
{
	return guc_update_doorbell_id(guc, client, db_id);
227 228 229 230 231
}

static void guc_disable_doorbell(struct intel_guc *guc,
				 struct i915_guc_client *client)
{
232
	(void)guc_update_doorbell_id(guc, client, GUC_INVALID_DOORBELL_ID);
233 234 235 236 237

	/* XXX: wait for any interrupts */
	/* XXX: wait for workqueue to drain */
}

238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
static uint16_t
select_doorbell_register(struct intel_guc *guc, uint32_t priority)
{
	/*
	 * The bitmap tracks which doorbell registers are currently in use.
	 * It is split into two halves; the first half is used for normal
	 * priority contexts, the second half for high-priority ones.
	 * Note that logically higher priorities are numerically less than
	 * normal ones, so the test below means "is it high-priority?"
	 */
	const bool hi_pri = (priority <= GUC_CTX_PRIORITY_HIGH);
	const uint16_t half = GUC_MAX_DOORBELLS / 2;
	const uint16_t start = hi_pri ? half : 0;
	const uint16_t end = start + half;
	uint16_t id;

	id = find_next_zero_bit(guc->doorbell_bitmap, end, start);
	if (id == end)
		id = GUC_INVALID_DOORBELL_ID;

	DRM_DEBUG_DRIVER("assigned %s priority doorbell id 0x%x\n",
			hi_pri ? "high" : "normal", id);

	return id;
}

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
/*
 * Select, assign and relase doorbell cachelines
 *
 * These functions track which doorbell cachelines are in use.
 * The data they manipulate is protected by the host2guc lock.
 */

static uint32_t select_doorbell_cacheline(struct intel_guc *guc)
{
	const uint32_t cacheline_size = cache_line_size();
	uint32_t offset;

	/* Doorbell uses a single cache line within a page */
	offset = offset_in_page(guc->db_cacheline);

	/* Moving to next cache line to reduce contention */
	guc->db_cacheline += cacheline_size;

	DRM_DEBUG_DRIVER("selected doorbell cacheline 0x%x, next 0x%x, linesize %u\n",
			offset, guc->db_cacheline, cacheline_size);

	return offset;
}

/*
 * Initialise the process descriptor shared with the GuC firmware.
 */
291
static void guc_proc_desc_init(struct intel_guc *guc,
292 293 294 295
			       struct i915_guc_client *client)
{
	struct guc_process_desc *desc;

296
	desc = client->client_base + client->proc_desc_offset;
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322

	memset(desc, 0, sizeof(*desc));

	/*
	 * XXX: pDoorbell and WQVBaseAddress are pointers in process address
	 * space for ring3 clients (set them as in mmap_ioctl) or kernel
	 * space for kernel clients (map on demand instead? May make debug
	 * easier to have it mapped).
	 */
	desc->wq_base_addr = 0;
	desc->db_base_addr = 0;

	desc->context_id = client->ctx_index;
	desc->wq_size_bytes = client->wq_size;
	desc->wq_status = WQ_STATUS_ACTIVE;
	desc->priority = client->priority;
}

/*
 * Initialise/clear the context descriptor shared with the GuC firmware.
 *
 * This descriptor tells the GuC where (in GGTT space) to find the important
 * data structures relating to this client (doorbell, process descriptor,
 * write queue, etc).
 */

323
static void guc_ctx_desc_init(struct intel_guc *guc,
324 325
			      struct i915_guc_client *client)
{
326
	struct drm_i915_private *dev_priv = guc_to_i915(guc);
327
	struct intel_engine_cs *engine;
328
	struct i915_gem_context *ctx = client->owner;
329 330
	struct guc_context_desc desc;
	struct sg_table *sg;
331
	unsigned int tmp;
332
	u32 gfx_addr;
333 334 335 336 337 338 339 340

	memset(&desc, 0, sizeof(desc));

	desc.attribute = GUC_CTX_DESC_ATTR_ACTIVE | GUC_CTX_DESC_ATTR_KERNEL;
	desc.context_id = client->ctx_index;
	desc.priority = client->priority;
	desc.db_id = client->doorbell_id;

341
	for_each_engine_masked(engine, dev_priv, client->engines, tmp) {
342
		struct intel_context *ce = &ctx->engine[engine->id];
343 344
		uint32_t guc_engine_id = engine->guc_id;
		struct guc_execlist_context *lrc = &desc.lrc[guc_engine_id];
345 346 347 348 349 350 351 352

		/* TODO: We have a design issue to be solved here. Only when we
		 * receive the first batch, we know which engine is used by the
		 * user. But here GuC expects the lrc and ring to be pinned. It
		 * is not an issue for default context, which is the only one
		 * for now who owns a GuC client. But for future owner of GuC
		 * client, need to make sure lrc is pinned prior to enter here.
		 */
353
		if (!ce->state)
354 355
			break;	/* XXX: continue? */

356
		lrc->context_desc = lower_32_bits(ce->lrc_desc);
357 358

		/* The state page is after PPHWSP */
359
		lrc->ring_lcra =
360
			i915_ggtt_offset(ce->state) + LRC_STATE_PN * PAGE_SIZE;
361
		lrc->context_id = (client->ctx_index << GUC_ELC_CTXID_OFFSET) |
362
				(guc_engine_id << GUC_ELC_ENGINE_OFFSET);
363

364
		lrc->ring_begin = i915_ggtt_offset(ce->ring->vma);
365 366
		lrc->ring_end = lrc->ring_begin + ce->ring->size - 1;
		lrc->ring_next_free_location = lrc->ring_begin;
367 368
		lrc->ring_current_tail_pointer_value = 0;

369
		desc.engines_used |= (1 << guc_engine_id);
370 371
	}

372 373
	DRM_DEBUG_DRIVER("Host engines 0x%x => GuC engines used 0x%x\n",
			client->engines, desc.engines_used);
374 375
	WARN_ON(desc.engines_used == 0);

376
	/*
377 378
	 * The doorbell, process descriptor, and workqueue are all parts
	 * of the client object, which the GuC will reference via the GGTT
379
	 */
380
	gfx_addr = i915_ggtt_offset(client->vma);
381
	desc.db_trigger_phy = sg_dma_address(client->vma->pages->sgl) +
382 383 384 385 386 387
				client->doorbell_offset;
	desc.db_trigger_cpu = (uintptr_t)client->client_base +
				client->doorbell_offset;
	desc.db_trigger_uk = gfx_addr + client->doorbell_offset;
	desc.process_desc = gfx_addr + client->proc_desc_offset;
	desc.wq_addr = gfx_addr + client->wq_offset;
388 389 390
	desc.wq_size = client->wq_size;

	/*
391
	 * XXX: Take LRCs from an existing context if this is not an
392 393 394 395 396
	 * IsKMDCreatedContext client
	 */
	desc.desc_private = (uintptr_t)client;

	/* Pool context is pinned already */
397
	sg = guc->ctx_pool_vma->pages;
398 399 400 401
	sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc),
			     sizeof(desc) * client->ctx_index);
}

402
static void guc_ctx_desc_fini(struct intel_guc *guc,
403 404 405 406 407 408 409
			      struct i915_guc_client *client)
{
	struct guc_context_desc desc;
	struct sg_table *sg;

	memset(&desc, 0, sizeof(desc));

410
	sg = guc->ctx_pool_vma->pages;
411 412 413 414
	sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc),
			     sizeof(desc) * client->ctx_index);
}

415
/**
416
 * i915_guc_wq_reserve() - reserve space in the GuC's workqueue
417 418 419 420 421 422 423
 * @request:	request associated with the commands
 *
 * Return:	0 if space is available
 *		-EAGAIN if space is not currently available
 *
 * This function must be called (and must return 0) before a request
 * is submitted to the GuC via i915_guc_submit() below. Once a result
424 425
 * of 0 has been returned, it must be balanced by a corresponding
 * call to submit().
426
 *
427
 * Reservation allows the caller to determine in advance that space
428 429 430
 * will be available for the next submission before committing resources
 * to it, and helps avoid late failures with complicated recovery paths.
 */
431
int i915_guc_wq_reserve(struct drm_i915_gem_request *request)
432
{
433
	const size_t wqi_size = sizeof(struct guc_wq_item);
434
	struct i915_guc_client *gc = request->i915->guc.execbuf_client;
435
	struct guc_process_desc *desc = gc->client_base + gc->proc_desc_offset;
436
	u32 freespace;
437
	int ret;
438

439
	spin_lock(&gc->wq_lock);
440
	freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size);
441 442 443 444 445 446 447 448 449
	freespace -= gc->wq_rsvd;
	if (likely(freespace >= wqi_size)) {
		gc->wq_rsvd += wqi_size;
		ret = 0;
	} else {
		gc->no_wq_space++;
		ret = -EAGAIN;
	}
	spin_unlock(&gc->wq_lock);
450

451
	return ret;
452 453
}

454 455 456 457 458 459 460 461 462 463 464 465
void i915_guc_wq_unreserve(struct drm_i915_gem_request *request)
{
	const size_t wqi_size = sizeof(struct guc_wq_item);
	struct i915_guc_client *gc = request->i915->guc.execbuf_client;

	GEM_BUG_ON(READ_ONCE(gc->wq_rsvd) < wqi_size);

	spin_lock(&gc->wq_lock);
	gc->wq_rsvd -= wqi_size;
	spin_unlock(&gc->wq_lock);
}

466 467 468
/* Construct a Work Item and append it to the GuC's Work Queue */
static void guc_wq_item_append(struct i915_guc_client *gc,
			       struct drm_i915_gem_request *rq)
469
{
470 471 472
	/* wqi_len is in DWords, and does not include the one-word header */
	const size_t wqi_size = sizeof(struct guc_wq_item);
	const u32 wqi_len = wqi_size/sizeof(u32) - 1;
473
	struct intel_engine_cs *engine = rq->engine;
474
	struct guc_process_desc *desc;
475 476
	struct guc_wq_item *wqi;
	void *base;
477
	u32 freespace, tail, wq_off, wq_page;
478

479
	desc = gc->client_base + gc->proc_desc_offset;
480

481
	/* Free space is guaranteed, see i915_guc_wq_reserve() above */
482 483 484 485 486 487 488 489
	freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size);
	GEM_BUG_ON(freespace < wqi_size);

	/* The GuC firmware wants the tail index in QWords, not bytes */
	tail = rq->tail;
	GEM_BUG_ON(tail & 7);
	tail >>= 3;
	GEM_BUG_ON(tail > WQ_RING_TAIL_MAX);
490 491 492 493 494 495 496 497

	/* For now workqueue item is 4 DWs; workqueue buffer is 2 pages. So we
	 * should not have the case where structure wqi is across page, neither
	 * wrapped to the beginning. This simplifies the implementation below.
	 *
	 * XXX: if not the case, we need save data to a temp wqi and copy it to
	 * workqueue buffer dw by dw.
	 */
498
	BUILD_BUG_ON(wqi_size != 16);
499
	GEM_BUG_ON(gc->wq_rsvd < wqi_size);
500

501 502
	/* postincrement WQ tail for next time */
	wq_off = gc->wq_tail;
503
	GEM_BUG_ON(wq_off & (wqi_size - 1));
504 505
	gc->wq_tail += wqi_size;
	gc->wq_tail &= gc->wq_size - 1;
506
	gc->wq_rsvd -= wqi_size;
507 508 509

	/* WQ starts from the page after doorbell / process_desc */
	wq_page = (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT;
510
	wq_off &= PAGE_SIZE - 1;
511
	base = kmap_atomic(i915_gem_object_get_page(gc->vma->obj, wq_page));
512 513
	wqi = (struct guc_wq_item *)((char *)base + wq_off);

514
	/* Now fill in the 4-word work queue item */
515
	wqi->header = WQ_TYPE_INORDER |
516
			(wqi_len << WQ_LEN_SHIFT) |
517
			(engine->guc_id << WQ_TARGET_SHIFT) |
518 519 520
			WQ_NO_WCFLUSH_WAIT;

	/* The GuC wants only the low-order word of the context descriptor */
521
	wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx, engine);
522 523

	wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT;
524
	wqi->fence_id = rq->fence.seqno;
525 526 527 528

	kunmap_atomic(base);
}

529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570
static int guc_ring_doorbell(struct i915_guc_client *gc)
{
	struct guc_process_desc *desc;
	union guc_doorbell_qw db_cmp, db_exc, db_ret;
	union guc_doorbell_qw *db;
	int attempt = 2, ret = -EAGAIN;

	desc = gc->client_base + gc->proc_desc_offset;

	/* Update the tail so it is visible to GuC */
	desc->tail = gc->wq_tail;

	/* current cookie */
	db_cmp.db_status = GUC_DOORBELL_ENABLED;
	db_cmp.cookie = gc->cookie;

	/* cookie to be updated */
	db_exc.db_status = GUC_DOORBELL_ENABLED;
	db_exc.cookie = gc->cookie + 1;
	if (db_exc.cookie == 0)
		db_exc.cookie = 1;

	/* pointer of current doorbell cacheline */
	db = gc->client_base + gc->doorbell_offset;

	while (attempt--) {
		/* lets ring the doorbell */
		db_ret.value_qw = atomic64_cmpxchg((atomic64_t *)db,
			db_cmp.value_qw, db_exc.value_qw);

		/* if the exchange was successfully executed */
		if (db_ret.value_qw == db_cmp.value_qw) {
			/* db was successfully rung */
			gc->cookie = db_exc.cookie;
			ret = 0;
			break;
		}

		/* XXX: doorbell was lost and need to acquire it again */
		if (db_ret.db_status == GUC_DOORBELL_DISABLED)
			break;

571 572
		DRM_WARN("Cookie mismatch. Expected %d, found %d\n",
			 db_cmp.cookie, db_ret.cookie);
573 574 575 576 577 578 579 580 581 582 583

		/* update the cookie to newly read cookie from GuC */
		db_cmp.cookie = db_ret.cookie;
		db_exc.cookie = db_ret.cookie + 1;
		if (db_exc.cookie == 0)
			db_exc.cookie = 1;
	}

	return ret;
}

584 585
/**
 * i915_guc_submit() - Submit commands through GuC
A
Alex Dai 已提交
586
 * @rq:		request associated with the commands
587
 *
588 589 590
 * Return:	0 on success, otherwise an errno.
 * 		(Note: nonzero really shouldn't happen!)
 *
591 592 593
 * The caller must have already called i915_guc_wq_reserve() above with
 * a result of 0 (success), guaranteeing that there is space in the work
 * queue for the new request, so enqueuing the item cannot fail.
594 595
 *
 * Bad Things Will Happen if the caller violates this protocol e.g. calls
596 597
 * submit() when _reserve() says there's no space, or calls _submit()
 * a different number of times from (successful) calls to _reserve().
598 599 600
 *
 * The only error here arises if the doorbell hardware isn't functioning
 * as expected, which really shouln't happen.
601
 */
602
static void i915_guc_submit(struct drm_i915_gem_request *rq)
603
{
604
	unsigned int engine_id = rq->engine->id;
605 606
	struct intel_guc *guc = &rq->i915->guc;
	struct i915_guc_client *client = guc->execbuf_client;
607
	int b_ret;
608

609
	spin_lock(&client->wq_lock);
610
	guc_wq_item_append(client, rq);
611
	b_ret = guc_ring_doorbell(client);
612

613
	client->submissions[engine_id] += 1;
614 615
	client->retcode = b_ret;
	if (b_ret)
616
		client->b_fail += 1;
617

618
	guc->submissions[engine_id] += 1;
619
	guc->last_seqno[engine_id] = rq->fence.seqno;
620
	spin_unlock(&client->wq_lock);
621 622 623 624 625 626 627 628
}

/*
 * Everything below here is concerned with setup & teardown, and is
 * therefore not part of the somewhat time-critical batch-submission
 * path of i915_guc_submit() above.
 */

629
/**
630 631 632
 * guc_allocate_vma() - Allocate a GGTT VMA for GuC usage
 * @guc:	the guc
 * @size:	size of area to allocate (both virtual space and memory)
633
 *
634 635 636 637 638
 * This is a wrapper to create an object for use with the GuC. In order to
 * use it inside the GuC, an object needs to be pinned lifetime, so we allocate
 * both some backing storage and a range inside the Global GTT. We must pin
 * it in the GGTT somewhere other than than [0, GUC_WOPCM_TOP) because that
 * range is reserved inside GuC.
639
 *
640
 * Return:	A i915_vma if successful, otherwise an ERR_PTR.
641
 */
642
static struct i915_vma *guc_allocate_vma(struct intel_guc *guc, u32 size)
643
{
644
	struct drm_i915_private *dev_priv = guc_to_i915(guc);
645
	struct drm_i915_gem_object *obj;
646 647
	struct i915_vma *vma;
	int ret;
648

649
	obj = i915_gem_object_create(&dev_priv->drm, size);
650
	if (IS_ERR(obj))
651
		return ERR_CAST(obj);
652

653 654 655
	vma = i915_vma_create(obj, &dev_priv->ggtt.base, NULL);
	if (IS_ERR(vma))
		goto err;
656

657 658 659 660 661
	ret = i915_vma_pin(vma, 0, PAGE_SIZE,
			   PIN_GLOBAL | PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
	if (ret) {
		vma = ERR_PTR(ret);
		goto err;
662 663 664 665 666
	}

	/* Invalidate GuC TLB to let GuC take the latest updates to GTT. */
	I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);

667 668 669 670 671
	return vma;

err:
	i915_gem_object_put(obj);
	return vma;
672 673
}

674 675 676
static void
guc_client_free(struct drm_i915_private *dev_priv,
		struct i915_guc_client *client)
677 678 679 680 681 682 683 684 685 686 687
{
	struct intel_guc *guc = &dev_priv->guc;

	if (!client)
		return;

	/*
	 * XXX: wait for any outstanding submissions before freeing memory.
	 * Be sure to drop any locks
	 */

688 689
	if (client->client_base) {
		/*
690 691
		 * If we got as far as setting up a doorbell, make sure we
		 * shut it down before unmapping & deallocating the memory.
692
		 */
693
		guc_disable_doorbell(guc, client);
694 695 696 697

		kunmap(kmap_to_page(client->client_base));
	}

698
	i915_vma_unpin_and_release(&client->vma);
699 700

	if (client->ctx_index != GUC_INVALID_CTX_ID) {
701
		guc_ctx_desc_fini(guc, client);
702 703 704 705 706 707
		ida_simple_remove(&guc->ctx_ids, client->ctx_index);
	}

	kfree(client);
}

708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
/* Check that a doorbell register is in the expected state */
static bool guc_doorbell_check(struct intel_guc *guc, uint16_t db_id)
{
	struct drm_i915_private *dev_priv = guc_to_i915(guc);
	i915_reg_t drbreg = GEN8_DRBREGL(db_id);
	uint32_t value = I915_READ(drbreg);
	bool enabled = (value & GUC_DOORBELL_ENABLED) != 0;
	bool expected = test_bit(db_id, guc->doorbell_bitmap);

	if (enabled == expected)
		return true;

	DRM_DEBUG_DRIVER("Doorbell %d (reg 0x%x) 0x%x, should be %s\n",
			 db_id, drbreg.reg, value,
			 expected ? "active" : "inactive");

	return false;
}

727
/*
728
 * Borrow the first client to set up & tear down each unused doorbell
729 730 731 732 733
 * in turn, to ensure that all doorbell h/w is (re)initialised.
 */
static void guc_init_doorbell_hw(struct intel_guc *guc)
{
	struct i915_guc_client *client = guc->execbuf_client;
734 735
	uint16_t db_id;
	int i, err;
736

737
	/* Save client's original doorbell selection */
738 739 740
	db_id = client->doorbell_id;

	for (i = 0; i < GUC_MAX_DOORBELLS; ++i) {
741 742
		/* Skip if doorbell is OK */
		if (guc_doorbell_check(guc, i))
743 744
			continue;

745
		err = guc_update_doorbell_id(guc, client, i);
746 747 748
		if (err)
			DRM_DEBUG_DRIVER("Doorbell %d update failed, err %d\n",
					i, err);
749 750 751 752 753
	}

	/* Restore to original value */
	err = guc_update_doorbell_id(guc, client, db_id);
	if (err)
754 755
		DRM_WARN("Failed to restore doorbell to %d, err %d\n",
			 db_id, err);
756

757 758 759
	/* Read back & verify all doorbell registers */
	for (i = 0; i < GUC_MAX_DOORBELLS; ++i)
		(void)guc_doorbell_check(guc, i);
760 761
}

762 763
/**
 * guc_client_alloc() - Allocate an i915_guc_client
764
 * @dev_priv:	driver private data structure
765
 * @engines:	The set of engines to enable for this client
766 767 768 769
 * @priority:	four levels priority _CRITICAL, _HIGH, _NORMAL and _LOW
 * 		The kernel client to replace ExecList submission is created with
 * 		NORMAL priority. Priority of a client for scheduler can be HIGH,
 * 		while a preemption context can use CRITICAL.
A
Alex Dai 已提交
770 771
 * @ctx:	the context that owns the client (we use the default render
 * 		context)
772
 *
773
 * Return:	An i915_guc_client object if success, else NULL.
774
 */
775 776
static struct i915_guc_client *
guc_client_alloc(struct drm_i915_private *dev_priv,
777
		 uint32_t engines,
778 779
		 uint32_t priority,
		 struct i915_gem_context *ctx)
780 781 782
{
	struct i915_guc_client *client;
	struct intel_guc *guc = &dev_priv->guc;
783
	struct i915_vma *vma;
784
	uint16_t db_id;
785 786 787 788 789

	client = kzalloc(sizeof(*client), GFP_KERNEL);
	if (!client)
		return NULL;

790
	client->owner = ctx;
791
	client->guc = guc;
792 793 794
	client->engines = engines;
	client->priority = priority;
	client->doorbell_id = GUC_INVALID_DOORBELL_ID;
795 796 797 798 799 800 801 802 803

	client->ctx_index = (uint32_t)ida_simple_get(&guc->ctx_ids, 0,
			GUC_MAX_GPU_CONTEXTS, GFP_KERNEL);
	if (client->ctx_index >= GUC_MAX_GPU_CONTEXTS) {
		client->ctx_index = GUC_INVALID_CTX_ID;
		goto err;
	}

	/* The first page is doorbell/proc_desc. Two followed pages are wq. */
804 805
	vma = guc_allocate_vma(guc, GUC_DB_SIZE + GUC_WQ_SIZE);
	if (IS_ERR(vma))
806 807
		goto err;

808
	/* We'll keep just the first (doorbell/proc) page permanently kmap'd. */
809 810
	client->vma = vma;
	client->client_base = kmap(i915_vma_first_page(vma));
811 812

	spin_lock_init(&client->wq_lock);
813 814 815
	client->wq_offset = GUC_DB_SIZE;
	client->wq_size = GUC_WQ_SIZE;

816 817 818 819 820
	db_id = select_doorbell_register(guc, client->priority);
	if (db_id == GUC_INVALID_DOORBELL_ID)
		/* XXX: evict a doorbell instead? */
		goto err;

821 822 823 824 825 826 827 828 829 830 831 832
	client->doorbell_offset = select_doorbell_cacheline(guc);

	/*
	 * Since the doorbell only requires a single cacheline, we can save
	 * space by putting the application process descriptor in the same
	 * page. Use the half of the page that doesn't include the doorbell.
	 */
	if (client->doorbell_offset >= (GUC_DB_SIZE / 2))
		client->proc_desc_offset = 0;
	else
		client->proc_desc_offset = (GUC_DB_SIZE / 2);

833 834
	guc_proc_desc_init(guc, client);
	guc_ctx_desc_init(guc, client);
835
	if (guc_init_doorbell(guc, client, db_id))
836 837
		goto err;

838 839
	DRM_DEBUG_DRIVER("new priority %u client %p for engine(s) 0x%x: ctx_index %u\n",
		priority, client, client->engines, client->ctx_index);
840 841
	DRM_DEBUG_DRIVER("doorbell id %u, cacheline offset 0x%x\n",
		client->doorbell_id, client->doorbell_offset);
842 843 844 845

	return client;

err:
846
	guc_client_free(dev_priv, client);
847 848 849
	return NULL;
}

850
static void guc_log_create(struct intel_guc *guc)
A
Alex Dai 已提交
851
{
852
	struct i915_vma *vma;
A
Alex Dai 已提交
853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
	unsigned long offset;
	uint32_t size, flags;

	if (i915.guc_log_level < GUC_LOG_VERBOSITY_MIN)
		return;

	if (i915.guc_log_level > GUC_LOG_VERBOSITY_MAX)
		i915.guc_log_level = GUC_LOG_VERBOSITY_MAX;

	/* The first page is to save log buffer state. Allocate one
	 * extra page for others in case for overlap */
	size = (1 + GUC_LOG_DPC_PAGES + 1 +
		GUC_LOG_ISR_PAGES + 1 +
		GUC_LOG_CRASH_PAGES + 1) << PAGE_SHIFT;

868 869 870 871
	vma = guc->log_vma;
	if (!vma) {
		vma = guc_allocate_vma(guc, size);
		if (IS_ERR(vma)) {
A
Alex Dai 已提交
872 873 874 875 876
			/* logging will be off */
			i915.guc_log_level = -1;
			return;
		}

877
		guc->log_vma = vma;
A
Alex Dai 已提交
878 879 880 881 882 883 884 885
	}

	/* each allocated unit is a page */
	flags = GUC_LOG_VALID | GUC_LOG_NOTIFY_ON_HALF_FULL |
		(GUC_LOG_DPC_PAGES << GUC_LOG_DPC_SHIFT) |
		(GUC_LOG_ISR_PAGES << GUC_LOG_ISR_SHIFT) |
		(GUC_LOG_CRASH_PAGES << GUC_LOG_CRASH_SHIFT);

886
	offset = i915_ggtt_offset(vma) >> PAGE_SHIFT; /* in pages */
A
Alex Dai 已提交
887 888 889
	guc->log_flags = (offset << GUC_LOG_BUF_ADDR_SHIFT) | flags;
}

890
static void guc_policies_init(struct guc_policies *policies)
891 892 893 894 895 896 897 898
{
	struct guc_policy *policy;
	u32 p, i;

	policies->dpc_promote_time = 500000;
	policies->max_num_work_items = POLICY_MAX_NUM_WI;

	for (p = 0; p < GUC_CTX_PRIORITY_NUM; p++) {
899
		for (i = GUC_RENDER_ENGINE; i < GUC_MAX_ENGINES_NUM; i++) {
900 901 902 903 904 905 906 907 908 909 910 911
			policy = &policies->policy[p][i];

			policy->execution_quantum = 1000000;
			policy->preemption_time = 500000;
			policy->fault_time = 250000;
			policy->policy_flags = 0;
		}
	}

	policies->is_valid = 1;
}

912
static void guc_addon_create(struct intel_guc *guc)
913 914
{
	struct drm_i915_private *dev_priv = guc_to_i915(guc);
915
	struct i915_vma *vma;
916
	struct guc_ads *ads;
917
	struct guc_policies *policies;
918
	struct guc_mmio_reg_state *reg_state;
919
	struct intel_engine_cs *engine;
920
	struct page *page;
921
	u32 size;
922 923

	/* The ads obj includes the struct itself and buffers passed to GuC */
924 925 926
	size = sizeof(struct guc_ads) + sizeof(struct guc_policies) +
			sizeof(struct guc_mmio_reg_state) +
			GUC_S3_SAVE_SPACE_PAGES * PAGE_SIZE;
927

928 929 930 931
	vma = guc->ads_vma;
	if (!vma) {
		vma = guc_allocate_vma(guc, PAGE_ALIGN(size));
		if (IS_ERR(vma))
932 933
			return;

934
		guc->ads_vma = vma;
935 936
	}

937
	page = i915_vma_first_page(vma);
938 939 940 941 942 943 944 945 946
	ads = kmap(page);

	/*
	 * The GuC requires a "Golden Context" when it reinitialises
	 * engines after a reset. Here we use the Render ring default
	 * context, which must already exist and be pinned in the GGTT,
	 * so its address won't change after we've told the GuC where
	 * to find it.
	 */
947
	engine = &dev_priv->engine[RCS];
948
	ads->golden_context_lrca = engine->status_page.ggtt_offset;
949

950
	for_each_engine(engine, dev_priv)
951
		ads->eng_state_size[engine->guc_id] = intel_lr_context_size(engine);
952

953 954
	/* GuC scheduling policies */
	policies = (void *)ads + sizeof(struct guc_ads);
955
	guc_policies_init(policies);
956

957 958
	ads->scheduler_policies =
		i915_ggtt_offset(vma) + sizeof(struct guc_ads);
959

960 961 962
	/* MMIO reg state */
	reg_state = (void *)policies + sizeof(struct guc_policies);

963
	for_each_engine(engine, dev_priv) {
964 965
		reg_state->mmio_white_list[engine->guc_id].mmio_start =
			engine->mmio_base + GUC_MMIO_WHITE_LIST_START;
966 967

		/* Nothing to be saved or restored for now. */
968
		reg_state->mmio_white_list[engine->guc_id].count = 0;
969 970 971 972 973 974 975 976
	}

	ads->reg_state_addr = ads->scheduler_policies +
			sizeof(struct guc_policies);

	ads->reg_state_buffer = ads->reg_state_addr +
			sizeof(struct guc_mmio_reg_state);

977 978 979
	kunmap(page);
}

980 981 982 983
/*
 * Set up the memory resources to be shared with the GuC.  At this point,
 * we require just one object that can be mapped through the GGTT.
 */
984
int i915_guc_submission_init(struct drm_i915_private *dev_priv)
985
{
986 987 988
	const size_t ctxsize = sizeof(struct guc_context_desc);
	const size_t poolsize = GUC_MAX_GPU_CONTEXTS * ctxsize;
	const size_t gemsize = round_up(poolsize, PAGE_SIZE);
989
	struct intel_guc *guc = &dev_priv->guc;
990
	struct i915_vma *vma;
991

992 993
	/* Wipe bitmap & delete client in case of reinitialisation */
	bitmap_clear(guc->doorbell_bitmap, 0, GUC_MAX_DOORBELLS);
994
	i915_guc_submission_disable(dev_priv);
995

996 997 998
	if (!i915.enable_guc_submission)
		return 0; /* not enabled  */

999
	if (guc->ctx_pool_vma)
1000 1001
		return 0; /* already allocated */

1002
	vma = guc_allocate_vma(guc, gemsize);
1003 1004
	if (IS_ERR(vma))
		return PTR_ERR(vma);
1005

1006
	guc->ctx_pool_vma = vma;
1007
	ida_init(&guc->ctx_ids);
1008 1009
	guc_log_create(guc);
	guc_addon_create(guc);
1010

1011 1012 1013
	return 0;
}

1014
int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
1015 1016 1017
{
	struct intel_guc *guc = &dev_priv->guc;
	struct i915_guc_client *client;
1018
	struct intel_engine_cs *engine;
1019
	struct drm_i915_gem_request *request;
1020 1021

	/* client for execbuf submission */
1022
	client = guc_client_alloc(dev_priv,
1023
				  INTEL_INFO(dev_priv)->ring_mask,
1024 1025
				  GUC_CTX_PRIORITY_KMD_NORMAL,
				  dev_priv->kernel_context);
1026
	if (!client) {
1027
		DRM_ERROR("Failed to create normal GuC client!\n");
1028 1029 1030 1031
		return -ENOMEM;
	}

	guc->execbuf_client = client;
A
Alex Dai 已提交
1032
	host2guc_sample_forcewake(guc, client);
1033
	guc_init_doorbell_hw(guc);
A
Alex Dai 已提交
1034

1035
	/* Take over from manual control of ELSP (execlists) */
1036
	for_each_engine(engine, dev_priv) {
1037 1038
		engine->submit_request = i915_guc_submit;

1039
		/* Replay the current set of previously submitted requests */
1040 1041
		list_for_each_entry(request, &engine->request_list, link) {
			client->wq_rsvd += sizeof(struct guc_wq_item);
1042 1043
			if (i915_sw_fence_done(&request->submit))
				i915_guc_submit(request);
1044
		}
1045 1046
	}

1047 1048 1049
	return 0;
}

1050
void i915_guc_submission_disable(struct drm_i915_private *dev_priv)
1051 1052 1053
{
	struct intel_guc *guc = &dev_priv->guc;

1054 1055 1056 1057 1058
	if (!guc->execbuf_client)
		return;

	/* Revert back to manual ELSP submission */
	intel_execlists_enable_submission(dev_priv);
1059 1060 1061

	guc_client_free(dev_priv, guc->execbuf_client);
	guc->execbuf_client = NULL;
1062 1063
}

1064
void i915_guc_submission_fini(struct drm_i915_private *dev_priv)
1065 1066 1067
{
	struct intel_guc *guc = &dev_priv->guc;

1068 1069
	i915_vma_unpin_and_release(&guc->ads_vma);
	i915_vma_unpin_and_release(&guc->log_vma);
A
Alex Dai 已提交
1070

1071
	if (guc->ctx_pool_vma)
1072
		ida_destroy(&guc->ctx_ids);
1073
	i915_vma_unpin_and_release(&guc->ctx_pool_vma);
1074
}
1075 1076 1077 1078 1079 1080 1081

/**
 * intel_guc_suspend() - notify GuC entering suspend state
 * @dev:	drm device
 */
int intel_guc_suspend(struct drm_device *dev)
{
1082
	struct drm_i915_private *dev_priv = to_i915(dev);
1083
	struct intel_guc *guc = &dev_priv->guc;
1084
	struct i915_gem_context *ctx;
1085 1086
	u32 data[3];

1087
	if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS)
1088 1089
		return 0;

1090
	ctx = dev_priv->kernel_context;
1091 1092 1093 1094 1095

	data[0] = HOST2GUC_ACTION_ENTER_S_STATE;
	/* any value greater than GUC_POWER_D0 */
	data[1] = GUC_POWER_D1;
	/* first page is shared data with GuC */
1096
	data[2] = i915_ggtt_offset(ctx->engine[RCS].state);
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107

	return host2guc_action(guc, data, ARRAY_SIZE(data));
}


/**
 * intel_guc_resume() - notify GuC resuming from suspend state
 * @dev:	drm device
 */
int intel_guc_resume(struct drm_device *dev)
{
1108
	struct drm_i915_private *dev_priv = to_i915(dev);
1109
	struct intel_guc *guc = &dev_priv->guc;
1110
	struct i915_gem_context *ctx;
1111 1112
	u32 data[3];

1113
	if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS)
1114 1115
		return 0;

1116
	ctx = dev_priv->kernel_context;
1117 1118 1119 1120

	data[0] = HOST2GUC_ACTION_EXIT_S_STATE;
	data[1] = GUC_POWER_D0;
	/* first page is shared data with GuC */
1121
	data[2] = i915_ggtt_offset(ctx->engine[RCS].state);
1122 1123 1124

	return host2guc_action(guc, data, ARRAY_SIZE(data));
}