intel_lrc.c 72.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 * Copyright © 2014 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Ben Widawsky <ben@bwidawsk.net>
 *    Michel Thierry <michel.thierry@intel.com>
 *    Thomas Daniel <thomas.daniel@intel.com>
 *    Oscar Mateo <oscar.mateo@intel.com>
 *
 */

31 32 33 34
/**
 * DOC: Logical Rings, Logical Ring Contexts and Execlists
 *
 * Motivation:
35 36 37 38
 * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
 * These expanded contexts enable a number of new abilities, especially
 * "Execlists" (also implemented in this file).
 *
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
 * One of the main differences with the legacy HW contexts is that logical
 * ring contexts incorporate many more things to the context's state, like
 * PDPs or ringbuffer control registers:
 *
 * The reason why PDPs are included in the context is straightforward: as
 * PPGTTs (per-process GTTs) are actually per-context, having the PDPs
 * contained there mean you don't need to do a ppgtt->switch_mm yourself,
 * instead, the GPU will do it for you on the context switch.
 *
 * But, what about the ringbuffer control registers (head, tail, etc..)?
 * shouldn't we just need a set of those per engine command streamer? This is
 * where the name "Logical Rings" starts to make sense: by virtualizing the
 * rings, the engine cs shifts to a new "ring buffer" with every context
 * switch. When you want to submit a workload to the GPU you: A) choose your
 * context, B) find its appropriate virtualized ring, C) write commands to it
 * and then, finally, D) tell the GPU to switch to that context.
 *
 * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch
 * to a contexts is via a context execution list, ergo "Execlists".
 *
 * LRC implementation:
 * Regarding the creation of contexts, we have:
 *
 * - One global default context.
 * - One local default context for each opened fd.
 * - One local extra context for each context create ioctl call.
 *
 * Now that ringbuffers belong per-context (and not per-engine, like before)
 * and that contexts are uniquely tied to a given engine (and not reusable,
 * like before) we need:
 *
 * - One ringbuffer per-engine inside each context.
 * - One backing object per-engine inside each context.
 *
 * The global default context starts its life with these new objects fully
 * allocated and populated. The local default context for each opened fd is
 * more complex, because we don't know at creation time which engine is going
 * to use them. To handle this, we have implemented a deferred creation of LR
 * contexts:
 *
 * The local context starts its life as a hollow or blank holder, that only
 * gets populated for a given engine once we receive an execbuffer. If later
 * on we receive another execbuffer ioctl for the same context but a different
 * engine, we allocate/populate a new ringbuffer and context backing object and
 * so on.
 *
 * Finally, regarding local contexts created using the ioctl call: as they are
 * only allowed with the render ring, we can allocate & populate them right
 * away (no need to defer anything, at least for now).
 *
 * Execlists implementation:
90 91
 * Execlists are the new method by which, on gen8+ hardware, workloads are
 * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
 * This method works as follows:
 *
 * When a request is committed, its commands (the BB start and any leading or
 * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer
 * for the appropriate context. The tail pointer in the hardware context is not
 * updated at this time, but instead, kept by the driver in the ringbuffer
 * structure. A structure representing this request is added to a request queue
 * for the appropriate engine: this structure contains a copy of the context's
 * tail after the request was written to the ring buffer and a pointer to the
 * context itself.
 *
 * If the engine's request queue was empty before the request was added, the
 * queue is processed immediately. Otherwise the queue will be processed during
 * a context switch interrupt. In any case, elements on the queue will get sent
 * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a
 * globally unique 20-bits submission ID.
 *
 * When execution of a request completes, the GPU updates the context status
 * buffer with a context complete event and generates a context switch interrupt.
 * During the interrupt handling, the driver examines the events in the buffer:
 * for each context complete event, if the announced ID matches that on the head
 * of the request queue, then that request is retired and removed from the queue.
 *
 * After processing, if any requests were retired and the queue is not empty
 * then a new execution list can be submitted. The two requests at the front of
 * the queue are next to be submitted but since a context may not occur twice in
 * an execution list, if subsequent requests have the same ID as the first then
 * the two requests must be combined. This is done simply by discarding requests
 * at the head of the queue until either only one requests is left (in which case
 * we use a NULL second context) or the first two requests have unique IDs.
 *
 * By always executing the first two requests in the queue the driver ensures
 * that the GPU is kept as busy as possible. In the case where a single context
 * completes but a second context is still executing, the request for this second
 * context will be at the head of the queue when we remove the first one. This
 * request will then be resubmitted along with a new request for a different context,
 * which will cause the hardware to continue executing the second request and queue
 * the new request (the GPU detects the condition of a context getting preempted
 * with the same context and optimizes the context switch flow by not doing
 * preemption, but just sampling the new tail pointer).
 *
133
 */
134
#include <linux/interrupt.h>
135 136 137 138

#include <drm/drmP.h>
#include <drm/i915_drm.h>
#include "i915_drv.h"
139
#include "i915_gem_render_state.h"
140
#include "intel_mocs.h"
141

142 143 144 145 146 147 148 149 150 151 152 153 154
#define RING_EXECLIST_QFULL		(1 << 0x2)
#define RING_EXECLIST1_VALID		(1 << 0x3)
#define RING_EXECLIST0_VALID		(1 << 0x4)
#define RING_EXECLIST_ACTIVE_STATUS	(3 << 0xE)
#define RING_EXECLIST1_ACTIVE		(1 << 0x11)
#define RING_EXECLIST0_ACTIVE		(1 << 0x12)

#define GEN8_CTX_STATUS_IDLE_ACTIVE	(1 << 0)
#define GEN8_CTX_STATUS_PREEMPTED	(1 << 1)
#define GEN8_CTX_STATUS_ELEMENT_SWITCH	(1 << 2)
#define GEN8_CTX_STATUS_ACTIVE_IDLE	(1 << 3)
#define GEN8_CTX_STATUS_COMPLETE	(1 << 4)
#define GEN8_CTX_STATUS_LITE_RESTORE	(1 << 15)
155

156
#define GEN8_CTX_STATUS_COMPLETED_MASK \
157
	 (GEN8_CTX_STATUS_COMPLETE | GEN8_CTX_STATUS_PREEMPTED)
158

159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
#define CTX_LRI_HEADER_0		0x01
#define CTX_CONTEXT_CONTROL		0x02
#define CTX_RING_HEAD			0x04
#define CTX_RING_TAIL			0x06
#define CTX_RING_BUFFER_START		0x08
#define CTX_RING_BUFFER_CONTROL		0x0a
#define CTX_BB_HEAD_U			0x0c
#define CTX_BB_HEAD_L			0x0e
#define CTX_BB_STATE			0x10
#define CTX_SECOND_BB_HEAD_U		0x12
#define CTX_SECOND_BB_HEAD_L		0x14
#define CTX_SECOND_BB_STATE		0x16
#define CTX_BB_PER_CTX_PTR		0x18
#define CTX_RCS_INDIRECT_CTX		0x1a
#define CTX_RCS_INDIRECT_CTX_OFFSET	0x1c
#define CTX_LRI_HEADER_1		0x21
#define CTX_CTX_TIMESTAMP		0x22
#define CTX_PDP3_UDW			0x24
#define CTX_PDP3_LDW			0x26
#define CTX_PDP2_UDW			0x28
#define CTX_PDP2_LDW			0x2a
#define CTX_PDP1_UDW			0x2c
#define CTX_PDP1_LDW			0x2e
#define CTX_PDP0_UDW			0x30
#define CTX_PDP0_LDW			0x32
#define CTX_LRI_HEADER_2		0x41
#define CTX_R_PWR_CLK_STATE		0x42
#define CTX_GPGPU_CSR_BASE_ADDRESS	0x44

188
#define CTX_REG(reg_state, pos, reg, val) do { \
189
	(reg_state)[(pos)+0] = i915_mmio_reg_offset(reg); \
190 191 192 193
	(reg_state)[(pos)+1] = (val); \
} while (0)

#define ASSIGN_CTX_PDP(ppgtt, reg_state, n) do {		\
194
	const u64 _addr = i915_page_dir_dma_addr((ppgtt), (n));	\
195 196
	reg_state[CTX_PDP ## n ## _UDW+1] = upper_32_bits(_addr); \
	reg_state[CTX_PDP ## n ## _LDW+1] = lower_32_bits(_addr); \
197
} while (0)
198

199
#define ASSIGN_CTX_PML4(ppgtt, reg_state) do { \
200 201
	reg_state[CTX_PDP0_UDW + 1] = upper_32_bits(px_dma(&ppgtt->pml4)); \
	reg_state[CTX_PDP0_LDW + 1] = lower_32_bits(px_dma(&ppgtt->pml4)); \
202
} while (0)
203

204 205
#define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT	0x17
#define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT	0x26
206
#define GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT	0x19
207

208 209
/* Typical size of the average request (2 pipecontrols and a MI_BB) */
#define EXECLISTS_REQUEST_SIZE 64 /* bytes */
210
#define WA_TAIL_DWORDS 2
211
#define WA_TAIL_BYTES (sizeof(u32) * WA_TAIL_DWORDS)
C
Chris Wilson 已提交
212
#define PREEMPT_ID 0x1
213

214
static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
215
					    struct intel_engine_cs *engine);
216 217 218 219
static void execlists_init_reg_state(u32 *reg_state,
				     struct i915_gem_context *ctx,
				     struct intel_engine_cs *engine,
				     struct intel_ring *ring);
220

221
/**
222 223 224
 * intel_lr_context_descriptor_update() - calculate & cache the descriptor
 * 					  descriptor for a pinned context
 * @ctx: Context to work on
225
 * @engine: Engine the descriptor will be used with
226
 *
227 228 229 230 231
 * The context descriptor encodes various attributes of a context,
 * including its GTT address and some flags. Because it's fairly
 * expensive to calculate, we'll just do it once and cache the result,
 * which remains valid until the context is unpinned.
 *
232 233
 * This is what a descriptor looks like, from LSB to MSB::
 *
234
 *      bits  0-11:    flags, GEN8_CTX_* (cached in ctx->desc_template)
235 236 237 238
 *      bits 12-31:    LRCA, GTT address of (the HWSP of) this context
 *      bits 32-52:    ctx ID, a globally unique tag
 *      bits 53-54:    mbz, reserved for use by hardware
 *      bits 55-63:    group ID, currently unused and set to 0
239
 */
240
static void
241
intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
242
				   struct intel_engine_cs *engine)
243
{
244
	struct intel_context *ce = &ctx->engine[engine->id];
245
	u64 desc;
246

247
	BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (1<<GEN8_CTX_ID_WIDTH));
248

249
	desc = ctx->desc_template;				/* bits  0-11 */
250
	desc |= i915_ggtt_offset(ce->state) + LRC_HEADER_PAGES * PAGE_SIZE;
251
								/* bits 12-31 */
252
	desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT;		/* bits 32-52 */
253

254
	ce->lrc_desc = desc;
255 256
}

257 258 259 260
static struct i915_priolist *
lookup_priolist(struct intel_engine_cs *engine,
		struct i915_priotree *pt,
		int prio)
261
{
262
	struct intel_engine_execlists * const execlists = &engine->execlists;
263 264 265 266
	struct i915_priolist *p;
	struct rb_node **parent, *rb;
	bool first = true;

267
	if (unlikely(execlists->no_priolist))
268 269 270 271 272
		prio = I915_PRIORITY_NORMAL;

find_priolist:
	/* most positive priority is scheduled first, equal priorities fifo */
	rb = NULL;
273
	parent = &execlists->queue.rb_node;
274 275 276 277 278 279 280 281 282
	while (*parent) {
		rb = *parent;
		p = rb_entry(rb, typeof(*p), node);
		if (prio > p->priority) {
			parent = &rb->rb_left;
		} else if (prio < p->priority) {
			parent = &rb->rb_right;
			first = false;
		} else {
283
			return p;
284 285 286 287
		}
	}

	if (prio == I915_PRIORITY_NORMAL) {
288
		p = &execlists->default_priolist;
289 290 291 292 293 294 295 296 297 298 299 300 301 302
	} else {
		p = kmem_cache_alloc(engine->i915->priorities, GFP_ATOMIC);
		/* Convert an allocation failure to a priority bump */
		if (unlikely(!p)) {
			prio = I915_PRIORITY_NORMAL; /* recurses just once */

			/* To maintain ordering with all rendering, after an
			 * allocation failure we have to disable all scheduling.
			 * Requests will then be executed in fifo, and schedule
			 * will ensure that dependencies are emitted in fifo.
			 * There will be still some reordering with existing
			 * requests, so if userspace lied about their
			 * dependencies that reordering may be visible.
			 */
303
			execlists->no_priolist = true;
304 305 306 307 308
			goto find_priolist;
		}
	}

	p->priority = prio;
309
	INIT_LIST_HEAD(&p->requests);
310
	rb_link_node(&p->node, rb, parent);
311
	rb_insert_color(&p->node, &execlists->queue);
312 313

	if (first)
314
		execlists->first = &p->node;
315

316
	return ptr_pack_bits(p, first, 1);
317 318
}

319 320 321 322 323 324
static void unwind_wa_tail(struct drm_i915_gem_request *rq)
{
	rq->tail = intel_ring_wrap(rq->ring, rq->wa_tail - WA_TAIL_BYTES);
	assert_ring_tail_valid(rq->ring, rq->tail);
}

325
static void __unwind_incomplete_requests(struct intel_engine_cs *engine)
326 327
{
	struct drm_i915_gem_request *rq, *rn;
328 329
	struct i915_priolist *uninitialized_var(p);
	int last_prio = I915_PRIORITY_INVALID;
330 331 332 333 334 335 336 337 338 339 340 341

	lockdep_assert_held(&engine->timeline->lock);

	list_for_each_entry_safe_reverse(rq, rn,
					 &engine->timeline->requests,
					 link) {
		if (i915_gem_request_completed(rq))
			return;

		__i915_gem_request_unsubmit(rq);
		unwind_wa_tail(rq);

342 343 344 345 346 347 348 349 350 351 352
		GEM_BUG_ON(rq->priotree.priority == I915_PRIORITY_INVALID);
		if (rq->priotree.priority != last_prio) {
			p = lookup_priolist(engine,
					    &rq->priotree,
					    rq->priotree.priority);
			p = ptr_mask_bits(p, 1);

			last_prio = rq->priotree.priority;
		}

		list_add(&rq->priotree.link, &p->requests);
353 354 355
	}
}

356
void
357 358 359 360 361 362 363 364 365 366
execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
{
	struct intel_engine_cs *engine =
		container_of(execlists, typeof(*engine), execlists);

	spin_lock_irq(&engine->timeline->lock);
	__unwind_incomplete_requests(engine);
	spin_unlock_irq(&engine->timeline->lock);
}

367 368 369
static inline void
execlists_context_status_change(struct drm_i915_gem_request *rq,
				unsigned long status)
370
{
371 372 373 374 375 376
	/*
	 * Only used when GVT-g is enabled now. When GVT-g is disabled,
	 * The compiler should eliminate this function as dead-code.
	 */
	if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
		return;
377

378 379
	atomic_notifier_call_chain(&rq->engine->context_status_notifier,
				   status, rq);
380 381
}

382 383 384 385
static inline void
execlists_context_schedule_in(struct drm_i915_gem_request *rq)
{
	execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN);
386
	intel_engine_context_in(rq->engine);
387 388 389 390 391
}

static inline void
execlists_context_schedule_out(struct drm_i915_gem_request *rq)
{
392
	intel_engine_context_out(rq->engine);
393 394 395
	execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT);
}

396 397 398 399 400 401 402 403 404
static void
execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state)
{
	ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
	ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
	ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
	ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
}

405
static u64 execlists_update_context(struct drm_i915_gem_request *rq)
406
{
407
	struct intel_context *ce = &rq->ctx->engine[rq->engine->id];
408 409
	struct i915_hw_ppgtt *ppgtt =
		rq->ctx->ppgtt ?: rq->i915->mm.aliasing_ppgtt;
410
	u32 *reg_state = ce->lrc_reg_state;
411

412
	reg_state[CTX_RING_TAIL+1] = intel_ring_set_tail(rq->ring, rq->tail);
413

414 415 416 417 418
	/* True 32b PPGTT with dynamic page allocation: update PDP
	 * registers and point the unallocated PDPs to scratch page.
	 * PML4 is allocated during ppgtt init, so this is not needed
	 * in 48-bit mode.
	 */
419
	if (ppgtt && !i915_vm_is_48bit(&ppgtt->base))
420
		execlists_update_context_pdps(ppgtt, reg_state);
421 422

	return ce->lrc_desc;
423 424
}

C
Chris Wilson 已提交
425 426 427 428 429 430
static inline void elsp_write(u64 desc, u32 __iomem *elsp)
{
	writel(upper_32_bits(desc), elsp);
	writel(lower_32_bits(desc), elsp);
}

431
static void execlists_submit_ports(struct intel_engine_cs *engine)
432
{
433
	struct execlist_port *port = engine->execlists.port;
434
	unsigned int n;
435

436
	for (n = execlists_num_ports(&engine->execlists); n--; ) {
437 438 439 440 441 442 443 444
		struct drm_i915_gem_request *rq;
		unsigned int count;
		u64 desc;

		rq = port_unpack(&port[n], &count);
		if (rq) {
			GEM_BUG_ON(count > !n);
			if (!count++)
445
				execlists_context_schedule_in(rq);
446 447 448
			port_set(&port[n], port_pack(rq, count));
			desc = execlists_update_context(rq);
			GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc));
449 450 451

			GEM_TRACE("%s in[%d]:  ctx=%d.%d, seqno=%x\n",
				  engine->name, n,
452
				  port[n].context_id, count,
453
				  rq->global_seqno);
454 455 456 457
		} else {
			GEM_BUG_ON(!n);
			desc = 0;
		}
458

459
		elsp_write(desc, engine->execlists.elsp);
460
	}
461
	execlists_clear_active(&engine->execlists, EXECLISTS_ACTIVE_HWACK);
462 463
}

464
static bool ctx_single_port_submission(const struct i915_gem_context *ctx)
465
{
466
	return (IS_ENABLED(CONFIG_DRM_I915_GVT) &&
467
		i915_gem_context_force_single_submission(ctx));
468
}
469

470 471 472 473 474
static bool can_merge_ctx(const struct i915_gem_context *prev,
			  const struct i915_gem_context *next)
{
	if (prev != next)
		return false;
475

476 477
	if (ctx_single_port_submission(prev))
		return false;
478

479
	return true;
480 481
}

482 483 484 485 486 487 488 489 490 491 492
static void port_assign(struct execlist_port *port,
			struct drm_i915_gem_request *rq)
{
	GEM_BUG_ON(rq == port_request(port));

	if (port_isset(port))
		i915_gem_request_put(port_request(port));

	port_set(port, port_pack(i915_gem_request_get(rq), port_count(port)));
}

C
Chris Wilson 已提交
493 494 495 496 497 498 499 500 501 502 503 504 505 506
static void inject_preempt_context(struct intel_engine_cs *engine)
{
	struct intel_context *ce =
		&engine->i915->preempt_context->engine[engine->id];
	unsigned int n;

	GEM_BUG_ON(engine->i915->preempt_context->hw_id != PREEMPT_ID);
	GEM_BUG_ON(!IS_ALIGNED(ce->ring->size, WA_TAIL_BYTES));

	memset(ce->ring->vaddr + ce->ring->tail, 0, WA_TAIL_BYTES);
	ce->ring->tail += WA_TAIL_BYTES;
	ce->ring->tail &= (ce->ring->size - 1);
	ce->lrc_reg_state[CTX_RING_TAIL+1] = ce->ring->tail;

507
	GEM_TRACE("%s\n", engine->name);
C
Chris Wilson 已提交
508
	for (n = execlists_num_ports(&engine->execlists); --n; )
509
		elsp_write(0, engine->execlists.elsp);
C
Chris Wilson 已提交
510

511
	elsp_write(ce->lrc_desc, engine->execlists.elsp);
512
	execlists_clear_active(&engine->execlists, EXECLISTS_ACTIVE_HWACK);
C
Chris Wilson 已提交
513 514
}

515
static void execlists_dequeue(struct intel_engine_cs *engine)
516
{
517 518
	struct intel_engine_execlists * const execlists = &engine->execlists;
	struct execlist_port *port = execlists->port;
519 520
	const struct execlist_port * const last_port =
		&execlists->port[execlists->port_mask];
C
Chris Wilson 已提交
521
	struct drm_i915_gem_request *last = port_request(port);
522
	struct rb_node *rb;
523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
	bool submit = false;

	/* Hardware submission is through 2 ports. Conceptually each port
	 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
	 * static for a context, and unique to each, so we only execute
	 * requests belonging to a single context from each ring. RING_HEAD
	 * is maintained by the CS in the context image, it marks the place
	 * where it got up to last time, and through RING_TAIL we tell the CS
	 * where we want to execute up to this time.
	 *
	 * In this list the requests are in order of execution. Consecutive
	 * requests from the same context are adjacent in the ringbuffer. We
	 * can combine these requests into a single RING_TAIL update:
	 *
	 *              RING_HEAD...req1...req2
	 *                                    ^- RING_TAIL
	 * since to execute req2 the CS must first execute req1.
	 *
	 * Our goal then is to point each port to the end of a consecutive
	 * sequence of requests as being the most optimal (fewest wake ups
	 * and context switches) submission.
544
	 */
545

546
	spin_lock_irq(&engine->timeline->lock);
547 548
	rb = execlists->first;
	GEM_BUG_ON(rb_first(&execlists->queue) != rb);
C
Chris Wilson 已提交
549 550 551 552 553 554 555 556 557 558
	if (!rb)
		goto unlock;

	if (last) {
		/*
		 * Don't resubmit or switch until all outstanding
		 * preemptions (lite-restore) are seen. Then we
		 * know the next preemption status we see corresponds
		 * to this ELSP update.
		 */
559
		GEM_BUG_ON(!port_count(&port[0]));
C
Chris Wilson 已提交
560 561 562
		if (port_count(&port[0]) > 1)
			goto unlock;

563 564 565 566 567 568 569 570 571 572
		/*
		 * If we write to ELSP a second time before the HW has had
		 * a chance to respond to the previous write, we can confuse
		 * the HW and hit "undefined behaviour". After writing to ELSP,
		 * we must then wait until we see a context-switch event from
		 * the HW to indicate that it has had a chance to respond.
		 */
		if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK))
			goto unlock;

573
		if (HAS_LOGICAL_RING_PREEMPTION(engine->i915) &&
C
Chris Wilson 已提交
574 575 576 577 578 579 580
		    rb_entry(rb, struct i915_priolist, node)->priority >
		    max(last->priotree.priority, 0)) {
			/*
			 * Switch to our empty preempt context so
			 * the state of the GPU is known (idle).
			 */
			inject_preempt_context(engine);
581 582
			execlists_set_active(execlists,
					     EXECLISTS_ACTIVE_PREEMPT);
C
Chris Wilson 已提交
583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
			goto unlock;
		} else {
			/*
			 * In theory, we could coalesce more requests onto
			 * the second port (the first port is active, with
			 * no preemptions pending). However, that means we
			 * then have to deal with the possible lite-restore
			 * of the second port (as we submit the ELSP, there
			 * may be a context-switch) but also we may complete
			 * the resubmission before the context-switch. Ergo,
			 * coalescing onto the second port will cause a
			 * preemption event, but we cannot predict whether
			 * that will affect port[0] or port[1].
			 *
			 * If the second port is already active, we can wait
			 * until the next context-switch before contemplating
			 * new requests. The GPU will be busy and we should be
			 * able to resubmit the new ELSP before it idles,
			 * avoiding pipeline bubbles (momentary pauses where
			 * the driver is unable to keep up the supply of new
			 * work).
			 */
			if (port_count(&port[1]))
				goto unlock;

			/* WaIdleLiteRestore:bdw,skl
			 * Apply the wa NOOPs to prevent
			 * ring:HEAD == req:TAIL as we resubmit the
			 * request. See gen8_emit_breadcrumb() for
			 * where we prepare the padding after the
			 * end of the request.
			 */
			last->tail = last->wa_tail;
		}
	}

	do {
620 621 622 623 624 625 626 627 628 629 630 631 632 633
		struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
		struct drm_i915_gem_request *rq, *rn;

		list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
			/*
			 * Can we combine this request with the current port?
			 * It has to be the same context/ringbuffer and not
			 * have any exceptions (e.g. GVT saying never to
			 * combine contexts).
			 *
			 * If we can combine the requests, we can execute both
			 * by updating the RING_TAIL to point to the end of the
			 * second request, and so we never need to tell the
			 * hardware about the first.
634
			 */
635 636 637 638 639 640
			if (last && !can_merge_ctx(rq->ctx, last->ctx)) {
				/*
				 * If we are on the second port and cannot
				 * combine this request with the last, then we
				 * are done.
				 */
641
				if (port == last_port) {
642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665
					__list_del_many(&p->requests,
							&rq->priotree.link);
					goto done;
				}

				/*
				 * If GVT overrides us we only ever submit
				 * port[0], leaving port[1] empty. Note that we
				 * also have to be careful that we don't queue
				 * the same context (even though a different
				 * request) to the second port.
				 */
				if (ctx_single_port_submission(last->ctx) ||
				    ctx_single_port_submission(rq->ctx)) {
					__list_del_many(&p->requests,
							&rq->priotree.link);
					goto done;
				}

				GEM_BUG_ON(last->ctx == rq->ctx);

				if (submit)
					port_assign(port, last);
				port++;
666 667

				GEM_BUG_ON(port_isset(port));
668
			}
669

670 671
			INIT_LIST_HEAD(&rq->priotree.link);
			__i915_gem_request_submit(rq);
672
			trace_i915_gem_request_in(rq, port_index(port, execlists));
673 674
			last = rq;
			submit = true;
675
		}
676

677
		rb = rb_next(rb);
678
		rb_erase(&p->node, &execlists->queue);
679 680
		INIT_LIST_HEAD(&p->requests);
		if (p->priority != I915_PRIORITY_NORMAL)
681
			kmem_cache_free(engine->i915->priorities, p);
C
Chris Wilson 已提交
682
	} while (rb);
683
done:
684
	execlists->first = rb;
685
	if (submit)
686
		port_assign(port, last);
C
Chris Wilson 已提交
687
unlock:
688
	spin_unlock_irq(&engine->timeline->lock);
689

690 691
	if (submit) {
		execlists_set_active(execlists, EXECLISTS_ACTIVE_USER);
692
		execlists_submit_ports(engine);
693
	}
694 695
}

696
void
697
execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
698
{
699
	struct execlist_port *port = execlists->port;
700
	unsigned int num_ports = execlists_num_ports(execlists);
701

702
	while (num_ports-- && port_isset(port)) {
703 704
		struct drm_i915_gem_request *rq = port_request(port);

705
		GEM_BUG_ON(!execlists->active);
706
		intel_engine_context_out(rq->engine);
707
		execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_PREEMPTED);
708 709
		i915_gem_request_put(rq);

710 711 712
		memset(port, 0, sizeof(*port));
		port++;
	}
713 714
}

715 716
static void execlists_cancel_requests(struct intel_engine_cs *engine)
{
717
	struct intel_engine_execlists * const execlists = &engine->execlists;
718 719 720 721 722 723 724
	struct drm_i915_gem_request *rq, *rn;
	struct rb_node *rb;
	unsigned long flags;

	spin_lock_irqsave(&engine->timeline->lock, flags);

	/* Cancel the requests on the HW and clear the ELSP tracker. */
725
	execlists_cancel_port_requests(execlists);
726 727 728 729 730 731 732 733 734

	/* Mark all executing requests as skipped. */
	list_for_each_entry(rq, &engine->timeline->requests, link) {
		GEM_BUG_ON(!rq->global_seqno);
		if (!i915_gem_request_completed(rq))
			dma_fence_set_error(&rq->fence, -EIO);
	}

	/* Flush the queued requests to the timeline list (for retiring). */
735
	rb = execlists->first;
736 737 738 739 740 741 742 743 744 745 746
	while (rb) {
		struct i915_priolist *p = rb_entry(rb, typeof(*p), node);

		list_for_each_entry_safe(rq, rn, &p->requests, priotree.link) {
			INIT_LIST_HEAD(&rq->priotree.link);

			dma_fence_set_error(&rq->fence, -EIO);
			__i915_gem_request_submit(rq);
		}

		rb = rb_next(rb);
747
		rb_erase(&p->node, &execlists->queue);
748 749 750 751 752 753 754
		INIT_LIST_HEAD(&p->requests);
		if (p->priority != I915_PRIORITY_NORMAL)
			kmem_cache_free(engine->i915->priorities, p);
	}

	/* Remaining _unready_ requests will be nop'ed when submitted */

755

756 757
	execlists->queue = RB_ROOT;
	execlists->first = NULL;
758
	GEM_BUG_ON(port_isset(execlists->port));
759 760 761 762 763 764 765 766 767 768 769 770

	/*
	 * The port is checked prior to scheduling a tasklet, but
	 * just in case we have suspended the tasklet to do the
	 * wedging make sure that when it wakes, it decides there
	 * is no work to do by clearing the irq_posted bit.
	 */
	clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);

	spin_unlock_irqrestore(&engine->timeline->lock, flags);
}

771
/*
772 773 774
 * Check the unread Context Status Buffers and manage the submission of new
 * contexts to the ELSP accordingly.
 */
775
static void execlists_submission_tasklet(unsigned long data)
776
{
777 778
	struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
	struct intel_engine_execlists * const execlists = &engine->execlists;
C
Chris Wilson 已提交
779
	struct execlist_port * const port = execlists->port;
780
	struct drm_i915_private *dev_priv = engine->i915;
781

782 783 784 785 786 787 788 789 790
	/* We can skip acquiring intel_runtime_pm_get() here as it was taken
	 * on our behalf by the request (see i915_gem_mark_busy()) and it will
	 * not be relinquished until the device is idle (see
	 * i915_gem_idle_work_handler()). As a precaution, we make sure
	 * that all ELSP are drained i.e. we have processed the CSB,
	 * before allowing ourselves to idle and calling intel_runtime_pm_put().
	 */
	GEM_BUG_ON(!dev_priv->gt.awake);

791
	intel_uncore_forcewake_get(dev_priv, execlists->fw_domains);
792

793 794 795 796 797
	/* Prefer doing test_and_clear_bit() as a two stage operation to avoid
	 * imposing the cost of a locked atomic transaction when submitting a
	 * new request (outside of the context-switch interrupt).
	 */
	while (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted)) {
798 799 800
		/* The HWSP contains a (cacheable) mirror of the CSB */
		const u32 *buf =
			&engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
801
		unsigned int head, tail;
802

803
		if (unlikely(execlists->csb_use_mmio)) {
804 805
			buf = (u32 * __force)
				(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0)));
806
			execlists->csb_head = -1; /* force mmio read of CSB ptrs */
807 808
		}

809 810 811 812 813 814 815 816 817 818 819
		/* The write will be ordered by the uncached read (itself
		 * a memory barrier), so we do not need another in the form
		 * of a locked instruction. The race between the interrupt
		 * handler and the split test/clear is harmless as we order
		 * our clear before the CSB read. If the interrupt arrived
		 * first between the test and the clear, we read the updated
		 * CSB and clear the bit. If the interrupt arrives as we read
		 * the CSB or later (i.e. after we had cleared the bit) the bit
		 * is set and we do a new loop.
		 */
		__clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
820
		if (unlikely(execlists->csb_head == -1)) { /* following a reset */
821 822 823
			head = readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)));
			tail = GEN8_CSB_WRITE_PTR(head);
			head = GEN8_CSB_READ_PTR(head);
824
			execlists->csb_head = head;
825 826 827 828 829
		} else {
			const int write_idx =
				intel_hws_csb_write_index(dev_priv) -
				I915_HWS_CSB_BUF0_INDEX;

830
			head = execlists->csb_head;
831 832
			tail = READ_ONCE(buf[write_idx]);
		}
833 834 835 836
		GEM_TRACE("%s cs-irq head=%d [%d], tail=%d [%d]\n",
			  engine->name,
			  head, GEN8_CSB_READ_PTR(readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))),
			  tail, GEN8_CSB_WRITE_PTR(readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))));
837

838
		while (head != tail) {
839
			struct drm_i915_gem_request *rq;
840
			unsigned int status;
841
			unsigned int count;
842 843 844

			if (++head == GEN8_CSB_ENTRIES)
				head = 0;
845

846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862
			/* We are flying near dragons again.
			 *
			 * We hold a reference to the request in execlist_port[]
			 * but no more than that. We are operating in softirq
			 * context and so cannot hold any mutex or sleep. That
			 * prevents us stopping the requests we are processing
			 * in port[] from being retired simultaneously (the
			 * breadcrumb will be complete before we see the
			 * context-switch). As we only hold the reference to the
			 * request, any pointer chasing underneath the request
			 * is subject to a potential use-after-free. Thus we
			 * store all of the bookkeeping within port[] as
			 * required, and avoid using unguarded pointers beneath
			 * request itself. The same applies to the atomic
			 * status notifier.
			 */

863
			status = READ_ONCE(buf[2 * head]); /* maybe mmio! */
864
			GEM_TRACE("%s csb[%d]: status=0x%08x:0x%08x, active=0x%x\n",
865
				  engine->name, head,
866 867
				  status, buf[2*head + 1],
				  execlists->active);
868 869 870 871 872 873 874 875 876

			if (status & (GEN8_CTX_STATUS_IDLE_ACTIVE |
				      GEN8_CTX_STATUS_PREEMPTED))
				execlists_set_active(execlists,
						     EXECLISTS_ACTIVE_HWACK);
			if (status & GEN8_CTX_STATUS_ACTIVE_IDLE)
				execlists_clear_active(execlists,
						       EXECLISTS_ACTIVE_HWACK);

877 878 879
			if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
				continue;

880 881 882
			/* We should never get a COMPLETED | IDLE_ACTIVE! */
			GEM_BUG_ON(status & GEN8_CTX_STATUS_IDLE_ACTIVE);

883
			if (status & GEN8_CTX_STATUS_COMPLETE &&
C
Chris Wilson 已提交
884
			    buf[2*head + 1] == PREEMPT_ID) {
885 886
				GEM_TRACE("%s preempt-idle\n", engine->name);

887 888
				execlists_cancel_port_requests(execlists);
				execlists_unwind_incomplete_requests(execlists);
C
Chris Wilson 已提交
889

890 891 892 893
				GEM_BUG_ON(!execlists_is_active(execlists,
								EXECLISTS_ACTIVE_PREEMPT));
				execlists_clear_active(execlists,
						       EXECLISTS_ACTIVE_PREEMPT);
C
Chris Wilson 已提交
894 895 896 897
				continue;
			}

			if (status & GEN8_CTX_STATUS_PREEMPTED &&
898 899
			    execlists_is_active(execlists,
						EXECLISTS_ACTIVE_PREEMPT))
C
Chris Wilson 已提交
900 901
				continue;

902 903 904
			GEM_BUG_ON(!execlists_is_active(execlists,
							EXECLISTS_ACTIVE_USER));

905
			/* Check the context/desc id for this event matches */
906
			GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
907

908
			rq = port_unpack(port, &count);
909 910
			GEM_TRACE("%s out[0]: ctx=%d.%d, seqno=%x\n",
				  engine->name,
911
				  port->context_id, count,
912
				  rq ? rq->global_seqno : 0);
913 914
			GEM_BUG_ON(count == 0);
			if (--count == 0) {
915
				GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
916 917
				GEM_BUG_ON(port_isset(&port[1]) &&
					   !(status & GEN8_CTX_STATUS_ELEMENT_SWITCH));
918
				GEM_BUG_ON(!i915_gem_request_completed(rq));
919
				execlists_context_schedule_out(rq);
920 921
				trace_i915_gem_request_out(rq);
				i915_gem_request_put(rq);
922

923
				execlists_port_complete(execlists, port);
924 925
			} else {
				port_set(port, port_pack(rq, count));
926
			}
927

928 929
			/* After the final element, the hw should be idle */
			GEM_BUG_ON(port_count(port) == 0 &&
930
				   !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
931 932 933
			if (port_count(port) == 0)
				execlists_clear_active(execlists,
						       EXECLISTS_ACTIVE_USER);
934
		}
935

936 937
		if (head != execlists->csb_head) {
			execlists->csb_head = head;
938 939 940
			writel(_MASKED_FIELD(GEN8_CSB_READ_PTR_MASK, head << 8),
			       dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)));
		}
941 942
	}

943
	if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT))
944
		execlists_dequeue(engine);
945

946
	intel_uncore_forcewake_put(dev_priv, execlists->fw_domains);
947 948
}

949 950 951 952 953 954 955
static void insert_request(struct intel_engine_cs *engine,
			   struct i915_priotree *pt,
			   int prio)
{
	struct i915_priolist *p = lookup_priolist(engine, pt, prio);

	list_add_tail(&pt->link, &ptr_mask_bits(p, 1)->requests);
C
Chris Wilson 已提交
956
	if (ptr_unmask_bits(p, 1))
957
		tasklet_hi_schedule(&engine->execlists.tasklet);
958 959
}

960
static void execlists_submit_request(struct drm_i915_gem_request *request)
961
{
962
	struct intel_engine_cs *engine = request->engine;
963
	unsigned long flags;
964

965 966
	/* Will be called from irq-context when using foreign fences. */
	spin_lock_irqsave(&engine->timeline->lock, flags);
967

968
	insert_request(engine, &request->priotree, request->priotree.priority);
969

970
	GEM_BUG_ON(!engine->execlists.first);
971 972
	GEM_BUG_ON(list_empty(&request->priotree.link));

973
	spin_unlock_irqrestore(&engine->timeline->lock, flags);
974 975
}

976 977 978 979 980
static struct drm_i915_gem_request *pt_to_request(struct i915_priotree *pt)
{
	return container_of(pt, struct drm_i915_gem_request, priotree);
}

981 982 983
static struct intel_engine_cs *
pt_lock_engine(struct i915_priotree *pt, struct intel_engine_cs *locked)
{
984
	struct intel_engine_cs *engine = pt_to_request(pt)->engine;
985 986

	GEM_BUG_ON(!locked);
987 988

	if (engine != locked) {
989 990
		spin_unlock(&locked->timeline->lock);
		spin_lock(&engine->timeline->lock);
991 992 993 994 995 996 997
	}

	return engine;
}

static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
{
998
	struct intel_engine_cs *engine;
999 1000 1001 1002
	struct i915_dependency *dep, *p;
	struct i915_dependency stack;
	LIST_HEAD(dfs);

1003 1004
	GEM_BUG_ON(prio == I915_PRIORITY_INVALID);

1005 1006 1007
	if (i915_gem_request_completed(request))
		return;

1008 1009 1010
	if (prio <= READ_ONCE(request->priotree.priority))
		return;

1011 1012
	/* Need BKL in order to use the temporary link inside i915_dependency */
	lockdep_assert_held(&request->i915->drm.struct_mutex);
1013 1014 1015 1016

	stack.signaler = &request->priotree;
	list_add(&stack.dfs_link, &dfs);

1017 1018
	/*
	 * Recursively bump all dependent priorities to match the new request.
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
	 *
	 * A naive approach would be to use recursion:
	 * static void update_priorities(struct i915_priotree *pt, prio) {
	 *	list_for_each_entry(dep, &pt->signalers_list, signal_link)
	 *		update_priorities(dep->signal, prio)
	 *	insert_request(pt);
	 * }
	 * but that may have unlimited recursion depth and so runs a very
	 * real risk of overunning the kernel stack. Instead, we build
	 * a flat list of all dependencies starting with the current request.
	 * As we walk the list of dependencies, we add all of its dependencies
	 * to the end of the list (this may include an already visited
	 * request) and continue to walk onwards onto the new dependencies. The
	 * end result is a topological list of requests in reverse order, the
	 * last element in the list is the request we must execute first.
	 */
1035
	list_for_each_entry(dep, &dfs, dfs_link) {
1036 1037
		struct i915_priotree *pt = dep->signaler;

1038 1039
		/*
		 * Within an engine, there can be no cycle, but we may
1040 1041 1042 1043 1044
		 * refer to the same dependency chain multiple times
		 * (redundant dependencies are not eliminated) and across
		 * engines.
		 */
		list_for_each_entry(p, &pt->signalers_list, signal_link) {
1045 1046
			GEM_BUG_ON(p == dep); /* no cycles! */

1047
			if (i915_priotree_signaled(p->signaler))
1048 1049
				continue;

1050
			GEM_BUG_ON(p->signaler->priority < pt->priority);
1051 1052
			if (prio > READ_ONCE(p->signaler->priority))
				list_move_tail(&p->dfs_link, &dfs);
1053
		}
1054 1055
	}

1056 1057
	/*
	 * If we didn't need to bump any existing priorities, and we haven't
1058 1059 1060 1061
	 * yet submitted this request (i.e. there is no potential race with
	 * execlists_submit_request()), we can set our own priority and skip
	 * acquiring the engine locks.
	 */
1062
	if (request->priotree.priority == I915_PRIORITY_INVALID) {
1063 1064 1065 1066 1067 1068 1069
		GEM_BUG_ON(!list_empty(&request->priotree.link));
		request->priotree.priority = prio;
		if (stack.dfs_link.next == stack.dfs_link.prev)
			return;
		__list_del_entry(&stack.dfs_link);
	}

1070 1071 1072
	engine = request->engine;
	spin_lock_irq(&engine->timeline->lock);

1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
	/* Fifo and depth-first replacement ensure our deps execute before us */
	list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) {
		struct i915_priotree *pt = dep->signaler;

		INIT_LIST_HEAD(&dep->dfs_link);

		engine = pt_lock_engine(pt, engine);

		if (prio <= pt->priority)
			continue;

		pt->priority = prio;
1085 1086 1087
		if (!list_empty(&pt->link)) {
			__list_del_entry(&pt->link);
			insert_request(engine, pt, prio);
1088
		}
1089 1090
	}

1091
	spin_unlock_irq(&engine->timeline->lock);
1092 1093
}

1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
static int __context_pin(struct i915_gem_context *ctx, struct i915_vma *vma)
{
	unsigned int flags;
	int err;

	/*
	 * Clear this page out of any CPU caches for coherent swap-in/out.
	 * We only want to do this on the first bind so that we do not stall
	 * on an active context (which by nature is already on the GPU).
	 */
	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
		err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
		if (err)
			return err;
	}

	flags = PIN_GLOBAL | PIN_HIGH;
	if (ctx->ggtt_offset_bias)
		flags |= PIN_OFFSET_BIAS | ctx->ggtt_offset_bias;

	return i915_vma_pin(vma, 0, GEN8_LR_CONTEXT_ALIGN, flags);
}

1117 1118 1119
static struct intel_ring *
execlists_context_pin(struct intel_engine_cs *engine,
		      struct i915_gem_context *ctx)
1120
{
1121
	struct intel_context *ce = &ctx->engine[engine->id];
1122
	void *vaddr;
1123
	int ret;
1124

1125
	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
1126

1127 1128
	if (likely(ce->pin_count++))
		goto out;
1129
	GEM_BUG_ON(!ce->pin_count); /* no overflow please! */
1130

1131 1132 1133 1134 1135
	if (!ce->state) {
		ret = execlists_context_deferred_alloc(ctx, engine);
		if (ret)
			goto err;
	}
1136
	GEM_BUG_ON(!ce->state);
1137

1138
	ret = __context_pin(ctx, ce->state);
1139
	if (ret)
1140
		goto err;
1141

1142
	vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB);
1143 1144
	if (IS_ERR(vaddr)) {
		ret = PTR_ERR(vaddr);
1145
		goto unpin_vma;
1146 1147
	}

1148
	ret = intel_ring_pin(ce->ring, ctx->i915, ctx->ggtt_offset_bias);
1149
	if (ret)
1150
		goto unpin_map;
1151

1152
	intel_lr_context_descriptor_update(ctx, engine);
1153

1154 1155
	ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
1156
		i915_ggtt_offset(ce->ring->vma);
1157

1158
	ce->state->obj->pin_global++;
1159
	i915_gem_context_get(ctx);
1160 1161
out:
	return ce->ring;
1162

1163
unpin_map:
1164 1165 1166
	i915_gem_object_unpin_map(ce->state->obj);
unpin_vma:
	__i915_vma_unpin(ce->state);
1167
err:
1168
	ce->pin_count = 0;
1169
	return ERR_PTR(ret);
1170 1171
}

1172 1173
static void execlists_context_unpin(struct intel_engine_cs *engine,
				    struct i915_gem_context *ctx)
1174
{
1175
	struct intel_context *ce = &ctx->engine[engine->id];
1176

1177
	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
1178
	GEM_BUG_ON(ce->pin_count == 0);
1179

1180
	if (--ce->pin_count)
1181
		return;
1182

1183
	intel_ring_unpin(ce->ring);
1184

1185
	ce->state->obj->pin_global--;
1186 1187
	i915_gem_object_unpin_map(ce->state->obj);
	i915_vma_unpin(ce->state);
1188

1189
	i915_gem_context_put(ctx);
1190 1191
}

1192
static int execlists_request_alloc(struct drm_i915_gem_request *request)
1193 1194 1195
{
	struct intel_engine_cs *engine = request->engine;
	struct intel_context *ce = &request->ctx->engine[engine->id];
1196
	int ret;
1197

1198 1199
	GEM_BUG_ON(!ce->pin_count);

1200 1201 1202 1203 1204 1205
	/* Flush enough space to reduce the likelihood of waiting after
	 * we start building the request - in which case we will just
	 * have to repeat work.
	 */
	request->reserved_space += EXECLISTS_REQUEST_SIZE;

1206 1207 1208
	ret = intel_ring_wait_for_space(request->ring, request->reserved_space);
	if (ret)
		return ret;
1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220

	/* Note that after this point, we have committed to using
	 * this request as it is being used to both track the
	 * state of engine initialisation and liveness of the
	 * golden renderstate above. Think twice before you try
	 * to cancel/unwind this request now.
	 */

	request->reserved_space -= EXECLISTS_REQUEST_SIZE;
	return 0;
}

1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
/*
 * In this WA we need to set GEN8_L3SQCREG4[21:21] and reset it after
 * PIPE_CONTROL instruction. This is required for the flush to happen correctly
 * but there is a slight complication as this is applied in WA batch where the
 * values are only initialized once so we cannot take register value at the
 * beginning and reuse it further; hence we save its value to memory, upload a
 * constant value with bit21 set and then we restore it back with the saved value.
 * To simplify the WA, a constant value is formed by using the default value
 * of this register. This shouldn't be a problem because we are only modifying
 * it for a short period and this batch in non-premptible. We can ofcourse
 * use additional instructions that read the actual value of the register
 * at that time and set our bit of interest but it makes the WA complicated.
 *
 * This WA is also required for Gen9 so extracting as a function avoids
 * code duplication.
 */
1237 1238
static u32 *
gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, u32 *batch)
1239
{
1240 1241 1242 1243 1244 1245 1246 1247 1248
	*batch++ = MI_STORE_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT;
	*batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
	*batch++ = i915_ggtt_offset(engine->scratch) + 256;
	*batch++ = 0;

	*batch++ = MI_LOAD_REGISTER_IMM(1);
	*batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
	*batch++ = 0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES;

1249 1250 1251 1252
	batch = gen8_emit_pipe_control(batch,
				       PIPE_CONTROL_CS_STALL |
				       PIPE_CONTROL_DC_FLUSH_ENABLE,
				       0);
1253 1254 1255 1256 1257 1258 1259

	*batch++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT;
	*batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
	*batch++ = i915_ggtt_offset(engine->scratch) + 256;
	*batch++ = 0;

	return batch;
1260 1261
}

1262 1263 1264 1265 1266 1267
/*
 * Typically we only have one indirect_ctx and per_ctx batch buffer which are
 * initialized at the beginning and shared across all contexts but this field
 * helps us to have multiple batches at different offsets and select them based
 * on a criteria. At the moment this batch always start at the beginning of the page
 * and at this point we don't have multiple wa_ctx batch buffers.
1268
 *
1269 1270
 * The number of WA applied are not known at the beginning; we use this field
 * to return the no of DWORDS written.
1271
 *
1272 1273 1274 1275
 * It is to be noted that this batch does not contain MI_BATCH_BUFFER_END
 * so it adds NOOPs as padding to make it cacheline aligned.
 * MI_BATCH_BUFFER_END will be added to perctx batch and both of them together
 * makes a complete batch buffer.
1276
 */
1277
static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
1278
{
1279
	/* WaDisableCtxRestoreArbitration:bdw,chv */
1280
	*batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1281

1282
	/* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */
1283 1284
	if (IS_BROADWELL(engine->i915))
		batch = gen8_emit_flush_coherentl3_wa(engine, batch);
1285

1286 1287
	/* WaClearSlmSpaceAtContextSwitch:bdw,chv */
	/* Actual scratch location is at 128 bytes offset */
1288 1289 1290 1291 1292 1293 1294
	batch = gen8_emit_pipe_control(batch,
				       PIPE_CONTROL_FLUSH_L3 |
				       PIPE_CONTROL_GLOBAL_GTT_IVB |
				       PIPE_CONTROL_CS_STALL |
				       PIPE_CONTROL_QW_WRITE,
				       i915_ggtt_offset(engine->scratch) +
				       2 * CACHELINE_BYTES);
1295

C
Chris Wilson 已提交
1296 1297
	*batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;

1298
	/* Pad to end of cacheline */
1299 1300
	while ((unsigned long)batch % CACHELINE_BYTES)
		*batch++ = MI_NOOP;
1301 1302 1303 1304 1305 1306 1307

	/*
	 * MI_BATCH_BUFFER_END is not required in Indirect ctx BB because
	 * execution depends on the length specified in terms of cache lines
	 * in the register CTX_RCS_INDIRECT_CTX
	 */

1308
	return batch;
1309 1310
}

1311
static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
1312
{
C
Chris Wilson 已提交
1313 1314
	*batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;

1315
	/* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt,glk */
1316
	batch = gen8_emit_flush_coherentl3_wa(engine, batch);
1317

1318
	/* WaDisableGatherAtSetShaderCommonSlice:skl,bxt,kbl,glk */
1319 1320 1321 1322 1323
	*batch++ = MI_LOAD_REGISTER_IMM(1);
	*batch++ = i915_mmio_reg_offset(COMMON_SLICE_CHICKEN2);
	*batch++ = _MASKED_BIT_DISABLE(
			GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE);
	*batch++ = MI_NOOP;
1324

1325 1326
	/* WaClearSlmSpaceAtContextSwitch:kbl */
	/* Actual scratch location is at 128 bytes offset */
1327
	if (IS_KBL_REVID(engine->i915, 0, KBL_REVID_A0)) {
1328 1329 1330 1331 1332 1333 1334
		batch = gen8_emit_pipe_control(batch,
					       PIPE_CONTROL_FLUSH_L3 |
					       PIPE_CONTROL_GLOBAL_GTT_IVB |
					       PIPE_CONTROL_CS_STALL |
					       PIPE_CONTROL_QW_WRITE,
					       i915_ggtt_offset(engine->scratch)
					       + 2 * CACHELINE_BYTES);
1335
	}
1336

1337
	/* WaMediaPoolStateCmdInWABB:bxt,glk */
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351
	if (HAS_POOLED_EU(engine->i915)) {
		/*
		 * EU pool configuration is setup along with golden context
		 * during context initialization. This value depends on
		 * device type (2x6 or 3x6) and needs to be updated based
		 * on which subslice is disabled especially for 2x6
		 * devices, however it is safe to load default
		 * configuration of 3x6 device instead of masking off
		 * corresponding bits because HW ignores bits of a disabled
		 * subslice and drops down to appropriate config. Please
		 * see render_state_setup() in i915_gem_render_state.c for
		 * possible configurations, to avoid duplication they are
		 * not shown here again.
		 */
1352 1353 1354 1355 1356 1357
		*batch++ = GEN9_MEDIA_POOL_STATE;
		*batch++ = GEN9_MEDIA_POOL_ENABLE;
		*batch++ = 0x00777000;
		*batch++ = 0;
		*batch++ = 0;
		*batch++ = 0;
1358 1359
	}

C
Chris Wilson 已提交
1360 1361
	*batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;

1362
	/* Pad to end of cacheline */
1363 1364
	while ((unsigned long)batch % CACHELINE_BYTES)
		*batch++ = MI_NOOP;
1365

1366
	return batch;
1367 1368
}

1369 1370 1371
#define CTX_WA_BB_OBJ_SIZE (PAGE_SIZE)

static int lrc_setup_wa_ctx(struct intel_engine_cs *engine)
1372
{
1373 1374 1375
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma;
	int err;
1376

1377
	obj = i915_gem_object_create(engine->i915, CTX_WA_BB_OBJ_SIZE);
1378 1379
	if (IS_ERR(obj))
		return PTR_ERR(obj);
1380

1381
	vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
1382 1383 1384
	if (IS_ERR(vma)) {
		err = PTR_ERR(vma);
		goto err;
1385 1386
	}

1387 1388 1389 1390 1391
	err = i915_vma_pin(vma, 0, PAGE_SIZE, PIN_GLOBAL | PIN_HIGH);
	if (err)
		goto err;

	engine->wa_ctx.vma = vma;
1392
	return 0;
1393 1394 1395 1396

err:
	i915_gem_object_put(obj);
	return err;
1397 1398
}

1399
static void lrc_destroy_wa_ctx(struct intel_engine_cs *engine)
1400
{
1401
	i915_vma_unpin_and_release(&engine->wa_ctx.vma);
1402 1403
}

1404 1405
typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch);

1406
static int intel_init_workaround_bb(struct intel_engine_cs *engine)
1407
{
1408
	struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
1409 1410 1411
	struct i915_wa_ctx_bb *wa_bb[2] = { &wa_ctx->indirect_ctx,
					    &wa_ctx->per_ctx };
	wa_bb_func_t wa_bb_fn[2];
1412
	struct page *page;
1413 1414
	void *batch, *batch_ptr;
	unsigned int i;
1415
	int ret;
1416

1417
	if (GEM_WARN_ON(engine->id != RCS))
1418
		return -EINVAL;
1419

1420
	switch (INTEL_GEN(engine->i915)) {
1421 1422
	case 10:
		return 0;
1423 1424
	case 9:
		wa_bb_fn[0] = gen9_init_indirectctx_bb;
1425
		wa_bb_fn[1] = NULL;
1426 1427 1428
		break;
	case 8:
		wa_bb_fn[0] = gen8_init_indirectctx_bb;
1429
		wa_bb_fn[1] = NULL;
1430 1431 1432
		break;
	default:
		MISSING_CASE(INTEL_GEN(engine->i915));
1433
		return 0;
1434
	}
1435

1436
	ret = lrc_setup_wa_ctx(engine);
1437 1438 1439 1440 1441
	if (ret) {
		DRM_DEBUG_DRIVER("Failed to setup context WA page: %d\n", ret);
		return ret;
	}

1442
	page = i915_gem_object_get_dirty_page(wa_ctx->vma->obj, 0);
1443
	batch = batch_ptr = kmap_atomic(page);
1444

1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455
	/*
	 * Emit the two workaround batch buffers, recording the offset from the
	 * start of the workaround batch buffer object for each and their
	 * respective sizes.
	 */
	for (i = 0; i < ARRAY_SIZE(wa_bb_fn); i++) {
		wa_bb[i]->offset = batch_ptr - batch;
		if (WARN_ON(!IS_ALIGNED(wa_bb[i]->offset, CACHELINE_BYTES))) {
			ret = -EINVAL;
			break;
		}
1456 1457
		if (wa_bb_fn[i])
			batch_ptr = wa_bb_fn[i](engine, batch_ptr);
1458
		wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset);
1459 1460
	}

1461 1462
	BUG_ON(batch_ptr - batch > CTX_WA_BB_OBJ_SIZE);

1463 1464
	kunmap_atomic(batch);
	if (ret)
1465
		lrc_destroy_wa_ctx(engine);
1466 1467 1468 1469

	return ret;
}

1470 1471 1472 1473 1474 1475 1476 1477
static u8 gtiir[] = {
	[RCS] = 0,
	[BCS] = 0,
	[VCS] = 1,
	[VCS2] = 1,
	[VECS] = 3,
};

1478
static void enable_execlists(struct intel_engine_cs *engine)
1479
{
1480
	struct drm_i915_private *dev_priv = engine->i915;
1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491

	I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff);
	I915_WRITE(RING_MODE_GEN7(engine),
		   _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
	I915_WRITE(RING_HWS_PGA(engine->mmio_base),
		   engine->status_page.ggtt_offset);
	POSTING_READ(RING_HWS_PGA(engine->mmio_base));
}

static int gen8_init_common_ring(struct intel_engine_cs *engine)
{
1492
	struct intel_engine_execlists * const execlists = &engine->execlists;
1493 1494 1495 1496 1497
	int ret;

	ret = intel_mocs_init_engine(engine);
	if (ret)
		return ret;
1498

1499
	intel_engine_reset_breadcrumbs(engine);
1500
	intel_engine_init_hangcheck(engine);
1501

1502
	enable_execlists(engine);
1503
	DRM_DEBUG_DRIVER("Execlists enabled for %s\n", engine->name);
1504

1505 1506
	GEM_BUG_ON(engine->id >= ARRAY_SIZE(gtiir));

1507
	execlists->csb_head = -1;
1508
	execlists->active = 0;
1509

1510
	/* After a GPU reset, we may have requests to replay */
1511
	if (execlists->first)
1512
		tasklet_schedule(&execlists->tasklet);
1513

1514
	return 0;
1515 1516
}

1517
static int gen8_init_render_ring(struct intel_engine_cs *engine)
1518
{
1519
	struct drm_i915_private *dev_priv = engine->i915;
1520 1521
	int ret;

1522
	ret = gen8_init_common_ring(engine);
1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
	if (ret)
		return ret;

	/* We need to disable the AsyncFlip performance optimisations in order
	 * to use MI_WAIT_FOR_EVENT within the CS. It should already be
	 * programmed to '1' on all products.
	 *
	 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
	 */
	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));

	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));

1536
	return init_workarounds_ring(engine);
1537 1538
}

1539
static int gen9_init_render_ring(struct intel_engine_cs *engine)
1540 1541 1542
{
	int ret;

1543
	ret = gen8_init_common_ring(engine);
1544 1545 1546
	if (ret)
		return ret;

1547
	return init_workarounds_ring(engine);
1548 1549
}

1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567
static void reset_irq(struct intel_engine_cs *engine)
{
	struct drm_i915_private *dev_priv = engine->i915;

	/*
	 * Clear any pending interrupt state.
	 *
	 * We do it twice out of paranoia that some of the IIR are double
	 * buffered, and if we only reset it once there may still be
	 * an interrupt pending.
	 */
	I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]),
		   GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift);
	I915_WRITE(GEN8_GT_IIR(gtiir[engine->id]),
		   GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift);
	clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
}

1568 1569 1570
static void reset_common_ring(struct intel_engine_cs *engine,
			      struct drm_i915_gem_request *request)
{
1571
	struct intel_engine_execlists * const execlists = &engine->execlists;
1572
	struct intel_context *ce;
1573
	unsigned long flags;
1574

1575 1576
	GEM_TRACE("%s seqno=%x\n",
		  engine->name, request ? request->global_seqno : 0);
1577 1578 1579

	reset_irq(engine);

1580 1581
	spin_lock_irqsave(&engine->timeline->lock, flags);

1582 1583 1584 1585 1586 1587 1588 1589 1590
	/*
	 * Catch up with any missed context-switch interrupts.
	 *
	 * Ideally we would just read the remaining CSB entries now that we
	 * know the gpu is idle. However, the CSB registers are sometimes^W
	 * often trashed across a GPU reset! Instead we have to rely on
	 * guessing the missed context-switch events by looking at what
	 * requests were completed.
	 */
1591
	execlists_cancel_port_requests(execlists);
1592

1593
	/* Push back any incomplete requests for replay after the reset. */
1594
	__unwind_incomplete_requests(engine);
1595

1596
	spin_unlock_irqrestore(&engine->timeline->lock, flags);
1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607

	/* If the request was innocent, we leave the request in the ELSP
	 * and will try to replay it on restarting. The context image may
	 * have been corrupted by the reset, in which case we may have
	 * to service a new GPU hang, but more likely we can continue on
	 * without impact.
	 *
	 * If the request was guilty, we presume the context is corrupt
	 * and have to at least restore the RING register in the context
	 * image back to the expected values to skip over the guilty request.
	 */
1608
	if (!request || request->fence.error != -EIO)
1609
		return;
1610

1611 1612 1613 1614 1615 1616 1617
	/* We want a simple context + ring to execute the breadcrumb update.
	 * We cannot rely on the context being intact across the GPU hang,
	 * so clear it and rebuild just what we need for the breadcrumb.
	 * All pending requests for this context will be zapped, and any
	 * future request will be after userspace has had the opportunity
	 * to recreate its own state.
	 */
1618
	ce = &request->ctx->engine[engine->id];
1619 1620 1621
	execlists_init_reg_state(ce->lrc_reg_state,
				 request->ctx, engine, ce->ring);

1622
	/* Move the RING_HEAD onto the breadcrumb, past the hanging batch */
1623 1624
	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
		i915_ggtt_offset(ce->ring->vma);
1625
	ce->lrc_reg_state[CTX_RING_HEAD+1] = request->postfix;
1626

1627 1628 1629
	request->ring->head = request->postfix;
	intel_ring_update_space(request->ring);

1630
	/* Reset WaIdleLiteRestore:bdw,skl as well */
1631
	unwind_wa_tail(request);
1632 1633
}

1634 1635 1636
static int intel_logical_ring_emit_pdps(struct drm_i915_gem_request *req)
{
	struct i915_hw_ppgtt *ppgtt = req->ctx->ppgtt;
1637
	struct intel_engine_cs *engine = req->engine;
1638
	const int num_lri_cmds = GEN8_3LVL_PDPES * 2;
1639 1640
	u32 *cs;
	int i;
1641

1642 1643 1644
	cs = intel_ring_begin(req, num_lri_cmds * 2 + 2);
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1645

1646
	*cs++ = MI_LOAD_REGISTER_IMM(num_lri_cmds);
1647
	for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) {
1648 1649
		const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);

1650 1651 1652 1653
		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, i));
		*cs++ = upper_32_bits(pd_daddr);
		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, i));
		*cs++ = lower_32_bits(pd_daddr);
1654 1655
	}

1656 1657
	*cs++ = MI_NOOP;
	intel_ring_advance(req, cs);
1658 1659 1660 1661

	return 0;
}

1662
static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
1663
			      u64 offset, u32 len,
1664
			      const unsigned int flags)
1665
{
1666
	u32 *cs;
1667 1668
	int ret;

1669 1670 1671 1672
	/* Don't rely in hw updating PDPs, specially in lite-restore.
	 * Ideally, we should set Force PD Restore in ctx descriptor,
	 * but we can't. Force Restore would be a second option, but
	 * it is unsafe in case of lite-restore (because the ctx is
1673 1674
	 * not idle). PML4 is allocated during ppgtt init so this is
	 * not needed in 48-bit.*/
1675
	if (req->ctx->ppgtt &&
1676 1677 1678 1679 1680 1681
	    (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings) &&
	    !i915_vm_is_48bit(&req->ctx->ppgtt->base) &&
	    !intel_vgpu_active(req->i915)) {
		ret = intel_logical_ring_emit_pdps(req);
		if (ret)
			return ret;
1682

1683
		req->ctx->ppgtt->pd_dirty_rings &= ~intel_engine_flag(req->engine);
1684 1685
	}

1686 1687 1688
	cs = intel_ring_begin(req, 4);
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1689

1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706
	/*
	 * WaDisableCtxRestoreArbitration:bdw,chv
	 *
	 * We don't need to perform MI_ARB_ENABLE as often as we do (in
	 * particular all the gen that do not need the w/a at all!), if we
	 * took care to make sure that on every switch into this context
	 * (both ordinary and for preemption) that arbitrartion was enabled
	 * we would be fine. However, there doesn't seem to be a downside to
	 * being paranoid and making sure it is set before each batch and
	 * every context-switch.
	 *
	 * Note that if we fail to enable arbitration before the request
	 * is complete, then we do not see the context-switch interrupt and
	 * the engine hangs (with RING_HEAD == RING_TAIL).
	 *
	 * That satisfies both the GPGPU w/a and our heavy-handed paranoia.
	 */
1707 1708
	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;

1709
	/* FIXME(BDW): Address space and security selectors. */
1710 1711 1712
	*cs++ = MI_BATCH_BUFFER_START_GEN8 |
		(flags & I915_DISPATCH_SECURE ? 0 : BIT(8)) |
		(flags & I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0);
1713 1714 1715
	*cs++ = lower_32_bits(offset);
	*cs++ = upper_32_bits(offset);
	intel_ring_advance(req, cs);
1716 1717 1718 1719

	return 0;
}

1720
static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
1721
{
1722
	struct drm_i915_private *dev_priv = engine->i915;
1723 1724 1725
	I915_WRITE_IMR(engine,
		       ~(engine->irq_enable_mask | engine->irq_keep_mask));
	POSTING_READ_FW(RING_IMR(engine->mmio_base));
1726 1727
}

1728
static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
1729
{
1730
	struct drm_i915_private *dev_priv = engine->i915;
1731
	I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1732 1733
}

1734
static int gen8_emit_flush(struct drm_i915_gem_request *request, u32 mode)
1735
{
1736
	u32 cmd, *cs;
1737

1738 1739 1740
	cs = intel_ring_begin(request, 4);
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1741 1742 1743

	cmd = MI_FLUSH_DW + 1;

1744 1745 1746 1747 1748 1749 1750
	/* We always require a command barrier so that subsequent
	 * commands, such as breadcrumb interrupts, are strictly ordered
	 * wrt the contents of the write cache being flushed to memory
	 * (and thus being coherent from the CPU).
	 */
	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;

1751
	if (mode & EMIT_INVALIDATE) {
1752
		cmd |= MI_INVALIDATE_TLB;
1753
		if (request->engine->id == VCS)
1754
			cmd |= MI_INVALIDATE_BSD;
1755 1756
	}

1757 1758 1759 1760 1761
	*cs++ = cmd;
	*cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
	*cs++ = 0; /* upper addr */
	*cs++ = 0; /* value */
	intel_ring_advance(request, cs);
1762 1763 1764 1765

	return 0;
}

1766
static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
1767
				  u32 mode)
1768
{
1769
	struct intel_engine_cs *engine = request->engine;
1770 1771
	u32 scratch_addr =
		i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES;
M
Mika Kuoppala 已提交
1772
	bool vf_flush_wa = false, dc_flush_wa = false;
1773
	u32 *cs, flags = 0;
M
Mika Kuoppala 已提交
1774
	int len;
1775 1776 1777

	flags |= PIPE_CONTROL_CS_STALL;

1778
	if (mode & EMIT_FLUSH) {
1779 1780
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
1781
		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
1782
		flags |= PIPE_CONTROL_FLUSH_ENABLE;
1783 1784
	}

1785
	if (mode & EMIT_INVALIDATE) {
1786 1787 1788 1789 1790 1791 1792 1793 1794
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_QW_WRITE;
		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;

1795 1796 1797 1798
		/*
		 * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL
		 * pipe control.
		 */
1799
		if (IS_GEN9(request->i915))
1800
			vf_flush_wa = true;
M
Mika Kuoppala 已提交
1801 1802 1803 1804

		/* WaForGAMHang:kbl */
		if (IS_KBL_REVID(request->i915, 0, KBL_REVID_B0))
			dc_flush_wa = true;
1805
	}
1806

M
Mika Kuoppala 已提交
1807 1808 1809 1810 1811 1812 1813 1814
	len = 6;

	if (vf_flush_wa)
		len += 6;

	if (dc_flush_wa)
		len += 12;

1815 1816 1817
	cs = intel_ring_begin(request, len);
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1818

1819 1820
	if (vf_flush_wa)
		cs = gen8_emit_pipe_control(cs, 0, 0);
1821

1822 1823 1824
	if (dc_flush_wa)
		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE,
					    0);
M
Mika Kuoppala 已提交
1825

1826
	cs = gen8_emit_pipe_control(cs, flags, scratch_addr);
M
Mika Kuoppala 已提交
1827

1828 1829
	if (dc_flush_wa)
		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0);
M
Mika Kuoppala 已提交
1830

1831
	intel_ring_advance(request, cs);
1832 1833 1834 1835

	return 0;
}

1836 1837 1838 1839 1840
/*
 * Reserve space for 2 NOOPs at the end of each request to be
 * used as a workaround for not being allowed to do lite
 * restore with HEAD==TAIL (WaIdleLiteRestore).
 */
1841
static void gen8_emit_wa_tail(struct drm_i915_gem_request *request, u32 *cs)
1842
{
C
Chris Wilson 已提交
1843 1844
	/* Ensure there's always at least one preemption point per-request. */
	*cs++ = MI_ARB_CHECK;
1845 1846
	*cs++ = MI_NOOP;
	request->wa_tail = intel_ring_offset(request, cs);
C
Chris Wilson 已提交
1847
}
1848

1849
static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, u32 *cs)
C
Chris Wilson 已提交
1850
{
1851 1852
	/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
	BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
1853

1854 1855
	cs = gen8_emit_ggtt_write(cs, request->global_seqno,
				  intel_hws_seqno_address(request->engine));
1856 1857 1858
	*cs++ = MI_USER_INTERRUPT;
	*cs++ = MI_NOOP;
	request->tail = intel_ring_offset(request, cs);
1859
	assert_ring_tail_valid(request->ring, request->tail);
C
Chris Wilson 已提交
1860

1861
	gen8_emit_wa_tail(request, cs);
1862
}
1863 1864
static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS;

1865
static void gen8_emit_breadcrumb_rcs(struct drm_i915_gem_request *request,
1866
					u32 *cs)
1867
{
1868 1869 1870
	/* We're using qword write, seqno should be aligned to 8 bytes. */
	BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1);

1871 1872
	cs = gen8_emit_ggtt_write_rcs(cs, request->global_seqno,
				      intel_hws_seqno_address(request->engine));
1873 1874 1875
	*cs++ = MI_USER_INTERRUPT;
	*cs++ = MI_NOOP;
	request->tail = intel_ring_offset(request, cs);
1876
	assert_ring_tail_valid(request->ring, request->tail);
C
Chris Wilson 已提交
1877

1878
	gen8_emit_wa_tail(request, cs);
1879
}
1880
static const int gen8_emit_breadcrumb_rcs_sz = 8 + WA_TAIL_DWORDS;
1881

1882
static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
1883 1884 1885
{
	int ret;

1886
	ret = intel_ring_workarounds_emit(req);
1887 1888 1889
	if (ret)
		return ret;

1890 1891 1892 1893 1894 1895 1896 1897
	ret = intel_rcs_context_init_mocs(req);
	/*
	 * Failing to program the MOCS is non-fatal.The system will not
	 * run at peak performance. So generate an error and carry on.
	 */
	if (ret)
		DRM_ERROR("MOCS failed to program: expect performance issues.\n");

1898
	return i915_gem_render_state_emit(req);
1899 1900
}

1901 1902
/**
 * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer
1903
 * @engine: Engine Command Streamer.
1904
 */
1905
void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
1906
{
1907
	struct drm_i915_private *dev_priv;
1908

1909 1910 1911 1912
	/*
	 * Tasklet cannot be active at this point due intel_mark_active/idle
	 * so this is just for documentation.
	 */
1913 1914 1915
	if (WARN_ON(test_bit(TASKLET_STATE_SCHED,
			     &engine->execlists.tasklet.state)))
		tasklet_kill(&engine->execlists.tasklet);
1916

1917
	dev_priv = engine->i915;
1918

1919 1920
	if (engine->buffer) {
		WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
1921
	}
1922

1923 1924
	if (engine->cleanup)
		engine->cleanup(engine);
1925

1926
	intel_engine_cleanup_common(engine);
1927

1928
	lrc_destroy_wa_ctx(engine);
1929

1930
	engine->i915 = NULL;
1931 1932
	dev_priv->engine[engine->id] = NULL;
	kfree(engine);
1933 1934
}

1935
static void execlists_set_default_submission(struct intel_engine_cs *engine)
1936
{
1937
	engine->submit_request = execlists_submit_request;
1938
	engine->cancel_requests = execlists_cancel_requests;
1939
	engine->schedule = execlists_schedule;
1940
	engine->execlists.tasklet.func = execlists_submission_tasklet;
1941 1942 1943

	engine->park = NULL;
	engine->unpark = NULL;
1944 1945

	engine->flags |= I915_ENGINE_SUPPORTS_STATS;
1946 1947
}

1948
static void
1949
logical_ring_default_vfuncs(struct intel_engine_cs *engine)
1950 1951
{
	/* Default vfuncs which can be overriden by each engine. */
1952
	engine->init_hw = gen8_init_common_ring;
1953
	engine->reset_hw = reset_common_ring;
1954 1955 1956 1957

	engine->context_pin = execlists_context_pin;
	engine->context_unpin = execlists_context_unpin;

1958 1959
	engine->request_alloc = execlists_request_alloc;

1960
	engine->emit_flush = gen8_emit_flush;
1961
	engine->emit_breadcrumb = gen8_emit_breadcrumb;
1962
	engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz;
1963 1964

	engine->set_default_submission = execlists_set_default_submission;
1965

1966 1967
	engine->irq_enable = gen8_logical_ring_enable_irq;
	engine->irq_disable = gen8_logical_ring_disable_irq;
1968
	engine->emit_bb_start = gen8_emit_bb_start;
1969 1970
}

1971
static inline void
1972
logical_ring_default_irqs(struct intel_engine_cs *engine)
1973
{
1974
	unsigned shift = engine->irq_shift;
1975 1976
	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
	engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
1977 1978
}

1979 1980 1981 1982 1983 1984
static void
logical_ring_setup(struct intel_engine_cs *engine)
{
	struct drm_i915_private *dev_priv = engine->i915;
	enum forcewake_domains fw_domains;

1985 1986
	intel_engine_setup_common(engine);

1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001
	/* Intentionally left blank. */
	engine->buffer = NULL;

	fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
						    RING_ELSP(engine),
						    FW_REG_WRITE);

	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
						     RING_CONTEXT_STATUS_PTR(engine),
						     FW_REG_READ | FW_REG_WRITE);

	fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
						     RING_CONTEXT_STATUS_BUF_BASE(engine),
						     FW_REG_READ);

2002
	engine->execlists.fw_domains = fw_domains;
2003

2004 2005
	tasklet_init(&engine->execlists.tasklet,
		     execlists_submission_tasklet, (unsigned long)engine);
2006 2007 2008 2009 2010

	logical_ring_default_vfuncs(engine);
	logical_ring_default_irqs(engine);
}

2011
static int logical_ring_init(struct intel_engine_cs *engine)
2012 2013 2014
{
	int ret;

2015
	ret = intel_engine_init_common(engine);
2016 2017 2018
	if (ret)
		goto error;

2019 2020 2021
	engine->execlists.elsp =
		engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine));

2022 2023 2024 2025 2026 2027 2028
	return 0;

error:
	intel_logical_ring_cleanup(engine);
	return ret;
}

2029
int logical_render_ring_init(struct intel_engine_cs *engine)
2030 2031 2032 2033
{
	struct drm_i915_private *dev_priv = engine->i915;
	int ret;

2034 2035
	logical_ring_setup(engine);

2036 2037 2038 2039 2040 2041 2042 2043 2044 2045
	if (HAS_L3_DPF(dev_priv))
		engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;

	/* Override some for render ring. */
	if (INTEL_GEN(dev_priv) >= 9)
		engine->init_hw = gen9_init_render_ring;
	else
		engine->init_hw = gen8_init_render_ring;
	engine->init_context = gen8_init_rcs_context;
	engine->emit_flush = gen8_emit_flush_render;
2046 2047
	engine->emit_breadcrumb = gen8_emit_breadcrumb_rcs;
	engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_rcs_sz;
2048

2049
	ret = intel_engine_create_scratch(engine, PAGE_SIZE);
2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063
	if (ret)
		return ret;

	ret = intel_init_workaround_bb(engine);
	if (ret) {
		/*
		 * We continue even if we fail to initialize WA batch
		 * because we only expect rare glitches but nothing
		 * critical to prevent us from using GPU
		 */
		DRM_ERROR("WA batch buffer initialization failed: %d\n",
			  ret);
	}

2064
	return logical_ring_init(engine);
2065 2066
}

2067
int logical_xcs_ring_init(struct intel_engine_cs *engine)
2068 2069 2070 2071
{
	logical_ring_setup(engine);

	return logical_ring_init(engine);
2072 2073
}

2074
static u32
2075
make_rpcs(struct drm_i915_private *dev_priv)
2076 2077 2078 2079 2080 2081 2082
{
	u32 rpcs = 0;

	/*
	 * No explicit RPCS request is needed to ensure full
	 * slice/subslice/EU enablement prior to Gen9.
	*/
2083
	if (INTEL_GEN(dev_priv) < 9)
2084 2085 2086 2087 2088 2089 2090 2091
		return 0;

	/*
	 * Starting in Gen9, render power gating can leave
	 * slice/subslice/EU in a partially enabled state. We
	 * must make an explicit request through RPCS for full
	 * enablement.
	*/
2092
	if (INTEL_INFO(dev_priv)->sseu.has_slice_pg) {
2093
		rpcs |= GEN8_RPCS_S_CNT_ENABLE;
2094
		rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.slice_mask) <<
2095 2096 2097 2098
			GEN8_RPCS_S_CNT_SHIFT;
		rpcs |= GEN8_RPCS_ENABLE;
	}

2099
	if (INTEL_INFO(dev_priv)->sseu.has_subslice_pg) {
2100
		rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
2101
		rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.subslice_mask) <<
2102 2103 2104 2105
			GEN8_RPCS_SS_CNT_SHIFT;
		rpcs |= GEN8_RPCS_ENABLE;
	}

2106 2107
	if (INTEL_INFO(dev_priv)->sseu.has_eu_pg) {
		rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
2108
			GEN8_RPCS_EU_MIN_SHIFT;
2109
		rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
2110 2111 2112 2113 2114 2115 2116
			GEN8_RPCS_EU_MAX_SHIFT;
		rpcs |= GEN8_RPCS_ENABLE;
	}

	return rpcs;
}

2117
static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
2118 2119 2120
{
	u32 indirect_ctx_offset;

2121
	switch (INTEL_GEN(engine->i915)) {
2122
	default:
2123
		MISSING_CASE(INTEL_GEN(engine->i915));
2124
		/* fall through */
2125 2126 2127 2128
	case 10:
		indirect_ctx_offset =
			GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
		break;
2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141
	case 9:
		indirect_ctx_offset =
			GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
		break;
	case 8:
		indirect_ctx_offset =
			GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
		break;
	}

	return indirect_ctx_offset;
}

2142
static void execlists_init_reg_state(u32 *regs,
2143 2144 2145
				     struct i915_gem_context *ctx,
				     struct intel_engine_cs *engine,
				     struct intel_ring *ring)
2146
{
2147 2148
	struct drm_i915_private *dev_priv = engine->i915;
	struct i915_hw_ppgtt *ppgtt = ctx->ppgtt ?: dev_priv->mm.aliasing_ppgtt;
2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177
	u32 base = engine->mmio_base;
	bool rcs = engine->id == RCS;

	/* A context is actually a big batch buffer with several
	 * MI_LOAD_REGISTER_IMM commands followed by (reg, value) pairs. The
	 * values we are setting here are only for the first context restore:
	 * on a subsequent save, the GPU will recreate this batchbuffer with new
	 * values (including all the missing MI_LOAD_REGISTER_IMM commands that
	 * we are not initializing here).
	 */
	regs[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(rcs ? 14 : 11) |
				 MI_LRI_FORCE_POSTED;

	CTX_REG(regs, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(engine),
		_MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
				   (HAS_RESOURCE_STREAMER(dev_priv) ?
				   CTX_CTRL_RS_CTX_ENABLE : 0)));
	CTX_REG(regs, CTX_RING_HEAD, RING_HEAD(base), 0);
	CTX_REG(regs, CTX_RING_TAIL, RING_TAIL(base), 0);
	CTX_REG(regs, CTX_RING_BUFFER_START, RING_START(base), 0);
	CTX_REG(regs, CTX_RING_BUFFER_CONTROL, RING_CTL(base),
		RING_CTL_SIZE(ring->size) | RING_VALID);
	CTX_REG(regs, CTX_BB_HEAD_U, RING_BBADDR_UDW(base), 0);
	CTX_REG(regs, CTX_BB_HEAD_L, RING_BBADDR(base), 0);
	CTX_REG(regs, CTX_BB_STATE, RING_BBSTATE(base), RING_BB_PPGTT);
	CTX_REG(regs, CTX_SECOND_BB_HEAD_U, RING_SBBADDR_UDW(base), 0);
	CTX_REG(regs, CTX_SECOND_BB_HEAD_L, RING_SBBADDR(base), 0);
	CTX_REG(regs, CTX_SECOND_BB_STATE, RING_SBBSTATE(base), 0);
	if (rcs) {
2178 2179
		struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;

2180 2181 2182
		CTX_REG(regs, CTX_RCS_INDIRECT_CTX, RING_INDIRECT_CTX(base), 0);
		CTX_REG(regs, CTX_RCS_INDIRECT_CTX_OFFSET,
			RING_INDIRECT_CTX_OFFSET(base), 0);
2183
		if (wa_ctx->indirect_ctx.size) {
2184
			u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
2185

2186
			regs[CTX_RCS_INDIRECT_CTX + 1] =
2187 2188
				(ggtt_offset + wa_ctx->indirect_ctx.offset) |
				(wa_ctx->indirect_ctx.size / CACHELINE_BYTES);
2189

2190
			regs[CTX_RCS_INDIRECT_CTX_OFFSET + 1] =
2191
				intel_lr_indirect_ctx_offset(engine) << 6;
2192 2193 2194 2195 2196
		}

		CTX_REG(regs, CTX_BB_PER_CTX_PTR, RING_BB_PER_CTX_PTR(base), 0);
		if (wa_ctx->per_ctx.size) {
			u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
2197

2198
			regs[CTX_BB_PER_CTX_PTR + 1] =
2199
				(ggtt_offset + wa_ctx->per_ctx.offset) | 0x01;
2200
		}
2201
	}
2202 2203 2204 2205

	regs[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9) | MI_LRI_FORCE_POSTED;

	CTX_REG(regs, CTX_CTX_TIMESTAMP, RING_CTX_TIMESTAMP(base), 0);
2206
	/* PDP values well be assigned later if needed */
2207 2208 2209 2210 2211 2212 2213 2214
	CTX_REG(regs, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(engine, 3), 0);
	CTX_REG(regs, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(engine, 3), 0);
	CTX_REG(regs, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(engine, 2), 0);
	CTX_REG(regs, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(engine, 2), 0);
	CTX_REG(regs, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(engine, 1), 0);
	CTX_REG(regs, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(engine, 1), 0);
	CTX_REG(regs, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(engine, 0), 0);
	CTX_REG(regs, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(engine, 0), 0);
2215

2216
	if (ppgtt && i915_vm_is_48bit(&ppgtt->base)) {
2217 2218 2219 2220
		/* 64b PPGTT (48bit canonical)
		 * PDP0_DESCRIPTOR contains the base address to PML4 and
		 * other PDP Descriptors are ignored.
		 */
2221
		ASSIGN_CTX_PML4(ppgtt, regs);
2222 2223
	}

2224 2225 2226 2227
	if (rcs) {
		regs[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
		CTX_REG(regs, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
			make_rpcs(dev_priv));
2228 2229

		i915_oa_init_reg_state(engine, ctx, regs);
2230
	}
2231 2232 2233 2234 2235 2236 2237 2238 2239
}

static int
populate_lr_context(struct i915_gem_context *ctx,
		    struct drm_i915_gem_object *ctx_obj,
		    struct intel_engine_cs *engine,
		    struct intel_ring *ring)
{
	void *vaddr;
2240
	u32 *regs;
2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254
	int ret;

	ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
	if (ret) {
		DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
		return ret;
	}

	vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB);
	if (IS_ERR(vaddr)) {
		ret = PTR_ERR(vaddr);
		DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
		return ret;
	}
C
Chris Wilson 已提交
2255
	ctx_obj->mm.dirty = true;
2256

2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274
	if (engine->default_state) {
		/*
		 * We only want to copy over the template context state;
		 * skipping over the headers reserved for GuC communication,
		 * leaving those as zero.
		 */
		const unsigned long start = LRC_HEADER_PAGES * PAGE_SIZE;
		void *defaults;

		defaults = i915_gem_object_pin_map(engine->default_state,
						   I915_MAP_WB);
		if (IS_ERR(defaults))
			return PTR_ERR(defaults);

		memcpy(vaddr + start, defaults + start, engine->context_size);
		i915_gem_object_unpin_map(engine->default_state);
	}

2275 2276
	/* The second page of the context object contains some fields which must
	 * be set up prior to the first execution. */
2277 2278 2279 2280 2281
	regs = vaddr + LRC_STATE_PN * PAGE_SIZE;
	execlists_init_reg_state(regs, ctx, engine, ring);
	if (!engine->default_state)
		regs[CTX_CONTEXT_CONTROL + 1] |=
			_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
2282

2283
	i915_gem_object_unpin_map(ctx_obj);
2284 2285 2286 2287

	return 0;
}

2288
static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
2289
					    struct intel_engine_cs *engine)
2290
{
2291
	struct drm_i915_gem_object *ctx_obj;
2292
	struct intel_context *ce = &ctx->engine[engine->id];
2293
	struct i915_vma *vma;
2294
	uint32_t context_size;
2295
	struct intel_ring *ring;
2296 2297
	int ret;

2298
	WARN_ON(ce->state);
2299

2300
	context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE);
2301

2302 2303 2304 2305 2306
	/*
	 * Before the actual start of the context image, we insert a few pages
	 * for our own use and for sharing with the GuC.
	 */
	context_size += LRC_HEADER_PAGES * PAGE_SIZE;
2307

2308
	ctx_obj = i915_gem_object_create(ctx->i915, context_size);
2309
	if (IS_ERR(ctx_obj)) {
2310
		DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n");
2311
		return PTR_ERR(ctx_obj);
2312 2313
	}

2314
	vma = i915_vma_instance(ctx_obj, &ctx->i915->ggtt.base, NULL);
2315 2316 2317 2318 2319
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
		goto error_deref_obj;
	}

2320
	ring = intel_engine_create_ring(engine, ctx->ring_size);
2321 2322
	if (IS_ERR(ring)) {
		ret = PTR_ERR(ring);
2323
		goto error_deref_obj;
2324 2325
	}

2326
	ret = populate_lr_context(ctx, ctx_obj, engine, ring);
2327 2328
	if (ret) {
		DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
2329
		goto error_ring_free;
2330 2331
	}

2332
	ce->ring = ring;
2333
	ce->state = vma;
2334 2335

	return 0;
2336

2337
error_ring_free:
2338
	intel_ring_free(ring);
2339
error_deref_obj:
2340
	i915_gem_object_put(ctx_obj);
2341
	return ret;
2342
}
2343

2344
void intel_lr_context_resume(struct drm_i915_private *dev_priv)
2345
{
2346
	struct intel_engine_cs *engine;
2347
	struct i915_gem_context *ctx;
2348
	enum intel_engine_id id;
2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359

	/* Because we emit WA_TAIL_DWORDS there may be a disparity
	 * between our bookkeeping in ce->ring->head and ce->ring->tail and
	 * that stored in context. As we only write new commands from
	 * ce->ring->tail onwards, everything before that is junk. If the GPU
	 * starts reading from its RING_HEAD from the context, it may try to
	 * execute that junk and die.
	 *
	 * So to avoid that we reset the context images upon resume. For
	 * simplicity, we just zero everything out.
	 */
2360
	list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
2361
		for_each_engine(engine, dev_priv, id) {
2362 2363
			struct intel_context *ce = &ctx->engine[engine->id];
			u32 *reg;
2364

2365 2366
			if (!ce->state)
				continue;
2367

2368 2369 2370 2371
			reg = i915_gem_object_pin_map(ce->state->obj,
						      I915_MAP_WB);
			if (WARN_ON(IS_ERR(reg)))
				continue;
2372

2373 2374 2375
			reg += LRC_STATE_PN * PAGE_SIZE / sizeof(*reg);
			reg[CTX_RING_HEAD+1] = 0;
			reg[CTX_RING_TAIL+1] = 0;
2376

C
Chris Wilson 已提交
2377
			ce->state->obj->mm.dirty = true;
2378
			i915_gem_object_unpin_map(ce->state->obj);
2379

2380
			intel_ring_reset(ce->ring, 0);
2381
		}
2382 2383
	}
}