intel_lrc.c 85.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/*
 * Copyright © 2014 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Ben Widawsky <ben@bwidawsk.net>
 *    Michel Thierry <michel.thierry@intel.com>
 *    Thomas Daniel <thomas.daniel@intel.com>
 *    Oscar Mateo <oscar.mateo@intel.com>
 *
 */

31 32 33 34
/**
 * DOC: Logical Rings, Logical Ring Contexts and Execlists
 *
 * Motivation:
35 36 37 38
 * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
 * These expanded contexts enable a number of new abilities, especially
 * "Execlists" (also implemented in this file).
 *
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
 * One of the main differences with the legacy HW contexts is that logical
 * ring contexts incorporate many more things to the context's state, like
 * PDPs or ringbuffer control registers:
 *
 * The reason why PDPs are included in the context is straightforward: as
 * PPGTTs (per-process GTTs) are actually per-context, having the PDPs
 * contained there mean you don't need to do a ppgtt->switch_mm yourself,
 * instead, the GPU will do it for you on the context switch.
 *
 * But, what about the ringbuffer control registers (head, tail, etc..)?
 * shouldn't we just need a set of those per engine command streamer? This is
 * where the name "Logical Rings" starts to make sense: by virtualizing the
 * rings, the engine cs shifts to a new "ring buffer" with every context
 * switch. When you want to submit a workload to the GPU you: A) choose your
 * context, B) find its appropriate virtualized ring, C) write commands to it
 * and then, finally, D) tell the GPU to switch to that context.
 *
 * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch
 * to a contexts is via a context execution list, ergo "Execlists".
 *
 * LRC implementation:
 * Regarding the creation of contexts, we have:
 *
 * - One global default context.
 * - One local default context for each opened fd.
 * - One local extra context for each context create ioctl call.
 *
 * Now that ringbuffers belong per-context (and not per-engine, like before)
 * and that contexts are uniquely tied to a given engine (and not reusable,
 * like before) we need:
 *
 * - One ringbuffer per-engine inside each context.
 * - One backing object per-engine inside each context.
 *
 * The global default context starts its life with these new objects fully
 * allocated and populated. The local default context for each opened fd is
 * more complex, because we don't know at creation time which engine is going
 * to use them. To handle this, we have implemented a deferred creation of LR
 * contexts:
 *
 * The local context starts its life as a hollow or blank holder, that only
 * gets populated for a given engine once we receive an execbuffer. If later
 * on we receive another execbuffer ioctl for the same context but a different
 * engine, we allocate/populate a new ringbuffer and context backing object and
 * so on.
 *
 * Finally, regarding local contexts created using the ioctl call: as they are
 * only allowed with the render ring, we can allocate & populate them right
 * away (no need to defer anything, at least for now).
 *
 * Execlists implementation:
90 91
 * Execlists are the new method by which, on gen8+ hardware, workloads are
 * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
 * This method works as follows:
 *
 * When a request is committed, its commands (the BB start and any leading or
 * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer
 * for the appropriate context. The tail pointer in the hardware context is not
 * updated at this time, but instead, kept by the driver in the ringbuffer
 * structure. A structure representing this request is added to a request queue
 * for the appropriate engine: this structure contains a copy of the context's
 * tail after the request was written to the ring buffer and a pointer to the
 * context itself.
 *
 * If the engine's request queue was empty before the request was added, the
 * queue is processed immediately. Otherwise the queue will be processed during
 * a context switch interrupt. In any case, elements on the queue will get sent
 * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a
 * globally unique 20-bits submission ID.
 *
 * When execution of a request completes, the GPU updates the context status
 * buffer with a context complete event and generates a context switch interrupt.
 * During the interrupt handling, the driver examines the events in the buffer:
 * for each context complete event, if the announced ID matches that on the head
 * of the request queue, then that request is retired and removed from the queue.
 *
 * After processing, if any requests were retired and the queue is not empty
 * then a new execution list can be submitted. The two requests at the front of
 * the queue are next to be submitted but since a context may not occur twice in
 * an execution list, if subsequent requests have the same ID as the first then
 * the two requests must be combined. This is done simply by discarding requests
 * at the head of the queue until either only one requests is left (in which case
 * we use a NULL second context) or the first two requests have unique IDs.
 *
 * By always executing the first two requests in the queue the driver ensures
 * that the GPU is kept as busy as possible. In the case where a single context
 * completes but a second context is still executing, the request for this second
 * context will be at the head of the queue when we remove the first one. This
 * request will then be resubmitted along with a new request for a different context,
 * which will cause the hardware to continue executing the second request and queue
 * the new request (the GPU detects the condition of a context getting preempted
 * with the same context and optimizes the context switch flow by not doing
 * preemption, but just sampling the new tail pointer).
 *
133
 */
134
#include <linux/interrupt.h>
135 136 137 138

#include <drm/drmP.h>
#include <drm/i915_drm.h>
#include "i915_drv.h"
139
#include "i915_gem_render_state.h"
140
#include "i915_vgpu.h"
141
#include "intel_lrc_reg.h"
142
#include "intel_mocs.h"
143
#include "intel_workarounds.h"
144

145 146 147 148 149 150 151 152 153 154 155 156 157
#define RING_EXECLIST_QFULL		(1 << 0x2)
#define RING_EXECLIST1_VALID		(1 << 0x3)
#define RING_EXECLIST0_VALID		(1 << 0x4)
#define RING_EXECLIST_ACTIVE_STATUS	(3 << 0xE)
#define RING_EXECLIST1_ACTIVE		(1 << 0x11)
#define RING_EXECLIST0_ACTIVE		(1 << 0x12)

#define GEN8_CTX_STATUS_IDLE_ACTIVE	(1 << 0)
#define GEN8_CTX_STATUS_PREEMPTED	(1 << 1)
#define GEN8_CTX_STATUS_ELEMENT_SWITCH	(1 << 2)
#define GEN8_CTX_STATUS_ACTIVE_IDLE	(1 << 3)
#define GEN8_CTX_STATUS_COMPLETE	(1 << 4)
#define GEN8_CTX_STATUS_LITE_RESTORE	(1 << 15)
158

159
#define GEN8_CTX_STATUS_COMPLETED_MASK \
160
	 (GEN8_CTX_STATUS_COMPLETE | GEN8_CTX_STATUS_PREEMPTED)
161

162 163
/* Typical size of the average request (2 pipecontrols and a MI_BB) */
#define EXECLISTS_REQUEST_SIZE 64 /* bytes */
164
#define WA_TAIL_DWORDS 2
165
#define WA_TAIL_BYTES (sizeof(u32) * WA_TAIL_DWORDS)
166

167
static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
168 169
					    struct intel_engine_cs *engine,
					    struct intel_context *ce);
170 171 172 173
static void execlists_init_reg_state(u32 *reg_state,
				     struct i915_gem_context *ctx,
				     struct intel_engine_cs *engine,
				     struct intel_ring *ring);
174

175 176 177 178 179 180 181
static inline struct i915_priolist *to_priolist(struct rb_node *rb)
{
	return rb_entry(rb, struct i915_priolist, node);
}

static inline int rq_prio(const struct i915_request *rq)
{
182
	return rq->sched.attr.priority;
183 184 185 186 187 188
}

static inline bool need_preempt(const struct intel_engine_cs *engine,
				const struct i915_request *last,
				int prio)
{
189
	return (intel_engine_has_preemption(engine) &&
190 191
		__execlists_need_preempt(prio, rq_prio(last)) &&
		!i915_request_completed(last));
192 193
}

194
/*
195 196 197 198 199
 * The context descriptor encodes various attributes of a context,
 * including its GTT address and some flags. Because it's fairly
 * expensive to calculate, we'll just do it once and cache the result,
 * which remains valid until the context is unpinned.
 *
200 201
 * This is what a descriptor looks like, from LSB to MSB::
 *
202
 *      bits  0-11:    flags, GEN8_CTX_* (cached in ctx->desc_template)
203
 *      bits 12-31:    LRCA, GTT address of (the HWSP of) this context
204
 *      bits 32-52:    ctx ID, a globally unique tag (highest bit used by GuC)
205 206
 *      bits 53-54:    mbz, reserved for use by hardware
 *      bits 55-63:    group ID, currently unused and set to 0
207 208 209 210 211 212 213 214 215 216 217 218
 *
 * Starting from Gen11, the upper dword of the descriptor has a new format:
 *
 *      bits 32-36:    reserved
 *      bits 37-47:    SW context ID
 *      bits 48:53:    engine instance
 *      bit 54:        mbz, reserved for use by hardware
 *      bits 55-60:    SW counter
 *      bits 61-63:    engine class
 *
 * engine info, SW context ID and SW counter need to form a unique number
 * (Context ID) per lrc.
219
 */
220
static void
221
intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
222 223
				   struct intel_engine_cs *engine,
				   struct intel_context *ce)
224
{
225
	u64 desc;
226

227 228
	BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (BIT(GEN8_CTX_ID_WIDTH)));
	BUILD_BUG_ON(GEN11_MAX_CONTEXT_HW_ID > (BIT(GEN11_SW_CTX_ID_WIDTH)));
229

230
	desc = ctx->desc_template;				/* bits  0-11 */
231 232
	GEM_BUG_ON(desc & GENMASK_ULL(63, 12));

233
	desc |= i915_ggtt_offset(ce->state) + LRC_HEADER_PAGES * PAGE_SIZE;
234
								/* bits 12-31 */
235 236
	GEM_BUG_ON(desc & GENMASK_ULL(63, 32));

237 238 239 240 241
	/*
	 * The following 32bits are copied into the OA reports (dword 2).
	 * Consider updating oa_get_render_ctx_id in i915_perf.c when changing
	 * anything below.
	 */
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
	if (INTEL_GEN(ctx->i915) >= 11) {
		GEM_BUG_ON(ctx->hw_id >= BIT(GEN11_SW_CTX_ID_WIDTH));
		desc |= (u64)ctx->hw_id << GEN11_SW_CTX_ID_SHIFT;
								/* bits 37-47 */

		desc |= (u64)engine->instance << GEN11_ENGINE_INSTANCE_SHIFT;
								/* bits 48-53 */

		/* TODO: decide what to do with SW counter (bits 55-60) */

		desc |= (u64)engine->class << GEN11_ENGINE_CLASS_SHIFT;
								/* bits 61-63 */
	} else {
		GEM_BUG_ON(ctx->hw_id >= BIT(GEN8_CTX_ID_WIDTH));
		desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT;	/* bits 32-52 */
	}
258

259
	ce->lrc_desc = desc;
260 261
}

262
static struct i915_priolist *
263
lookup_priolist(struct intel_engine_cs *engine, int prio)
264
{
265
	struct intel_engine_execlists * const execlists = &engine->execlists;
266 267 268 269
	struct i915_priolist *p;
	struct rb_node **parent, *rb;
	bool first = true;

270
	if (unlikely(execlists->no_priolist))
271 272 273 274 275
		prio = I915_PRIORITY_NORMAL;

find_priolist:
	/* most positive priority is scheduled first, equal priorities fifo */
	rb = NULL;
276
	parent = &execlists->queue.rb_root.rb_node;
277 278
	while (*parent) {
		rb = *parent;
279
		p = to_priolist(rb);
280 281 282 283 284 285
		if (prio > p->priority) {
			parent = &rb->rb_left;
		} else if (prio < p->priority) {
			parent = &rb->rb_right;
			first = false;
		} else {
286
			return p;
287 288 289 290
		}
	}

	if (prio == I915_PRIORITY_NORMAL) {
291
		p = &execlists->default_priolist;
292 293 294 295 296 297 298 299 300 301 302 303 304 305
	} else {
		p = kmem_cache_alloc(engine->i915->priorities, GFP_ATOMIC);
		/* Convert an allocation failure to a priority bump */
		if (unlikely(!p)) {
			prio = I915_PRIORITY_NORMAL; /* recurses just once */

			/* To maintain ordering with all rendering, after an
			 * allocation failure we have to disable all scheduling.
			 * Requests will then be executed in fifo, and schedule
			 * will ensure that dependencies are emitted in fifo.
			 * There will be still some reordering with existing
			 * requests, so if userspace lied about their
			 * dependencies that reordering may be visible.
			 */
306
			execlists->no_priolist = true;
307 308 309 310 311
			goto find_priolist;
		}
	}

	p->priority = prio;
312
	INIT_LIST_HEAD(&p->requests);
313
	rb_link_node(&p->node, rb, parent);
314
	rb_insert_color_cached(&p->node, &execlists->queue, first);
315

316
	return p;
317 318
}

319
static void unwind_wa_tail(struct i915_request *rq)
320 321 322 323 324
{
	rq->tail = intel_ring_wrap(rq->ring, rq->wa_tail - WA_TAIL_BYTES);
	assert_ring_tail_valid(rq->ring, rq->tail);
}

325
static void __unwind_incomplete_requests(struct intel_engine_cs *engine)
326
{
327
	struct i915_request *rq, *rn;
328 329
	struct i915_priolist *uninitialized_var(p);
	int last_prio = I915_PRIORITY_INVALID;
330

331
	lockdep_assert_held(&engine->timeline.lock);
332 333

	list_for_each_entry_safe_reverse(rq, rn,
334
					 &engine->timeline.requests,
335
					 link) {
336
		if (i915_request_completed(rq))
337 338
			return;

339
		__i915_request_unsubmit(rq);
340 341
		unwind_wa_tail(rq);

342 343 344
		GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID);
		if (rq_prio(rq) != last_prio) {
			last_prio = rq_prio(rq);
345
			p = lookup_priolist(engine, last_prio);
346 347
		}

348
		GEM_BUG_ON(p->priority != rq_prio(rq));
349
		list_add(&rq->sched.link, &p->requests);
350 351 352
	}
}

353
void
354 355 356 357
execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
{
	struct intel_engine_cs *engine =
		container_of(execlists, typeof(*engine), execlists);
358 359 360
	unsigned long flags;

	spin_lock_irqsave(&engine->timeline.lock, flags);
361 362

	__unwind_incomplete_requests(engine);
363 364

	spin_unlock_irqrestore(&engine->timeline.lock, flags);
365 366
}

367
static inline void
368
execlists_context_status_change(struct i915_request *rq, unsigned long status)
369
{
370 371 372 373 374 375
	/*
	 * Only used when GVT-g is enabled now. When GVT-g is disabled,
	 * The compiler should eliminate this function as dead-code.
	 */
	if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
		return;
376

377 378
	atomic_notifier_call_chain(&rq->engine->context_status_notifier,
				   status, rq);
379 380
}

381 382 383 384 385 386 387 388 389 390 391 392 393
inline void
execlists_user_begin(struct intel_engine_execlists *execlists,
		     const struct execlist_port *port)
{
	execlists_set_active_once(execlists, EXECLISTS_ACTIVE_USER);
}

inline void
execlists_user_end(struct intel_engine_execlists *execlists)
{
	execlists_clear_active(execlists, EXECLISTS_ACTIVE_USER);
}

394
static inline void
395
execlists_context_schedule_in(struct i915_request *rq)
396 397
{
	execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN);
398
	intel_engine_context_in(rq->engine);
399 400 401
}

static inline void
402
execlists_context_schedule_out(struct i915_request *rq, unsigned long status)
403
{
404
	intel_engine_context_out(rq->engine);
405 406
	execlists_context_status_change(rq, status);
	trace_i915_request_out(rq);
407 408
}

409 410 411 412 413 414 415 416 417
static void
execlists_update_context_pdps(struct i915_hw_ppgtt *ppgtt, u32 *reg_state)
{
	ASSIGN_CTX_PDP(ppgtt, reg_state, 3);
	ASSIGN_CTX_PDP(ppgtt, reg_state, 2);
	ASSIGN_CTX_PDP(ppgtt, reg_state, 1);
	ASSIGN_CTX_PDP(ppgtt, reg_state, 0);
}

418
static u64 execlists_update_context(struct i915_request *rq)
419
{
420
	struct intel_context *ce = rq->hw_context;
421
	struct i915_hw_ppgtt *ppgtt =
C
Chris Wilson 已提交
422
		rq->gem_context->ppgtt ?: rq->i915->mm.aliasing_ppgtt;
423
	u32 *reg_state = ce->lrc_reg_state;
424

425
	reg_state[CTX_RING_TAIL+1] = intel_ring_set_tail(rq->ring, rq->tail);
426

427 428 429 430 431
	/* True 32b PPGTT with dynamic page allocation: update PDP
	 * registers and point the unallocated PDPs to scratch page.
	 * PML4 is allocated during ppgtt init, so this is not needed
	 * in 48-bit mode.
	 */
432
	if (ppgtt && !i915_vm_is_48bit(&ppgtt->vm))
433
		execlists_update_context_pdps(ppgtt, reg_state);
434 435

	return ce->lrc_desc;
436 437
}

438
static inline void write_desc(struct intel_engine_execlists *execlists, u64 desc, u32 port)
C
Chris Wilson 已提交
439
{
440 441 442 443 444 445 446
	if (execlists->ctrl_reg) {
		writel(lower_32_bits(desc), execlists->submit_reg + port * 2);
		writel(upper_32_bits(desc), execlists->submit_reg + port * 2 + 1);
	} else {
		writel(upper_32_bits(desc), execlists->submit_reg);
		writel(lower_32_bits(desc), execlists->submit_reg);
	}
C
Chris Wilson 已提交
447 448
}

449
static void execlists_submit_ports(struct intel_engine_cs *engine)
450
{
451 452
	struct intel_engine_execlists *execlists = &engine->execlists;
	struct execlist_port *port = execlists->port;
453
	unsigned int n;
454

455 456 457 458 459 460 461
	/*
	 * ELSQ note: the submit queue is not cleared after being submitted
	 * to the HW so we need to make sure we always clean it up. This is
	 * currently ensured by the fact that we always write the same number
	 * of elsq entries, keep this in mind before changing the loop below.
	 */
	for (n = execlists_num_ports(execlists); n--; ) {
462
		struct i915_request *rq;
463 464 465 466 467 468 469
		unsigned int count;
		u64 desc;

		rq = port_unpack(&port[n], &count);
		if (rq) {
			GEM_BUG_ON(count > !n);
			if (!count++)
470
				execlists_context_schedule_in(rq);
471 472 473
			port_set(&port[n], port_pack(rq, count));
			desc = execlists_update_context(rq);
			GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc));
474

475
			GEM_TRACE("%s in[%d]:  ctx=%d.%d, global=%d (fence %llx:%d) (current %d), prio=%d\n",
476
				  engine->name, n,
477
				  port[n].context_id, count,
478
				  rq->global_seqno,
479
				  rq->fence.context, rq->fence.seqno,
480
				  intel_engine_get_seqno(engine),
481
				  rq_prio(rq));
482 483 484 485
		} else {
			GEM_BUG_ON(!n);
			desc = 0;
		}
486

487
		write_desc(execlists, desc, n);
488
	}
489 490 491 492 493 494

	/* we need to manually load the submit queue */
	if (execlists->ctrl_reg)
		writel(EL_CTRL_LOAD, execlists->ctrl_reg);

	execlists_clear_active(execlists, EXECLISTS_ACTIVE_HWACK);
495 496
}

497
static bool ctx_single_port_submission(const struct intel_context *ce)
498
{
499
	return (IS_ENABLED(CONFIG_DRM_I915_GVT) &&
500
		i915_gem_context_force_single_submission(ce->gem_context));
501
}
502

503 504
static bool can_merge_ctx(const struct intel_context *prev,
			  const struct intel_context *next)
505 506 507
{
	if (prev != next)
		return false;
508

509 510
	if (ctx_single_port_submission(prev))
		return false;
511

512
	return true;
513 514
}

515
static void port_assign(struct execlist_port *port, struct i915_request *rq)
516 517 518 519
{
	GEM_BUG_ON(rq == port_request(port));

	if (port_isset(port))
520
		i915_request_put(port_request(port));
521

522
	port_set(port, port_pack(i915_request_get(rq), port_count(port)));
523 524
}

C
Chris Wilson 已提交
525 526
static void inject_preempt_context(struct intel_engine_cs *engine)
{
527
	struct intel_engine_execlists *execlists = &engine->execlists;
C
Chris Wilson 已提交
528
	struct intel_context *ce =
529
		to_intel_context(engine->i915->preempt_context, engine);
C
Chris Wilson 已提交
530 531
	unsigned int n;

532
	GEM_BUG_ON(execlists->preempt_complete_status !=
533
		   upper_32_bits(ce->lrc_desc));
534 535 536 537 538 539
	GEM_BUG_ON((ce->lrc_reg_state[CTX_CONTEXT_CONTROL + 1] &
		    _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
				       CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT)) !=
		   _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
				      CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT));

540 541 542 543
	/*
	 * Switch to our empty preempt context so
	 * the state of the GPU is known (idle).
	 */
544
	GEM_TRACE("%s\n", engine->name);
545 546 547 548 549 550 551 552
	for (n = execlists_num_ports(execlists); --n; )
		write_desc(execlists, 0, n);

	write_desc(execlists, ce->lrc_desc, n);

	/* we need to manually load the submit queue */
	if (execlists->ctrl_reg)
		writel(EL_CTRL_LOAD, execlists->ctrl_reg);
C
Chris Wilson 已提交
553

554 555 556 557 558 559 560 561 562
	execlists_clear_active(execlists, EXECLISTS_ACTIVE_HWACK);
	execlists_set_active(execlists, EXECLISTS_ACTIVE_PREEMPT);
}

static void complete_preempt_context(struct intel_engine_execlists *execlists)
{
	GEM_BUG_ON(!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT));

	execlists_cancel_port_requests(execlists);
563 564 565
	__unwind_incomplete_requests(container_of(execlists,
						  struct intel_engine_cs,
						  execlists));
566 567

	execlists_clear_active(execlists, EXECLISTS_ACTIVE_PREEMPT);
C
Chris Wilson 已提交
568 569
}

570
static void execlists_dequeue(struct intel_engine_cs *engine)
571
{
572 573
	struct intel_engine_execlists * const execlists = &engine->execlists;
	struct execlist_port *port = execlists->port;
574 575
	const struct execlist_port * const last_port =
		&execlists->port[execlists->port_mask];
576
	struct i915_request *last = port_request(port);
577
	struct rb_node *rb;
578 579
	bool submit = false;

580 581
	/*
	 * Hardware submission is through 2 ports. Conceptually each port
582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
	 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
	 * static for a context, and unique to each, so we only execute
	 * requests belonging to a single context from each ring. RING_HEAD
	 * is maintained by the CS in the context image, it marks the place
	 * where it got up to last time, and through RING_TAIL we tell the CS
	 * where we want to execute up to this time.
	 *
	 * In this list the requests are in order of execution. Consecutive
	 * requests from the same context are adjacent in the ringbuffer. We
	 * can combine these requests into a single RING_TAIL update:
	 *
	 *              RING_HEAD...req1...req2
	 *                                    ^- RING_TAIL
	 * since to execute req2 the CS must first execute req1.
	 *
	 * Our goal then is to point each port to the end of a consecutive
	 * sequence of requests as being the most optimal (fewest wake ups
	 * and context switches) submission.
600
	 */
601

C
Chris Wilson 已提交
602 603 604 605 606 607 608
	if (last) {
		/*
		 * Don't resubmit or switch until all outstanding
		 * preemptions (lite-restore) are seen. Then we
		 * know the next preemption status we see corresponds
		 * to this ELSP update.
		 */
609 610
		GEM_BUG_ON(!execlists_is_active(execlists,
						EXECLISTS_ACTIVE_USER));
611
		GEM_BUG_ON(!port_count(&port[0]));
C
Chris Wilson 已提交
612

613 614 615 616 617 618 619 620
		/*
		 * If we write to ELSP a second time before the HW has had
		 * a chance to respond to the previous write, we can confuse
		 * the HW and hit "undefined behaviour". After writing to ELSP,
		 * we must then wait until we see a context-switch event from
		 * the HW to indicate that it has had a chance to respond.
		 */
		if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK))
621
			return;
622

623
		if (need_preempt(engine, last, execlists->queue_priority)) {
C
Chris Wilson 已提交
624
			inject_preempt_context(engine);
625
			return;
C
Chris Wilson 已提交
626
		}
627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649

		/*
		 * In theory, we could coalesce more requests onto
		 * the second port (the first port is active, with
		 * no preemptions pending). However, that means we
		 * then have to deal with the possible lite-restore
		 * of the second port (as we submit the ELSP, there
		 * may be a context-switch) but also we may complete
		 * the resubmission before the context-switch. Ergo,
		 * coalescing onto the second port will cause a
		 * preemption event, but we cannot predict whether
		 * that will affect port[0] or port[1].
		 *
		 * If the second port is already active, we can wait
		 * until the next context-switch before contemplating
		 * new requests. The GPU will be busy and we should be
		 * able to resubmit the new ELSP before it idles,
		 * avoiding pipeline bubbles (momentary pauses where
		 * the driver is unable to keep up the supply of new
		 * work). However, we have to double check that the
		 * priorities of the ports haven't been switch.
		 */
		if (port_count(&port[1]))
650
			return;
651 652 653 654 655 656 657 658 659 660

		/*
		 * WaIdleLiteRestore:bdw,skl
		 * Apply the wa NOOPs to prevent
		 * ring:HEAD == rq:TAIL as we resubmit the
		 * request. See gen8_emit_breadcrumb() for
		 * where we prepare the padding after the
		 * end of the request.
		 */
		last->tail = last->wa_tail;
C
Chris Wilson 已提交
661 662
	}

663
	while ((rb = rb_first_cached(&execlists->queue))) {
664
		struct i915_priolist *p = to_priolist(rb);
665
		struct i915_request *rq, *rn;
666

667
		list_for_each_entry_safe(rq, rn, &p->requests, sched.link) {
668 669 670 671 672 673 674 675 676 677
			/*
			 * Can we combine this request with the current port?
			 * It has to be the same context/ringbuffer and not
			 * have any exceptions (e.g. GVT saying never to
			 * combine contexts).
			 *
			 * If we can combine the requests, we can execute both
			 * by updating the RING_TAIL to point to the end of the
			 * second request, and so we never need to tell the
			 * hardware about the first.
678
			 */
679 680
			if (last &&
			    !can_merge_ctx(rq->hw_context, last->hw_context)) {
681 682 683 684 685
				/*
				 * If we are on the second port and cannot
				 * combine this request with the last, then we
				 * are done.
				 */
686
				if (port == last_port) {
687
					__list_del_many(&p->requests,
688
							&rq->sched.link);
689 690 691 692 693 694 695 696 697 698
					goto done;
				}

				/*
				 * If GVT overrides us we only ever submit
				 * port[0], leaving port[1] empty. Note that we
				 * also have to be careful that we don't queue
				 * the same context (even though a different
				 * request) to the second port.
				 */
699 700
				if (ctx_single_port_submission(last->hw_context) ||
				    ctx_single_port_submission(rq->hw_context)) {
701
					__list_del_many(&p->requests,
702
							&rq->sched.link);
703 704 705
					goto done;
				}

706
				GEM_BUG_ON(last->hw_context == rq->hw_context);
707 708 709 710

				if (submit)
					port_assign(port, last);
				port++;
711 712

				GEM_BUG_ON(port_isset(port));
713
			}
714

715
			INIT_LIST_HEAD(&rq->sched.link);
716 717
			__i915_request_submit(rq);
			trace_i915_request_in(rq, port_index(port, execlists));
718 719
			last = rq;
			submit = true;
720
		}
721

722
		rb_erase_cached(&p->node, &execlists->queue);
723 724
		INIT_LIST_HEAD(&p->requests);
		if (p->priority != I915_PRIORITY_NORMAL)
725
			kmem_cache_free(engine->i915->priorities, p);
726
	}
727

728
done:
729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
	/*
	 * Here be a bit of magic! Or sleight-of-hand, whichever you prefer.
	 *
	 * We choose queue_priority such that if we add a request of greater
	 * priority than this, we kick the submission tasklet to decide on
	 * the right order of submitting the requests to hardware. We must
	 * also be prepared to reorder requests as they are in-flight on the
	 * HW. We derive the queue_priority then as the first "hole" in
	 * the HW submission ports and if there are no available slots,
	 * the priority of the lowest executing request, i.e. last.
	 *
	 * When we do receive a higher priority request ready to run from the
	 * user, see queue_request(), the queue_priority is bumped to that
	 * request triggering preemption on the next dequeue (or subsequent
	 * interrupt for secondary ports).
	 */
	execlists->queue_priority =
		port != execlists->port ? rq_prio(last) : INT_MIN;

748
	if (submit) {
749
		port_assign(port, last);
750 751
		execlists_submit_ports(engine);
	}
752 753

	/* We must always keep the beast fed if we have work piled up */
754 755
	GEM_BUG_ON(rb_first_cached(&execlists->queue) &&
		   !port_isset(execlists->port));
756

757 758
	/* Re-evaluate the executing context setup after each preemptive kick */
	if (last)
759
		execlists_user_begin(execlists, execlists->port);
760

761 762 763 764
	/* If the engine is now idle, so should be the flag; and vice versa. */
	GEM_BUG_ON(execlists_is_active(&engine->execlists,
				       EXECLISTS_ACTIVE_USER) ==
		   !port_isset(engine->execlists.port));
765 766
}

767
void
768
execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
769
{
770
	struct execlist_port *port = execlists->port;
771
	unsigned int num_ports = execlists_num_ports(execlists);
772

773
	while (num_ports-- && port_isset(port)) {
774
		struct i915_request *rq = port_request(port);
775

776 777 778 779 780 781 782
		GEM_TRACE("%s:port%u global=%d (fence %llx:%d), (current %d)\n",
			  rq->engine->name,
			  (unsigned int)(port - execlists->port),
			  rq->global_seqno,
			  rq->fence.context, rq->fence.seqno,
			  intel_engine_get_seqno(rq->engine));

783
		GEM_BUG_ON(!execlists->active);
784 785 786 787
		execlists_context_schedule_out(rq,
					       i915_request_completed(rq) ?
					       INTEL_CONTEXT_SCHEDULE_OUT :
					       INTEL_CONTEXT_SCHEDULE_PREEMPTED);
788

789
		i915_request_put(rq);
790

791 792 793
		memset(port, 0, sizeof(*port));
		port++;
	}
794

795
	execlists_clear_active(execlists, EXECLISTS_ACTIVE_USER);
796
	execlists_user_end(execlists);
797 798
}

799 800 801 802 803 804 805 806 807 808 809 810 811 812 813
static void reset_csb_pointers(struct intel_engine_execlists *execlists)
{
	/*
	 * After a reset, the HW starts writing into CSB entry [0]. We
	 * therefore have to set our HEAD pointer back one entry so that
	 * the *first* entry we check is entry 0. To complicate this further,
	 * as we don't wait for the first interrupt after reset, we have to
	 * fake the HW write to point back to the last entry so that our
	 * inline comparison of our cached head position against the last HW
	 * write works even before the first interrupt.
	 */
	execlists->csb_head = execlists->csb_write_reset;
	WRITE_ONCE(*execlists->csb_write, execlists->csb_write_reset);
}

814 815 816 817 818
static void nop_submission_tasklet(unsigned long data)
{
	/* The driver is wedged; don't process any more events. */
}

819 820
static void execlists_cancel_requests(struct intel_engine_cs *engine)
{
821
	struct intel_engine_execlists * const execlists = &engine->execlists;
822
	struct i915_request *rq, *rn;
823 824 825
	struct rb_node *rb;
	unsigned long flags;

826 827
	GEM_TRACE("%s current %d\n",
		  engine->name, intel_engine_get_seqno(engine));
828

829 830 831 832 833 834 835 836 837 838 839 840 841 842
	/*
	 * Before we call engine->cancel_requests(), we should have exclusive
	 * access to the submission state. This is arranged for us by the
	 * caller disabling the interrupt generation, the tasklet and other
	 * threads that may then access the same state, giving us a free hand
	 * to reset state. However, we still need to let lockdep be aware that
	 * we know this state may be accessed in hardirq context, so we
	 * disable the irq around this manipulation and we want to keep
	 * the spinlock focused on its duties and not accidentally conflate
	 * coverage to the submission's irq state. (Similarly, although we
	 * shouldn't need to disable irq around the manipulation of the
	 * submission's irq state, we also wish to remind ourselves that
	 * it is irq state.)
	 */
843
	spin_lock_irqsave(&engine->timeline.lock, flags);
844 845

	/* Cancel the requests on the HW and clear the ELSP tracker. */
846
	execlists_cancel_port_requests(execlists);
847 848

	/* Mark all executing requests as skipped. */
849
	list_for_each_entry(rq, &engine->timeline.requests, link) {
850
		GEM_BUG_ON(!rq->global_seqno);
851
		if (!i915_request_completed(rq))
852 853 854 855
			dma_fence_set_error(&rq->fence, -EIO);
	}

	/* Flush the queued requests to the timeline list (for retiring). */
856
	while ((rb = rb_first_cached(&execlists->queue))) {
857
		struct i915_priolist *p = to_priolist(rb);
858

859 860
		list_for_each_entry_safe(rq, rn, &p->requests, sched.link) {
			INIT_LIST_HEAD(&rq->sched.link);
861 862

			dma_fence_set_error(&rq->fence, -EIO);
863
			__i915_request_submit(rq);
864 865
		}

866
		rb_erase_cached(&p->node, &execlists->queue);
867 868 869 870 871 872 873
		INIT_LIST_HEAD(&p->requests);
		if (p->priority != I915_PRIORITY_NORMAL)
			kmem_cache_free(engine->i915->priorities, p);
	}

	/* Remaining _unready_ requests will be nop'ed when submitted */

874
	execlists->queue_priority = INT_MIN;
875
	execlists->queue = RB_ROOT_CACHED;
876
	GEM_BUG_ON(port_isset(execlists->port));
877

878 879 880
	GEM_BUG_ON(__tasklet_is_enabled(&execlists->tasklet));
	execlists->tasklet.func = nop_submission_tasklet;

881
	spin_unlock_irqrestore(&engine->timeline.lock, flags);
882 883
}

884 885 886 887 888 889
static inline bool
reset_in_progress(const struct intel_engine_execlists *execlists)
{
	return unlikely(!__tasklet_is_enabled(&execlists->tasklet));
}

890
static void process_csb(struct intel_engine_cs *engine)
891
{
892
	struct intel_engine_execlists * const execlists = &engine->execlists;
893
	struct execlist_port *port = execlists->port;
894 895
	const u32 * const buf = execlists->csb_status;
	u8 head, tail;
896

897 898 899 900 901 902 903 904 905 906 907 908 909 910 911
	/*
	 * Note that csb_write, csb_status may be either in HWSP or mmio.
	 * When reading from the csb_write mmio register, we have to be
	 * careful to only use the GEN8_CSB_WRITE_PTR portion, which is
	 * the low 4bits. As it happens we know the next 4bits are always
	 * zero and so we can simply masked off the low u8 of the register
	 * and treat it identically to reading from the HWSP (without having
	 * to use explicit shifting and masking, and probably bifurcating
	 * the code to handle the legacy mmio read).
	 */
	head = execlists->csb_head;
	tail = READ_ONCE(*execlists->csb_write);
	GEM_TRACE("%s cs-irq head=%d, tail=%d\n", engine->name, head, tail);
	if (unlikely(head == tail))
		return;
912

913 914 915 916 917 918 919 920 921
	/*
	 * Hopefully paired with a wmb() in HW!
	 *
	 * We must complete the read of the write pointer before any reads
	 * from the CSB, so that we do not see stale values. Without an rmb
	 * (lfence) the HW may speculatively perform the CSB[] reads *before*
	 * we perform the READ_ONCE(*csb_write).
	 */
	rmb();
922

923
	do {
924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950
		struct i915_request *rq;
		unsigned int status;
		unsigned int count;

		if (++head == GEN8_CSB_ENTRIES)
			head = 0;

		/*
		 * We are flying near dragons again.
		 *
		 * We hold a reference to the request in execlist_port[]
		 * but no more than that. We are operating in softirq
		 * context and so cannot hold any mutex or sleep. That
		 * prevents us stopping the requests we are processing
		 * in port[] from being retired simultaneously (the
		 * breadcrumb will be complete before we see the
		 * context-switch). As we only hold the reference to the
		 * request, any pointer chasing underneath the request
		 * is subject to a potential use-after-free. Thus we
		 * store all of the bookkeeping within port[] as
		 * required, and avoid using unguarded pointers beneath
		 * request itself. The same applies to the atomic
		 * status notifier.
		 */

		GEM_TRACE("%s csb[%d]: status=0x%08x:0x%08x, active=0x%x\n",
			  engine->name, head,
951
			  buf[2 * head + 0], buf[2 * head + 1],
952 953
			  execlists->active);

954
		status = buf[2 * head];
955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973
		if (status & (GEN8_CTX_STATUS_IDLE_ACTIVE |
			      GEN8_CTX_STATUS_PREEMPTED))
			execlists_set_active(execlists,
					     EXECLISTS_ACTIVE_HWACK);
		if (status & GEN8_CTX_STATUS_ACTIVE_IDLE)
			execlists_clear_active(execlists,
					       EXECLISTS_ACTIVE_HWACK);

		if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
			continue;

		/* We should never get a COMPLETED | IDLE_ACTIVE! */
		GEM_BUG_ON(status & GEN8_CTX_STATUS_IDLE_ACTIVE);

		if (status & GEN8_CTX_STATUS_COMPLETE &&
		    buf[2*head + 1] == execlists->preempt_complete_status) {
			GEM_TRACE("%s preempt-idle\n", engine->name);
			complete_preempt_context(execlists);
			continue;
974
		}
975

976 977 978 979
		if (status & GEN8_CTX_STATUS_PREEMPTED &&
		    execlists_is_active(execlists,
					EXECLISTS_ACTIVE_PREEMPT))
			continue;
980

981 982
		GEM_BUG_ON(!execlists_is_active(execlists,
						EXECLISTS_ACTIVE_USER));
983

984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
		rq = port_unpack(port, &count);
		GEM_TRACE("%s out[0]: ctx=%d.%d, global=%d (fence %llx:%d) (current %d), prio=%d\n",
			  engine->name,
			  port->context_id, count,
			  rq ? rq->global_seqno : 0,
			  rq ? rq->fence.context : 0,
			  rq ? rq->fence.seqno : 0,
			  intel_engine_get_seqno(engine),
			  rq ? rq_prio(rq) : 0);

		/* Check the context/desc id for this event matches */
		GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);

		GEM_BUG_ON(count == 0);
		if (--count == 0) {
999
			/*
1000 1001 1002 1003 1004 1005
			 * On the final event corresponding to the
			 * submission of this context, we expect either
			 * an element-switch event or a completion
			 * event (and on completion, the active-idle
			 * marker). No more preemptions, lite-restore
			 * or otherwise.
1006
			 */
1007 1008 1009 1010 1011
			GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
			GEM_BUG_ON(port_isset(&port[1]) &&
				   !(status & GEN8_CTX_STATUS_ELEMENT_SWITCH));
			GEM_BUG_ON(!port_isset(&port[1]) &&
				   !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
1012

1013 1014 1015 1016 1017 1018 1019
			/*
			 * We rely on the hardware being strongly
			 * ordered, that the breadcrumb write is
			 * coherent (visible from the CPU) before the
			 * user interrupt and CSB is processed.
			 */
			GEM_BUG_ON(!i915_request_completed(rq));
C
Chris Wilson 已提交
1020

1021 1022 1023
			execlists_context_schedule_out(rq,
						       INTEL_CONTEXT_SCHEDULE_OUT);
			i915_request_put(rq);
1024

1025 1026
			GEM_TRACE("%s completed ctx=%d\n",
				  engine->name, port->context_id);
1027

1028 1029 1030 1031 1032 1033 1034
			port = execlists_port_complete(execlists, port);
			if (port_isset(port))
				execlists_user_begin(execlists, port);
			else
				execlists_user_end(execlists);
		} else {
			port_set(port, port_pack(rq, count));
1035
		}
1036
	} while (head != tail);
1037

1038
	execlists->csb_head = head;
1039
}
1040

1041
static void __execlists_submission_tasklet(struct intel_engine_cs *const engine)
1042
{
1043
	lockdep_assert_held(&engine->timeline.lock);
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054

	/*
	 * We can skip acquiring intel_runtime_pm_get() here as it was taken
	 * on our behalf by the request (see i915_gem_mark_busy()) and it will
	 * not be relinquished until the device is idle (see
	 * i915_gem_idle_work_handler()). As a precaution, we make sure
	 * that all ELSP are drained i.e. we have processed the CSB,
	 * before allowing ourselves to idle and calling intel_runtime_pm_put().
	 */
	GEM_BUG_ON(!engine->i915->gt.awake);

C
Chris Wilson 已提交
1055
	process_csb(engine);
1056 1057
	if (!execlists_is_active(&engine->execlists, EXECLISTS_ACTIVE_PREEMPT))
		execlists_dequeue(engine);
1058 1059
}

1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
/*
 * Check the unread Context Status Buffers and manage the submission of new
 * contexts to the ELSP accordingly.
 */
static void execlists_submission_tasklet(unsigned long data)
{
	struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
	unsigned long flags;

	GEM_TRACE("%s awake?=%d, active=%x\n",
		  engine->name,
		  engine->i915->gt.awake,
		  engine->execlists.active);

	spin_lock_irqsave(&engine->timeline.lock, flags);

	if (engine->i915->gt.awake) /* we may be delayed until after we idle! */
		__execlists_submission_tasklet(engine);

	spin_unlock_irqrestore(&engine->timeline.lock, flags);
}

1082
static void queue_request(struct intel_engine_cs *engine,
1083
			  struct i915_sched_node *node,
1084
			  int prio)
1085
{
1086
	list_add_tail(&node->link,
1087
		      &lookup_priolist(engine, prio)->requests);
1088
}
1089

1090
static void __update_queue(struct intel_engine_cs *engine, int prio)
1091 1092
{
	engine->execlists.queue_priority = prio;
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
}

static void __submit_queue_imm(struct intel_engine_cs *engine)
{
	struct intel_engine_execlists * const execlists = &engine->execlists;

	if (reset_in_progress(execlists))
		return; /* defer until we restart the engine following reset */

	if (execlists->tasklet.func == execlists_submission_tasklet)
		__execlists_submission_tasklet(engine);
	else
		tasklet_hi_schedule(&execlists->tasklet);
1106 1107
}

1108 1109
static void submit_queue(struct intel_engine_cs *engine, int prio)
{
1110 1111 1112 1113
	if (prio > engine->execlists.queue_priority) {
		__update_queue(engine, prio);
		__submit_queue_imm(engine);
	}
1114 1115
}

1116
static void execlists_submit_request(struct i915_request *request)
1117
{
1118
	struct intel_engine_cs *engine = request->engine;
1119
	unsigned long flags;
1120

1121
	/* Will be called from irq-context when using foreign fences. */
1122
	spin_lock_irqsave(&engine->timeline.lock, flags);
1123

1124
	queue_request(engine, &request->sched, rq_prio(request));
1125

1126
	GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root));
1127
	GEM_BUG_ON(list_empty(&request->sched.link));
1128

1129 1130
	submit_queue(engine, rq_prio(request));

1131
	spin_unlock_irqrestore(&engine->timeline.lock, flags);
1132 1133
}

1134
static struct i915_request *sched_to_request(struct i915_sched_node *node)
1135
{
1136
	return container_of(node, struct i915_request, sched);
1137 1138
}

1139
static struct intel_engine_cs *
1140
sched_lock_engine(struct i915_sched_node *node, struct intel_engine_cs *locked)
1141
{
1142
	struct intel_engine_cs *engine = sched_to_request(node)->engine;
1143 1144

	GEM_BUG_ON(!locked);
1145 1146

	if (engine != locked) {
1147 1148
		spin_unlock(&locked->timeline.lock);
		spin_lock(&engine->timeline.lock);
1149 1150 1151 1152 1153
	}

	return engine;
}

1154 1155
static void execlists_schedule(struct i915_request *request,
			       const struct i915_sched_attr *attr)
1156
{
1157 1158
	struct i915_priolist *uninitialized_var(pl);
	struct intel_engine_cs *engine, *last;
1159 1160
	struct i915_dependency *dep, *p;
	struct i915_dependency stack;
1161
	const int prio = attr->priority;
1162 1163
	LIST_HEAD(dfs);

1164 1165
	GEM_BUG_ON(prio == I915_PRIORITY_INVALID);

1166
	if (i915_request_completed(request))
1167 1168
		return;

1169
	if (prio <= READ_ONCE(request->sched.attr.priority))
1170 1171
		return;

1172 1173
	/* Need BKL in order to use the temporary link inside i915_dependency */
	lockdep_assert_held(&request->i915->drm.struct_mutex);
1174

1175
	stack.signaler = &request->sched;
1176 1177
	list_add(&stack.dfs_link, &dfs);

1178 1179
	/*
	 * Recursively bump all dependent priorities to match the new request.
1180 1181
	 *
	 * A naive approach would be to use recursion:
1182 1183
	 * static void update_priorities(struct i915_sched_node *node, prio) {
	 *	list_for_each_entry(dep, &node->signalers_list, signal_link)
1184
	 *		update_priorities(dep->signal, prio)
1185
	 *	queue_request(node);
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195
	 * }
	 * but that may have unlimited recursion depth and so runs a very
	 * real risk of overunning the kernel stack. Instead, we build
	 * a flat list of all dependencies starting with the current request.
	 * As we walk the list of dependencies, we add all of its dependencies
	 * to the end of the list (this may include an already visited
	 * request) and continue to walk onwards onto the new dependencies. The
	 * end result is a topological list of requests in reverse order, the
	 * last element in the list is the request we must execute first.
	 */
1196
	list_for_each_entry(dep, &dfs, dfs_link) {
1197
		struct i915_sched_node *node = dep->signaler;
1198

1199 1200
		/*
		 * Within an engine, there can be no cycle, but we may
1201 1202 1203 1204
		 * refer to the same dependency chain multiple times
		 * (redundant dependencies are not eliminated) and across
		 * engines.
		 */
1205
		list_for_each_entry(p, &node->signalers_list, signal_link) {
1206 1207
			GEM_BUG_ON(p == dep); /* no cycles! */

1208
			if (i915_sched_node_signaled(p->signaler))
1209 1210
				continue;

1211 1212
			GEM_BUG_ON(p->signaler->attr.priority < node->attr.priority);
			if (prio > READ_ONCE(p->signaler->attr.priority))
1213
				list_move_tail(&p->dfs_link, &dfs);
1214
		}
1215 1216
	}

1217 1218
	/*
	 * If we didn't need to bump any existing priorities, and we haven't
1219 1220 1221 1222
	 * yet submitted this request (i.e. there is no potential race with
	 * execlists_submit_request()), we can set our own priority and skip
	 * acquiring the engine locks.
	 */
1223
	if (request->sched.attr.priority == I915_PRIORITY_INVALID) {
1224
		GEM_BUG_ON(!list_empty(&request->sched.link));
1225
		request->sched.attr = *attr;
1226 1227 1228 1229 1230
		if (stack.dfs_link.next == stack.dfs_link.prev)
			return;
		__list_del_entry(&stack.dfs_link);
	}

1231
	last = NULL;
1232
	engine = request->engine;
1233
	spin_lock_irq(&engine->timeline.lock);
1234

1235 1236
	/* Fifo and depth-first replacement ensure our deps execute before us */
	list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) {
1237
		struct i915_sched_node *node = dep->signaler;
1238 1239 1240

		INIT_LIST_HEAD(&dep->dfs_link);

1241
		engine = sched_lock_engine(node, engine);
1242

1243
		if (prio <= node->attr.priority)
1244 1245
			continue;

1246
		node->attr.priority = prio;
1247
		if (!list_empty(&node->link)) {
1248 1249 1250 1251 1252 1253
			if (last != engine) {
				pl = lookup_priolist(engine, prio);
				last = engine;
			}
			GEM_BUG_ON(pl->priority != prio);
			list_move_tail(&node->link, &pl->requests);
1254
		}
1255 1256

		if (prio > engine->execlists.queue_priority &&
1257 1258 1259 1260 1261
		    i915_sw_fence_done(&sched_to_request(node)->submit)) {
			/* defer submission until after all of our updates */
			__update_queue(engine, prio);
			tasklet_hi_schedule(&engine->execlists.tasklet);
		}
1262 1263
	}

1264
	spin_unlock_irq(&engine->timeline.lock);
1265 1266
}

1267 1268 1269 1270
static void execlists_context_destroy(struct intel_context *ce)
{
	GEM_BUG_ON(ce->pin_count);

1271 1272 1273
	if (!ce->state)
		return;

1274
	intel_ring_free(ce->ring);
1275 1276 1277

	GEM_BUG_ON(i915_gem_object_is_active(ce->state->obj));
	i915_gem_object_put(ce->state->obj);
1278 1279
}

1280
static void execlists_context_unpin(struct intel_context *ce)
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
{
	intel_ring_unpin(ce->ring);

	ce->state->obj->pin_global--;
	i915_gem_object_unpin_map(ce->state->obj);
	i915_vma_unpin(ce->state);

	i915_gem_context_put(ce->gem_context);
}

1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
static int __context_pin(struct i915_gem_context *ctx, struct i915_vma *vma)
{
	unsigned int flags;
	int err;

	/*
	 * Clear this page out of any CPU caches for coherent swap-in/out.
	 * We only want to do this on the first bind so that we do not stall
	 * on an active context (which by nature is already on the GPU).
	 */
	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
		err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
		if (err)
			return err;
	}

	flags = PIN_GLOBAL | PIN_HIGH;
	if (ctx->ggtt_offset_bias)
		flags |= PIN_OFFSET_BIAS | ctx->ggtt_offset_bias;

	return i915_vma_pin(vma, 0, GEN8_LR_CONTEXT_ALIGN, flags);
}

1314 1315 1316 1317
static struct intel_context *
__execlists_context_pin(struct intel_engine_cs *engine,
			struct i915_gem_context *ctx,
			struct intel_context *ce)
1318
{
1319
	void *vaddr;
1320
	int ret;
1321

1322
	ret = execlists_context_deferred_alloc(ctx, engine, ce);
1323 1324
	if (ret)
		goto err;
1325
	GEM_BUG_ON(!ce->state);
1326

1327
	ret = __context_pin(ctx, ce->state);
1328
	if (ret)
1329
		goto err;
1330

1331
	vaddr = i915_gem_object_pin_map(ce->state->obj, I915_MAP_WB);
1332 1333
	if (IS_ERR(vaddr)) {
		ret = PTR_ERR(vaddr);
1334
		goto unpin_vma;
1335 1336
	}

1337
	ret = intel_ring_pin(ce->ring, ctx->i915, ctx->ggtt_offset_bias);
1338
	if (ret)
1339
		goto unpin_map;
1340

1341
	intel_lr_context_descriptor_update(ctx, engine, ce);
1342

1343 1344
	ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
1345
		i915_ggtt_offset(ce->ring->vma);
1346
	GEM_BUG_ON(!intel_ring_offset_valid(ce->ring, ce->ring->head));
1347
	ce->lrc_reg_state[CTX_RING_HEAD+1] = ce->ring->head;
1348

1349
	ce->state->obj->pin_global++;
1350
	i915_gem_context_get(ctx);
1351
	return ce;
1352

1353
unpin_map:
1354 1355 1356
	i915_gem_object_unpin_map(ce->state->obj);
unpin_vma:
	__i915_vma_unpin(ce->state);
1357
err:
1358
	ce->pin_count = 0;
1359
	return ERR_PTR(ret);
1360 1361
}

1362 1363 1364 1365 1366 1367 1368 1369
static const struct intel_context_ops execlists_context_ops = {
	.unpin = execlists_context_unpin,
	.destroy = execlists_context_destroy,
};

static struct intel_context *
execlists_context_pin(struct intel_engine_cs *engine,
		      struct i915_gem_context *ctx)
1370
{
1371
	struct intel_context *ce = to_intel_context(ctx, engine);
1372

1373
	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
1374

1375 1376 1377
	if (likely(ce->pin_count++))
		return ce;
	GEM_BUG_ON(!ce->pin_count); /* no overflow please! */
1378

1379
	ce->ops = &execlists_context_ops;
1380

1381
	return __execlists_context_pin(engine, ctx, ce);
1382 1383
}

1384
static int execlists_request_alloc(struct i915_request *request)
1385
{
1386
	int ret;
1387

1388
	GEM_BUG_ON(!request->hw_context->pin_count);
1389

1390 1391 1392 1393 1394 1395
	/* Flush enough space to reduce the likelihood of waiting after
	 * we start building the request - in which case we will just
	 * have to repeat work.
	 */
	request->reserved_space += EXECLISTS_REQUEST_SIZE;

1396 1397 1398
	ret = intel_ring_wait_for_space(request->ring, request->reserved_space);
	if (ret)
		return ret;
1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410

	/* Note that after this point, we have committed to using
	 * this request as it is being used to both track the
	 * state of engine initialisation and liveness of the
	 * golden renderstate above. Think twice before you try
	 * to cancel/unwind this request now.
	 */

	request->reserved_space -= EXECLISTS_REQUEST_SIZE;
	return 0;
}

1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426
/*
 * In this WA we need to set GEN8_L3SQCREG4[21:21] and reset it after
 * PIPE_CONTROL instruction. This is required for the flush to happen correctly
 * but there is a slight complication as this is applied in WA batch where the
 * values are only initialized once so we cannot take register value at the
 * beginning and reuse it further; hence we save its value to memory, upload a
 * constant value with bit21 set and then we restore it back with the saved value.
 * To simplify the WA, a constant value is formed by using the default value
 * of this register. This shouldn't be a problem because we are only modifying
 * it for a short period and this batch in non-premptible. We can ofcourse
 * use additional instructions that read the actual value of the register
 * at that time and set our bit of interest but it makes the WA complicated.
 *
 * This WA is also required for Gen9 so extracting as a function avoids
 * code duplication.
 */
1427 1428
static u32 *
gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine, u32 *batch)
1429
{
1430 1431 1432 1433 1434 1435 1436 1437 1438
	*batch++ = MI_STORE_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT;
	*batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
	*batch++ = i915_ggtt_offset(engine->scratch) + 256;
	*batch++ = 0;

	*batch++ = MI_LOAD_REGISTER_IMM(1);
	*batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
	*batch++ = 0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES;

1439 1440 1441 1442
	batch = gen8_emit_pipe_control(batch,
				       PIPE_CONTROL_CS_STALL |
				       PIPE_CONTROL_DC_FLUSH_ENABLE,
				       0);
1443 1444 1445 1446 1447 1448 1449

	*batch++ = MI_LOAD_REGISTER_MEM_GEN8 | MI_SRM_LRM_GLOBAL_GTT;
	*batch++ = i915_mmio_reg_offset(GEN8_L3SQCREG4);
	*batch++ = i915_ggtt_offset(engine->scratch) + 256;
	*batch++ = 0;

	return batch;
1450 1451
}

1452 1453 1454 1455 1456 1457
/*
 * Typically we only have one indirect_ctx and per_ctx batch buffer which are
 * initialized at the beginning and shared across all contexts but this field
 * helps us to have multiple batches at different offsets and select them based
 * on a criteria. At the moment this batch always start at the beginning of the page
 * and at this point we don't have multiple wa_ctx batch buffers.
1458
 *
1459 1460
 * The number of WA applied are not known at the beginning; we use this field
 * to return the no of DWORDS written.
1461
 *
1462 1463 1464 1465
 * It is to be noted that this batch does not contain MI_BATCH_BUFFER_END
 * so it adds NOOPs as padding to make it cacheline aligned.
 * MI_BATCH_BUFFER_END will be added to perctx batch and both of them together
 * makes a complete batch buffer.
1466
 */
1467
static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
1468
{
1469
	/* WaDisableCtxRestoreArbitration:bdw,chv */
1470
	*batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1471

1472
	/* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */
1473 1474
	if (IS_BROADWELL(engine->i915))
		batch = gen8_emit_flush_coherentl3_wa(engine, batch);
1475

1476 1477
	/* WaClearSlmSpaceAtContextSwitch:bdw,chv */
	/* Actual scratch location is at 128 bytes offset */
1478 1479 1480 1481 1482 1483 1484
	batch = gen8_emit_pipe_control(batch,
				       PIPE_CONTROL_FLUSH_L3 |
				       PIPE_CONTROL_GLOBAL_GTT_IVB |
				       PIPE_CONTROL_CS_STALL |
				       PIPE_CONTROL_QW_WRITE,
				       i915_ggtt_offset(engine->scratch) +
				       2 * CACHELINE_BYTES);
1485

C
Chris Wilson 已提交
1486 1487
	*batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;

1488
	/* Pad to end of cacheline */
1489 1490
	while ((unsigned long)batch % CACHELINE_BYTES)
		*batch++ = MI_NOOP;
1491 1492 1493 1494 1495 1496 1497

	/*
	 * MI_BATCH_BUFFER_END is not required in Indirect ctx BB because
	 * execution depends on the length specified in terms of cache lines
	 * in the register CTX_RCS_INDIRECT_CTX
	 */

1498
	return batch;
1499 1500
}

1501 1502 1503 1504 1505 1506
struct lri {
	i915_reg_t reg;
	u32 value;
};

static u32 *emit_lri(u32 *batch, const struct lri *lri, unsigned int count)
1507
{
1508
	GEM_BUG_ON(!count || count > 63);
C
Chris Wilson 已提交
1509

1510 1511 1512 1513 1514 1515
	*batch++ = MI_LOAD_REGISTER_IMM(count);
	do {
		*batch++ = i915_mmio_reg_offset(lri->reg);
		*batch++ = lri->value;
	} while (lri++, --count);
	*batch++ = MI_NOOP;
1516

1517 1518
	return batch;
}
1519

1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
{
	static const struct lri lri[] = {
		/* WaDisableGatherAtSetShaderCommonSlice:skl,bxt,kbl,glk */
		{
			COMMON_SLICE_CHICKEN2,
			__MASKED_FIELD(GEN9_DISABLE_GATHER_AT_SET_SHADER_COMMON_SLICE,
				       0),
		},

		/* BSpec: 11391 */
		{
			FF_SLICE_CHICKEN,
			__MASKED_FIELD(FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX,
				       FF_SLICE_CHICKEN_CL_PROVOKING_VERTEX_FIX),
		},

		/* BSpec: 11299 */
		{
			_3D_CHICKEN3,
			__MASKED_FIELD(_3D_CHICKEN_SF_PROVOKING_VERTEX_FIX,
				       _3D_CHICKEN_SF_PROVOKING_VERTEX_FIX),
		}
	};
1544

1545
	*batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1546

1547 1548
	/* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt,glk */
	batch = gen8_emit_flush_coherentl3_wa(engine, batch);
1549

1550
	batch = emit_lri(batch, lri, ARRAY_SIZE(lri));
1551

1552 1553
	/* WaClearSlmSpaceAtContextSwitch:kbl */
	/* Actual scratch location is at 128 bytes offset */
1554
	if (IS_KBL_REVID(engine->i915, 0, KBL_REVID_A0)) {
1555 1556 1557 1558 1559 1560 1561
		batch = gen8_emit_pipe_control(batch,
					       PIPE_CONTROL_FLUSH_L3 |
					       PIPE_CONTROL_GLOBAL_GTT_IVB |
					       PIPE_CONTROL_CS_STALL |
					       PIPE_CONTROL_QW_WRITE,
					       i915_ggtt_offset(engine->scratch)
					       + 2 * CACHELINE_BYTES);
1562
	}
1563

1564
	/* WaMediaPoolStateCmdInWABB:bxt,glk */
1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578
	if (HAS_POOLED_EU(engine->i915)) {
		/*
		 * EU pool configuration is setup along with golden context
		 * during context initialization. This value depends on
		 * device type (2x6 or 3x6) and needs to be updated based
		 * on which subslice is disabled especially for 2x6
		 * devices, however it is safe to load default
		 * configuration of 3x6 device instead of masking off
		 * corresponding bits because HW ignores bits of a disabled
		 * subslice and drops down to appropriate config. Please
		 * see render_state_setup() in i915_gem_render_state.c for
		 * possible configurations, to avoid duplication they are
		 * not shown here again.
		 */
1579 1580 1581 1582 1583 1584
		*batch++ = GEN9_MEDIA_POOL_STATE;
		*batch++ = GEN9_MEDIA_POOL_ENABLE;
		*batch++ = 0x00777000;
		*batch++ = 0;
		*batch++ = 0;
		*batch++ = 0;
1585 1586
	}

C
Chris Wilson 已提交
1587 1588
	*batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;

1589
	/* Pad to end of cacheline */
1590 1591
	while ((unsigned long)batch % CACHELINE_BYTES)
		*batch++ = MI_NOOP;
1592

1593
	return batch;
1594 1595
}

1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629
static u32 *
gen10_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
{
	int i;

	/*
	 * WaPipeControlBefore3DStateSamplePattern: cnl
	 *
	 * Ensure the engine is idle prior to programming a
	 * 3DSTATE_SAMPLE_PATTERN during a context restore.
	 */
	batch = gen8_emit_pipe_control(batch,
				       PIPE_CONTROL_CS_STALL,
				       0);
	/*
	 * WaPipeControlBefore3DStateSamplePattern says we need 4 dwords for
	 * the PIPE_CONTROL followed by 12 dwords of 0x0, so 16 dwords in
	 * total. However, a PIPE_CONTROL is 6 dwords long, not 4, which is
	 * confusing. Since gen8_emit_pipe_control() already advances the
	 * batch by 6 dwords, we advance the other 10 here, completing a
	 * cacheline. It's not clear if the workaround requires this padding
	 * before other commands, or if it's just the regular padding we would
	 * already have for the workaround bb, so leave it here for now.
	 */
	for (i = 0; i < 10; i++)
		*batch++ = MI_NOOP;

	/* Pad to end of cacheline */
	while ((unsigned long)batch % CACHELINE_BYTES)
		*batch++ = MI_NOOP;

	return batch;
}

1630 1631 1632
#define CTX_WA_BB_OBJ_SIZE (PAGE_SIZE)

static int lrc_setup_wa_ctx(struct intel_engine_cs *engine)
1633
{
1634 1635 1636
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma;
	int err;
1637

1638
	obj = i915_gem_object_create(engine->i915, CTX_WA_BB_OBJ_SIZE);
1639 1640
	if (IS_ERR(obj))
		return PTR_ERR(obj);
1641

1642
	vma = i915_vma_instance(obj, &engine->i915->ggtt.vm, NULL);
1643 1644 1645
	if (IS_ERR(vma)) {
		err = PTR_ERR(vma);
		goto err;
1646 1647
	}

1648 1649 1650 1651 1652
	err = i915_vma_pin(vma, 0, PAGE_SIZE, PIN_GLOBAL | PIN_HIGH);
	if (err)
		goto err;

	engine->wa_ctx.vma = vma;
1653
	return 0;
1654 1655 1656 1657

err:
	i915_gem_object_put(obj);
	return err;
1658 1659
}

1660
static void lrc_destroy_wa_ctx(struct intel_engine_cs *engine)
1661
{
1662
	i915_vma_unpin_and_release(&engine->wa_ctx.vma);
1663 1664
}

1665 1666
typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch);

1667
static int intel_init_workaround_bb(struct intel_engine_cs *engine)
1668
{
1669
	struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;
1670 1671 1672
	struct i915_wa_ctx_bb *wa_bb[2] = { &wa_ctx->indirect_ctx,
					    &wa_ctx->per_ctx };
	wa_bb_func_t wa_bb_fn[2];
1673
	struct page *page;
1674 1675
	void *batch, *batch_ptr;
	unsigned int i;
1676
	int ret;
1677

1678
	if (GEM_WARN_ON(engine->id != RCS))
1679
		return -EINVAL;
1680

1681
	switch (INTEL_GEN(engine->i915)) {
1682 1683
	case 11:
		return 0;
1684
	case 10:
1685 1686 1687
		wa_bb_fn[0] = gen10_init_indirectctx_bb;
		wa_bb_fn[1] = NULL;
		break;
1688 1689
	case 9:
		wa_bb_fn[0] = gen9_init_indirectctx_bb;
1690
		wa_bb_fn[1] = NULL;
1691 1692 1693
		break;
	case 8:
		wa_bb_fn[0] = gen8_init_indirectctx_bb;
1694
		wa_bb_fn[1] = NULL;
1695 1696 1697
		break;
	default:
		MISSING_CASE(INTEL_GEN(engine->i915));
1698
		return 0;
1699
	}
1700

1701
	ret = lrc_setup_wa_ctx(engine);
1702 1703 1704 1705 1706
	if (ret) {
		DRM_DEBUG_DRIVER("Failed to setup context WA page: %d\n", ret);
		return ret;
	}

1707
	page = i915_gem_object_get_dirty_page(wa_ctx->vma->obj, 0);
1708
	batch = batch_ptr = kmap_atomic(page);
1709

1710 1711 1712 1713 1714 1715 1716
	/*
	 * Emit the two workaround batch buffers, recording the offset from the
	 * start of the workaround batch buffer object for each and their
	 * respective sizes.
	 */
	for (i = 0; i < ARRAY_SIZE(wa_bb_fn); i++) {
		wa_bb[i]->offset = batch_ptr - batch;
1717 1718
		if (GEM_WARN_ON(!IS_ALIGNED(wa_bb[i]->offset,
					    CACHELINE_BYTES))) {
1719 1720 1721
			ret = -EINVAL;
			break;
		}
1722 1723
		if (wa_bb_fn[i])
			batch_ptr = wa_bb_fn[i](engine, batch_ptr);
1724
		wa_bb[i]->size = batch_ptr - (batch + wa_bb[i]->offset);
1725 1726
	}

1727 1728
	BUG_ON(batch_ptr - batch > CTX_WA_BB_OBJ_SIZE);

1729 1730
	kunmap_atomic(batch);
	if (ret)
1731
		lrc_destroy_wa_ctx(engine);
1732 1733 1734 1735

	return ret;
}

1736
static void enable_execlists(struct intel_engine_cs *engine)
1737
{
1738
	struct drm_i915_private *dev_priv = engine->i915;
1739 1740

	I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff);
1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756

	/*
	 * Make sure we're not enabling the new 12-deep CSB
	 * FIFO as that requires a slightly updated handling
	 * in the ctx switch irq. Since we're currently only
	 * using only 2 elements of the enhanced execlists the
	 * deeper FIFO it's not needed and it's not worth adding
	 * more statements to the irq handler to support it.
	 */
	if (INTEL_GEN(dev_priv) >= 11)
		I915_WRITE(RING_MODE_GEN7(engine),
			   _MASKED_BIT_DISABLE(GEN11_GFX_DISABLE_LEGACY_MODE));
	else
		I915_WRITE(RING_MODE_GEN7(engine),
			   _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));

1757 1758 1759
	I915_WRITE(RING_MI_MODE(engine->mmio_base),
		   _MASKED_BIT_DISABLE(STOP_RING));

1760 1761 1762 1763 1764
	I915_WRITE(RING_HWS_PGA(engine->mmio_base),
		   engine->status_page.ggtt_offset);
	POSTING_READ(RING_HWS_PGA(engine->mmio_base));
}

1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
static bool unexpected_starting_state(struct intel_engine_cs *engine)
{
	struct drm_i915_private *dev_priv = engine->i915;
	bool unexpected = false;

	if (I915_READ(RING_MI_MODE(engine->mmio_base)) & STOP_RING) {
		DRM_DEBUG_DRIVER("STOP_RING still set in RING_MI_MODE\n");
		unexpected = true;
	}

	return unexpected;
}

1778 1779
static int gen8_init_common_ring(struct intel_engine_cs *engine)
{
1780 1781 1782 1783 1784
	int ret;

	ret = intel_mocs_init_engine(engine);
	if (ret)
		return ret;
1785

1786
	intel_engine_reset_breadcrumbs(engine);
1787

1788 1789 1790 1791 1792 1793
	if (GEM_SHOW_DEBUG() && unexpected_starting_state(engine)) {
		struct drm_printer p = drm_debug_printer(__func__);

		intel_engine_dump(engine, &p, NULL);
	}

1794
	enable_execlists(engine);
1795

1796
	return 0;
1797 1798
}

1799
static int gen8_init_render_ring(struct intel_engine_cs *engine)
1800
{
1801
	struct drm_i915_private *dev_priv = engine->i915;
1802 1803
	int ret;

1804
	ret = gen8_init_common_ring(engine);
1805 1806 1807
	if (ret)
		return ret;

1808
	intel_whitelist_workarounds_apply(engine);
1809

1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
	/* We need to disable the AsyncFlip performance optimisations in order
	 * to use MI_WAIT_FOR_EVENT within the CS. It should already be
	 * programmed to '1' on all products.
	 *
	 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
	 */
	I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));

	I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));

1820
	return 0;
1821 1822
}

1823
static int gen9_init_render_ring(struct intel_engine_cs *engine)
1824 1825 1826
{
	int ret;

1827
	ret = gen8_init_common_ring(engine);
1828 1829 1830
	if (ret)
		return ret;

1831
	intel_whitelist_workarounds_apply(engine);
1832 1833

	return 0;
1834 1835
}

1836 1837 1838 1839
static struct i915_request *
execlists_reset_prepare(struct intel_engine_cs *engine)
{
	struct intel_engine_execlists * const execlists = &engine->execlists;
1840
	struct i915_request *request, *active;
1841
	unsigned long flags;
1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855

	GEM_TRACE("%s\n", engine->name);

	/*
	 * Prevent request submission to the hardware until we have
	 * completed the reset in i915_gem_reset_finish(). If a request
	 * is completed by one engine, it may then queue a request
	 * to a second via its execlists->tasklet *just* as we are
	 * calling engine->init_hw() and also writing the ELSP.
	 * Turning off the execlists->tasklet until the reset is over
	 * prevents the race.
	 */
	__tasklet_disable_sync_once(&execlists->tasklet);

1856 1857
	spin_lock_irqsave(&engine->timeline.lock, flags);

1858 1859 1860 1861 1862 1863 1864
	/*
	 * We want to flush the pending context switches, having disabled
	 * the tasklet above, we can assume exclusive access to the execlists.
	 * For this allows us to catch up with an inflight preemption event,
	 * and avoid blaming an innocent request if the stall was due to the
	 * preemption itself.
	 */
C
Chris Wilson 已提交
1865
	process_csb(engine);
1866 1867 1868 1869 1870 1871 1872 1873 1874

	/*
	 * The last active request can then be no later than the last request
	 * now in ELSP[0]. So search backwards from there, so that if the GPU
	 * has advanced beyond the last CSB update, it will be pardoned.
	 */
	active = NULL;
	request = port_request(execlists->port);
	if (request) {
1875 1876 1877 1878 1879 1880
		/*
		 * Prevent the breadcrumb from advancing before we decide
		 * which request is currently active.
		 */
		intel_engine_stop_cs(engine);

1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891
		list_for_each_entry_from_reverse(request,
						 &engine->timeline.requests,
						 link) {
			if (__i915_request_completed(request,
						     request->global_seqno))
				break;

			active = request;
		}
	}

1892 1893
	spin_unlock_irqrestore(&engine->timeline.lock, flags);

1894
	return active;
1895 1896 1897 1898
}

static void execlists_reset(struct intel_engine_cs *engine,
			    struct i915_request *request)
1899
{
1900
	struct intel_engine_execlists * const execlists = &engine->execlists;
1901
	unsigned long flags;
1902
	u32 *regs;
1903

1904 1905 1906
	GEM_TRACE("%s request global=%x, current=%d\n",
		  engine->name, request ? request->global_seqno : 0,
		  intel_engine_get_seqno(engine));
1907

1908
	spin_lock_irqsave(&engine->timeline.lock, flags);
1909

1910 1911 1912 1913 1914 1915 1916 1917 1918
	/*
	 * Catch up with any missed context-switch interrupts.
	 *
	 * Ideally we would just read the remaining CSB entries now that we
	 * know the gpu is idle. However, the CSB registers are sometimes^W
	 * often trashed across a GPU reset! Instead we have to rely on
	 * guessing the missed context-switch events by looking at what
	 * requests were completed.
	 */
1919
	execlists_cancel_port_requests(execlists);
1920

1921
	/* Push back any incomplete requests for replay after the reset. */
1922
	__unwind_incomplete_requests(engine);
1923

1924
	/* Following the reset, we need to reload the CSB read/write pointers */
1925
	reset_csb_pointers(&engine->execlists);
1926

1927
	spin_unlock_irqrestore(&engine->timeline.lock, flags);
1928

1929 1930
	/*
	 * If the request was innocent, we leave the request in the ELSP
1931 1932 1933 1934 1935 1936 1937 1938 1939
	 * and will try to replay it on restarting. The context image may
	 * have been corrupted by the reset, in which case we may have
	 * to service a new GPU hang, but more likely we can continue on
	 * without impact.
	 *
	 * If the request was guilty, we presume the context is corrupt
	 * and have to at least restore the RING register in the context
	 * image back to the expected values to skip over the guilty request.
	 */
1940
	if (!request || request->fence.error != -EIO)
1941
		return;
1942

1943 1944
	/*
	 * We want a simple context + ring to execute the breadcrumb update.
1945 1946 1947 1948 1949 1950
	 * We cannot rely on the context being intact across the GPU hang,
	 * so clear it and rebuild just what we need for the breadcrumb.
	 * All pending requests for this context will be zapped, and any
	 * future request will be after userspace has had the opportunity
	 * to recreate its own state.
	 */
1951
	regs = request->hw_context->lrc_reg_state;
1952 1953 1954 1955
	if (engine->pinned_default_state) {
		memcpy(regs, /* skip restoring the vanilla PPHWSP */
		       engine->pinned_default_state + LRC_STATE_PN * PAGE_SIZE,
		       engine->context_size - PAGE_SIZE);
1956
	}
C
Chris Wilson 已提交
1957 1958
	execlists_init_reg_state(regs,
				 request->gem_context, engine, request->ring);
1959

1960
	/* Move the RING_HEAD onto the breadcrumb, past the hanging batch */
1961
	regs[CTX_RING_BUFFER_START + 1] = i915_ggtt_offset(request->ring->vma);
1962

1963 1964 1965
	request->ring->head = intel_ring_wrap(request->ring, request->postfix);
	regs[CTX_RING_HEAD + 1] = request->ring->head;

1966 1967
	intel_ring_update_space(request->ring);

1968
	/* Reset WaIdleLiteRestore:bdw,skl as well */
1969
	unwind_wa_tail(request);
1970 1971
}

1972 1973
static void execlists_reset_finish(struct intel_engine_cs *engine)
{
1974 1975 1976
	struct intel_engine_execlists * const execlists = &engine->execlists;

	/* After a GPU reset, we may have requests to replay */
1977
	if (!RB_EMPTY_ROOT(&execlists->queue.rb_root))
1978 1979
		tasklet_schedule(&execlists->tasklet);

1980 1981 1982 1983 1984 1985 1986 1987 1988
	/*
	 * Flush the tasklet while we still have the forcewake to be sure
	 * that it is not allowed to sleep before we restart and reload a
	 * context.
	 *
	 * As before (with execlists_reset_prepare) we rely on the caller
	 * serialising multiple attempts to reset so that we know that we
	 * are the only one manipulating tasklet state.
	 */
1989
	__tasklet_enable_sync_once(&execlists->tasklet);
1990 1991 1992 1993

	GEM_TRACE("%s\n", engine->name);
}

1994
static int intel_logical_ring_emit_pdps(struct i915_request *rq)
1995
{
C
Chris Wilson 已提交
1996
	struct i915_hw_ppgtt *ppgtt = rq->gem_context->ppgtt;
1997
	struct intel_engine_cs *engine = rq->engine;
1998
	const int num_lri_cmds = GEN8_3LVL_PDPES * 2;
1999 2000
	u32 *cs;
	int i;
2001

2002
	cs = intel_ring_begin(rq, num_lri_cmds * 2 + 2);
2003 2004
	if (IS_ERR(cs))
		return PTR_ERR(cs);
2005

2006
	*cs++ = MI_LOAD_REGISTER_IMM(num_lri_cmds);
2007
	for (i = GEN8_3LVL_PDPES - 1; i >= 0; i--) {
2008 2009
		const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);

2010 2011 2012 2013
		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(engine, i));
		*cs++ = upper_32_bits(pd_daddr);
		*cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(engine, i));
		*cs++ = lower_32_bits(pd_daddr);
2014 2015
	}

2016
	*cs++ = MI_NOOP;
2017
	intel_ring_advance(rq, cs);
2018 2019 2020 2021

	return 0;
}

2022
static int gen8_emit_bb_start(struct i915_request *rq,
2023
			      u64 offset, u32 len,
2024
			      const unsigned int flags)
2025
{
2026
	u32 *cs;
2027 2028
	int ret;

2029 2030 2031 2032
	/* Don't rely in hw updating PDPs, specially in lite-restore.
	 * Ideally, we should set Force PD Restore in ctx descriptor,
	 * but we can't. Force Restore would be a second option, but
	 * it is unsafe in case of lite-restore (because the ctx is
2033 2034
	 * not idle). PML4 is allocated during ppgtt init so this is
	 * not needed in 48-bit.*/
C
Chris Wilson 已提交
2035 2036
	if (rq->gem_context->ppgtt &&
	    (intel_engine_flag(rq->engine) & rq->gem_context->ppgtt->pd_dirty_rings) &&
2037
	    !i915_vm_is_48bit(&rq->gem_context->ppgtt->vm) &&
2038 2039
	    !intel_vgpu_active(rq->i915)) {
		ret = intel_logical_ring_emit_pdps(rq);
2040 2041
		if (ret)
			return ret;
2042

C
Chris Wilson 已提交
2043
		rq->gem_context->ppgtt->pd_dirty_rings &= ~intel_engine_flag(rq->engine);
2044 2045
	}

2046
	cs = intel_ring_begin(rq, 6);
2047 2048
	if (IS_ERR(cs))
		return PTR_ERR(cs);
2049

2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066
	/*
	 * WaDisableCtxRestoreArbitration:bdw,chv
	 *
	 * We don't need to perform MI_ARB_ENABLE as often as we do (in
	 * particular all the gen that do not need the w/a at all!), if we
	 * took care to make sure that on every switch into this context
	 * (both ordinary and for preemption) that arbitrartion was enabled
	 * we would be fine. However, there doesn't seem to be a downside to
	 * being paranoid and making sure it is set before each batch and
	 * every context-switch.
	 *
	 * Note that if we fail to enable arbitration before the request
	 * is complete, then we do not see the context-switch interrupt and
	 * the engine hangs (with RING_HEAD == RING_TAIL).
	 *
	 * That satisfies both the GPGPU w/a and our heavy-handed paranoia.
	 */
2067 2068
	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;

2069
	/* FIXME(BDW): Address space and security selectors. */
2070 2071 2072
	*cs++ = MI_BATCH_BUFFER_START_GEN8 |
		(flags & I915_DISPATCH_SECURE ? 0 : BIT(8)) |
		(flags & I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0);
2073 2074
	*cs++ = lower_32_bits(offset);
	*cs++ = upper_32_bits(offset);
2075 2076 2077

	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
	*cs++ = MI_NOOP;
2078
	intel_ring_advance(rq, cs);
2079 2080 2081 2082

	return 0;
}

2083
static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
2084
{
2085
	struct drm_i915_private *dev_priv = engine->i915;
2086 2087 2088
	I915_WRITE_IMR(engine,
		       ~(engine->irq_enable_mask | engine->irq_keep_mask));
	POSTING_READ_FW(RING_IMR(engine->mmio_base));
2089 2090
}

2091
static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
2092
{
2093
	struct drm_i915_private *dev_priv = engine->i915;
2094
	I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
2095 2096
}

2097
static int gen8_emit_flush(struct i915_request *request, u32 mode)
2098
{
2099
	u32 cmd, *cs;
2100

2101 2102 2103
	cs = intel_ring_begin(request, 4);
	if (IS_ERR(cs))
		return PTR_ERR(cs);
2104 2105 2106

	cmd = MI_FLUSH_DW + 1;

2107 2108 2109 2110 2111 2112 2113
	/* We always require a command barrier so that subsequent
	 * commands, such as breadcrumb interrupts, are strictly ordered
	 * wrt the contents of the write cache being flushed to memory
	 * (and thus being coherent from the CPU).
	 */
	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;

2114
	if (mode & EMIT_INVALIDATE) {
2115
		cmd |= MI_INVALIDATE_TLB;
2116
		if (request->engine->id == VCS)
2117
			cmd |= MI_INVALIDATE_BSD;
2118 2119
	}

2120 2121 2122 2123 2124
	*cs++ = cmd;
	*cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
	*cs++ = 0; /* upper addr */
	*cs++ = 0; /* value */
	intel_ring_advance(request, cs);
2125 2126 2127 2128

	return 0;
}

2129
static int gen8_emit_flush_render(struct i915_request *request,
2130
				  u32 mode)
2131
{
2132
	struct intel_engine_cs *engine = request->engine;
2133 2134
	u32 scratch_addr =
		i915_ggtt_offset(engine->scratch) + 2 * CACHELINE_BYTES;
M
Mika Kuoppala 已提交
2135
	bool vf_flush_wa = false, dc_flush_wa = false;
2136
	u32 *cs, flags = 0;
M
Mika Kuoppala 已提交
2137
	int len;
2138 2139 2140

	flags |= PIPE_CONTROL_CS_STALL;

2141
	if (mode & EMIT_FLUSH) {
2142 2143
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
2144
		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
2145
		flags |= PIPE_CONTROL_FLUSH_ENABLE;
2146 2147
	}

2148
	if (mode & EMIT_INVALIDATE) {
2149 2150 2151 2152 2153 2154 2155 2156 2157
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_QW_WRITE;
		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;

2158 2159 2160 2161
		/*
		 * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL
		 * pipe control.
		 */
2162
		if (IS_GEN9(request->i915))
2163
			vf_flush_wa = true;
M
Mika Kuoppala 已提交
2164 2165 2166 2167

		/* WaForGAMHang:kbl */
		if (IS_KBL_REVID(request->i915, 0, KBL_REVID_B0))
			dc_flush_wa = true;
2168
	}
2169

M
Mika Kuoppala 已提交
2170 2171 2172 2173 2174 2175 2176 2177
	len = 6;

	if (vf_flush_wa)
		len += 6;

	if (dc_flush_wa)
		len += 12;

2178 2179 2180
	cs = intel_ring_begin(request, len);
	if (IS_ERR(cs))
		return PTR_ERR(cs);
2181

2182 2183
	if (vf_flush_wa)
		cs = gen8_emit_pipe_control(cs, 0, 0);
2184

2185 2186 2187
	if (dc_flush_wa)
		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_DC_FLUSH_ENABLE,
					    0);
M
Mika Kuoppala 已提交
2188

2189
	cs = gen8_emit_pipe_control(cs, flags, scratch_addr);
M
Mika Kuoppala 已提交
2190

2191 2192
	if (dc_flush_wa)
		cs = gen8_emit_pipe_control(cs, PIPE_CONTROL_CS_STALL, 0);
M
Mika Kuoppala 已提交
2193

2194
	intel_ring_advance(request, cs);
2195 2196 2197 2198

	return 0;
}

2199 2200 2201 2202 2203
/*
 * Reserve space for 2 NOOPs at the end of each request to be
 * used as a workaround for not being allowed to do lite
 * restore with HEAD==TAIL (WaIdleLiteRestore).
 */
2204
static void gen8_emit_wa_tail(struct i915_request *request, u32 *cs)
2205
{
C
Chris Wilson 已提交
2206 2207
	/* Ensure there's always at least one preemption point per-request. */
	*cs++ = MI_ARB_CHECK;
2208 2209
	*cs++ = MI_NOOP;
	request->wa_tail = intel_ring_offset(request, cs);
C
Chris Wilson 已提交
2210
}
2211

2212
static void gen8_emit_breadcrumb(struct i915_request *request, u32 *cs)
C
Chris Wilson 已提交
2213
{
2214 2215
	/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
	BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
2216

2217 2218
	cs = gen8_emit_ggtt_write(cs, request->global_seqno,
				  intel_hws_seqno_address(request->engine));
2219
	*cs++ = MI_USER_INTERRUPT;
2220
	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2221
	request->tail = intel_ring_offset(request, cs);
2222
	assert_ring_tail_valid(request->ring, request->tail);
C
Chris Wilson 已提交
2223

2224
	gen8_emit_wa_tail(request, cs);
2225
}
2226 2227
static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS;

2228
static void gen8_emit_breadcrumb_rcs(struct i915_request *request, u32 *cs)
2229
{
2230 2231 2232
	/* We're using qword write, seqno should be aligned to 8 bytes. */
	BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1);

2233 2234
	cs = gen8_emit_ggtt_write_rcs(cs, request->global_seqno,
				      intel_hws_seqno_address(request->engine));
2235
	*cs++ = MI_USER_INTERRUPT;
2236
	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2237
	request->tail = intel_ring_offset(request, cs);
2238
	assert_ring_tail_valid(request->ring, request->tail);
C
Chris Wilson 已提交
2239

2240
	gen8_emit_wa_tail(request, cs);
2241
}
2242
static const int gen8_emit_breadcrumb_rcs_sz = 8 + WA_TAIL_DWORDS;
2243

2244
static int gen8_init_rcs_context(struct i915_request *rq)
2245 2246 2247
{
	int ret;

2248
	ret = intel_ctx_workarounds_emit(rq);
2249 2250 2251
	if (ret)
		return ret;

2252
	ret = intel_rcs_context_init_mocs(rq);
2253 2254 2255 2256 2257 2258 2259
	/*
	 * Failing to program the MOCS is non-fatal.The system will not
	 * run at peak performance. So generate an error and carry on.
	 */
	if (ret)
		DRM_ERROR("MOCS failed to program: expect performance issues.\n");

2260
	return i915_gem_render_state_emit(rq);
2261 2262
}

2263 2264
/**
 * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer
2265
 * @engine: Engine Command Streamer.
2266
 */
2267
void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
2268
{
2269
	struct drm_i915_private *dev_priv;
2270

2271 2272 2273 2274
	/*
	 * Tasklet cannot be active at this point due intel_mark_active/idle
	 * so this is just for documentation.
	 */
2275 2276 2277
	if (WARN_ON(test_bit(TASKLET_STATE_SCHED,
			     &engine->execlists.tasklet.state)))
		tasklet_kill(&engine->execlists.tasklet);
2278

2279
	dev_priv = engine->i915;
2280

2281 2282
	if (engine->buffer) {
		WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
2283
	}
2284

2285 2286
	if (engine->cleanup)
		engine->cleanup(engine);
2287

2288
	intel_engine_cleanup_common(engine);
2289

2290
	lrc_destroy_wa_ctx(engine);
2291

2292
	engine->i915 = NULL;
2293 2294
	dev_priv->engine[engine->id] = NULL;
	kfree(engine);
2295 2296
}

2297
static void execlists_set_default_submission(struct intel_engine_cs *engine)
2298
{
2299
	engine->submit_request = execlists_submit_request;
2300
	engine->cancel_requests = execlists_cancel_requests;
2301
	engine->schedule = execlists_schedule;
2302
	engine->execlists.tasklet.func = execlists_submission_tasklet;
2303

2304 2305
	engine->reset.prepare = execlists_reset_prepare;

2306 2307
	engine->park = NULL;
	engine->unpark = NULL;
2308 2309

	engine->flags |= I915_ENGINE_SUPPORTS_STATS;
2310 2311
	if (engine->i915->preempt_context)
		engine->flags |= I915_ENGINE_HAS_PREEMPTION;
2312 2313 2314 2315

	engine->i915->caps.scheduler =
		I915_SCHEDULER_CAP_ENABLED |
		I915_SCHEDULER_CAP_PRIORITY;
2316
	if (intel_engine_has_preemption(engine))
2317
		engine->i915->caps.scheduler |= I915_SCHEDULER_CAP_PREEMPTION;
2318 2319
}

2320
static void
2321
logical_ring_default_vfuncs(struct intel_engine_cs *engine)
2322 2323
{
	/* Default vfuncs which can be overriden by each engine. */
2324
	engine->init_hw = gen8_init_common_ring;
2325 2326 2327 2328

	engine->reset.prepare = execlists_reset_prepare;
	engine->reset.reset = execlists_reset;
	engine->reset.finish = execlists_reset_finish;
2329 2330

	engine->context_pin = execlists_context_pin;
2331 2332
	engine->request_alloc = execlists_request_alloc;

2333
	engine->emit_flush = gen8_emit_flush;
2334
	engine->emit_breadcrumb = gen8_emit_breadcrumb;
2335
	engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_sz;
2336 2337

	engine->set_default_submission = execlists_set_default_submission;
2338

2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349
	if (INTEL_GEN(engine->i915) < 11) {
		engine->irq_enable = gen8_logical_ring_enable_irq;
		engine->irq_disable = gen8_logical_ring_disable_irq;
	} else {
		/*
		 * TODO: On Gen11 interrupt masks need to be clear
		 * to allow C6 entry. Keep interrupts enabled at
		 * and take the hit of generating extra interrupts
		 * until a more refined solution exists.
		 */
	}
2350
	engine->emit_bb_start = gen8_emit_bb_start;
2351 2352
}

2353
static inline void
2354
logical_ring_default_irqs(struct intel_engine_cs *engine)
2355
{
2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369
	unsigned int shift = 0;

	if (INTEL_GEN(engine->i915) < 11) {
		const u8 irq_shifts[] = {
			[RCS]  = GEN8_RCS_IRQ_SHIFT,
			[BCS]  = GEN8_BCS_IRQ_SHIFT,
			[VCS]  = GEN8_VCS1_IRQ_SHIFT,
			[VCS2] = GEN8_VCS2_IRQ_SHIFT,
			[VECS] = GEN8_VECS_IRQ_SHIFT,
		};

		shift = irq_shifts[engine->id];
	}

2370 2371
	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift;
	engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift;
2372 2373
}

2374 2375 2376
static void
logical_ring_setup(struct intel_engine_cs *engine)
{
2377 2378
	intel_engine_setup_common(engine);

2379 2380 2381
	/* Intentionally left blank. */
	engine->buffer = NULL;

2382 2383
	tasklet_init(&engine->execlists.tasklet,
		     execlists_submission_tasklet, (unsigned long)engine);
2384 2385 2386 2387 2388

	logical_ring_default_vfuncs(engine);
	logical_ring_default_irqs(engine);
}

2389 2390 2391 2392 2393 2394
static bool csb_force_mmio(struct drm_i915_private *i915)
{
	/* Older GVT emulation depends upon intercepting CSB mmio */
	return intel_vgpu_active(i915) && !intel_vgpu_has_hwsp_emulation(i915);
}

2395
static int logical_ring_init(struct intel_engine_cs *engine)
2396
{
2397 2398
	struct drm_i915_private *i915 = engine->i915;
	struct intel_engine_execlists * const execlists = &engine->execlists;
2399 2400
	int ret;

2401
	ret = intel_engine_init_common(engine);
2402 2403 2404
	if (ret)
		goto error;

2405 2406
	if (HAS_LOGICAL_RING_ELSQ(i915)) {
		execlists->submit_reg = i915->regs +
2407
			i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(engine));
2408
		execlists->ctrl_reg = i915->regs +
2409 2410
			i915_mmio_reg_offset(RING_EXECLIST_CONTROL(engine));
	} else {
2411
		execlists->submit_reg = i915->regs +
2412 2413
			i915_mmio_reg_offset(RING_ELSP(engine));
	}
2414

2415 2416
	execlists->preempt_complete_status = ~0u;
	if (i915->preempt_context) {
2417
		struct intel_context *ce =
2418
			to_intel_context(i915->preempt_context, engine);
2419

2420
		execlists->preempt_complete_status =
2421 2422
			upper_32_bits(ce->lrc_desc);
	}
2423

2424 2425 2426 2427 2428 2429 2430
	execlists->csb_read =
		i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine));
	if (csb_force_mmio(i915)) {
		execlists->csb_status = (u32 __force *)
			(i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0)));

		execlists->csb_write = (u32 __force *)execlists->csb_read;
2431 2432 2433
		execlists->csb_write_reset =
			_MASKED_FIELD(GEN8_CSB_WRITE_PTR_MASK,
				      GEN8_CSB_ENTRIES - 1);
2434 2435 2436 2437 2438 2439
	} else {
		execlists->csb_status =
			&engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];

		execlists->csb_write =
			&engine->status_page.page_addr[intel_hws_csb_write_index(i915)];
2440
		execlists->csb_write_reset = GEN8_CSB_ENTRIES - 1;
2441
	}
2442
	reset_csb_pointers(execlists);
2443

2444 2445 2446 2447 2448 2449 2450
	return 0;

error:
	intel_logical_ring_cleanup(engine);
	return ret;
}

2451
int logical_render_ring_init(struct intel_engine_cs *engine)
2452 2453 2454 2455
{
	struct drm_i915_private *dev_priv = engine->i915;
	int ret;

2456 2457
	logical_ring_setup(engine);

2458 2459 2460 2461 2462 2463 2464 2465 2466 2467
	if (HAS_L3_DPF(dev_priv))
		engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;

	/* Override some for render ring. */
	if (INTEL_GEN(dev_priv) >= 9)
		engine->init_hw = gen9_init_render_ring;
	else
		engine->init_hw = gen8_init_render_ring;
	engine->init_context = gen8_init_rcs_context;
	engine->emit_flush = gen8_emit_flush_render;
2468 2469
	engine->emit_breadcrumb = gen8_emit_breadcrumb_rcs;
	engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_rcs_sz;
2470

2471
	ret = intel_engine_create_scratch(engine, PAGE_SIZE);
2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485
	if (ret)
		return ret;

	ret = intel_init_workaround_bb(engine);
	if (ret) {
		/*
		 * We continue even if we fail to initialize WA batch
		 * because we only expect rare glitches but nothing
		 * critical to prevent us from using GPU
		 */
		DRM_ERROR("WA batch buffer initialization failed: %d\n",
			  ret);
	}

2486
	return logical_ring_init(engine);
2487 2488
}

2489
int logical_xcs_ring_init(struct intel_engine_cs *engine)
2490 2491 2492 2493
{
	logical_ring_setup(engine);

	return logical_ring_init(engine);
2494 2495
}

2496
static u32
2497
make_rpcs(struct drm_i915_private *dev_priv)
2498 2499 2500 2501 2502 2503 2504
{
	u32 rpcs = 0;

	/*
	 * No explicit RPCS request is needed to ensure full
	 * slice/subslice/EU enablement prior to Gen9.
	*/
2505
	if (INTEL_GEN(dev_priv) < 9)
2506 2507 2508 2509 2510 2511 2512 2513
		return 0;

	/*
	 * Starting in Gen9, render power gating can leave
	 * slice/subslice/EU in a partially enabled state. We
	 * must make an explicit request through RPCS for full
	 * enablement.
	*/
2514
	if (INTEL_INFO(dev_priv)->sseu.has_slice_pg) {
2515
		rpcs |= GEN8_RPCS_S_CNT_ENABLE;
2516
		rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.slice_mask) <<
2517 2518 2519 2520
			GEN8_RPCS_S_CNT_SHIFT;
		rpcs |= GEN8_RPCS_ENABLE;
	}

2521
	if (INTEL_INFO(dev_priv)->sseu.has_subslice_pg) {
2522
		rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
2523
		rpcs |= hweight8(INTEL_INFO(dev_priv)->sseu.subslice_mask[0]) <<
2524 2525 2526 2527
			GEN8_RPCS_SS_CNT_SHIFT;
		rpcs |= GEN8_RPCS_ENABLE;
	}

2528 2529
	if (INTEL_INFO(dev_priv)->sseu.has_eu_pg) {
		rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
2530
			GEN8_RPCS_EU_MIN_SHIFT;
2531
		rpcs |= INTEL_INFO(dev_priv)->sseu.eu_per_subslice <<
2532 2533 2534 2535 2536 2537 2538
			GEN8_RPCS_EU_MAX_SHIFT;
		rpcs |= GEN8_RPCS_ENABLE;
	}

	return rpcs;
}

2539
static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
2540 2541 2542
{
	u32 indirect_ctx_offset;

2543
	switch (INTEL_GEN(engine->i915)) {
2544
	default:
2545
		MISSING_CASE(INTEL_GEN(engine->i915));
2546
		/* fall through */
2547 2548 2549 2550
	case 11:
		indirect_ctx_offset =
			GEN11_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
		break;
2551 2552 2553 2554
	case 10:
		indirect_ctx_offset =
			GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
		break;
2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567
	case 9:
		indirect_ctx_offset =
			GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
		break;
	case 8:
		indirect_ctx_offset =
			GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT;
		break;
	}

	return indirect_ctx_offset;
}

2568
static void execlists_init_reg_state(u32 *regs,
2569 2570 2571
				     struct i915_gem_context *ctx,
				     struct intel_engine_cs *engine,
				     struct intel_ring *ring)
2572
{
2573 2574
	struct drm_i915_private *dev_priv = engine->i915;
	struct i915_hw_ppgtt *ppgtt = ctx->ppgtt ?: dev_priv->mm.aliasing_ppgtt;
2575
	u32 base = engine->mmio_base;
2576
	bool rcs = engine->class == RENDER_CLASS;
2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588

	/* A context is actually a big batch buffer with several
	 * MI_LOAD_REGISTER_IMM commands followed by (reg, value) pairs. The
	 * values we are setting here are only for the first context restore:
	 * on a subsequent save, the GPU will recreate this batchbuffer with new
	 * values (including all the missing MI_LOAD_REGISTER_IMM commands that
	 * we are not initializing here).
	 */
	regs[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(rcs ? 14 : 11) |
				 MI_LRI_FORCE_POSTED;

	CTX_REG(regs, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(engine),
2589 2590
		_MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
				    CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT) |
2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605
		_MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
				   (HAS_RESOURCE_STREAMER(dev_priv) ?
				   CTX_CTRL_RS_CTX_ENABLE : 0)));
	CTX_REG(regs, CTX_RING_HEAD, RING_HEAD(base), 0);
	CTX_REG(regs, CTX_RING_TAIL, RING_TAIL(base), 0);
	CTX_REG(regs, CTX_RING_BUFFER_START, RING_START(base), 0);
	CTX_REG(regs, CTX_RING_BUFFER_CONTROL, RING_CTL(base),
		RING_CTL_SIZE(ring->size) | RING_VALID);
	CTX_REG(regs, CTX_BB_HEAD_U, RING_BBADDR_UDW(base), 0);
	CTX_REG(regs, CTX_BB_HEAD_L, RING_BBADDR(base), 0);
	CTX_REG(regs, CTX_BB_STATE, RING_BBSTATE(base), RING_BB_PPGTT);
	CTX_REG(regs, CTX_SECOND_BB_HEAD_U, RING_SBBADDR_UDW(base), 0);
	CTX_REG(regs, CTX_SECOND_BB_HEAD_L, RING_SBBADDR(base), 0);
	CTX_REG(regs, CTX_SECOND_BB_STATE, RING_SBBSTATE(base), 0);
	if (rcs) {
2606 2607
		struct i915_ctx_workarounds *wa_ctx = &engine->wa_ctx;

2608 2609 2610
		CTX_REG(regs, CTX_RCS_INDIRECT_CTX, RING_INDIRECT_CTX(base), 0);
		CTX_REG(regs, CTX_RCS_INDIRECT_CTX_OFFSET,
			RING_INDIRECT_CTX_OFFSET(base), 0);
2611
		if (wa_ctx->indirect_ctx.size) {
2612
			u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
2613

2614
			regs[CTX_RCS_INDIRECT_CTX + 1] =
2615 2616
				(ggtt_offset + wa_ctx->indirect_ctx.offset) |
				(wa_ctx->indirect_ctx.size / CACHELINE_BYTES);
2617

2618
			regs[CTX_RCS_INDIRECT_CTX_OFFSET + 1] =
2619
				intel_lr_indirect_ctx_offset(engine) << 6;
2620 2621 2622 2623 2624
		}

		CTX_REG(regs, CTX_BB_PER_CTX_PTR, RING_BB_PER_CTX_PTR(base), 0);
		if (wa_ctx->per_ctx.size) {
			u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma);
2625

2626
			regs[CTX_BB_PER_CTX_PTR + 1] =
2627
				(ggtt_offset + wa_ctx->per_ctx.offset) | 0x01;
2628
		}
2629
	}
2630 2631 2632 2633

	regs[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9) | MI_LRI_FORCE_POSTED;

	CTX_REG(regs, CTX_CTX_TIMESTAMP, RING_CTX_TIMESTAMP(base), 0);
2634
	/* PDP values well be assigned later if needed */
2635 2636 2637 2638 2639 2640 2641 2642
	CTX_REG(regs, CTX_PDP3_UDW, GEN8_RING_PDP_UDW(engine, 3), 0);
	CTX_REG(regs, CTX_PDP3_LDW, GEN8_RING_PDP_LDW(engine, 3), 0);
	CTX_REG(regs, CTX_PDP2_UDW, GEN8_RING_PDP_UDW(engine, 2), 0);
	CTX_REG(regs, CTX_PDP2_LDW, GEN8_RING_PDP_LDW(engine, 2), 0);
	CTX_REG(regs, CTX_PDP1_UDW, GEN8_RING_PDP_UDW(engine, 1), 0);
	CTX_REG(regs, CTX_PDP1_LDW, GEN8_RING_PDP_LDW(engine, 1), 0);
	CTX_REG(regs, CTX_PDP0_UDW, GEN8_RING_PDP_UDW(engine, 0), 0);
	CTX_REG(regs, CTX_PDP0_LDW, GEN8_RING_PDP_LDW(engine, 0), 0);
2643

2644
	if (ppgtt && i915_vm_is_48bit(&ppgtt->vm)) {
2645 2646 2647 2648
		/* 64b PPGTT (48bit canonical)
		 * PDP0_DESCRIPTOR contains the base address to PML4 and
		 * other PDP Descriptors are ignored.
		 */
2649
		ASSIGN_CTX_PML4(ppgtt, regs);
2650 2651
	}

2652 2653 2654 2655
	if (rcs) {
		regs[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
		CTX_REG(regs, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
			make_rpcs(dev_priv));
2656 2657

		i915_oa_init_reg_state(engine, ctx, regs);
2658
	}
2659 2660 2661 2662 2663 2664 2665 2666 2667
}

static int
populate_lr_context(struct i915_gem_context *ctx,
		    struct drm_i915_gem_object *ctx_obj,
		    struct intel_engine_cs *engine,
		    struct intel_ring *ring)
{
	void *vaddr;
2668
	u32 *regs;
2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682
	int ret;

	ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
	if (ret) {
		DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
		return ret;
	}

	vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB);
	if (IS_ERR(vaddr)) {
		ret = PTR_ERR(vaddr);
		DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
		return ret;
	}
C
Chris Wilson 已提交
2683
	ctx_obj->mm.dirty = true;
2684

2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695
	if (engine->default_state) {
		/*
		 * We only want to copy over the template context state;
		 * skipping over the headers reserved for GuC communication,
		 * leaving those as zero.
		 */
		const unsigned long start = LRC_HEADER_PAGES * PAGE_SIZE;
		void *defaults;

		defaults = i915_gem_object_pin_map(engine->default_state,
						   I915_MAP_WB);
2696 2697 2698 2699
		if (IS_ERR(defaults)) {
			ret = PTR_ERR(defaults);
			goto err_unpin_ctx;
		}
2700 2701 2702 2703 2704

		memcpy(vaddr + start, defaults + start, engine->context_size);
		i915_gem_object_unpin_map(engine->default_state);
	}

2705 2706
	/* The second page of the context object contains some fields which must
	 * be set up prior to the first execution. */
2707 2708 2709 2710 2711
	regs = vaddr + LRC_STATE_PN * PAGE_SIZE;
	execlists_init_reg_state(regs, ctx, engine, ring);
	if (!engine->default_state)
		regs[CTX_CONTEXT_CONTROL + 1] |=
			_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
2712
	if (ctx == ctx->i915->preempt_context && INTEL_GEN(engine->i915) < 11)
2713 2714 2715
		regs[CTX_CONTEXT_CONTROL + 1] |=
			_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
					   CTX_CTRL_ENGINE_CTX_SAVE_INHIBIT);
2716

2717
err_unpin_ctx:
2718
	i915_gem_object_unpin_map(ctx_obj);
2719
	return ret;
2720 2721
}

2722
static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
2723 2724
					    struct intel_engine_cs *engine,
					    struct intel_context *ce)
2725
{
2726
	struct drm_i915_gem_object *ctx_obj;
2727
	struct i915_vma *vma;
2728
	uint32_t context_size;
2729
	struct intel_ring *ring;
2730
	struct i915_timeline *timeline;
2731 2732
	int ret;

2733 2734
	if (ce->state)
		return 0;
2735

2736
	context_size = round_up(engine->context_size, I915_GTT_PAGE_SIZE);
2737

2738 2739 2740 2741 2742
	/*
	 * Before the actual start of the context image, we insert a few pages
	 * for our own use and for sharing with the GuC.
	 */
	context_size += LRC_HEADER_PAGES * PAGE_SIZE;
2743

2744
	ctx_obj = i915_gem_object_create(ctx->i915, context_size);
2745 2746
	if (IS_ERR(ctx_obj))
		return PTR_ERR(ctx_obj);
2747

2748
	vma = i915_vma_instance(ctx_obj, &ctx->i915->ggtt.vm, NULL);
2749 2750 2751 2752 2753
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
		goto error_deref_obj;
	}

2754 2755 2756 2757 2758 2759 2760 2761
	timeline = i915_timeline_create(ctx->i915, ctx->name);
	if (IS_ERR(timeline)) {
		ret = PTR_ERR(timeline);
		goto error_deref_obj;
	}

	ring = intel_engine_create_ring(engine, timeline, ctx->ring_size);
	i915_timeline_put(timeline);
2762 2763
	if (IS_ERR(ring)) {
		ret = PTR_ERR(ring);
2764
		goto error_deref_obj;
2765 2766
	}

2767
	ret = populate_lr_context(ctx, ctx_obj, engine, ring);
2768 2769
	if (ret) {
		DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
2770
		goto error_ring_free;
2771 2772
	}

2773
	ce->ring = ring;
2774
	ce->state = vma;
2775 2776

	return 0;
2777

2778
error_ring_free:
2779
	intel_ring_free(ring);
2780
error_deref_obj:
2781
	i915_gem_object_put(ctx_obj);
2782
	return ret;
2783
}
2784

2785
void intel_lr_context_resume(struct drm_i915_private *dev_priv)
2786
{
2787
	struct intel_engine_cs *engine;
2788
	struct i915_gem_context *ctx;
2789
	enum intel_engine_id id;
2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800

	/* Because we emit WA_TAIL_DWORDS there may be a disparity
	 * between our bookkeeping in ce->ring->head and ce->ring->tail and
	 * that stored in context. As we only write new commands from
	 * ce->ring->tail onwards, everything before that is junk. If the GPU
	 * starts reading from its RING_HEAD from the context, it may try to
	 * execute that junk and die.
	 *
	 * So to avoid that we reset the context images upon resume. For
	 * simplicity, we just zero everything out.
	 */
2801
	list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
2802
		for_each_engine(engine, dev_priv, id) {
2803 2804
			struct intel_context *ce =
				to_intel_context(ctx, engine);
2805
			u32 *reg;
2806

2807 2808
			if (!ce->state)
				continue;
2809

2810 2811 2812 2813
			reg = i915_gem_object_pin_map(ce->state->obj,
						      I915_MAP_WB);
			if (WARN_ON(IS_ERR(reg)))
				continue;
2814

2815 2816 2817
			reg += LRC_STATE_PN * PAGE_SIZE / sizeof(*reg);
			reg[CTX_RING_HEAD+1] = 0;
			reg[CTX_RING_TAIL+1] = 0;
2818

C
Chris Wilson 已提交
2819
			ce->state->obj->mm.dirty = true;
2820
			i915_gem_object_unpin_map(ce->state->obj);
2821

2822
			intel_ring_reset(ce->ring, 0);
2823
		}
2824 2825
	}
}
2826 2827 2828 2829

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/intel_lrc.c"
#endif