intel_ring_submission.c 53.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 * Copyright © 2008-2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *    Zou Nan hai <nanhai.zou@intel.com>
 *    Xiang Hai hao<haihao.xiang@intel.com>
 *
 */

30
#include <linux/log2.h>
31

32
#include <drm/i915_drm.h>
33

34 35
#include "gem/i915_gem_context.h"

36
#include "i915_drv.h"
37
#include "i915_trace.h"
38
#include "intel_context.h"
39
#include "intel_gt.h"
40
#include "intel_gt_irq.h"
41
#include "intel_gt_pm_irq.h"
42
#include "intel_reset.h"
43
#include "intel_ring.h"
44
#include "intel_workarounds.h"
45

46 47 48 49 50
/* Rough estimate of the typical request size, performing a flush,
 * set-context and then emitting the batch.
 */
#define LEGACY_REQUEST_SIZE 200

51
static int
52
gen2_render_ring_flush(struct i915_request *rq, u32 mode)
53
{
54
	unsigned int num_store_dw;
55
	u32 cmd, *cs;
56 57

	cmd = MI_FLUSH;
58
	num_store_dw = 0;
59
	if (mode & EMIT_INVALIDATE)
60
		cmd |= MI_READ_FLUSH;
61 62
	if (mode & EMIT_FLUSH)
		num_store_dw = 4;
63

64
	cs = intel_ring_begin(rq, 2 + 3 * num_store_dw);
65 66
	if (IS_ERR(cs))
		return PTR_ERR(cs);
67

68
	*cs++ = cmd;
69 70
	while (num_store_dw--) {
		*cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
71 72
		*cs++ = intel_gt_scratch_offset(rq->engine->gt,
						INTEL_GT_SCRATCH_FIELD_DEFAULT);
73 74 75 76
		*cs++ = 0;
	}
	*cs++ = MI_FLUSH | MI_NO_WRITE_FLUSH;

77
	intel_ring_advance(rq, cs);
78 79 80 81 82

	return 0;
}

static int
83
gen4_render_ring_flush(struct i915_request *rq, u32 mode)
84
{
85
	u32 cmd, *cs;
86
	int i;
87

88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
	/*
	 * read/write caches:
	 *
	 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
	 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
	 * also flushed at 2d versus 3d pipeline switches.
	 *
	 * read-only caches:
	 *
	 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
	 * MI_READ_FLUSH is set, and is always flushed on 965.
	 *
	 * I915_GEM_DOMAIN_COMMAND may not exist?
	 *
	 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
	 * invalidated when MI_EXE_FLUSH is set.
	 *
	 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
	 * invalidated with every MI_FLUSH.
	 *
	 * TLBs:
	 *
	 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
	 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
	 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
	 * are flushed at any MI_FLUSH.
	 */

116
	cmd = MI_FLUSH;
117
	if (mode & EMIT_INVALIDATE) {
118
		cmd |= MI_EXE_FLUSH;
119
		if (IS_G4X(rq->i915) || IS_GEN(rq->i915, 5))
120 121
			cmd |= MI_INVALIDATE_ISP;
	}
122

123 124 125 126 127
	i = 2;
	if (mode & EMIT_INVALIDATE)
		i += 20;

	cs = intel_ring_begin(rq, i);
128 129
	if (IS_ERR(cs))
		return PTR_ERR(cs);
130

131
	*cs++ = cmd;
132 133 134 135 136 137 138 139 140 141 142 143 144

	/*
	 * A random delay to let the CS invalidate take effect? Without this
	 * delay, the GPU relocation path fails as the CS does not see
	 * the updated contents. Just as important, if we apply the flushes
	 * to the EMIT_FLUSH branch (i.e. immediately after the relocation
	 * write and before the invalidate on the next batch), the relocations
	 * still fail. This implies that is a delay following invalidation
	 * that is required to reset the caches as opposed to a delay to
	 * ensure the memory is written.
	 */
	if (mode & EMIT_INVALIDATE) {
		*cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE;
145 146
		*cs++ = intel_gt_scratch_offset(rq->engine->gt,
						INTEL_GT_SCRATCH_FIELD_DEFAULT) |
147
			PIPE_CONTROL_GLOBAL_GTT;
148 149 150 151 152 153 154
		*cs++ = 0;
		*cs++ = 0;

		for (i = 0; i < 12; i++)
			*cs++ = MI_FLUSH;

		*cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE;
155 156
		*cs++ = intel_gt_scratch_offset(rq->engine->gt,
						INTEL_GT_SCRATCH_FIELD_DEFAULT) |
157
			PIPE_CONTROL_GLOBAL_GTT;
158 159 160 161 162 163
		*cs++ = 0;
		*cs++ = 0;
	}

	*cs++ = cmd;

164
	intel_ring_advance(rq, cs);
165 166

	return 0;
167 168
}

169
/*
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 * implementing two workarounds on gen6.  From section 1.4.7.1
 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 *
 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 * produced by non-pipelined state commands), software needs to first
 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 * 0.
 *
 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 *
 * And the workaround for these two requires this workaround first:
 *
 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 * BEFORE the pipe-control with a post-sync op and no write-cache
 * flushes.
 *
 * And this last workaround is tricky because of the requirements on
 * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 * volume 2 part 1:
 *
 *     "1 of the following must also be set:
 *      - Render Target Cache Flush Enable ([12] of DW1)
 *      - Depth Cache Flush Enable ([0] of DW1)
 *      - Stall at Pixel Scoreboard ([1] of DW1)
 *      - Depth Stall ([13] of DW1)
 *      - Post-Sync Operation ([13] of DW1)
 *      - Notify Enable ([8] of DW1)"
 *
 * The cache flushes require the workaround flush that triggered this
 * one, so we can't use it.  Depth stall would trigger the same.
 * Post-sync nonzero is what triggered this second workaround, so we
 * can't use that one either.  Notify enable is IRQs, which aren't
 * really our business.  That leaves only stall at scoreboard.
 */
static int
207
gen6_emit_post_sync_nonzero_flush(struct i915_request *rq)
208
{
209
	u32 scratch_addr =
210 211
		intel_gt_scratch_offset(rq->engine->gt,
					INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH);
212 213
	u32 *cs;

214
	cs = intel_ring_begin(rq, 6);
215 216 217 218 219 220 221 222 223
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	*cs++ = GFX_OP_PIPE_CONTROL(5);
	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;
	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
	*cs++ = 0; /* low dword */
	*cs++ = 0; /* high dword */
	*cs++ = MI_NOOP;
224
	intel_ring_advance(rq, cs);
225

226
	cs = intel_ring_begin(rq, 6);
227 228 229 230 231 232 233 234 235
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	*cs++ = GFX_OP_PIPE_CONTROL(5);
	*cs++ = PIPE_CONTROL_QW_WRITE;
	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
	*cs++ = 0;
	*cs++ = 0;
	*cs++ = MI_NOOP;
236
	intel_ring_advance(rq, cs);
237 238 239 240 241

	return 0;
}

static int
242
gen6_render_ring_flush(struct i915_request *rq, u32 mode)
243
{
244
	u32 scratch_addr =
245 246
		intel_gt_scratch_offset(rq->engine->gt,
					INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH);
247
	u32 *cs, flags = 0;
248 249
	int ret;

250
	/* Force SNB workarounds for PIPE_CONTROL flushes */
251
	ret = gen6_emit_post_sync_nonzero_flush(rq);
252 253 254
	if (ret)
		return ret;

255 256 257 258
	/* Just flush everything.  Experiments have shown that reducing the
	 * number of bits based on the write domains has little performance
	 * impact.
	 */
259
	if (mode & EMIT_FLUSH) {
260 261 262 263 264 265
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		/*
		 * Ensure that any following seqno writes only happen
		 * when the render cache is indeed flushed.
		 */
266
		flags |= PIPE_CONTROL_CS_STALL;
267
	}
268
	if (mode & EMIT_INVALIDATE) {
269 270 271 272 273 274 275 276 277
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		/*
		 * TLB invalidate requires a post-sync write.
		 */
278
		flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
279
	}
280

281
	cs = intel_ring_begin(rq, 4);
282 283
	if (IS_ERR(cs))
		return PTR_ERR(cs);
284

285 286 287 288
	*cs++ = GFX_OP_PIPE_CONTROL(4);
	*cs++ = flags;
	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
	*cs++ = 0;
289
	intel_ring_advance(rq, cs);
290 291 292 293

	return 0;
}

294
static u32 *gen6_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
295 296 297 298 299 300 301 302 303
{
	/* First we do the gen6_emit_post_sync_nonzero_flush w/a */
	*cs++ = GFX_OP_PIPE_CONTROL(4);
	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;
	*cs++ = 0;
	*cs++ = 0;

	*cs++ = GFX_OP_PIPE_CONTROL(4);
	*cs++ = PIPE_CONTROL_QW_WRITE;
304 305
	*cs++ = intel_gt_scratch_offset(rq->engine->gt,
					INTEL_GT_SCRATCH_FIELD_DEFAULT) |
306
		PIPE_CONTROL_GLOBAL_GTT;
307 308 309 310 311 312 313 314 315
	*cs++ = 0;

	/* Finally we can flush and with it emit the breadcrumb */
	*cs++ = GFX_OP_PIPE_CONTROL(4);
	*cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
		 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
		 PIPE_CONTROL_DC_FLUSH_ENABLE |
		 PIPE_CONTROL_QW_WRITE |
		 PIPE_CONTROL_CS_STALL);
316 317
	*cs++ = i915_request_active_timeline(rq)->hwsp_offset |
		PIPE_CONTROL_GLOBAL_GTT;
318 319
	*cs++ = rq->fence.seqno;

320 321 322 323 324
	*cs++ = MI_USER_INTERRUPT;
	*cs++ = MI_NOOP;

	rq->tail = intel_ring_offset(rq, cs);
	assert_ring_tail_valid(rq->ring, rq->tail);
325 326

	return cs;
327 328
}

329
static int
330
gen7_render_ring_cs_stall_wa(struct i915_request *rq)
331
{
332
	u32 *cs;
333

334
	cs = intel_ring_begin(rq, 4);
335 336
	if (IS_ERR(cs))
		return PTR_ERR(cs);
337

338 339 340 341
	*cs++ = GFX_OP_PIPE_CONTROL(4);
	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;
	*cs++ = 0;
	*cs++ = 0;
342
	intel_ring_advance(rq, cs);
343 344 345 346

	return 0;
}

347
static int
348
gen7_render_ring_flush(struct i915_request *rq, u32 mode)
349
{
350
	u32 scratch_addr =
351 352
		intel_gt_scratch_offset(rq->engine->gt,
					INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH);
353
	u32 *cs, flags = 0;
354

355 356 357 358 359 360 361 362 363 364
	/*
	 * Ensure that any following seqno writes only happen when the render
	 * cache is indeed flushed.
	 *
	 * Workaround: 4th PIPE_CONTROL command (except the ones with only
	 * read-cache invalidate bits set) must have the CS_STALL bit set. We
	 * don't try to be clever and just set it unconditionally.
	 */
	flags |= PIPE_CONTROL_CS_STALL;

365 366 367 368
	/* Just flush everything.  Experiments have shown that reducing the
	 * number of bits based on the write domains has little performance
	 * impact.
	 */
369
	if (mode & EMIT_FLUSH) {
370 371
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
372
		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
373
		flags |= PIPE_CONTROL_FLUSH_ENABLE;
374
	}
375
	if (mode & EMIT_INVALIDATE) {
376 377 378 379 380 381
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
382
		flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;
383 384 385 386
		/*
		 * TLB invalidate requires a post-sync write.
		 */
		flags |= PIPE_CONTROL_QW_WRITE;
387
		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
388

389 390
		flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;

391 392 393
		/* Workaround: we must issue a pipe_control with CS-stall bit
		 * set before a pipe_control command that has the state cache
		 * invalidate bit set. */
394
		gen7_render_ring_cs_stall_wa(rq);
395 396
	}

397
	cs = intel_ring_begin(rq, 4);
398 399
	if (IS_ERR(cs))
		return PTR_ERR(cs);
400

401 402 403 404
	*cs++ = GFX_OP_PIPE_CONTROL(4);
	*cs++ = flags;
	*cs++ = scratch_addr;
	*cs++ = 0;
405
	intel_ring_advance(rq, cs);
406 407 408 409

	return 0;
}

410
static u32 *gen7_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
411 412 413 414 415 416 417 418 419
{
	*cs++ = GFX_OP_PIPE_CONTROL(4);
	*cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
		 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
		 PIPE_CONTROL_DC_FLUSH_ENABLE |
		 PIPE_CONTROL_FLUSH_ENABLE |
		 PIPE_CONTROL_QW_WRITE |
		 PIPE_CONTROL_GLOBAL_GTT_IVB |
		 PIPE_CONTROL_CS_STALL);
420
	*cs++ = i915_request_active_timeline(rq)->hwsp_offset;
421 422
	*cs++ = rq->fence.seqno;

423 424 425 426 427
	*cs++ = MI_USER_INTERRUPT;
	*cs++ = MI_NOOP;

	rq->tail = intel_ring_offset(rq, cs);
	assert_ring_tail_valid(rq->ring, rq->tail);
428 429

	return cs;
430 431
}

432
static u32 *gen6_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
433
{
434 435
	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
	GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
436 437 438 439 440

	*cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
	*cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT;
	*cs++ = rq->fence.seqno;

441 442 443 444
	*cs++ = MI_USER_INTERRUPT;

	rq->tail = intel_ring_offset(rq, cs);
	assert_ring_tail_valid(rq->ring, rq->tail);
445 446

	return cs;
447 448
}

449
#define GEN7_XCS_WA 32
450
static u32 *gen7_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs)
451 452 453
{
	int i;

454 455
	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
	GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
456

457 458
	*cs++ = MI_FLUSH_DW | MI_INVALIDATE_TLB |
		MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
459 460 461
	*cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT;
	*cs++ = rq->fence.seqno;

462 463
	for (i = 0; i < GEN7_XCS_WA; i++) {
		*cs++ = MI_STORE_DWORD_INDEX;
464 465
		*cs++ = I915_GEM_HWS_SEQNO_ADDR;
		*cs++ = rq->fence.seqno;
466 467 468 469 470 471 472
	}

	*cs++ = MI_FLUSH_DW;
	*cs++ = 0;
	*cs++ = 0;

	*cs++ = MI_USER_INTERRUPT;
473
	*cs++ = MI_NOOP;
474 475 476

	rq->tail = intel_ring_offset(rq, cs);
	assert_ring_tail_valid(rq->ring, rq->tail);
477 478

	return cs;
479 480 481
}
#undef GEN7_XCS_WA

482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
static void set_hwstam(struct intel_engine_cs *engine, u32 mask)
{
	/*
	 * Keep the render interrupt unmasked as this papers over
	 * lost interrupts following a reset.
	 */
	if (engine->class == RENDER_CLASS) {
		if (INTEL_GEN(engine->i915) >= 6)
			mask &= ~BIT(0);
		else
			mask &= ~I915_USER_INTERRUPT;
	}

	intel_engine_set_hwsp_writemask(engine, mask);
}

static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys)
499 500 501
{
	u32 addr;

502
	addr = lower_32_bits(phys);
503
	if (INTEL_GEN(engine->i915) >= 4)
504 505
		addr |= (phys >> 28) & 0xf0;

506
	intel_uncore_write(engine->uncore, HWS_PGA, addr);
507 508
}

509
static struct page *status_page(struct intel_engine_cs *engine)
510
{
511
	struct drm_i915_gem_object *obj = engine->status_page.vma->obj;
512

513 514 515 516 517 518 519
	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
	return sg_page(obj->mm.pages->sgl);
}

static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
{
	set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine))));
520 521 522 523
	set_hwstam(engine, ~0u);
}

static void set_hwsp(struct intel_engine_cs *engine, u32 offset)
524
{
525
	i915_reg_t hwsp;
526

527 528
	/*
	 * The ring status page addresses are no longer next to the rest of
529 530
	 * the ring registers as of gen7.
	 */
531
	if (IS_GEN(engine->i915, 7)) {
532
		switch (engine->id) {
533 534 535 536 537 538
		/*
		 * No more rings exist on Gen7. Default case is only to shut up
		 * gcc switch check warning.
		 */
		default:
			GEM_BUG_ON(engine->id);
539 540
			/* fallthrough */
		case RCS0:
541
			hwsp = RENDER_HWS_PGA_GEN7;
542
			break;
543
		case BCS0:
544
			hwsp = BLT_HWS_PGA_GEN7;
545
			break;
546
		case VCS0:
547
			hwsp = BSD_HWS_PGA_GEN7;
548
			break;
549
		case VECS0:
550
			hwsp = VEBOX_HWS_PGA_GEN7;
551 552
			break;
		}
553
	} else if (IS_GEN(engine->i915, 6)) {
554
		hwsp = RING_HWS_PGA_GEN6(engine->mmio_base);
555
	} else {
556
		hwsp = RING_HWS_PGA(engine->mmio_base);
557
	}
558

559 560
	intel_uncore_write(engine->uncore, hwsp, offset);
	intel_uncore_posting_read(engine->uncore, hwsp);
561
}
562

563 564 565 566 567 568 569 570
static void flush_cs_tlb(struct intel_engine_cs *engine)
{
	struct drm_i915_private *dev_priv = engine->i915;

	if (!IS_GEN_RANGE(dev_priv, 6, 7))
		return;

	/* ring should be idle before issuing a sync flush*/
571 572 573 574 575 576 577 578
	WARN_ON((ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);

	ENGINE_WRITE(engine, RING_INSTPM,
		     _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
					INSTPM_SYNC_FLUSH));
	if (intel_wait_for_register(engine->uncore,
				    RING_INSTPM(engine->mmio_base),
				    INSTPM_SYNC_FLUSH, 0,
579 580 581 582
				    1000))
		DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
			  engine->name);
}
583

584 585
static void ring_setup_status_page(struct intel_engine_cs *engine)
{
586
	set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma));
587
	set_hwstam(engine, ~0u);
588

589
	flush_cs_tlb(engine);
590 591
}

592
static bool stop_ring(struct intel_engine_cs *engine)
593
{
594
	struct drm_i915_private *dev_priv = engine->i915;
595

596
	if (INTEL_GEN(dev_priv) > 2) {
597 598 599
		ENGINE_WRITE(engine,
			     RING_MI_MODE, _MASKED_BIT_ENABLE(STOP_RING));
		if (intel_wait_for_register(engine->uncore,
600 601 602 603
					    RING_MI_MODE(engine->mmio_base),
					    MODE_IDLE,
					    MODE_IDLE,
					    1000)) {
604 605
			DRM_ERROR("%s : timed out trying to stop ring\n",
				  engine->name);
606 607 608

			/*
			 * Sometimes we observe that the idle flag is not
609 610 611
			 * set even though the ring is empty. So double
			 * check before giving up.
			 */
612 613
			if (ENGINE_READ(engine, RING_HEAD) !=
			    ENGINE_READ(engine, RING_TAIL))
614
				return false;
615 616
		}
	}
617

618
	ENGINE_WRITE(engine, RING_HEAD, ENGINE_READ(engine, RING_TAIL));
619

620 621
	ENGINE_WRITE(engine, RING_HEAD, 0);
	ENGINE_WRITE(engine, RING_TAIL, 0);
622

623
	/* The ring must be empty before it is disabled */
624
	ENGINE_WRITE(engine, RING_CTL, 0);
625

626
	return (ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) == 0;
627
}
628

629
static int xcs_resume(struct intel_engine_cs *engine)
630
{
631
	struct drm_i915_private *dev_priv = engine->i915;
632
	struct intel_ring *ring = engine->legacy.ring;
633 634
	int ret = 0;

635 636
	ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n",
		     ring->head, ring->tail);
637

638
	intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL);
639

640
	/* WaClearRingBufHeadRegAtInit:ctg,elk */
641
	if (!stop_ring(engine)) {
642
		/* G45 ring initialization often fails to reset head to zero */
643 644 645
		DRM_DEBUG_DRIVER("%s head not reset to zero "
				"ctl %08x head %08x tail %08x start %08x\n",
				engine->name,
646 647 648 649
				ENGINE_READ(engine, RING_CTL),
				ENGINE_READ(engine, RING_HEAD),
				ENGINE_READ(engine, RING_TAIL),
				ENGINE_READ(engine, RING_START));
650

651
		if (!stop_ring(engine)) {
652 653
			DRM_ERROR("failed to set %s head to zero "
				  "ctl %08x head %08x tail %08x start %08x\n",
654
				  engine->name,
655 656 657 658
				  ENGINE_READ(engine, RING_CTL),
				  ENGINE_READ(engine, RING_HEAD),
				  ENGINE_READ(engine, RING_TAIL),
				  ENGINE_READ(engine, RING_START));
659 660
			ret = -EIO;
			goto out;
661
		}
662 663
	}

664
	if (HWS_NEEDS_PHYSICAL(dev_priv))
665
		ring_setup_phys_status_page(engine);
666
	else
667
		ring_setup_status_page(engine);
668

669
	intel_engine_reset_breadcrumbs(engine);
670

671
	/* Enforce ordering by reading HEAD register back */
672
	ENGINE_POSTING_READ(engine, RING_HEAD);
673

674 675
	/*
	 * Initialize the ring. This must happen _after_ we've cleared the ring
676 677
	 * registers with the above sequence (the readback of the HEAD registers
	 * also enforces ordering), otherwise the hw might lose the new ring
678 679
	 * register values.
	 */
680
	ENGINE_WRITE(engine, RING_START, i915_ggtt_offset(ring->vma));
681

682 683 684
	/* Check that the ring offsets point within the ring! */
	GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head));
	GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail));
685
	intel_ring_update_space(ring);
C
Chris Wilson 已提交
686 687

	/* First wake the ring up to an empty/idle ring */
688 689 690
	ENGINE_WRITE(engine, RING_HEAD, ring->head);
	ENGINE_WRITE(engine, RING_TAIL, ring->head);
	ENGINE_POSTING_READ(engine, RING_TAIL);
691

692
	ENGINE_WRITE(engine, RING_CTL, RING_CTL_SIZE(ring->size) | RING_VALID);
693 694

	/* If the head is still not zero, the ring is dead */
695
	if (intel_wait_for_register(engine->uncore,
696
				    RING_CTL(engine->mmio_base),
697 698
				    RING_VALID, RING_VALID,
				    50)) {
699
		DRM_ERROR("%s initialization failed "
700
			  "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
701
			  engine->name,
702 703 704 705 706
			  ENGINE_READ(engine, RING_CTL),
			  ENGINE_READ(engine, RING_CTL) & RING_VALID,
			  ENGINE_READ(engine, RING_HEAD), ring->head,
			  ENGINE_READ(engine, RING_TAIL), ring->tail,
			  ENGINE_READ(engine, RING_START),
707
			  i915_ggtt_offset(ring->vma));
708 709
		ret = -EIO;
		goto out;
710 711
	}

712
	if (INTEL_GEN(dev_priv) > 2)
713 714
		ENGINE_WRITE(engine,
			     RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
715

C
Chris Wilson 已提交
716 717
	/* Now awake, let it get started */
	if (ring->tail != ring->head) {
718 719
		ENGINE_WRITE(engine, RING_TAIL, ring->tail);
		ENGINE_POSTING_READ(engine, RING_TAIL);
C
Chris Wilson 已提交
720 721
	}

722
	/* Papering over lost _interrupts_ immediately following the restart */
723
	intel_engine_queue_breadcrumbs(engine);
724
out:
725
	intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL);
726 727

	return ret;
728 729
}

730
static void reset_prepare(struct intel_engine_cs *engine)
731
{
732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
	struct intel_uncore *uncore = engine->uncore;
	const u32 base = engine->mmio_base;

	/*
	 * We stop engines, otherwise we might get failed reset and a
	 * dead gpu (on elk). Also as modern gpu as kbl can suffer
	 * from system hang if batchbuffer is progressing when
	 * the reset is issued, regardless of READY_TO_RESET ack.
	 * Thus assume it is best to stop engines on all gens
	 * where we have a gpu reset.
	 *
	 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES)
	 *
	 * WaMediaResetMainRingCleanup:ctg,elk (presumably)
	 *
	 * FIXME: Wa for more modern gens needs to be validated
	 */
749
	ENGINE_TRACE(engine, "\n");
750 751

	if (intel_engine_stop_cs(engine))
752
		ENGINE_TRACE(engine, "timed out on STOP_RING\n");
753 754 755 756 757 758 759 760 761 762 763 764 765 766 767

	intel_uncore_write_fw(uncore,
			      RING_HEAD(base),
			      intel_uncore_read_fw(uncore, RING_TAIL(base)));
	intel_uncore_posting_read_fw(uncore, RING_HEAD(base)); /* paranoia */

	intel_uncore_write_fw(uncore, RING_HEAD(base), 0);
	intel_uncore_write_fw(uncore, RING_TAIL(base), 0);
	intel_uncore_posting_read_fw(uncore, RING_TAIL(base));

	/* The ring must be empty before it is disabled */
	intel_uncore_write_fw(uncore, RING_CTL(base), 0);

	/* Check acts as a post */
	if (intel_uncore_read_fw(uncore, RING_HEAD(base)))
768 769
		ENGINE_TRACE(engine, "ring head [%x] not parked\n",
			     intel_uncore_read_fw(uncore, RING_HEAD(base)));
770 771
}

772
static void reset_ring(struct intel_engine_cs *engine, bool stalled)
773
{
774 775
	struct i915_request *pos, *rq;
	unsigned long flags;
776
	u32 head;
777

778
	rq = NULL;
779 780
	spin_lock_irqsave(&engine->active.lock, flags);
	list_for_each_entry(pos, &engine->active.requests, sched.link) {
781
		if (!i915_request_completed(pos)) {
782 783 784
			rq = pos;
			break;
		}
785
	}
786 787

	/*
788
	 * The guilty request will get skipped on a hung engine.
789
	 *
790 791 792 793 794 795 796 797 798 799 800 801 802
	 * Users of client default contexts do not rely on logical
	 * state preserved between batches so it is safe to execute
	 * queued requests following the hang. Non default contexts
	 * rely on preserved state, so skipping a batch loses the
	 * evolution of the state and it needs to be considered corrupted.
	 * Executing more queued batches on top of corrupted state is
	 * risky. But we take the risk by trying to advance through
	 * the queued requests in order to make the client behaviour
	 * more predictable around resets, by not throwing away random
	 * amount of batches it has prepared for execution. Sophisticated
	 * clients can use gem_reset_stats_ioctl and dma fence status
	 * (exported via sync_file info ioctl on explicit fences) to observe
	 * when it loses the context state and should rebuild accordingly.
803
	 *
804 805 806
	 * The context ban, and ultimately the client ban, mechanism are safety
	 * valves if client submission ends up resulting in nothing more than
	 * subsequent hangs.
807
	 */
808

809
	if (rq) {
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824
		/*
		 * Try to restore the logical GPU state to match the
		 * continuation of the request queue. If we skip the
		 * context/PD restore, then the next request may try to execute
		 * assuming that its context is valid and loaded on the GPU and
		 * so may try to access invalid memory, prompting repeated GPU
		 * hangs.
		 *
		 * If the request was guilty, we still restore the logical
		 * state in case the next request requires it (e.g. the
		 * aliasing ppgtt), but skip over the hung batch.
		 *
		 * If the request was innocent, we try to replay the request
		 * with the restored context.
		 */
825
		__i915_request_reset(rq, stalled);
826

827
		GEM_BUG_ON(rq->ring != engine->legacy.ring);
828 829
		head = rq->head;
	} else {
830
		head = engine->legacy.ring->tail;
831
	}
832
	engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head);
833

834
	spin_unlock_irqrestore(&engine->active.lock, flags);
835 836
}

837 838 839 840
static void reset_finish(struct intel_engine_cs *engine)
{
}

841
static int rcs_resume(struct intel_engine_cs *engine)
842
{
843 844
	struct drm_i915_private *i915 = engine->i915;
	struct intel_uncore *uncore = engine->uncore;
845

846 847 848 849 850 851 852 853 854 855
	/*
	 * Disable CONSTANT_BUFFER before it is loaded from the context
	 * image. For as it is loaded, it is executed and the stored
	 * address may no longer be valid, leading to a GPU hang.
	 *
	 * This imposes the requirement that userspace reload their
	 * CONSTANT_BUFFER on every batch, fortunately a requirement
	 * they are already accustomed to from before contexts were
	 * enabled.
	 */
856 857
	if (IS_GEN(i915, 4))
		intel_uncore_write(uncore, ECOSKPD,
858 859
			   _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE));

860
	/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
861 862 863
	if (IS_GEN_RANGE(i915, 4, 6))
		intel_uncore_write(uncore, MI_MODE,
				   _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
864 865 866 867

	/* We need to disable the AsyncFlip performance optimisations in order
	 * to use MI_WAIT_FOR_EVENT within the CS. It should already be
	 * programmed to '1' on all products.
868
	 *
869
	 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
870
	 */
871 872 873
	if (IS_GEN_RANGE(i915, 6, 7))
		intel_uncore_write(uncore, MI_MODE,
				   _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
874

875
	/* Required for the hardware to program scanline values for waiting */
876
	/* WaEnableFlushTlbInvalidationMode:snb */
877 878
	if (IS_GEN(i915, 6))
		intel_uncore_write(uncore, GFX_MODE,
879
			   _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
880

881
	/* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
882 883
	if (IS_GEN(i915, 7))
		intel_uncore_write(uncore, GFX_MODE_GEN7,
884
			   _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
885
			   _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
886

887
	if (IS_GEN(i915, 6)) {
888 889 890 891 892
		/* From the Sandybridge PRM, volume 1 part 3, page 24:
		 * "If this bit is set, STCunit will have LRA as replacement
		 *  policy. [...] This bit must be reset.  LRA replacement
		 *  policy is not supported."
		 */
893
		intel_uncore_write(uncore, CACHE_MODE_0,
894
			   _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
895 896
	}

897 898 899
	if (IS_GEN_RANGE(i915, 6, 7))
		intel_uncore_write(uncore, INSTPM,
				   _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
900

901
	return xcs_resume(engine);
902 903
}

904 905
static void cancel_requests(struct intel_engine_cs *engine)
{
906
	struct i915_request *request;
907 908
	unsigned long flags;

909
	spin_lock_irqsave(&engine->active.lock, flags);
910 911

	/* Mark all submitted requests as skipped. */
912
	list_for_each_entry(request, &engine->active.requests, sched.link) {
913 914
		if (!i915_request_signaled(request))
			dma_fence_set_error(&request->fence, -EIO);
915

916
		i915_request_mark_complete(request);
917
	}
918

919 920
	/* Remaining _unready_ requests will be nop'ed when submitted */

921
	spin_unlock_irqrestore(&engine->active.lock, flags);
922 923
}

924
static void i9xx_submit_request(struct i915_request *request)
925
{
926
	i915_request_submit(request);
927
	wmb(); /* paranoid flush writes out of the WCB before mmio */
928

929 930
	ENGINE_WRITE(request->engine, RING_TAIL,
		     intel_ring_set_tail(request->ring, request->tail));
931 932
}

933
static u32 *i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs)
934
{
935 936
	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
	GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
937

938 939
	*cs++ = MI_FLUSH;

940 941 942 943
	*cs++ = MI_STORE_DWORD_INDEX;
	*cs++ = I915_GEM_HWS_SEQNO_ADDR;
	*cs++ = rq->fence.seqno;

944
	*cs++ = MI_USER_INTERRUPT;
945
	*cs++ = MI_NOOP;
946

947 948
	rq->tail = intel_ring_offset(rq, cs);
	assert_ring_tail_valid(rq->ring, rq->tail);
949 950

	return cs;
951
}
952

953
#define GEN5_WA_STORES 8 /* must be at least 1! */
954
static u32 *gen5_emit_breadcrumb(struct i915_request *rq, u32 *cs)
955
{
956 957
	int i;

958 959
	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
	GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
960

961 962 963 964 965
	*cs++ = MI_FLUSH;

	BUILD_BUG_ON(GEN5_WA_STORES < 1);
	for (i = 0; i < GEN5_WA_STORES; i++) {
		*cs++ = MI_STORE_DWORD_INDEX;
966 967
		*cs++ = I915_GEM_HWS_SEQNO_ADDR;
		*cs++ = rq->fence.seqno;
968 969 970 971 972 973
	}

	*cs++ = MI_USER_INTERRUPT;

	rq->tail = intel_ring_offset(rq, cs);
	assert_ring_tail_valid(rq->ring, rq->tail);
974 975

	return cs;
976
}
977
#undef GEN5_WA_STORES
978

979 980
static void
gen5_irq_enable(struct intel_engine_cs *engine)
981
{
982
	gen5_gt_enable_irq(engine->gt, engine->irq_enable_mask);
983 984 985
}

static void
986
gen5_irq_disable(struct intel_engine_cs *engine)
987
{
988
	gen5_gt_disable_irq(engine->gt, engine->irq_enable_mask);
989 990
}

991 992
static void
i9xx_irq_enable(struct intel_engine_cs *engine)
993
{
994
	engine->i915->irq_mask &= ~engine->irq_enable_mask;
995 996
	intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask);
	intel_uncore_posting_read_fw(engine->uncore, GEN2_IMR);
997 998
}

999
static void
1000
i9xx_irq_disable(struct intel_engine_cs *engine)
1001
{
1002
	engine->i915->irq_mask |= engine->irq_enable_mask;
1003
	intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask);
1004 1005
}

1006 1007
static void
i8xx_irq_enable(struct intel_engine_cs *engine)
C
Chris Wilson 已提交
1008
{
T
Tvrtko Ursulin 已提交
1009
	struct drm_i915_private *i915 = engine->i915;
C
Chris Wilson 已提交
1010

T
Tvrtko Ursulin 已提交
1011 1012 1013
	i915->irq_mask &= ~engine->irq_enable_mask;
	intel_uncore_write16(&i915->uncore, GEN2_IMR, i915->irq_mask);
	ENGINE_POSTING_READ16(engine, RING_IMR);
C
Chris Wilson 已提交
1014 1015 1016
}

static void
1017
i8xx_irq_disable(struct intel_engine_cs *engine)
C
Chris Wilson 已提交
1018
{
1019
	struct drm_i915_private *i915 = engine->i915;
C
Chris Wilson 已提交
1020

1021 1022
	i915->irq_mask |= engine->irq_enable_mask;
	intel_uncore_write16(&i915->uncore, GEN2_IMR, i915->irq_mask);
C
Chris Wilson 已提交
1023 1024
}

1025
static int
1026
bsd_ring_flush(struct i915_request *rq, u32 mode)
1027
{
1028
	u32 *cs;
1029

1030
	cs = intel_ring_begin(rq, 2);
1031 1032
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1033

1034 1035
	*cs++ = MI_FLUSH;
	*cs++ = MI_NOOP;
1036
	intel_ring_advance(rq, cs);
1037
	return 0;
1038 1039
}

1040 1041
static void
gen6_irq_enable(struct intel_engine_cs *engine)
1042
{
1043 1044
	ENGINE_WRITE(engine, RING_IMR,
		     ~(engine->irq_enable_mask | engine->irq_keep_mask));
1045 1046

	/* Flush/delay to ensure the RING_IMR is active before the GT IMR */
1047
	ENGINE_POSTING_READ(engine, RING_IMR);
1048

1049
	gen5_gt_enable_irq(engine->gt, engine->irq_enable_mask);
1050 1051 1052
}

static void
1053
gen6_irq_disable(struct intel_engine_cs *engine)
1054
{
1055
	ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask);
1056
	gen5_gt_disable_irq(engine->gt, engine->irq_enable_mask);
1057 1058
}

1059 1060
static void
hsw_vebox_irq_enable(struct intel_engine_cs *engine)
B
Ben Widawsky 已提交
1061
{
1062
	ENGINE_WRITE(engine, RING_IMR, ~engine->irq_enable_mask);
1063 1064

	/* Flush/delay to ensure the RING_IMR is active before the GT IMR */
1065
	ENGINE_POSTING_READ(engine, RING_IMR);
1066

1067
	gen6_gt_pm_unmask_irq(engine->gt, engine->irq_enable_mask);
B
Ben Widawsky 已提交
1068 1069 1070
}

static void
1071
hsw_vebox_irq_disable(struct intel_engine_cs *engine)
B
Ben Widawsky 已提交
1072
{
1073
	ENGINE_WRITE(engine, RING_IMR, ~0);
1074
	gen6_gt_pm_mask_irq(engine->gt, engine->irq_enable_mask);
B
Ben Widawsky 已提交
1075 1076
}

1077
static int
1078
i965_emit_bb_start(struct i915_request *rq,
1079 1080
		   u64 offset, u32 length,
		   unsigned int dispatch_flags)
1081
{
1082
	u32 *cs;
1083

1084
	cs = intel_ring_begin(rq, 2);
1085 1086
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1087

1088 1089 1090
	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | (dispatch_flags &
		I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965);
	*cs++ = offset;
1091
	intel_ring_advance(rq, cs);
1092

1093 1094 1095
	return 0;
}

1096
/* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1097
#define I830_BATCH_LIMIT SZ_256K
1098 1099
#define I830_TLB_ENTRIES (2)
#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
1100
static int
1101
i830_emit_bb_start(struct i915_request *rq,
1102 1103
		   u64 offset, u32 len,
		   unsigned int dispatch_flags)
1104
{
1105 1106 1107
	u32 *cs, cs_offset =
		intel_gt_scratch_offset(rq->engine->gt,
					INTEL_GT_SCRATCH_FIELD_DEFAULT);
1108

1109
	GEM_BUG_ON(rq->engine->gt->scratch->size < I830_WA_SIZE);
1110

1111
	cs = intel_ring_begin(rq, 6);
1112 1113
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1114

1115
	/* Evict the invalid PTE TLBs */
1116 1117 1118 1119 1120 1121
	*cs++ = COLOR_BLT_CMD | BLT_WRITE_RGBA;
	*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096;
	*cs++ = I830_TLB_ENTRIES << 16 | 4; /* load each page */
	*cs++ = cs_offset;
	*cs++ = 0xdeadbeef;
	*cs++ = MI_NOOP;
1122
	intel_ring_advance(rq, cs);
1123

1124
	if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
1125 1126 1127
		if (len > I830_BATCH_LIMIT)
			return -ENOSPC;

1128
		cs = intel_ring_begin(rq, 6 + 2);
1129 1130
		if (IS_ERR(cs))
			return PTR_ERR(cs);
1131 1132 1133 1134 1135

		/* Blit the batch (which has now all relocs applied) to the
		 * stable batch scratch bo area (so that the CS never
		 * stumbles over its tlb invalidation bug) ...
		 */
1136
		*cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2);
1137 1138 1139 1140 1141 1142 1143 1144
		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096;
		*cs++ = DIV_ROUND_UP(len, 4096) << 16 | 4096;
		*cs++ = cs_offset;
		*cs++ = 4096;
		*cs++ = offset;

		*cs++ = MI_FLUSH;
		*cs++ = MI_NOOP;
1145
		intel_ring_advance(rq, cs);
1146 1147

		/* ... and execute it. */
1148
		offset = cs_offset;
1149
	}
1150

1151
	cs = intel_ring_begin(rq, 2);
1152 1153
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1154

1155 1156 1157
	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
	*cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :
		MI_BATCH_NON_SECURE);
1158
	intel_ring_advance(rq, cs);
1159

1160 1161 1162 1163
	return 0;
}

static int
1164
i915_emit_bb_start(struct i915_request *rq,
1165 1166
		   u64 offset, u32 len,
		   unsigned int dispatch_flags)
1167
{
1168
	u32 *cs;
1169

1170
	cs = intel_ring_begin(rq, 2);
1171 1172
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1173

1174 1175 1176
	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
	*cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :
		MI_BATCH_NON_SECURE);
1177
	intel_ring_advance(rq, cs);
1178 1179 1180 1181

	return 0;
}

1182 1183
static void __ring_context_fini(struct intel_context *ce)
{
1184
	i915_vma_put(ce->state);
1185 1186
}

1187
static void ring_context_destroy(struct kref *ref)
1188
{
1189 1190
	struct intel_context *ce = container_of(ref, typeof(*ce), ref);

1191
	GEM_BUG_ON(intel_context_is_pinned(ce));
1192

1193 1194
	if (ce->state)
		__ring_context_fini(ce);
1195

1196
	intel_context_fini(ce);
1197
	intel_context_free(ce);
1198 1199
}

1200 1201 1202 1203
static struct i915_address_space *vm_alias(struct intel_context *ce)
{
	struct i915_address_space *vm;

1204 1205 1206
	vm = ce->vm;
	if (i915_is_ggtt(vm))
		vm = &i915_vm_to_ggtt(vm)->alias->vm;
1207 1208 1209 1210 1211

	return vm;
}

static int __context_pin_ppgtt(struct intel_context *ce)
1212
{
1213
	struct i915_address_space *vm;
1214 1215
	int err = 0;

1216
	vm = vm_alias(ce);
1217 1218
	if (vm)
		err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)));
1219 1220 1221 1222

	return err;
}

1223
static void __context_unpin_ppgtt(struct intel_context *ce)
1224
{
1225
	struct i915_address_space *vm;
1226

1227
	vm = vm_alias(ce);
1228 1229
	if (vm)
		gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm));
1230 1231
}

1232
static void ring_context_unpin(struct intel_context *ce)
1233
{
1234
	__context_unpin_ppgtt(ce);
1235 1236
}

1237 1238 1239 1240 1241 1242
static struct i915_vma *
alloc_context_vma(struct intel_engine_cs *engine)
{
	struct drm_i915_private *i915 = engine->i915;
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma;
1243
	int err;
1244

1245
	obj = i915_gem_object_create_shmem(i915, engine->context_size);
1246 1247 1248
	if (IS_ERR(obj))
		return ERR_CAST(obj);

1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
	/*
	 * Try to make the context utilize L3 as well as LLC.
	 *
	 * On VLV we don't have L3 controls in the PTEs so we
	 * shouldn't touch the cache level, especially as that
	 * would make the object snooped which might have a
	 * negative performance impact.
	 *
	 * Snooping is required on non-llc platforms in execlist
	 * mode, but since all GGTT accesses use PAT entry 0 we
	 * get snooping anyway regardless of cache_level.
	 *
	 * This is only applicable for Ivy Bridge devices since
	 * later platforms don't have L3 control bits in the PTE.
	 */
	if (IS_IVYBRIDGE(i915))
		i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC);

1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285
	if (engine->default_state) {
		void *defaults, *vaddr;

		vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
		if (IS_ERR(vaddr)) {
			err = PTR_ERR(vaddr);
			goto err_obj;
		}

		defaults = i915_gem_object_pin_map(engine->default_state,
						   I915_MAP_WB);
		if (IS_ERR(defaults)) {
			err = PTR_ERR(defaults);
			goto err_map;
		}

		memcpy(vaddr, defaults, engine->context_size);
		i915_gem_object_unpin_map(engine->default_state);

1286 1287
		i915_gem_object_flush_map(obj);
		i915_gem_object_unpin_map(obj);
1288 1289
	}

1290
	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
1291 1292 1293 1294
	if (IS_ERR(vma)) {
		err = PTR_ERR(vma);
		goto err_obj;
	}
1295 1296

	return vma;
1297 1298 1299 1300 1301 1302

err_map:
	i915_gem_object_unpin_map(obj);
err_obj:
	i915_gem_object_put(obj);
	return ERR_PTR(err);
1303 1304
}

1305
static int ring_context_alloc(struct intel_context *ce)
1306
{
1307
	struct intel_engine_cs *engine = ce->engine;
1308

1309
	/* One ringbuffer to rule them all */
1310 1311 1312
	GEM_BUG_ON(!engine->legacy.ring);
	ce->ring = engine->legacy.ring;
	ce->timeline = intel_timeline_get(engine->legacy.timeline);
1313

1314 1315
	GEM_BUG_ON(ce->state);
	if (engine->context_size) {
1316 1317 1318
		struct i915_vma *vma;

		vma = alloc_context_vma(engine);
1319 1320
		if (IS_ERR(vma))
			return PTR_ERR(vma);
1321 1322

		ce->state = vma;
1323 1324
		if (engine->default_state)
			__set_bit(CONTEXT_VALID_BIT, &ce->flags);
1325 1326
	}

1327 1328 1329 1330 1331 1332 1333
	return 0;
}

static int ring_context_pin(struct intel_context *ce)
{
	int err;

1334
	err = intel_context_active_acquire(ce);
1335
	if (err)
1336
		return err;
1337

1338
	err = __context_pin_ppgtt(ce);
1339
	if (err)
1340
		goto err_active;
1341

1342
	return 0;
1343

1344 1345
err_active:
	intel_context_active_release(ce);
1346
	return err;
1347 1348
}

1349 1350 1351 1352 1353
static void ring_context_reset(struct intel_context *ce)
{
	intel_ring_reset(ce->ring, 0);
}

1354
static const struct intel_context_ops ring_context_ops = {
1355 1356
	.alloc = ring_context_alloc,

1357
	.pin = ring_context_pin,
1358
	.unpin = ring_context_unpin,
1359

1360 1361 1362
	.enter = intel_context_enter_engine,
	.exit = intel_context_exit_engine,

1363
	.reset = ring_context_reset,
1364 1365 1366
	.destroy = ring_context_destroy,
};

1367 1368 1369
static int load_pd_dir(struct i915_request *rq,
		       const struct i915_ppgtt *ppgtt,
		       u32 valid)
1370 1371 1372 1373
{
	const struct intel_engine_cs * const engine = rq->engine;
	u32 *cs;

1374
	cs = intel_ring_begin(rq, 12);
1375 1376 1377 1378
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	*cs++ = MI_LOAD_REGISTER_IMM(1);
1379
	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base));
1380
	*cs++ = valid;
1381

1382 1383 1384 1385 1386
	*cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base));
	*cs++ = intel_gt_scratch_offset(rq->engine->gt,
					INTEL_GT_SCRATCH_FIELD_DEFAULT);

1387
	*cs++ = MI_LOAD_REGISTER_IMM(1);
1388
	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
1389
	*cs++ = px_base(ppgtt->pd)->ggtt_offset << 10;
1390

1391
	/* Stall until the page table load is complete? */
1392
	*cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
1393
	*cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base));
1394 1395
	*cs++ = intel_gt_scratch_offset(rq->engine->gt,
					INTEL_GT_SCRATCH_FIELD_DEFAULT);
1396 1397

	intel_ring_advance(rq, cs);
1398

1399
	return rq->engine->emit_flush(rq, EMIT_FLUSH);
1400 1401
}

1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420
static int flush_tlb(struct i915_request *rq)
{
	const struct intel_engine_cs * const engine = rq->engine;
	u32 *cs;

	cs = intel_ring_begin(rq, 4);
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	*cs++ = MI_LOAD_REGISTER_IMM(1);
	*cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base));
	*cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE);

	*cs++ = MI_NOOP;
	intel_ring_advance(rq, cs);

	return 0;
}

1421
static inline int mi_set_context(struct i915_request *rq, u32 flags)
1422 1423 1424 1425
{
	struct drm_i915_private *i915 = rq->i915;
	struct intel_engine_cs *engine = rq->engine;
	enum intel_engine_id id;
1426
	const int num_engines =
1427
		IS_HASWELL(i915) ? RUNTIME_INFO(i915)->num_engines - 1 : 0;
1428
	bool force_restore = false;
1429 1430 1431 1432 1433 1434 1435 1436
	int len;
	u32 *cs;

	flags |= MI_MM_SPACE_GTT;
	if (IS_HASWELL(i915))
		/* These flags are for resource streamer on HSW+ */
		flags |= HSW_MI_RS_SAVE_STATE_EN | HSW_MI_RS_RESTORE_STATE_EN;
	else
1437
		/* We need to save the extended state for powersaving modes */
1438 1439 1440
		flags |= MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN;

	len = 4;
1441
	if (IS_GEN(i915, 7))
1442
		len += 2 + (num_engines ? 4 * num_engines + 6 : 0);
1443 1444
	else if (IS_GEN(i915, 5))
		len += 2;
1445 1446 1447 1448 1449 1450
	if (flags & MI_FORCE_RESTORE) {
		GEM_BUG_ON(flags & MI_RESTORE_INHIBIT);
		flags &= ~MI_FORCE_RESTORE;
		force_restore = true;
		len += 2;
	}
1451 1452 1453 1454 1455 1456

	cs = intel_ring_begin(rq, len);
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	/* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
1457
	if (IS_GEN(i915, 7)) {
1458
		*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1459
		if (num_engines) {
1460 1461
			struct intel_engine_cs *signaller;

1462
			*cs++ = MI_LOAD_REGISTER_IMM(num_engines);
1463
			for_each_engine(signaller, engine->gt, id) {
1464 1465 1466 1467 1468 1469 1470 1471 1472
				if (signaller == engine)
					continue;

				*cs++ = i915_mmio_reg_offset(
					   RING_PSMI_CTL(signaller->mmio_base));
				*cs++ = _MASKED_BIT_ENABLE(
						GEN6_PSMI_SLEEP_MSG_DISABLE);
			}
		}
1473 1474 1475 1476 1477 1478 1479 1480
	} else if (IS_GEN(i915, 5)) {
		/*
		 * This w/a is only listed for pre-production ilk a/b steppings,
		 * but is also mentioned for programming the powerctx. To be
		 * safe, just apply the workaround; we do not use SyncFlush so
		 * this should never take effect and so be a no-op!
		 */
		*cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN;
1481 1482
	}

1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496
	if (force_restore) {
		/*
		 * The HW doesn't handle being told to restore the current
		 * context very well. Quite often it likes goes to go off and
		 * sulk, especially when it is meant to be reloading PP_DIR.
		 * A very simple fix to force the reload is to simply switch
		 * away from the current context and back again.
		 *
		 * Note that the kernel_context will contain random state
		 * following the INHIBIT_RESTORE. We accept this since we
		 * never use the kernel_context state; it is merely a
		 * placeholder we use to flush other contexts.
		 */
		*cs++ = MI_SET_CONTEXT;
1497
		*cs++ = i915_ggtt_offset(engine->kernel_context->state) |
1498 1499 1500 1501
			MI_MM_SPACE_GTT |
			MI_RESTORE_INHIBIT;
	}

1502 1503
	*cs++ = MI_NOOP;
	*cs++ = MI_SET_CONTEXT;
1504
	*cs++ = i915_ggtt_offset(rq->hw_context->state) | flags;
1505 1506 1507 1508 1509 1510
	/*
	 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
	 * WaMiSetContext_Hang:snb,ivb,vlv
	 */
	*cs++ = MI_NOOP;

1511
	if (IS_GEN(i915, 7)) {
1512
		if (num_engines) {
1513 1514 1515
			struct intel_engine_cs *signaller;
			i915_reg_t last_reg = {}; /* keep gcc quiet */

1516
			*cs++ = MI_LOAD_REGISTER_IMM(num_engines);
1517
			for_each_engine(signaller, engine->gt, id) {
1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529
				if (signaller == engine)
					continue;

				last_reg = RING_PSMI_CTL(signaller->mmio_base);
				*cs++ = i915_mmio_reg_offset(last_reg);
				*cs++ = _MASKED_BIT_DISABLE(
						GEN6_PSMI_SLEEP_MSG_DISABLE);
			}

			/* Insert a delay before the next switch! */
			*cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
			*cs++ = i915_mmio_reg_offset(last_reg);
1530
			*cs++ = intel_gt_scratch_offset(engine->gt,
1531
							INTEL_GT_SCRATCH_FIELD_DEFAULT);
1532 1533 1534
			*cs++ = MI_NOOP;
		}
		*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1535 1536
	} else if (IS_GEN(i915, 5)) {
		*cs++ = MI_SUSPEND_FLUSH;
1537 1538 1539 1540 1541 1542 1543
	}

	intel_ring_advance(rq, cs);

	return 0;
}

1544
static int remap_l3_slice(struct i915_request *rq, int slice)
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571
{
	u32 *cs, *remap_info = rq->i915->l3_parity.remap_info[slice];
	int i;

	if (!remap_info)
		return 0;

	cs = intel_ring_begin(rq, GEN7_L3LOG_SIZE/4 * 2 + 2);
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	/*
	 * Note: We do not worry about the concurrent register cacheline hang
	 * here because no other code should access these registers other than
	 * at initialization time.
	 */
	*cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4);
	for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) {
		*cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i));
		*cs++ = remap_info[i];
	}
	*cs++ = MI_NOOP;
	intel_ring_advance(rq, cs);

	return 0;
}

1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592
static int remap_l3(struct i915_request *rq)
{
	struct i915_gem_context *ctx = rq->gem_context;
	int i, err;

	if (!ctx->remap_slice)
		return 0;

	for (i = 0; i < MAX_L3_SLICES; i++) {
		if (!(ctx->remap_slice & BIT(i)))
			continue;

		err = remap_l3_slice(rq, i);
		if (err)
			return err;
	}

	ctx->remap_slice = 0;
	return 0;
}

1593
static int switch_context(struct i915_request *rq)
1594
{
1595 1596
	struct intel_context *ce = rq->hw_context;
	struct i915_address_space *vm = vm_alias(ce);
1597
	u32 hw_flags = 0;
1598
	int ret;
1599 1600 1601

	GEM_BUG_ON(HAS_EXECLISTS(rq->i915));

1602
	if (vm) {
1603 1604 1605 1606 1607 1608 1609 1610
		/*
		 * Not only do we need a full barrier (post-sync write) after
		 * invalidating the TLBs, but we need to wait a little bit
		 * longer. Whether this is merely delaying us, or the
		 * subsequent flush is a key part of serialising with the
		 * post-sync op, this extra pass appears vital before a
		 * mm switch!
		 */
1611 1612 1613 1614 1615 1616 1617
		ret = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
		if (ret)
			return ret;

		ret = flush_tlb(rq);
		if (ret)
			return ret;
1618

1619 1620 1621
		ret = load_pd_dir(rq, i915_vm_to_ppgtt(vm), 0);
		if (ret)
			return ret;
1622

1623
		ret = load_pd_dir(rq, i915_vm_to_ppgtt(vm), PP_DIR_DCLV_2G);
1624 1625 1626
		if (ret)
			return ret;

1627
		ret = flush_tlb(rq);
1628 1629 1630
		if (ret)
			return ret;

1631
		ret = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
1632
		if (ret)
1633
			return ret;
1634 1635
	}

1636 1637 1638
	if (ce->state) {
		GEM_BUG_ON(rq->engine->id != RCS0);

1639
		if (!test_bit(CONTEXT_VALID_BIT, &ce->flags))
1640 1641 1642 1643 1644 1645 1646
			hw_flags = MI_RESTORE_INHIBIT;

		ret = mi_set_context(rq, hw_flags);
		if (ret)
			return ret;
	}

1647 1648
	ret = remap_l3(rq);
	if (ret)
1649
		return ret;
1650 1651 1652 1653

	return 0;
}

1654
static int ring_request_alloc(struct i915_request *request)
1655
{
1656
	int ret;
1657

1658
	GEM_BUG_ON(!intel_context_is_pinned(request->hw_context));
1659
	GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb);
1660

1661 1662
	/*
	 * Flush enough space to reduce the likelihood of waiting after
1663 1664 1665
	 * we start building the request - in which case we will just
	 * have to repeat work.
	 */
1666
	request->reserved_space += LEGACY_REQUEST_SIZE;
1667

1668 1669
	/* Unconditionally invalidate GPU caches and TLBs. */
	ret = request->engine->emit_flush(request, EMIT_INVALIDATE);
1670 1671
	if (ret)
		return ret;
1672

1673
	ret = switch_context(request);
1674 1675 1676
	if (ret)
		return ret;

1677
	request->reserved_space -= LEGACY_REQUEST_SIZE;
1678
	return 0;
1679 1680
}

1681
static void gen6_bsd_submit_request(struct i915_request *request)
1682
{
1683
	struct intel_uncore *uncore = request->engine->uncore;
1684

1685
	intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1686

1687
       /* Every tail move must follow the sequence below */
1688 1689 1690 1691

	/* Disable notification that the ring is IDLE. The GT
	 * will then assume that it is busy and bring it out of rc6.
	 */
1692 1693
	intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,
			      _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1694 1695

	/* Clear the context id. Here be magic! */
1696
	intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0);
1697

1698
	/* Wait for the ring not to be idle, i.e. for it to wake up. */
1699
	if (__intel_wait_for_register_fw(uncore,
1700 1701 1702 1703
					 GEN6_BSD_SLEEP_PSMI_CONTROL,
					 GEN6_BSD_SLEEP_INDICATOR,
					 0,
					 1000, 0, NULL))
1704
		DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
1705

1706
	/* Now that the ring is fully powered up, update the tail */
1707
	i9xx_submit_request(request);
1708 1709 1710 1711

	/* Let the ring send IDLE messages to the GT again,
	 * and so let it sleep to conserve power when idle.
	 */
1712 1713
	intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL,
			      _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1714

1715
	intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1716 1717
}

1718
static int mi_flush_dw(struct i915_request *rq, u32 flags)
1719
{
1720
	u32 cmd, *cs;
1721

1722
	cs = intel_ring_begin(rq, 4);
1723 1724
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1725

1726
	cmd = MI_FLUSH_DW;
1727

1728 1729
	/*
	 * We always require a command barrier so that subsequent
1730 1731 1732 1733 1734 1735
	 * commands, such as breadcrumb interrupts, are strictly ordered
	 * wrt the contents of the write cache being flushed to memory
	 * (and thus being coherent from the CPU).
	 */
	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;

1736
	/*
1737
	 * Bspec vol 1c.3 - blitter engine command streamer:
1738 1739 1740 1741
	 * "If ENABLED, all TLBs will be invalidated once the flush
	 * operation is complete. This bit is only valid when the
	 * Post-Sync Operation field is a value of 1h or 3h."
	 */
1742
	cmd |= flags;
1743

1744 1745
	*cs++ = cmd;
	*cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
1746
	*cs++ = 0;
1747
	*cs++ = MI_NOOP;
1748

1749
	intel_ring_advance(rq, cs);
1750

1751 1752 1753
	return 0;
}

1754 1755
static int gen6_flush_dw(struct i915_request *rq, u32 mode, u32 invflags)
{
1756
	return mi_flush_dw(rq, mode & EMIT_INVALIDATE ? invflags : 0);
1757 1758 1759 1760 1761 1762 1763
}

static int gen6_bsd_ring_flush(struct i915_request *rq, u32 mode)
{
	return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB | MI_INVALIDATE_BSD);
}

1764
static int
1765
hsw_emit_bb_start(struct i915_request *rq,
1766 1767
		  u64 offset, u32 len,
		  unsigned int dispatch_flags)
1768
{
1769
	u32 *cs;
1770

1771
	cs = intel_ring_begin(rq, 2);
1772 1773
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1774

1775
	*cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ?
1776
		0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW);
1777
	/* bit0-7 is the length on GEN6+ */
1778
	*cs++ = offset;
1779
	intel_ring_advance(rq, cs);
1780 1781 1782 1783

	return 0;
}

1784
static int
1785
gen6_emit_bb_start(struct i915_request *rq,
1786 1787
		   u64 offset, u32 len,
		   unsigned int dispatch_flags)
1788
{
1789
	u32 *cs;
1790

1791
	cs = intel_ring_begin(rq, 2);
1792 1793
	if (IS_ERR(cs))
		return PTR_ERR(cs);
1794

1795 1796
	*cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ?
		0 : MI_BATCH_NON_SECURE_I965);
1797
	/* bit0-7 is the length on GEN6+ */
1798
	*cs++ = offset;
1799
	intel_ring_advance(rq, cs);
1800

1801
	return 0;
1802 1803
}

1804 1805
/* Blitter support (SandyBridge+) */

1806
static int gen6_ring_flush(struct i915_request *rq, u32 mode)
Z
Zou Nan hai 已提交
1807
{
1808
	return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB);
Z
Zou Nan hai 已提交
1809 1810
}

1811 1812 1813
static void i9xx_set_default_submission(struct intel_engine_cs *engine)
{
	engine->submit_request = i9xx_submit_request;
1814
	engine->cancel_requests = cancel_requests;
1815 1816 1817

	engine->park = NULL;
	engine->unpark = NULL;
1818 1819 1820 1821
}

static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)
{
1822
	i9xx_set_default_submission(engine);
1823 1824 1825
	engine->submit_request = gen6_bsd_submit_request;
}

1826 1827 1828 1829 1830 1831 1832
static void ring_destroy(struct intel_engine_cs *engine)
{
	struct drm_i915_private *dev_priv = engine->i915;

	WARN_ON(INTEL_GEN(dev_priv) > 2 &&
		(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0);

1833 1834
	intel_engine_cleanup_common(engine);

1835 1836 1837 1838 1839
	intel_ring_unpin(engine->legacy.ring);
	intel_ring_put(engine->legacy.ring);

	intel_timeline_unpin(engine->legacy.timeline);
	intel_timeline_put(engine->legacy.timeline);
1840 1841 1842 1843

	kfree(engine);
}

1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863
static void setup_irq(struct intel_engine_cs *engine)
{
	struct drm_i915_private *i915 = engine->i915;

	if (INTEL_GEN(i915) >= 6) {
		engine->irq_enable = gen6_irq_enable;
		engine->irq_disable = gen6_irq_disable;
	} else if (INTEL_GEN(i915) >= 5) {
		engine->irq_enable = gen5_irq_enable;
		engine->irq_disable = gen5_irq_disable;
	} else if (INTEL_GEN(i915) >= 3) {
		engine->irq_enable = i9xx_irq_enable;
		engine->irq_disable = i9xx_irq_disable;
	} else {
		engine->irq_enable = i8xx_irq_enable;
		engine->irq_disable = i8xx_irq_disable;
	}
}

static void setup_common(struct intel_engine_cs *engine)
1864
{
1865 1866
	struct drm_i915_private *i915 = engine->i915;

1867
	/* gen8+ are only supported with execlists */
1868
	GEM_BUG_ON(INTEL_GEN(i915) >= 8);
1869

1870
	setup_irq(engine);
1871

1872 1873
	engine->destroy = ring_destroy;

1874
	engine->resume = xcs_resume;
1875 1876 1877
	engine->reset.prepare = reset_prepare;
	engine->reset.reset = reset_ring;
	engine->reset.finish = reset_finish;
1878

1879
	engine->cops = &ring_context_ops;
1880 1881
	engine->request_alloc = ring_request_alloc;

1882 1883 1884 1885 1886 1887
	/*
	 * Using a global execution timeline; the previous final breadcrumb is
	 * equivalent to our next initial bread so we can elide
	 * engine->emit_init_breadcrumb().
	 */
	engine->emit_fini_breadcrumb = i9xx_emit_breadcrumb;
1888
	if (IS_GEN(i915, 5))
1889
		engine->emit_fini_breadcrumb = gen5_emit_breadcrumb;
1890 1891

	engine->set_default_submission = i9xx_set_default_submission;
1892

1893
	if (INTEL_GEN(i915) >= 6)
1894
		engine->emit_bb_start = gen6_emit_bb_start;
1895
	else if (INTEL_GEN(i915) >= 4)
1896
		engine->emit_bb_start = i965_emit_bb_start;
1897
	else if (IS_I830(i915) || IS_I845G(i915))
1898
		engine->emit_bb_start = i830_emit_bb_start;
1899
	else
1900
		engine->emit_bb_start = i915_emit_bb_start;
1901 1902
}

1903
static void setup_rcs(struct intel_engine_cs *engine)
1904
{
1905
	struct drm_i915_private *i915 = engine->i915;
1906

1907
	if (HAS_L3_DPF(i915))
1908
		engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
1909

1910 1911
	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;

1912
	if (INTEL_GEN(i915) >= 7) {
1913
		engine->emit_flush = gen7_render_ring_flush;
1914
		engine->emit_fini_breadcrumb = gen7_rcs_emit_breadcrumb;
1915
	} else if (IS_GEN(i915, 6)) {
1916
		engine->emit_flush = gen6_render_ring_flush;
1917
		engine->emit_fini_breadcrumb = gen6_rcs_emit_breadcrumb;
1918
	} else if (IS_GEN(i915, 5)) {
1919
		engine->emit_flush = gen4_render_ring_flush;
1920
	} else {
1921
		if (INTEL_GEN(i915) < 4)
1922
			engine->emit_flush = gen2_render_ring_flush;
1923
		else
1924
			engine->emit_flush = gen4_render_ring_flush;
1925
		engine->irq_enable_mask = I915_USER_INTERRUPT;
1926
	}
B
Ben Widawsky 已提交
1927

1928
	if (IS_HASWELL(i915))
1929
		engine->emit_bb_start = hsw_emit_bb_start;
1930

1931
	engine->resume = rcs_resume;
1932 1933
}

1934
static void setup_vcs(struct intel_engine_cs *engine)
1935
{
1936
	struct drm_i915_private *i915 = engine->i915;
1937

1938
	if (INTEL_GEN(i915) >= 6) {
1939
		/* gen6 bsd needs a special wa for tail updates */
1940
		if (IS_GEN(i915, 6))
1941
			engine->set_default_submission = gen6_bsd_set_default_submission;
1942
		engine->emit_flush = gen6_bsd_ring_flush;
1943
		engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1944

1945
		if (IS_GEN(i915, 6))
1946
			engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb;
1947
		else
1948
			engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb;
1949
	} else {
1950
		engine->emit_flush = bsd_ring_flush;
1951
		if (IS_GEN(i915, 5))
1952
			engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
1953
		else
1954
			engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1955
	}
1956
}
1957

1958
static void setup_bcs(struct intel_engine_cs *engine)
1959
{
1960
	struct drm_i915_private *i915 = engine->i915;
1961

1962
	engine->emit_flush = gen6_ring_flush;
1963
	engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
1964

1965
	if (IS_GEN(i915, 6))
1966
		engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb;
1967
	else
1968
		engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb;
1969
}
1970

1971
static void setup_vecs(struct intel_engine_cs *engine)
B
Ben Widawsky 已提交
1972
{
1973
	struct drm_i915_private *i915 = engine->i915;
1974

1975
	GEM_BUG_ON(INTEL_GEN(i915) < 7);
1976

1977
	engine->emit_flush = gen6_ring_flush;
1978 1979 1980
	engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
	engine->irq_enable = hsw_vebox_irq_enable;
	engine->irq_disable = hsw_vebox_irq_disable;
B
Ben Widawsky 已提交
1981

1982
	engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb;
1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011
}

int intel_ring_submission_setup(struct intel_engine_cs *engine)
{
	setup_common(engine);

	switch (engine->class) {
	case RENDER_CLASS:
		setup_rcs(engine);
		break;
	case VIDEO_DECODE_CLASS:
		setup_vcs(engine);
		break;
	case COPY_ENGINE_CLASS:
		setup_bcs(engine);
		break;
	case VIDEO_ENHANCEMENT_CLASS:
		setup_vecs(engine);
		break;
	default:
		MISSING_CASE(engine->class);
		return -ENODEV;
	}

	return 0;
}

int intel_ring_submission_init(struct intel_engine_cs *engine)
{
2012
	struct intel_timeline *timeline;
2013 2014 2015
	struct intel_ring *ring;
	int err;

2016
	timeline = intel_timeline_create(engine->gt, engine->status_page.vma);
2017 2018 2019 2020 2021 2022
	if (IS_ERR(timeline)) {
		err = PTR_ERR(timeline);
		goto err;
	}
	GEM_BUG_ON(timeline->has_initial_breadcrumb);

2023 2024 2025 2026 2027
	err = intel_timeline_pin(timeline);
	if (err)
		goto err_timeline;

	ring = intel_engine_create_ring(engine, SZ_16K);
2028 2029
	if (IS_ERR(ring)) {
		err = PTR_ERR(ring);
2030
		goto err_timeline_unpin;
2031 2032 2033 2034 2035
	}

	err = intel_ring_pin(ring);
	if (err)
		goto err_ring;
2036

2037 2038 2039
	GEM_BUG_ON(engine->legacy.ring);
	engine->legacy.ring = ring;
	engine->legacy.timeline = timeline;
2040 2041 2042

	err = intel_engine_init_common(engine);
	if (err)
2043
		goto err_ring_unpin;
2044

2045
	GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);
2046 2047 2048

	return 0;

2049
err_ring_unpin:
2050 2051 2052
	intel_ring_unpin(ring);
err_ring:
	intel_ring_put(ring);
2053 2054 2055 2056
err_timeline_unpin:
	intel_timeline_unpin(timeline);
err_timeline:
	intel_timeline_put(timeline);
2057 2058 2059
err:
	intel_engine_cleanup_common(engine);
	return err;
B
Ben Widawsky 已提交
2060
}