intel_ringbuffer.c 73.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 * Copyright © 2008-2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *    Zou Nan hai <nanhai.zou@intel.com>
 *    Xiang Hai hao<haihao.xiang@intel.com>
 *
 */

30
#include <linux/log2.h>
31
#include <drm/drmP.h>
32
#include "i915_drv.h"
33
#include <drm/i915_drm.h>
34
#include "i915_trace.h"
35
#include "intel_drv.h"
36

37 38 39 40 41
/* Rough estimate of the typical request size, performing a flush,
 * set-context and then emitting the batch.
 */
#define LEGACY_REQUEST_SIZE 200

42
int __intel_ring_space(int head, int tail, int size)
43
{
44 45
	int space = head - tail;
	if (space <= 0)
46
		space += size;
47
	return space - I915_RING_FREE_SPACE;
48 49
}

50
void intel_ring_update_space(struct intel_ring *ring)
51
{
52 53 54
	if (ring->last_retired_head != -1) {
		ring->head = ring->last_retired_head;
		ring->last_retired_head = -1;
55 56
	}

57 58
	ring->space = __intel_ring_space(ring->head & HEAD_ADDR,
					 ring->tail, ring->size);
59 60
}

61
static int
62
gen2_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
63
{
64
	struct intel_ring *ring = req->ring;
65 66 67 68 69
	u32 cmd;
	int ret;

	cmd = MI_FLUSH;

70
	if (mode & EMIT_INVALIDATE)
71 72
		cmd |= MI_READ_FLUSH;

73
	ret = intel_ring_begin(req, 2);
74 75 76
	if (ret)
		return ret;

77 78 79
	intel_ring_emit(ring, cmd);
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
80 81 82 83 84

	return 0;
}

static int
85
gen4_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
86
{
87
	struct intel_ring *ring = req->ring;
88
	u32 cmd;
89
	int ret;
90

91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
	/*
	 * read/write caches:
	 *
	 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
	 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
	 * also flushed at 2d versus 3d pipeline switches.
	 *
	 * read-only caches:
	 *
	 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
	 * MI_READ_FLUSH is set, and is always flushed on 965.
	 *
	 * I915_GEM_DOMAIN_COMMAND may not exist?
	 *
	 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
	 * invalidated when MI_EXE_FLUSH is set.
	 *
	 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
	 * invalidated with every MI_FLUSH.
	 *
	 * TLBs:
	 *
	 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
	 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
	 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
	 * are flushed at any MI_FLUSH.
	 */

119
	cmd = MI_FLUSH;
120
	if (mode & EMIT_INVALIDATE) {
121
		cmd |= MI_EXE_FLUSH;
122 123 124
		if (IS_G4X(req->i915) || IS_GEN5(req->i915))
			cmd |= MI_INVALIDATE_ISP;
	}
125

126
	ret = intel_ring_begin(req, 2);
127 128
	if (ret)
		return ret;
129

130 131 132
	intel_ring_emit(ring, cmd);
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
133 134

	return 0;
135 136
}

137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
/**
 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 * implementing two workarounds on gen6.  From section 1.4.7.1
 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 *
 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 * produced by non-pipelined state commands), software needs to first
 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 * 0.
 *
 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 *
 * And the workaround for these two requires this workaround first:
 *
 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 * BEFORE the pipe-control with a post-sync op and no write-cache
 * flushes.
 *
 * And this last workaround is tricky because of the requirements on
 * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 * volume 2 part 1:
 *
 *     "1 of the following must also be set:
 *      - Render Target Cache Flush Enable ([12] of DW1)
 *      - Depth Cache Flush Enable ([0] of DW1)
 *      - Stall at Pixel Scoreboard ([1] of DW1)
 *      - Depth Stall ([13] of DW1)
 *      - Post-Sync Operation ([13] of DW1)
 *      - Notify Enable ([8] of DW1)"
 *
 * The cache flushes require the workaround flush that triggered this
 * one, so we can't use it.  Depth stall would trigger the same.
 * Post-sync nonzero is what triggered this second workaround, so we
 * can't use that one either.  Notify enable is IRQs, which aren't
 * really our business.  That leaves only stall at scoreboard.
 */
static int
175
intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
176
{
177
	struct intel_ring *ring = req->ring;
178
	u32 scratch_addr =
179
		i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
180 181
	int ret;

182
	ret = intel_ring_begin(req, 6);
183 184 185
	if (ret)
		return ret;

186 187
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
188
			PIPE_CONTROL_STALL_AT_SCOREBOARD);
189 190 191 192 193
	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
	intel_ring_emit(ring, 0); /* low dword */
	intel_ring_emit(ring, 0); /* high dword */
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
194

195
	ret = intel_ring_begin(req, 6);
196 197 198
	if (ret)
		return ret;

199 200 201 202 203 204 205
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
	intel_ring_emit(ring, 0);
	intel_ring_emit(ring, 0);
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
206 207 208 209 210

	return 0;
}

static int
211
gen6_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
212
{
213
	struct intel_ring *ring = req->ring;
214
	u32 scratch_addr =
215
		i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
216 217 218
	u32 flags = 0;
	int ret;

219
	/* Force SNB workarounds for PIPE_CONTROL flushes */
220
	ret = intel_emit_post_sync_nonzero_flush(req);
221 222 223
	if (ret)
		return ret;

224 225 226 227
	/* Just flush everything.  Experiments have shown that reducing the
	 * number of bits based on the write domains has little performance
	 * impact.
	 */
228
	if (mode & EMIT_FLUSH) {
229 230 231 232 233 234
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		/*
		 * Ensure that any following seqno writes only happen
		 * when the render cache is indeed flushed.
		 */
235
		flags |= PIPE_CONTROL_CS_STALL;
236
	}
237
	if (mode & EMIT_INVALIDATE) {
238 239 240 241 242 243 244 245 246
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		/*
		 * TLB invalidate requires a post-sync write.
		 */
247
		flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
248
	}
249

250
	ret = intel_ring_begin(req, 4);
251 252 253
	if (ret)
		return ret;

254 255 256 257 258
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
	intel_ring_emit(ring, flags);
	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
	intel_ring_emit(ring, 0);
	intel_ring_advance(ring);
259 260 261 262

	return 0;
}

263
static int
264
gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
265
{
266
	struct intel_ring *ring = req->ring;
267 268
	int ret;

269
	ret = intel_ring_begin(req, 4);
270 271 272
	if (ret)
		return ret;

273 274 275 276 277 278 279
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
	intel_ring_emit(ring,
			PIPE_CONTROL_CS_STALL |
			PIPE_CONTROL_STALL_AT_SCOREBOARD);
	intel_ring_emit(ring, 0);
	intel_ring_emit(ring, 0);
	intel_ring_advance(ring);
280 281 282 283

	return 0;
}

284
static int
285
gen7_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
286
{
287
	struct intel_ring *ring = req->ring;
288
	u32 scratch_addr =
289
		i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
290 291 292
	u32 flags = 0;
	int ret;

293 294 295 296 297 298 299 300 301 302
	/*
	 * Ensure that any following seqno writes only happen when the render
	 * cache is indeed flushed.
	 *
	 * Workaround: 4th PIPE_CONTROL command (except the ones with only
	 * read-cache invalidate bits set) must have the CS_STALL bit set. We
	 * don't try to be clever and just set it unconditionally.
	 */
	flags |= PIPE_CONTROL_CS_STALL;

303 304 305 306
	/* Just flush everything.  Experiments have shown that reducing the
	 * number of bits based on the write domains has little performance
	 * impact.
	 */
307
	if (mode & EMIT_FLUSH) {
308 309
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
310
		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
311
		flags |= PIPE_CONTROL_FLUSH_ENABLE;
312
	}
313
	if (mode & EMIT_INVALIDATE) {
314 315 316 317 318 319
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
320
		flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;
321 322 323 324
		/*
		 * TLB invalidate requires a post-sync write.
		 */
		flags |= PIPE_CONTROL_QW_WRITE;
325
		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
326

327 328
		flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;

329 330 331
		/* Workaround: we must issue a pipe_control with CS-stall bit
		 * set before a pipe_control command that has the state cache
		 * invalidate bit set. */
332
		gen7_render_ring_cs_stall_wa(req);
333 334
	}

335
	ret = intel_ring_begin(req, 4);
336 337 338
	if (ret)
		return ret;

339 340 341 342 343
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
	intel_ring_emit(ring, flags);
	intel_ring_emit(ring, scratch_addr);
	intel_ring_emit(ring, 0);
	intel_ring_advance(ring);
344 345 346 347

	return 0;
}

348
static int
349
gen8_emit_pipe_control(struct drm_i915_gem_request *req,
350 351
		       u32 flags, u32 scratch_addr)
{
352
	struct intel_ring *ring = req->ring;
353 354
	int ret;

355
	ret = intel_ring_begin(req, 6);
356 357 358
	if (ret)
		return ret;

359 360 361 362 363 364 365
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
	intel_ring_emit(ring, flags);
	intel_ring_emit(ring, scratch_addr);
	intel_ring_emit(ring, 0);
	intel_ring_emit(ring, 0);
	intel_ring_emit(ring, 0);
	intel_ring_advance(ring);
366 367 368 369

	return 0;
}

B
Ben Widawsky 已提交
370
static int
371
gen8_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
B
Ben Widawsky 已提交
372
{
373
	u32 scratch_addr =
374
		i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
375
	u32 flags = 0;
376
	int ret;
B
Ben Widawsky 已提交
377 378 379

	flags |= PIPE_CONTROL_CS_STALL;

380
	if (mode & EMIT_FLUSH) {
B
Ben Widawsky 已提交
381 382
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
383
		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
384
		flags |= PIPE_CONTROL_FLUSH_ENABLE;
B
Ben Widawsky 已提交
385
	}
386
	if (mode & EMIT_INVALIDATE) {
B
Ben Widawsky 已提交
387 388 389 390 391 392 393 394
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_QW_WRITE;
		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
395 396

		/* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
397
		ret = gen8_emit_pipe_control(req,
398 399 400 401 402
					     PIPE_CONTROL_CS_STALL |
					     PIPE_CONTROL_STALL_AT_SCOREBOARD,
					     0);
		if (ret)
			return ret;
B
Ben Widawsky 已提交
403 404
	}

405
	return gen8_emit_pipe_control(req, flags, scratch_addr);
B
Ben Widawsky 已提交
406 407
}

408
static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
409
{
410
	struct drm_i915_private *dev_priv = engine->i915;
411 412 413
	u32 addr;

	addr = dev_priv->status_page_dmah->busaddr;
414
	if (INTEL_GEN(dev_priv) >= 4)
415 416 417 418
		addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
	I915_WRITE(HWS_PGA, addr);
}

419
static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
420
{
421
	struct drm_i915_private *dev_priv = engine->i915;
422
	i915_reg_t mmio;
423 424 425 426

	/* The ring status page addresses are no longer next to the rest of
	 * the ring registers as of gen7.
	 */
427
	if (IS_GEN7(dev_priv)) {
428
		switch (engine->id) {
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446
		case RCS:
			mmio = RENDER_HWS_PGA_GEN7;
			break;
		case BCS:
			mmio = BLT_HWS_PGA_GEN7;
			break;
		/*
		 * VCS2 actually doesn't exist on Gen7. Only shut up
		 * gcc switch check warning
		 */
		case VCS2:
		case VCS:
			mmio = BSD_HWS_PGA_GEN7;
			break;
		case VECS:
			mmio = VEBOX_HWS_PGA_GEN7;
			break;
		}
447
	} else if (IS_GEN6(dev_priv)) {
448
		mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
449 450
	} else {
		/* XXX: gen8 returns to sanity */
451
		mmio = RING_HWS_PGA(engine->mmio_base);
452 453
	}

454
	I915_WRITE(mmio, engine->status_page.ggtt_offset);
455 456 457 458 459 460 461 462 463
	POSTING_READ(mmio);

	/*
	 * Flush the TLB for this page
	 *
	 * FIXME: These two bits have disappeared on gen8, so a question
	 * arises: do we still need this and if so how should we go about
	 * invalidating the TLB?
	 */
464
	if (IS_GEN(dev_priv, 6, 7)) {
465
		i915_reg_t reg = RING_INSTPM(engine->mmio_base);
466 467

		/* ring should be idle before issuing a sync flush*/
468
		WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
469 470 471 472

		I915_WRITE(reg,
			   _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
					      INSTPM_SYNC_FLUSH));
473 474 475
		if (intel_wait_for_register(dev_priv,
					    reg, INSTPM_SYNC_FLUSH, 0,
					    1000))
476
			DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
477
				  engine->name);
478 479 480
	}
}

481
static bool stop_ring(struct intel_engine_cs *engine)
482
{
483
	struct drm_i915_private *dev_priv = engine->i915;
484

485
	if (INTEL_GEN(dev_priv) > 2) {
486
		I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
487 488 489 490 491
		if (intel_wait_for_register(dev_priv,
					    RING_MI_MODE(engine->mmio_base),
					    MODE_IDLE,
					    MODE_IDLE,
					    1000)) {
492 493
			DRM_ERROR("%s : timed out trying to stop ring\n",
				  engine->name);
494 495 496 497
			/* Sometimes we observe that the idle flag is not
			 * set even though the ring is empty. So double
			 * check before giving up.
			 */
498
			if (I915_READ_HEAD(engine) != I915_READ_TAIL(engine))
499
				return false;
500 501
		}
	}
502

503 504
	I915_WRITE_CTL(engine, 0);
	I915_WRITE_HEAD(engine, 0);
505
	I915_WRITE_TAIL(engine, 0);
506

507
	if (INTEL_GEN(dev_priv) > 2) {
508 509
		(void)I915_READ_CTL(engine);
		I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
510
	}
511

512
	return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;
513
}
514

515
static int init_ring_common(struct intel_engine_cs *engine)
516
{
517
	struct drm_i915_private *dev_priv = engine->i915;
518
	struct intel_ring *ring = engine->buffer;
519 520
	int ret = 0;

521
	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
522

523
	if (!stop_ring(engine)) {
524
		/* G45 ring initialization often fails to reset head to zero */
525 526
		DRM_DEBUG_KMS("%s head not reset to zero "
			      "ctl %08x head %08x tail %08x start %08x\n",
527 528 529 530 531
			      engine->name,
			      I915_READ_CTL(engine),
			      I915_READ_HEAD(engine),
			      I915_READ_TAIL(engine),
			      I915_READ_START(engine));
532

533
		if (!stop_ring(engine)) {
534 535
			DRM_ERROR("failed to set %s head to zero "
				  "ctl %08x head %08x tail %08x start %08x\n",
536 537 538 539 540
				  engine->name,
				  I915_READ_CTL(engine),
				  I915_READ_HEAD(engine),
				  I915_READ_TAIL(engine),
				  I915_READ_START(engine));
541 542
			ret = -EIO;
			goto out;
543
		}
544 545
	}

546
	if (HWS_NEEDS_PHYSICAL(dev_priv))
547
		ring_setup_phys_status_page(engine);
548 549
	else
		intel_ring_setup_status_page(engine);
550

551
	intel_engine_reset_breadcrumbs(engine);
552

553
	/* Enforce ordering by reading HEAD register back */
554
	I915_READ_HEAD(engine);
555

556 557 558 559
	/* Initialize the ring. This must happen _after_ we've cleared the ring
	 * registers with the above sequence (the readback of the HEAD registers
	 * also enforces ordering), otherwise the hw might lose the new ring
	 * register values. */
560
	I915_WRITE_START(engine, i915_ggtt_offset(ring->vma));
561 562

	/* WaClearRingBufHeadRegAtInit:ctg,elk */
563
	if (I915_READ_HEAD(engine))
564
		DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
565
			  engine->name, I915_READ_HEAD(engine));
566 567 568 569 570

	intel_ring_update_space(ring);
	I915_WRITE_HEAD(engine, ring->head);
	I915_WRITE_TAIL(engine, ring->tail);
	(void)I915_READ_TAIL(engine);
571

572
	I915_WRITE_CTL(engine, RING_CTL_SIZE(ring->size) | RING_VALID);
573 574

	/* If the head is still not zero, the ring is dead */
575 576 577
	if (intel_wait_for_register_fw(dev_priv, RING_CTL(engine->mmio_base),
				       RING_VALID, RING_VALID,
				       50)) {
578
		DRM_ERROR("%s initialization failed "
579
			  "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
580 581 582
			  engine->name,
			  I915_READ_CTL(engine),
			  I915_READ_CTL(engine) & RING_VALID,
583 584
			  I915_READ_HEAD(engine), ring->head,
			  I915_READ_TAIL(engine), ring->tail,
585
			  I915_READ_START(engine),
586
			  i915_ggtt_offset(ring->vma));
587 588
		ret = -EIO;
		goto out;
589 590
	}

591
	intel_engine_init_hangcheck(engine);
592

593
out:
594
	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
595 596

	return ret;
597 598
}

599 600 601 602 603 604 605 606 607
static void reset_ring_common(struct intel_engine_cs *engine,
			      struct drm_i915_gem_request *request)
{
	struct intel_ring *ring = request->ring;

	ring->head = request->postfix;
	ring->last_retired_head = -1;
}

608
static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
609
{
610
	struct intel_ring *ring = req->ring;
611 612
	struct i915_workarounds *w = &req->i915->workarounds;
	int ret, i;
613

614
	if (w->count == 0)
615
		return 0;
616

617
	ret = req->engine->emit_flush(req, EMIT_BARRIER);
618 619
	if (ret)
		return ret;
620

621
	ret = intel_ring_begin(req, (w->count * 2 + 2));
622 623 624
	if (ret)
		return ret;

625
	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count));
626
	for (i = 0; i < w->count; i++) {
627 628
		intel_ring_emit_reg(ring, w->reg[i].addr);
		intel_ring_emit(ring, w->reg[i].value);
629
	}
630
	intel_ring_emit(ring, MI_NOOP);
631

632
	intel_ring_advance(ring);
633

634
	ret = req->engine->emit_flush(req, EMIT_BARRIER);
635 636
	if (ret)
		return ret;
637

638
	DRM_DEBUG_DRIVER("Number of Workarounds emitted: %d\n", w->count);
639

640
	return 0;
641 642
}

643
static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
644 645 646
{
	int ret;

647
	ret = intel_ring_workarounds_emit(req);
648 649 650
	if (ret != 0)
		return ret;

651
	ret = i915_gem_render_state_emit(req);
652
	if (ret)
653
		return ret;
654

655
	return 0;
656 657
}

658
static int wa_add(struct drm_i915_private *dev_priv,
659 660
		  i915_reg_t addr,
		  const u32 mask, const u32 val)
661 662 663 664 665 666 667 668 669 670 671 672 673
{
	const u32 idx = dev_priv->workarounds.count;

	if (WARN_ON(idx >= I915_MAX_WA_REGS))
		return -ENOSPC;

	dev_priv->workarounds.reg[idx].addr = addr;
	dev_priv->workarounds.reg[idx].value = val;
	dev_priv->workarounds.reg[idx].mask = mask;

	dev_priv->workarounds.count++;

	return 0;
674 675
}

676
#define WA_REG(addr, mask, val) do { \
677
		const int r = wa_add(dev_priv, (addr), (mask), (val)); \
678 679
		if (r) \
			return r; \
680
	} while (0)
681 682

#define WA_SET_BIT_MASKED(addr, mask) \
683
	WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
684 685

#define WA_CLR_BIT_MASKED(addr, mask) \
686
	WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
687

688
#define WA_SET_FIELD_MASKED(addr, mask, value) \
689
	WA_REG(addr, mask, _MASKED_FIELD(mask, value))
690

691 692
#define WA_SET_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) | (mask))
#define WA_CLR_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) & ~(mask))
693

694
#define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val)
695

696 697
static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
				 i915_reg_t reg)
698
{
699
	struct drm_i915_private *dev_priv = engine->i915;
700
	struct i915_workarounds *wa = &dev_priv->workarounds;
701
	const uint32_t index = wa->hw_whitelist_count[engine->id];
702 703 704 705

	if (WARN_ON(index >= RING_MAX_NONPRIV_SLOTS))
		return -EINVAL;

706
	WA_WRITE(RING_FORCE_TO_NONPRIV(engine->mmio_base, index),
707
		 i915_mmio_reg_offset(reg));
708
	wa->hw_whitelist_count[engine->id]++;
709 710 711 712

	return 0;
}

713
static int gen8_init_workarounds(struct intel_engine_cs *engine)
714
{
715
	struct drm_i915_private *dev_priv = engine->i915;
716 717

	WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
718

719 720 721
	/* WaDisableAsyncFlipPerfMode:bdw,chv */
	WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);

722 723 724 725
	/* WaDisablePartialInstShootdown:bdw,chv */
	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
			  PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);

726 727 728 729 730
	/* Use Force Non-Coherent whenever executing a 3D context. This is a
	 * workaround for for a possible hang in the unlikely event a TLB
	 * invalidation occurs during a PSD flush.
	 */
	/* WaForceEnableNonCoherent:bdw,chv */
731
	/* WaHdcDisableFetchWhenMasked:bdw,chv */
732
	WA_SET_BIT_MASKED(HDC_CHICKEN0,
733
			  HDC_DONOT_FETCH_MEM_WHEN_MASKED |
734 735
			  HDC_FORCE_NON_COHERENT);

736 737 738 739 740 741 742 743 744 745
	/* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
	 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
	 *  polygons in the same 8x4 pixel/sample area to be processed without
	 *  stalling waiting for the earlier ones to write to Hierarchical Z
	 *  buffer."
	 *
	 * This optimization is off by default for BDW and CHV; turn it on.
	 */
	WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);

746 747 748
	/* Wa4x4STCOptimizationDisable:bdw,chv */
	WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);

749 750 751 752 753 754 755 756 757 758 759 760
	/*
	 * BSpec recommends 8x4 when MSAA is used,
	 * however in practice 16x4 seems fastest.
	 *
	 * Note that PS/WM thread counts depend on the WIZ hashing
	 * disable bit, which we don't touch here, but it's good
	 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
	 */
	WA_SET_FIELD_MASKED(GEN7_GT_MODE,
			    GEN6_WIZ_HASHING_MASK,
			    GEN6_WIZ_HASHING_16x4);

761 762 763
	return 0;
}

764
static int bdw_init_workarounds(struct intel_engine_cs *engine)
765
{
766
	struct drm_i915_private *dev_priv = engine->i915;
767
	int ret;
768

769
	ret = gen8_init_workarounds(engine);
770 771 772
	if (ret)
		return ret;

773
	/* WaDisableThreadStallDopClockGating:bdw (pre-production) */
774
	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
775

776
	/* WaDisableDopClockGating:bdw */
777 778
	WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
			  DOP_CLOCK_GATING_DISABLE);
779

780 781
	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
			  GEN8_SAMPLER_POWER_BYPASS_DIS);
782

783
	WA_SET_BIT_MASKED(HDC_CHICKEN0,
784 785 786
			  /* WaForceContextSaveRestoreNonCoherent:bdw */
			  HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
			  /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
787
			  (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
788 789 790 791

	return 0;
}

792
static int chv_init_workarounds(struct intel_engine_cs *engine)
793
{
794
	struct drm_i915_private *dev_priv = engine->i915;
795
	int ret;
796

797
	ret = gen8_init_workarounds(engine);
798 799 800
	if (ret)
		return ret;

801
	/* WaDisableThreadStallDopClockGating:chv */
802
	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
803

804 805 806
	/* Improve HiZ throughput on CHV. */
	WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);

807 808 809
	return 0;
}

810
static int gen9_init_workarounds(struct intel_engine_cs *engine)
811
{
812
	struct drm_i915_private *dev_priv = engine->i915;
813
	int ret;
814

815 816 817
	/* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl */
	I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));

818
	/* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl */
819 820 821
	I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
		   GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);

822
	/* WaDisableKillLogic:bxt,skl,kbl */
823 824 825
	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
		   ECOCHK_DIS_TLB);

826 827
	/* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl */
	/* WaDisablePartialInstShootdown:skl,bxt,kbl */
828
	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
829
			  FLOW_CONTROL_ENABLE |
830 831
			  PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);

832
	/* Syncing dependencies between camera and graphics:skl,bxt,kbl */
833 834 835
	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
			  GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);

836 837
	/* WaDisableDgMirrorFixInHalfSliceChicken5:bxt */
	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
838 839
		WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
				  GEN9_DG_MIRROR_FIX_ENABLE);
840

841 842
	/* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
843 844
		WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
				  GEN9_RHWO_OPTIMIZATION_DISABLE);
845 846 847 848 849
		/*
		 * WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
		 * but we do that in per ctx batchbuffer as there is an issue
		 * with this register not getting restored on ctx restore
		 */
850 851
	}

852
	/* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl */
853 854
	WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
			  GEN9_ENABLE_GPGPU_PREEMPTION);
855

856 857
	/* Wa4x4STCOptimizationDisable:skl,bxt,kbl */
	/* WaDisablePartialResolveInVc:skl,bxt,kbl */
858 859
	WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
					 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
860

861
	/* WaCcsTlbPrefetchDisable:skl,bxt,kbl */
862 863 864
	WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
			  GEN9_CCS_TLB_PREFETCH_ENABLE);

865 866
	/* WaDisableMaskBasedCammingInRCC:bxt */
	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
867 868 869
		WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
				  PIXEL_MASK_CAMMING_DISABLE);

870 871 872 873
	/* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl */
	WA_SET_BIT_MASKED(HDC_CHICKEN0,
			  HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
			  HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
874

875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895
	/* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
	 * both tied to WaForceContextSaveRestoreNonCoherent
	 * in some hsds for skl. We keep the tie for all gen9. The
	 * documentation is a bit hazy and so we want to get common behaviour,
	 * even though there is no clear evidence we would need both on kbl/bxt.
	 * This area has been source of system hangs so we play it safe
	 * and mimic the skl regardless of what bspec says.
	 *
	 * Use Force Non-Coherent whenever executing a 3D context. This
	 * is a workaround for a possible hang in the unlikely event
	 * a TLB invalidation occurs during a PSD flush.
	 */

	/* WaForceEnableNonCoherent:skl,bxt,kbl */
	WA_SET_BIT_MASKED(HDC_CHICKEN0,
			  HDC_FORCE_NON_COHERENT);

	/* WaDisableHDCInvalidation:skl,bxt,kbl */
	I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
		   BDW_DISABLE_HDC_INVALIDATION);

896 897 898 899
	/* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl */
	if (IS_SKYLAKE(dev_priv) ||
	    IS_KABYLAKE(dev_priv) ||
	    IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
900 901 902
		WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
				  GEN8_SAMPLER_POWER_BYPASS_DIS);

903
	/* WaDisableSTUnitPowerOptimization:skl,bxt,kbl */
904 905
	WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);

906
	/* WaOCLCoherentLineFlush:skl,bxt,kbl */
907 908 909
	I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
				    GEN8_LQSC_FLUSH_COHERENT_LINES));

910 911 912 913 914
	/* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt */
	ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG);
	if (ret)
		return ret;

915
	/* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl */
916
	ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
917 918 919
	if (ret)
		return ret;

920
	/* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl */
921
	ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1);
922 923 924
	if (ret)
		return ret;

925 926 927
	return 0;
}

928
static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
929
{
930
	struct drm_i915_private *dev_priv = engine->i915;
931 932 933 934 935 936 937 938 939 940
	u8 vals[3] = { 0, 0, 0 };
	unsigned int i;

	for (i = 0; i < 3; i++) {
		u8 ss;

		/*
		 * Only consider slices where one, and only one, subslice has 7
		 * EUs
		 */
941
		if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]))
942 943 944 945 946 947 948 949
			continue;

		/*
		 * subslice_7eu[i] != 0 (because of the check above) and
		 * ss_max == 4 (maximum number of subslices possible per slice)
		 *
		 * ->    0 <= ss <= 3;
		 */
950
		ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1;
951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968
		vals[i] = 3 - ss;
	}

	if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
		return 0;

	/* Tune IZ hashing. See intel_device_info_runtime_init() */
	WA_SET_FIELD_MASKED(GEN7_GT_MODE,
			    GEN9_IZ_HASHING_MASK(2) |
			    GEN9_IZ_HASHING_MASK(1) |
			    GEN9_IZ_HASHING_MASK(0),
			    GEN9_IZ_HASHING(2, vals[2]) |
			    GEN9_IZ_HASHING(1, vals[1]) |
			    GEN9_IZ_HASHING(0, vals[0]));

	return 0;
}

969
static int skl_init_workarounds(struct intel_engine_cs *engine)
970
{
971
	struct drm_i915_private *dev_priv = engine->i915;
972
	int ret;
973

974
	ret = gen9_init_workarounds(engine);
975 976
	if (ret)
		return ret;
977

978 979 980 981 982
	/*
	 * Actual WA is to disable percontext preemption granularity control
	 * until D0 which is the default case so this is equivalent to
	 * !WaDisablePerCtxtPreemptionGranularityControl:skl
	 */
983 984
	I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
		   _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
985

986
	/* WaEnableGapsTsvCreditFix:skl */
987 988
	I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
				   GEN9_GAPS_TSV_CREDIT_DISABLE));
989

990 991 992
	/* WaDisableGafsUnitClkGating:skl */
	WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);

993 994 995 996 997
	/* WaInPlaceDecompressionHang:skl */
	if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
		WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
			   GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);

998
	/* WaDisableLSQCROPERFforOCL:skl */
999
	ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1000 1001 1002
	if (ret)
		return ret;

1003
	return skl_tune_iz_hashing(engine);
1004 1005
}

1006
static int bxt_init_workarounds(struct intel_engine_cs *engine)
1007
{
1008
	struct drm_i915_private *dev_priv = engine->i915;
1009
	int ret;
1010

1011
	ret = gen9_init_workarounds(engine);
1012 1013
	if (ret)
		return ret;
1014

1015 1016
	/* WaStoreMultiplePTEenable:bxt */
	/* This is a requirement according to Hardware specification */
1017
	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1018 1019 1020
		I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);

	/* WaSetClckGatingDisableMedia:bxt */
1021
	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1022 1023 1024 1025
		I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
					    ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
	}

1026 1027 1028 1029
	/* WaDisableThreadStallDopClockGating:bxt */
	WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
			  STALL_DOP_GATING_DISABLE);

1030 1031 1032 1033 1034 1035
	/* WaDisablePooledEuLoadBalancingFix:bxt */
	if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
		WA_SET_BIT_MASKED(FF_SLICE_CS_CHICKEN2,
				  GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
	}

1036
	/* WaDisableSbeCacheDispatchPortSharing:bxt */
1037
	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) {
1038 1039 1040 1041 1042
		WA_SET_BIT_MASKED(
			GEN7_HALF_SLICE_CHICKEN1,
			GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
	}

1043 1044 1045
	/* WaDisableObjectLevelPreemptionForTrifanOrPolygon:bxt */
	/* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */
	/* WaDisableObjectLevelPreemtionForInstanceId:bxt */
1046
	/* WaDisableLSQCROPERFforOCL:bxt */
1047
	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1048
		ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
1049 1050
		if (ret)
			return ret;
1051

1052
		ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1053 1054
		if (ret)
			return ret;
1055 1056
	}

1057
	/* WaProgramL3SqcReg1DefaultForPerf:bxt */
1058
	if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER))
1059 1060
		I915_WRITE(GEN8_L3SQCREG1, L3_GENERAL_PRIO_CREDITS(62) |
					   L3_HIGH_PRIO_CREDITS(2));
1061

1062 1063
	/* WaToEnableHwFixForPushConstHWBug:bxt */
	if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
1064 1065 1066
		WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
				  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);

1067 1068 1069 1070 1071
	/* WaInPlaceDecompressionHang:bxt */
	if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
		WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
			   GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);

1072 1073 1074
	return 0;
}

1075 1076
static int kbl_init_workarounds(struct intel_engine_cs *engine)
{
1077
	struct drm_i915_private *dev_priv = engine->i915;
1078 1079 1080 1081 1082 1083
	int ret;

	ret = gen9_init_workarounds(engine);
	if (ret)
		return ret;

1084 1085 1086 1087
	/* WaEnableGapsTsvCreditFix:kbl */
	I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
				   GEN9_GAPS_TSV_CREDIT_DISABLE));

1088 1089 1090 1091 1092
	/* WaDisableDynamicCreditSharing:kbl */
	if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
		WA_SET_BIT(GAMT_CHKN_BIT_REG,
			   GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);

1093 1094 1095 1096 1097
	/* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
	if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
		WA_SET_BIT_MASKED(HDC_CHICKEN0,
				  HDC_FENCE_DEST_SLM_DISABLE);

1098 1099 1100 1101 1102 1103 1104 1105
	/* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
	 * involving this register should also be added to WA batch as required.
	 */
	if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0))
		/* WaDisableLSQCROPERFforOCL:kbl */
		I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
			   GEN8_LQSC_RO_PERF_DIS);

1106 1107
	/* WaToEnableHwFixForPushConstHWBug:kbl */
	if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER))
1108 1109 1110
		WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
				  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);

1111 1112 1113
	/* WaDisableGafsUnitClkGating:kbl */
	WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);

1114 1115 1116 1117 1118
	/* WaDisableSbeCacheDispatchPortSharing:kbl */
	WA_SET_BIT_MASKED(
		GEN7_HALF_SLICE_CHICKEN1,
		GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);

1119 1120 1121 1122
	/* WaInPlaceDecompressionHang:kbl */
	WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
		   GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);

1123 1124 1125 1126 1127
	/* WaDisableLSQCROPERFforOCL:kbl */
	ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
	if (ret)
		return ret;

1128 1129 1130
	return 0;
}

1131
int init_workarounds_ring(struct intel_engine_cs *engine)
1132
{
1133
	struct drm_i915_private *dev_priv = engine->i915;
1134

1135
	WARN_ON(engine->id != RCS);
1136 1137

	dev_priv->workarounds.count = 0;
1138
	dev_priv->workarounds.hw_whitelist_count[RCS] = 0;
1139

1140
	if (IS_BROADWELL(dev_priv))
1141
		return bdw_init_workarounds(engine);
1142

1143
	if (IS_CHERRYVIEW(dev_priv))
1144
		return chv_init_workarounds(engine);
1145

1146
	if (IS_SKYLAKE(dev_priv))
1147
		return skl_init_workarounds(engine);
1148

1149
	if (IS_BROXTON(dev_priv))
1150
		return bxt_init_workarounds(engine);
1151

1152 1153 1154
	if (IS_KABYLAKE(dev_priv))
		return kbl_init_workarounds(engine);

1155 1156 1157
	return 0;
}

1158
static int init_render_ring(struct intel_engine_cs *engine)
1159
{
1160
	struct drm_i915_private *dev_priv = engine->i915;
1161
	int ret = init_ring_common(engine);
1162 1163
	if (ret)
		return ret;
1164

1165
	/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1166
	if (IS_GEN(dev_priv, 4, 6))
1167
		I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
1168 1169 1170 1171

	/* We need to disable the AsyncFlip performance optimisations in order
	 * to use MI_WAIT_FOR_EVENT within the CS. It should already be
	 * programmed to '1' on all products.
1172
	 *
1173
	 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1174
	 */
1175
	if (IS_GEN(dev_priv, 6, 7))
1176 1177
		I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));

1178
	/* Required for the hardware to program scanline values for waiting */
1179
	/* WaEnableFlushTlbInvalidationMode:snb */
1180
	if (IS_GEN6(dev_priv))
1181
		I915_WRITE(GFX_MODE,
1182
			   _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
1183

1184
	/* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1185
	if (IS_GEN7(dev_priv))
1186
		I915_WRITE(GFX_MODE_GEN7,
1187
			   _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
1188
			   _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
1189

1190
	if (IS_GEN6(dev_priv)) {
1191 1192 1193 1194 1195 1196
		/* From the Sandybridge PRM, volume 1 part 3, page 24:
		 * "If this bit is set, STCunit will have LRA as replacement
		 *  policy. [...] This bit must be reset.  LRA replacement
		 *  policy is not supported."
		 */
		I915_WRITE(CACHE_MODE_0,
1197
			   _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
1198 1199
	}

1200
	if (IS_GEN(dev_priv, 6, 7))
1201
		I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1202

1203 1204
	if (INTEL_INFO(dev_priv)->gen >= 6)
		I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1205

1206
	return init_workarounds_ring(engine);
1207 1208
}

1209
static void render_ring_cleanup(struct intel_engine_cs *engine)
1210
{
1211
	struct drm_i915_private *dev_priv = engine->i915;
1212

1213
	i915_vma_unpin_and_release(&dev_priv->semaphore);
1214 1215
}

C
Chris Wilson 已提交
1216
static u32 *gen8_rcs_signal(struct drm_i915_gem_request *req, u32 *out)
1217
{
1218
	struct drm_i915_private *dev_priv = req->i915;
1219
	struct intel_engine_cs *waiter;
1220
	enum intel_engine_id id;
1221

1222
	for_each_engine(waiter, dev_priv, id) {
1223
		u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
1224 1225 1226
		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
			continue;

C
Chris Wilson 已提交
1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237
		*out++ = GFX_OP_PIPE_CONTROL(6);
		*out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB |
			  PIPE_CONTROL_QW_WRITE |
			  PIPE_CONTROL_CS_STALL);
		*out++ = lower_32_bits(gtt_offset);
		*out++ = upper_32_bits(gtt_offset);
		*out++ = req->global_seqno;
		*out++ = 0;
		*out++ = (MI_SEMAPHORE_SIGNAL |
			  MI_SEMAPHORE_TARGET(waiter->hw_id));
		*out++ = 0;
1238 1239
	}

C
Chris Wilson 已提交
1240
	return out;
1241 1242
}

C
Chris Wilson 已提交
1243
static u32 *gen8_xcs_signal(struct drm_i915_gem_request *req, u32 *out)
1244
{
1245
	struct drm_i915_private *dev_priv = req->i915;
1246
	struct intel_engine_cs *waiter;
1247
	enum intel_engine_id id;
1248

1249
	for_each_engine(waiter, dev_priv, id) {
1250
		u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
1251 1252 1253
		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
			continue;

C
Chris Wilson 已提交
1254 1255 1256 1257 1258 1259 1260
		*out++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
		*out++ = lower_32_bits(gtt_offset) | MI_FLUSH_DW_USE_GTT;
		*out++ = upper_32_bits(gtt_offset);
		*out++ = req->global_seqno;
		*out++ = (MI_SEMAPHORE_SIGNAL |
			  MI_SEMAPHORE_TARGET(waiter->hw_id));
		*out++ = 0;
1261 1262
	}

C
Chris Wilson 已提交
1263
	return out;
1264 1265
}

C
Chris Wilson 已提交
1266
static u32 *gen6_signal(struct drm_i915_gem_request *req, u32 *out)
1267
{
1268
	struct drm_i915_private *dev_priv = req->i915;
1269
	struct intel_engine_cs *engine;
1270
	enum intel_engine_id id;
C
Chris Wilson 已提交
1271
	int num_rings = 0;
1272

1273
	for_each_engine(engine, dev_priv, id) {
1274 1275 1276 1277
		i915_reg_t mbox_reg;

		if (!(BIT(engine->hw_id) & GEN6_SEMAPHORES_MASK))
			continue;
1278

1279
		mbox_reg = req->engine->semaphore.mbox.signal[engine->hw_id];
1280
		if (i915_mmio_reg_valid(mbox_reg)) {
C
Chris Wilson 已提交
1281 1282 1283 1284
			*out++ = MI_LOAD_REGISTER_IMM(1);
			*out++ = i915_mmio_reg_offset(mbox_reg);
			*out++ = req->global_seqno;
			num_rings++;
1285 1286
		}
	}
C
Chris Wilson 已提交
1287 1288
	if (num_rings & 1)
		*out++ = MI_NOOP;
1289

C
Chris Wilson 已提交
1290
	return out;
1291 1292
}

1293 1294 1295 1296
static void i9xx_submit_request(struct drm_i915_gem_request *request)
{
	struct drm_i915_private *dev_priv = request->i915;

1297 1298
	i915_gem_request_submit(request);

C
Chris Wilson 已提交
1299
	I915_WRITE_TAIL(request->engine, request->tail);
1300 1301
}

C
Chris Wilson 已提交
1302 1303
static void i9xx_emit_breadcrumb(struct drm_i915_gem_request *req,
				 u32 *out)
1304
{
C
Chris Wilson 已提交
1305 1306 1307 1308
	*out++ = MI_STORE_DWORD_INDEX;
	*out++ = I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT;
	*out++ = req->global_seqno;
	*out++ = MI_USER_INTERRUPT;
1309

C
Chris Wilson 已提交
1310
	req->tail = intel_ring_offset(req->ring, out);
1311 1312
}

1313 1314
static const int i9xx_emit_breadcrumb_sz = 4;

1315
/**
1316
 * gen6_sema_emit_breadcrumb - Update the semaphore mailbox registers
1317 1318 1319 1320 1321 1322
 *
 * @request - request to write to the ring
 *
 * Update the mailbox registers in the *other* rings with the current seqno.
 * This acts like a signal in the canonical semaphore.
 */
C
Chris Wilson 已提交
1323 1324
static void gen6_sema_emit_breadcrumb(struct drm_i915_gem_request *req,
				      u32 *out)
1325
{
C
Chris Wilson 已提交
1326 1327
	return i9xx_emit_breadcrumb(req,
				    req->engine->semaphore.signal(req, out));
1328 1329
}

C
Chris Wilson 已提交
1330 1331
static void gen8_render_emit_breadcrumb(struct drm_i915_gem_request *req,
					u32 *out)
1332 1333
{
	struct intel_engine_cs *engine = req->engine;
1334

C
Chris Wilson 已提交
1335 1336
	if (engine->semaphore.signal)
		out = engine->semaphore.signal(req, out);
1337

C
Chris Wilson 已提交
1338 1339
	*out++ = GFX_OP_PIPE_CONTROL(6);
	*out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB |
1340
			       PIPE_CONTROL_CS_STALL |
C
Chris Wilson 已提交
1341 1342 1343 1344
			       PIPE_CONTROL_QW_WRITE);
	*out++ = intel_hws_seqno_address(engine);
	*out++ = 0;
	*out++ = req->global_seqno;
1345
	/* We're thrashing one dword of HWS. */
C
Chris Wilson 已提交
1346 1347 1348
	*out++ = 0;
	*out++ = MI_USER_INTERRUPT;
	*out++ = MI_NOOP;
1349

C
Chris Wilson 已提交
1350
	req->tail = intel_ring_offset(req->ring, out);
1351 1352
}

1353 1354
static const int gen8_render_emit_breadcrumb_sz = 8;

1355 1356 1357 1358 1359 1360 1361
/**
 * intel_ring_sync - sync the waiter to the signaller on seqno
 *
 * @waiter - ring that is waiting
 * @signaller - ring which has, or will signal
 * @seqno - seqno which the waiter will block on
 */
1362 1363

static int
1364 1365
gen8_ring_sync_to(struct drm_i915_gem_request *req,
		  struct drm_i915_gem_request *signal)
1366
{
1367 1368 1369
	struct intel_ring *ring = req->ring;
	struct drm_i915_private *dev_priv = req->i915;
	u64 offset = GEN8_WAIT_OFFSET(req->engine, signal->engine->id);
1370
	struct i915_hw_ppgtt *ppgtt;
1371 1372
	int ret;

1373
	ret = intel_ring_begin(req, 4);
1374 1375 1376
	if (ret)
		return ret;

1377 1378 1379 1380
	intel_ring_emit(ring,
			MI_SEMAPHORE_WAIT |
			MI_SEMAPHORE_GLOBAL_GTT |
			MI_SEMAPHORE_SAD_GTE_SDD);
1381
	intel_ring_emit(ring, signal->global_seqno);
1382 1383 1384
	intel_ring_emit(ring, lower_32_bits(offset));
	intel_ring_emit(ring, upper_32_bits(offset));
	intel_ring_advance(ring);
1385 1386 1387 1388 1389 1390

	/* When the !RCS engines idle waiting upon a semaphore, they lose their
	 * pagetables and we must reload them before executing the batch.
	 * We do this on the i915_switch_context() following the wait and
	 * before the dispatch.
	 */
1391 1392 1393
	ppgtt = req->ctx->ppgtt;
	if (ppgtt && req->engine->id != RCS)
		ppgtt->pd_dirty_rings |= intel_engine_flag(req->engine);
1394 1395 1396
	return 0;
}

1397
static int
1398 1399
gen6_ring_sync_to(struct drm_i915_gem_request *req,
		  struct drm_i915_gem_request *signal)
1400
{
1401
	struct intel_ring *ring = req->ring;
1402 1403 1404
	u32 dw1 = MI_SEMAPHORE_MBOX |
		  MI_SEMAPHORE_COMPARE |
		  MI_SEMAPHORE_REGISTER;
1405
	u32 wait_mbox = signal->engine->semaphore.mbox.wait[req->engine->hw_id];
1406
	int ret;
1407

1408
	WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
1409

1410
	ret = intel_ring_begin(req, 4);
1411 1412 1413
	if (ret)
		return ret;

1414
	intel_ring_emit(ring, dw1 | wait_mbox);
1415 1416 1417 1418
	/* Throughout all of the GEM code, seqno passed implies our current
	 * seqno is >= the last seqno executed. However for hardware the
	 * comparison is strictly greater than.
	 */
1419
	intel_ring_emit(ring, signal->global_seqno - 1);
1420 1421 1422
	intel_ring_emit(ring, 0);
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
1423 1424 1425 1426

	return 0;
}

1427
static void
1428
gen5_seqno_barrier(struct intel_engine_cs *engine)
1429
{
1430 1431 1432
	/* MI_STORE are internally buffered by the GPU and not flushed
	 * either by MI_FLUSH or SyncFlush or any other combination of
	 * MI commands.
1433
	 *
1434 1435 1436 1437 1438 1439 1440
	 * "Only the submission of the store operation is guaranteed.
	 * The write result will be complete (coherent) some time later
	 * (this is practically a finite period but there is no guaranteed
	 * latency)."
	 *
	 * Empirically, we observe that we need a delay of at least 75us to
	 * be sure that the seqno write is visible by the CPU.
1441
	 */
1442
	usleep_range(125, 250);
1443 1444
}

1445 1446
static void
gen6_seqno_barrier(struct intel_engine_cs *engine)
1447
{
1448
	struct drm_i915_private *dev_priv = engine->i915;
1449

1450 1451
	/* Workaround to force correct ordering between irq and seqno writes on
	 * ivb (and maybe also on snb) by reading from a CS register (like
1452 1453 1454 1455 1456 1457 1458 1459 1460
	 * ACTHD) before reading the status page.
	 *
	 * Note that this effectively stalls the read by the time it takes to
	 * do a memory transaction, which more or less ensures that the write
	 * from the GPU has sufficient time to invalidate the CPU cacheline.
	 * Alternatively we could delay the interrupt from the CS ring to give
	 * the write time to land, but that would incur a delay after every
	 * batch i.e. much more frequent than a delay when waiting for the
	 * interrupt (with the same net latency).
1461 1462 1463
	 *
	 * Also note that to prevent whole machine hangs on gen7, we have to
	 * take the spinlock to guard against concurrent cacheline access.
1464
	 */
1465
	spin_lock_irq(&dev_priv->uncore.lock);
1466
	POSTING_READ_FW(RING_ACTHD(engine->mmio_base));
1467
	spin_unlock_irq(&dev_priv->uncore.lock);
1468 1469
}

1470 1471
static void
gen5_irq_enable(struct intel_engine_cs *engine)
1472
{
1473
	gen5_enable_gt_irq(engine->i915, engine->irq_enable_mask);
1474 1475 1476
}

static void
1477
gen5_irq_disable(struct intel_engine_cs *engine)
1478
{
1479
	gen5_disable_gt_irq(engine->i915, engine->irq_enable_mask);
1480 1481
}

1482 1483
static void
i9xx_irq_enable(struct intel_engine_cs *engine)
1484
{
1485
	struct drm_i915_private *dev_priv = engine->i915;
1486

1487 1488 1489
	dev_priv->irq_mask &= ~engine->irq_enable_mask;
	I915_WRITE(IMR, dev_priv->irq_mask);
	POSTING_READ_FW(RING_IMR(engine->mmio_base));
1490 1491
}

1492
static void
1493
i9xx_irq_disable(struct intel_engine_cs *engine)
1494
{
1495
	struct drm_i915_private *dev_priv = engine->i915;
1496

1497 1498
	dev_priv->irq_mask |= engine->irq_enable_mask;
	I915_WRITE(IMR, dev_priv->irq_mask);
1499 1500
}

1501 1502
static void
i8xx_irq_enable(struct intel_engine_cs *engine)
C
Chris Wilson 已提交
1503
{
1504
	struct drm_i915_private *dev_priv = engine->i915;
C
Chris Wilson 已提交
1505

1506 1507 1508
	dev_priv->irq_mask &= ~engine->irq_enable_mask;
	I915_WRITE16(IMR, dev_priv->irq_mask);
	POSTING_READ16(RING_IMR(engine->mmio_base));
C
Chris Wilson 已提交
1509 1510 1511
}

static void
1512
i8xx_irq_disable(struct intel_engine_cs *engine)
C
Chris Wilson 已提交
1513
{
1514
	struct drm_i915_private *dev_priv = engine->i915;
C
Chris Wilson 已提交
1515

1516 1517
	dev_priv->irq_mask |= engine->irq_enable_mask;
	I915_WRITE16(IMR, dev_priv->irq_mask);
C
Chris Wilson 已提交
1518 1519
}

1520
static int
1521
bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
1522
{
1523
	struct intel_ring *ring = req->ring;
1524 1525
	int ret;

1526
	ret = intel_ring_begin(req, 2);
1527 1528 1529
	if (ret)
		return ret;

1530 1531 1532
	intel_ring_emit(ring, MI_FLUSH);
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
1533
	return 0;
1534 1535
}

1536 1537
static void
gen6_irq_enable(struct intel_engine_cs *engine)
1538
{
1539
	struct drm_i915_private *dev_priv = engine->i915;
1540

1541 1542 1543
	I915_WRITE_IMR(engine,
		       ~(engine->irq_enable_mask |
			 engine->irq_keep_mask));
1544
	gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1545 1546 1547
}

static void
1548
gen6_irq_disable(struct intel_engine_cs *engine)
1549
{
1550
	struct drm_i915_private *dev_priv = engine->i915;
1551

1552
	I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1553
	gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1554 1555
}

1556 1557
static void
hsw_vebox_irq_enable(struct intel_engine_cs *engine)
B
Ben Widawsky 已提交
1558
{
1559
	struct drm_i915_private *dev_priv = engine->i915;
B
Ben Widawsky 已提交
1560

1561
	I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1562
	gen6_unmask_pm_irq(dev_priv, engine->irq_enable_mask);
B
Ben Widawsky 已提交
1563 1564 1565
}

static void
1566
hsw_vebox_irq_disable(struct intel_engine_cs *engine)
B
Ben Widawsky 已提交
1567
{
1568
	struct drm_i915_private *dev_priv = engine->i915;
B
Ben Widawsky 已提交
1569

1570
	I915_WRITE_IMR(engine, ~0);
1571
	gen6_mask_pm_irq(dev_priv, engine->irq_enable_mask);
B
Ben Widawsky 已提交
1572 1573
}

1574 1575
static void
gen8_irq_enable(struct intel_engine_cs *engine)
1576
{
1577
	struct drm_i915_private *dev_priv = engine->i915;
1578

1579 1580 1581
	I915_WRITE_IMR(engine,
		       ~(engine->irq_enable_mask |
			 engine->irq_keep_mask));
1582
	POSTING_READ_FW(RING_IMR(engine->mmio_base));
1583 1584 1585
}

static void
1586
gen8_irq_disable(struct intel_engine_cs *engine)
1587
{
1588
	struct drm_i915_private *dev_priv = engine->i915;
1589

1590
	I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1591 1592
}

1593
static int
1594 1595 1596
i965_emit_bb_start(struct drm_i915_gem_request *req,
		   u64 offset, u32 length,
		   unsigned int dispatch_flags)
1597
{
1598
	struct intel_ring *ring = req->ring;
1599
	int ret;
1600

1601
	ret = intel_ring_begin(req, 2);
1602 1603 1604
	if (ret)
		return ret;

1605
	intel_ring_emit(ring,
1606 1607
			MI_BATCH_BUFFER_START |
			MI_BATCH_GTT |
1608 1609
			(dispatch_flags & I915_DISPATCH_SECURE ?
			 0 : MI_BATCH_NON_SECURE_I965));
1610 1611
	intel_ring_emit(ring, offset);
	intel_ring_advance(ring);
1612

1613 1614 1615
	return 0;
}

1616 1617
/* Just userspace ABI convention to limit the wa batch bo to a resonable size */
#define I830_BATCH_LIMIT (256*1024)
1618 1619
#define I830_TLB_ENTRIES (2)
#define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
1620
static int
1621 1622 1623
i830_emit_bb_start(struct drm_i915_gem_request *req,
		   u64 offset, u32 len,
		   unsigned int dispatch_flags)
1624
{
1625
	struct intel_ring *ring = req->ring;
1626
	u32 cs_offset = i915_ggtt_offset(req->engine->scratch);
1627
	int ret;
1628

1629
	ret = intel_ring_begin(req, 6);
1630 1631
	if (ret)
		return ret;
1632

1633
	/* Evict the invalid PTE TLBs */
1634 1635 1636 1637 1638 1639 1640
	intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA);
	intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
	intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */
	intel_ring_emit(ring, cs_offset);
	intel_ring_emit(ring, 0xdeadbeef);
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
1641

1642
	if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
1643 1644 1645
		if (len > I830_BATCH_LIMIT)
			return -ENOSPC;

1646
		ret = intel_ring_begin(req, 6 + 2);
1647 1648
		if (ret)
			return ret;
1649 1650 1651 1652 1653

		/* Blit the batch (which has now all relocs applied) to the
		 * stable batch scratch bo area (so that the CS never
		 * stumbles over its tlb invalidation bug) ...
		 */
1654 1655
		intel_ring_emit(ring, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA);
		intel_ring_emit(ring,
1656
				BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096);
1657 1658 1659 1660
		intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 | 4096);
		intel_ring_emit(ring, cs_offset);
		intel_ring_emit(ring, 4096);
		intel_ring_emit(ring, offset);
1661

1662 1663 1664
		intel_ring_emit(ring, MI_FLUSH);
		intel_ring_emit(ring, MI_NOOP);
		intel_ring_advance(ring);
1665 1666

		/* ... and execute it. */
1667
		offset = cs_offset;
1668
	}
1669

1670
	ret = intel_ring_begin(req, 2);
1671 1672 1673
	if (ret)
		return ret;

1674 1675 1676 1677
	intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
	intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
					0 : MI_BATCH_NON_SECURE));
	intel_ring_advance(ring);
1678

1679 1680 1681 1682
	return 0;
}

static int
1683 1684 1685
i915_emit_bb_start(struct drm_i915_gem_request *req,
		   u64 offset, u32 len,
		   unsigned int dispatch_flags)
1686
{
1687
	struct intel_ring *ring = req->ring;
1688 1689
	int ret;

1690
	ret = intel_ring_begin(req, 2);
1691 1692 1693
	if (ret)
		return ret;

1694 1695 1696 1697
	intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
	intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
					0 : MI_BATCH_NON_SECURE));
	intel_ring_advance(ring);
1698 1699 1700 1701

	return 0;
}

1702
static void cleanup_phys_status_page(struct intel_engine_cs *engine)
1703
{
1704
	struct drm_i915_private *dev_priv = engine->i915;
1705 1706 1707 1708

	if (!dev_priv->status_page_dmah)
		return;

1709
	drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah);
1710
	engine->status_page.page_addr = NULL;
1711 1712
}

1713
static void cleanup_status_page(struct intel_engine_cs *engine)
1714
{
1715
	struct i915_vma *vma;
1716
	struct drm_i915_gem_object *obj;
1717

1718 1719
	vma = fetch_and_zero(&engine->status_page.vma);
	if (!vma)
1720 1721
		return;

1722 1723
	obj = vma->obj;

1724
	i915_vma_unpin(vma);
1725 1726 1727 1728
	i915_vma_close(vma);

	i915_gem_object_unpin_map(obj);
	__i915_gem_object_release_unless_active(obj);
1729 1730
}

1731
static int init_status_page(struct intel_engine_cs *engine)
1732
{
1733 1734 1735
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma;
	unsigned int flags;
1736
	void *vaddr;
1737
	int ret;
1738

1739
	obj = i915_gem_object_create_internal(engine->i915, 4096);
1740 1741 1742 1743
	if (IS_ERR(obj)) {
		DRM_ERROR("Failed to allocate status page\n");
		return PTR_ERR(obj);
	}
1744

1745 1746 1747
	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
	if (ret)
		goto err;
1748

1749 1750 1751 1752
	vma = i915_vma_create(obj, &engine->i915->ggtt.base, NULL);
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
		goto err;
1753
	}
1754

1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770
	flags = PIN_GLOBAL;
	if (!HAS_LLC(engine->i915))
		/* On g33, we cannot place HWS above 256MiB, so
		 * restrict its pinning to the low mappable arena.
		 * Though this restriction is not documented for
		 * gen4, gen5, or byt, they also behave similarly
		 * and hang if the HWS is placed at the top of the
		 * GTT. To generalise, it appears that all !llc
		 * platforms have issues with us placing the HWS
		 * above the mappable region (even though we never
		 * actualy map it).
		 */
		flags |= PIN_MAPPABLE;
	ret = i915_vma_pin(vma, 0, 4096, flags);
	if (ret)
		goto err;
1771

1772 1773 1774 1775 1776 1777
	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
	if (IS_ERR(vaddr)) {
		ret = PTR_ERR(vaddr);
		goto err_unpin;
	}

1778
	engine->status_page.vma = vma;
1779
	engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
1780
	engine->status_page.page_addr = memset(vaddr, 0, 4096);
1781

1782 1783
	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
			 engine->name, i915_ggtt_offset(vma));
1784
	return 0;
1785

1786 1787
err_unpin:
	i915_vma_unpin(vma);
1788 1789 1790
err:
	i915_gem_object_put(obj);
	return ret;
1791 1792
}

1793
static int init_phys_status_page(struct intel_engine_cs *engine)
1794
{
1795
	struct drm_i915_private *dev_priv = engine->i915;
1796

1797 1798 1799 1800
	dev_priv->status_page_dmah =
		drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
	if (!dev_priv->status_page_dmah)
		return -ENOMEM;
1801

1802 1803
	engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
	memset(engine->status_page.page_addr, 0, PAGE_SIZE);
1804 1805 1806 1807

	return 0;
}

1808
int intel_ring_pin(struct intel_ring *ring, unsigned int offset_bias)
1809
{
1810
	unsigned int flags;
1811
	enum i915_map_type map;
1812
	struct i915_vma *vma = ring->vma;
1813
	void *addr;
1814 1815
	int ret;

1816
	GEM_BUG_ON(ring->vaddr);
1817

1818 1819
	map = HAS_LLC(ring->engine->i915) ? I915_MAP_WB : I915_MAP_WC;

1820 1821 1822
	flags = PIN_GLOBAL;
	if (offset_bias)
		flags |= PIN_OFFSET_BIAS | offset_bias;
1823
	if (vma->obj->stolen)
1824
		flags |= PIN_MAPPABLE;
1825

1826
	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
1827
		if (flags & PIN_MAPPABLE || map == I915_MAP_WC)
1828 1829 1830 1831
			ret = i915_gem_object_set_to_gtt_domain(vma->obj, true);
		else
			ret = i915_gem_object_set_to_cpu_domain(vma->obj, true);
		if (unlikely(ret))
1832
			return ret;
1833
	}
1834

1835 1836 1837
	ret = i915_vma_pin(vma, 0, PAGE_SIZE, flags);
	if (unlikely(ret))
		return ret;
1838

1839
	if (i915_vma_is_map_and_fenceable(vma))
1840 1841
		addr = (void __force *)i915_vma_pin_iomap(vma);
	else
1842
		addr = i915_gem_object_pin_map(vma->obj, map);
1843 1844
	if (IS_ERR(addr))
		goto err;
1845

1846
	ring->vaddr = addr;
1847
	return 0;
1848

1849 1850 1851
err:
	i915_vma_unpin(vma);
	return PTR_ERR(addr);
1852 1853
}

1854 1855 1856 1857 1858
void intel_ring_unpin(struct intel_ring *ring)
{
	GEM_BUG_ON(!ring->vma);
	GEM_BUG_ON(!ring->vaddr);

1859
	if (i915_vma_is_map_and_fenceable(ring->vma))
1860
		i915_vma_unpin_iomap(ring->vma);
1861 1862
	else
		i915_gem_object_unpin_map(ring->vma->obj);
1863 1864
	ring->vaddr = NULL;

1865
	i915_vma_unpin(ring->vma);
1866 1867
}

1868 1869
static struct i915_vma *
intel_ring_create_vma(struct drm_i915_private *dev_priv, int size)
1870
{
1871
	struct drm_i915_gem_object *obj;
1872
	struct i915_vma *vma;
1873

1874
	obj = i915_gem_object_create_stolen(dev_priv, size);
1875
	if (!obj)
1876
		obj = i915_gem_object_create(dev_priv, size);
1877 1878
	if (IS_ERR(obj))
		return ERR_CAST(obj);
1879

1880 1881 1882
	/* mark ring buffers as read-only from GPU side by default */
	obj->gt_ro = 1;

1883 1884 1885 1886 1887
	vma = i915_vma_create(obj, &dev_priv->ggtt.base, NULL);
	if (IS_ERR(vma))
		goto err;

	return vma;
1888

1889 1890 1891
err:
	i915_gem_object_put(obj);
	return vma;
1892 1893
}

1894 1895
struct intel_ring *
intel_engine_create_ring(struct intel_engine_cs *engine, int size)
1896
{
1897
	struct intel_ring *ring;
1898
	struct i915_vma *vma;
1899

1900
	GEM_BUG_ON(!is_power_of_2(size));
1901
	GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
1902

1903
	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1904
	if (!ring)
1905 1906
		return ERR_PTR(-ENOMEM);

1907
	ring->engine = engine;
1908

1909 1910
	INIT_LIST_HEAD(&ring->request_list);

1911 1912 1913 1914 1915 1916
	ring->size = size;
	/* Workaround an erratum on the i830 which causes a hang if
	 * the TAIL pointer points to within the last 2 cachelines
	 * of the buffer.
	 */
	ring->effective_size = size;
1917
	if (IS_I830(engine->i915) || IS_I845G(engine->i915))
1918 1919 1920 1921 1922
		ring->effective_size -= 2 * CACHELINE_BYTES;

	ring->last_retired_head = -1;
	intel_ring_update_space(ring);

1923 1924
	vma = intel_ring_create_vma(engine->i915, size);
	if (IS_ERR(vma)) {
1925
		kfree(ring);
1926
		return ERR_CAST(vma);
1927
	}
1928
	ring->vma = vma;
1929 1930 1931 1932 1933

	return ring;
}

void
1934
intel_ring_free(struct intel_ring *ring)
1935
{
1936 1937 1938 1939 1940
	struct drm_i915_gem_object *obj = ring->vma->obj;

	i915_vma_close(ring->vma);
	__i915_gem_object_release_unless_active(obj);

1941 1942 1943
	kfree(ring);
}

1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963
static int context_pin(struct i915_gem_context *ctx, unsigned int flags)
{
	struct i915_vma *vma = ctx->engine[RCS].state;
	int ret;

	/* Clear this page out of any CPU caches for coherent swap-in/out.
	 * We only want to do this on the first bind so that we do not stall
	 * on an active context (which by nature is already on the GPU).
	 */
	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
		ret = i915_gem_object_set_to_gtt_domain(vma->obj, false);
		if (ret)
			return ret;
	}

	return i915_vma_pin(vma, 0, ctx->ggtt_alignment, PIN_GLOBAL | flags);
}

static int intel_ring_context_pin(struct intel_engine_cs *engine,
				  struct i915_gem_context *ctx)
1964 1965 1966 1967
{
	struct intel_context *ce = &ctx->engine[engine->id];
	int ret;

1968
	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
1969 1970 1971 1972 1973

	if (ce->pin_count++)
		return 0;

	if (ce->state) {
1974 1975 1976 1977 1978
		unsigned int flags;

		flags = 0;
		if (ctx == ctx->i915->kernel_context)
			flags = PIN_HIGH;
1979

1980 1981
		ret = context_pin(ctx, flags);
		if (ret)
1982 1983 1984
			goto error;
	}

1985 1986 1987 1988 1989 1990 1991 1992 1993 1994
	/* The kernel context is only used as a placeholder for flushing the
	 * active context. It is never used for submitting user rendering and
	 * as such never requires the golden render context, and so we can skip
	 * emitting it when we switch to the kernel context. This is required
	 * as during eviction we cannot allocate and pin the renderstate in
	 * order to initialise the context.
	 */
	if (ctx == ctx->i915->kernel_context)
		ce->initialised = true;

1995
	i915_gem_context_get(ctx);
1996 1997 1998 1999 2000 2001 2002
	return 0;

error:
	ce->pin_count = 0;
	return ret;
}

2003 2004
static void intel_ring_context_unpin(struct intel_engine_cs *engine,
				     struct i915_gem_context *ctx)
2005 2006 2007
{
	struct intel_context *ce = &ctx->engine[engine->id];

2008
	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
2009
	GEM_BUG_ON(ce->pin_count == 0);
2010 2011 2012 2013 2014

	if (--ce->pin_count)
		return;

	if (ce->state)
2015
		i915_vma_unpin(ce->state);
2016

2017
	i915_gem_context_put(ctx);
2018 2019
}

2020
static int intel_init_ring_buffer(struct intel_engine_cs *engine)
2021
{
2022
	struct drm_i915_private *dev_priv = engine->i915;
2023
	struct intel_ring *ring;
2024 2025
	int ret;

2026
	WARN_ON(engine->buffer);
2027

2028 2029 2030
	intel_engine_setup_common(engine);

	ret = intel_engine_init_common(engine);
2031 2032
	if (ret)
		goto error;
2033

2034 2035 2036
	ring = intel_engine_create_ring(engine, 32 * PAGE_SIZE);
	if (IS_ERR(ring)) {
		ret = PTR_ERR(ring);
2037 2038
		goto error;
	}
2039

2040 2041 2042
	if (HWS_NEEDS_PHYSICAL(dev_priv)) {
		WARN_ON(engine->id != RCS);
		ret = init_phys_status_page(engine);
2043
		if (ret)
2044
			goto error;
2045
	} else {
2046
		ret = init_status_page(engine);
2047
		if (ret)
2048
			goto error;
2049 2050
	}

2051 2052
	/* Ring wraparound at offset 0 sometimes hangs. No idea why. */
	ret = intel_ring_pin(ring, 4096);
2053
	if (ret) {
2054
		intel_ring_free(ring);
2055
		goto error;
2056
	}
2057
	engine->buffer = ring;
2058

2059
	return 0;
2060

2061
error:
2062
	intel_engine_cleanup(engine);
2063
	return ret;
2064 2065
}

2066
void intel_engine_cleanup(struct intel_engine_cs *engine)
2067
{
2068
	struct drm_i915_private *dev_priv;
2069

2070
	dev_priv = engine->i915;
2071

2072
	if (engine->buffer) {
2073 2074
		WARN_ON(INTEL_GEN(dev_priv) > 2 &&
			(I915_READ_MODE(engine) & MODE_IDLE) == 0);
2075

2076
		intel_ring_unpin(engine->buffer);
2077
		intel_ring_free(engine->buffer);
2078
		engine->buffer = NULL;
2079
	}
2080

2081 2082
	if (engine->cleanup)
		engine->cleanup(engine);
Z
Zou Nan hai 已提交
2083

2084
	if (HWS_NEEDS_PHYSICAL(dev_priv)) {
2085 2086
		WARN_ON(engine->id != RCS);
		cleanup_phys_status_page(engine);
2087 2088
	} else {
		cleanup_status_page(engine);
2089
	}
2090

2091
	intel_engine_cleanup_common(engine);
2092

2093
	engine->i915 = NULL;
2094 2095
	dev_priv->engine[engine->id] = NULL;
	kfree(engine);
2096 2097
}

2098 2099 2100
void intel_legacy_submission_resume(struct drm_i915_private *dev_priv)
{
	struct intel_engine_cs *engine;
2101
	enum intel_engine_id id;
2102

2103
	for_each_engine(engine, dev_priv, id) {
2104 2105 2106 2107 2108
		engine->buffer->head = engine->buffer->tail;
		engine->buffer->last_retired_head = -1;
	}
}

2109
static int ring_request_alloc(struct drm_i915_gem_request *request)
2110
{
2111 2112
	int ret;

2113 2114
	GEM_BUG_ON(!request->ctx->engine[request->engine->id].pin_count);

2115 2116 2117 2118
	/* Flush enough space to reduce the likelihood of waiting after
	 * we start building the request - in which case we will just
	 * have to repeat work.
	 */
2119
	request->reserved_space += LEGACY_REQUEST_SIZE;
2120

2121
	GEM_BUG_ON(!request->engine->buffer);
2122
	request->ring = request->engine->buffer;
2123 2124 2125 2126 2127

	ret = intel_ring_begin(request, 0);
	if (ret)
		return ret;

2128
	request->reserved_space -= LEGACY_REQUEST_SIZE;
2129
	return 0;
2130 2131
}

2132 2133
static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
{
2134
	struct intel_ring *ring = req->ring;
2135
	struct drm_i915_gem_request *target;
2136 2137 2138
	long timeout;

	lockdep_assert_held(&req->i915->drm.struct_mutex);
2139

2140 2141
	intel_ring_update_space(ring);
	if (ring->space >= bytes)
2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152
		return 0;

	/*
	 * Space is reserved in the ringbuffer for finalising the request,
	 * as that cannot be allowed to fail. During request finalisation,
	 * reserved_space is set to 0 to stop the overallocation and the
	 * assumption is that then we never need to wait (which has the
	 * risk of failing with EINTR).
	 *
	 * See also i915_gem_request_alloc() and i915_add_request().
	 */
2153
	GEM_BUG_ON(!req->reserved_space);
2154

2155
	list_for_each_entry(target, &ring->request_list, ring_link) {
2156 2157 2158
		unsigned space;

		/* Would completion of this request free enough space? */
2159 2160
		space = __intel_ring_space(target->postfix, ring->tail,
					   ring->size);
2161 2162
		if (space >= bytes)
			break;
2163
	}
2164

2165
	if (WARN_ON(&target->ring_link == &ring->request_list))
2166 2167
		return -ENOSPC;

2168 2169 2170 2171 2172
	timeout = i915_wait_request(target,
				    I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
				    MAX_SCHEDULE_TIMEOUT);
	if (timeout < 0)
		return timeout;
2173 2174 2175 2176 2177 2178

	i915_gem_request_retire_upto(target);

	intel_ring_update_space(ring);
	GEM_BUG_ON(ring->space < bytes);
	return 0;
2179 2180
}

2181
int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
M
Mika Kuoppala 已提交
2182
{
2183
	struct intel_ring *ring = req->ring;
2184 2185
	int remain_actual = ring->size - ring->tail;
	int remain_usable = ring->effective_size - ring->tail;
2186 2187
	int bytes = num_dwords * sizeof(u32);
	int total_bytes, wait_bytes;
2188
	bool need_wrap = false;
2189

2190
	total_bytes = bytes + req->reserved_space;
2191

2192 2193 2194 2195 2196 2197 2198
	if (unlikely(bytes > remain_usable)) {
		/*
		 * Not enough space for the basic request. So need to flush
		 * out the remainder and then wait for base + reserved.
		 */
		wait_bytes = remain_actual + total_bytes;
		need_wrap = true;
2199 2200 2201 2202 2203 2204 2205
	} else if (unlikely(total_bytes > remain_usable)) {
		/*
		 * The base request will fit but the reserved space
		 * falls off the end. So we don't need an immediate wrap
		 * and only need to effectively wait for the reserved
		 * size space from the start of ringbuffer.
		 */
2206
		wait_bytes = remain_actual + req->reserved_space;
2207
	} else {
2208 2209
		/* No wrapping required, just waiting. */
		wait_bytes = total_bytes;
M
Mika Kuoppala 已提交
2210 2211
	}

2212
	if (wait_bytes > ring->space) {
2213
		int ret = wait_for_space(req, wait_bytes);
M
Mika Kuoppala 已提交
2214 2215 2216 2217
		if (unlikely(ret))
			return ret;
	}

2218
	if (unlikely(need_wrap)) {
2219 2220
		GEM_BUG_ON(remain_actual > ring->space);
		GEM_BUG_ON(ring->tail + remain_actual > ring->size);
2221

2222
		/* Fill the tail with MI_NOOP */
2223 2224 2225
		memset(ring->vaddr + ring->tail, 0, remain_actual);
		ring->tail = 0;
		ring->space -= remain_actual;
2226
	}
2227

2228 2229
	ring->space -= bytes;
	GEM_BUG_ON(ring->space < 0);
2230
	return 0;
2231
}
2232

2233
/* Align the ring tail to a cacheline boundary */
2234
int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
2235
{
2236
	struct intel_ring *ring = req->ring;
2237 2238
	int num_dwords =
		(ring->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
2239 2240 2241 2242 2243
	int ret;

	if (num_dwords == 0)
		return 0;

2244
	num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
2245
	ret = intel_ring_begin(req, num_dwords);
2246 2247 2248 2249
	if (ret)
		return ret;

	while (num_dwords--)
2250
		intel_ring_emit(ring, MI_NOOP);
2251

2252
	intel_ring_advance(ring);
2253 2254 2255 2256

	return 0;
}

2257
static void gen6_bsd_submit_request(struct drm_i915_gem_request *request)
2258
{
2259
	struct drm_i915_private *dev_priv = request->i915;
2260

2261 2262
	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);

2263
       /* Every tail move must follow the sequence below */
2264 2265 2266 2267

	/* Disable notification that the ring is IDLE. The GT
	 * will then assume that it is busy and bring it out of rc6.
	 */
2268 2269
	I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
		      _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2270 2271

	/* Clear the context id. Here be magic! */
2272
	I915_WRITE64_FW(GEN6_BSD_RNCID, 0x0);
2273

2274
	/* Wait for the ring not to be idle, i.e. for it to wake up. */
2275 2276 2277 2278 2279
	if (intel_wait_for_register_fw(dev_priv,
				       GEN6_BSD_SLEEP_PSMI_CONTROL,
				       GEN6_BSD_SLEEP_INDICATOR,
				       0,
				       50))
2280
		DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2281

2282
	/* Now that the ring is fully powered up, update the tail */
2283
	i9xx_submit_request(request);
2284 2285 2286 2287

	/* Let the ring send IDLE messages to the GT again,
	 * and so let it sleep to conserve power when idle.
	 */
2288 2289 2290 2291
	I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
		      _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));

	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
2292 2293
}

2294
static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
2295
{
2296
	struct intel_ring *ring = req->ring;
2297
	uint32_t cmd;
2298 2299
	int ret;

2300
	ret = intel_ring_begin(req, 4);
2301 2302 2303
	if (ret)
		return ret;

2304
	cmd = MI_FLUSH_DW;
2305
	if (INTEL_GEN(req->i915) >= 8)
B
Ben Widawsky 已提交
2306
		cmd += 1;
2307 2308 2309 2310 2311 2312 2313 2314

	/* We always require a command barrier so that subsequent
	 * commands, such as breadcrumb interrupts, are strictly ordered
	 * wrt the contents of the write cache being flushed to memory
	 * (and thus being coherent from the CPU).
	 */
	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;

2315 2316 2317 2318 2319 2320
	/*
	 * Bspec vol 1c.5 - video engine command streamer:
	 * "If ENABLED, all TLBs will be invalidated once the flush
	 * operation is complete. This bit is only valid when the
	 * Post-Sync Operation field is a value of 1h or 3h."
	 */
2321
	if (mode & EMIT_INVALIDATE)
2322 2323
		cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;

2324 2325
	intel_ring_emit(ring, cmd);
	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2326
	if (INTEL_GEN(req->i915) >= 8) {
2327 2328
		intel_ring_emit(ring, 0); /* upper addr */
		intel_ring_emit(ring, 0); /* value */
B
Ben Widawsky 已提交
2329
	} else  {
2330 2331
		intel_ring_emit(ring, 0);
		intel_ring_emit(ring, MI_NOOP);
B
Ben Widawsky 已提交
2332
	}
2333
	intel_ring_advance(ring);
2334
	return 0;
2335 2336
}

2337
static int
2338 2339 2340
gen8_emit_bb_start(struct drm_i915_gem_request *req,
		   u64 offset, u32 len,
		   unsigned int dispatch_flags)
2341
{
2342
	struct intel_ring *ring = req->ring;
2343
	bool ppgtt = USES_PPGTT(req->i915) &&
2344
			!(dispatch_flags & I915_DISPATCH_SECURE);
2345 2346
	int ret;

2347
	ret = intel_ring_begin(req, 4);
2348 2349 2350 2351
	if (ret)
		return ret;

	/* FIXME(BDW): Address space and security selectors. */
2352
	intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8) |
2353 2354
			(dispatch_flags & I915_DISPATCH_RS ?
			 MI_BATCH_RESOURCE_STREAMER : 0));
2355 2356 2357 2358
	intel_ring_emit(ring, lower_32_bits(offset));
	intel_ring_emit(ring, upper_32_bits(offset));
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
2359 2360 2361 2362

	return 0;
}

2363
static int
2364 2365 2366
hsw_emit_bb_start(struct drm_i915_gem_request *req,
		  u64 offset, u32 len,
		  unsigned int dispatch_flags)
2367
{
2368
	struct intel_ring *ring = req->ring;
2369 2370
	int ret;

2371
	ret = intel_ring_begin(req, 2);
2372 2373 2374
	if (ret)
		return ret;

2375
	intel_ring_emit(ring,
2376
			MI_BATCH_BUFFER_START |
2377
			(dispatch_flags & I915_DISPATCH_SECURE ?
2378 2379 2380
			 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW) |
			(dispatch_flags & I915_DISPATCH_RS ?
			 MI_BATCH_RESOURCE_STREAMER : 0));
2381
	/* bit0-7 is the length on GEN6+ */
2382 2383
	intel_ring_emit(ring, offset);
	intel_ring_advance(ring);
2384 2385 2386 2387

	return 0;
}

2388
static int
2389 2390 2391
gen6_emit_bb_start(struct drm_i915_gem_request *req,
		   u64 offset, u32 len,
		   unsigned int dispatch_flags)
2392
{
2393
	struct intel_ring *ring = req->ring;
2394
	int ret;
2395

2396
	ret = intel_ring_begin(req, 2);
2397 2398
	if (ret)
		return ret;
2399

2400
	intel_ring_emit(ring,
2401
			MI_BATCH_BUFFER_START |
2402 2403
			(dispatch_flags & I915_DISPATCH_SECURE ?
			 0 : MI_BATCH_NON_SECURE_I965));
2404
	/* bit0-7 is the length on GEN6+ */
2405 2406
	intel_ring_emit(ring, offset);
	intel_ring_advance(ring);
2407

2408
	return 0;
2409 2410
}

2411 2412
/* Blitter support (SandyBridge+) */

2413
static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 mode)
Z
Zou Nan hai 已提交
2414
{
2415
	struct intel_ring *ring = req->ring;
2416
	uint32_t cmd;
2417 2418
	int ret;

2419
	ret = intel_ring_begin(req, 4);
2420 2421 2422
	if (ret)
		return ret;

2423
	cmd = MI_FLUSH_DW;
2424
	if (INTEL_GEN(req->i915) >= 8)
B
Ben Widawsky 已提交
2425
		cmd += 1;
2426 2427 2428 2429 2430 2431 2432 2433

	/* We always require a command barrier so that subsequent
	 * commands, such as breadcrumb interrupts, are strictly ordered
	 * wrt the contents of the write cache being flushed to memory
	 * (and thus being coherent from the CPU).
	 */
	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;

2434 2435 2436 2437 2438 2439
	/*
	 * Bspec vol 1c.3 - blitter engine command streamer:
	 * "If ENABLED, all TLBs will be invalidated once the flush
	 * operation is complete. This bit is only valid when the
	 * Post-Sync Operation field is a value of 1h or 3h."
	 */
2440
	if (mode & EMIT_INVALIDATE)
2441
		cmd |= MI_INVALIDATE_TLB;
2442 2443
	intel_ring_emit(ring, cmd);
	intel_ring_emit(ring,
2444
			I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2445
	if (INTEL_GEN(req->i915) >= 8) {
2446 2447
		intel_ring_emit(ring, 0); /* upper addr */
		intel_ring_emit(ring, 0); /* value */
B
Ben Widawsky 已提交
2448
	} else  {
2449 2450
		intel_ring_emit(ring, 0);
		intel_ring_emit(ring, MI_NOOP);
B
Ben Widawsky 已提交
2451
	}
2452
	intel_ring_advance(ring);
R
Rodrigo Vivi 已提交
2453

2454
	return 0;
Z
Zou Nan hai 已提交
2455 2456
}

2457 2458 2459
static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
				       struct intel_engine_cs *engine)
{
2460
	struct drm_i915_gem_object *obj;
2461
	int ret, i;
2462

2463
	if (!i915.semaphores)
2464 2465
		return;

2466 2467 2468
	if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore) {
		struct i915_vma *vma;

2469
		obj = i915_gem_object_create(dev_priv, 4096);
2470 2471
		if (IS_ERR(obj))
			goto err;
2472

2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486
		vma = i915_vma_create(obj, &dev_priv->ggtt.base, NULL);
		if (IS_ERR(vma))
			goto err_obj;

		ret = i915_gem_object_set_to_gtt_domain(obj, false);
		if (ret)
			goto err_obj;

		ret = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
		if (ret)
			goto err_obj;

		dev_priv->semaphore = vma;
	}
2487 2488

	if (INTEL_GEN(dev_priv) >= 8) {
2489
		u32 offset = i915_ggtt_offset(dev_priv->semaphore);
2490

2491
		engine->semaphore.sync_to = gen8_ring_sync_to;
2492
		engine->semaphore.signal = gen8_xcs_signal;
2493 2494

		for (i = 0; i < I915_NUM_ENGINES; i++) {
2495
			u32 ring_offset;
2496 2497 2498 2499 2500 2501 2502 2503

			if (i != engine->id)
				ring_offset = offset + GEN8_SEMAPHORE_OFFSET(engine->id, i);
			else
				ring_offset = MI_SEMAPHORE_SYNC_INVALID;

			engine->semaphore.signal_ggtt[i] = ring_offset;
		}
2504
	} else if (INTEL_GEN(dev_priv) >= 6) {
2505
		engine->semaphore.sync_to = gen6_ring_sync_to;
2506
		engine->semaphore.signal = gen6_signal;
2507 2508 2509 2510 2511 2512 2513 2514

		/*
		 * The current semaphore is only applied on pre-gen8
		 * platform.  And there is no VCS2 ring on the pre-gen8
		 * platform. So the semaphore between RCS and VCS2 is
		 * initialized as INVALID.  Gen8 will initialize the
		 * sema between VCS2 and RCS later.
		 */
2515
		for (i = 0; i < GEN6_NUM_SEMAPHORES; i++) {
2516 2517 2518
			static const struct {
				u32 wait_mbox;
				i915_reg_t mbox_reg;
2519 2520 2521 2522 2523
			} sem_data[GEN6_NUM_SEMAPHORES][GEN6_NUM_SEMAPHORES] = {
				[RCS_HW] = {
					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RV,  .mbox_reg = GEN6_VRSYNC },
					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RB,  .mbox_reg = GEN6_BRSYNC },
					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC },
2524
				},
2525 2526 2527 2528
				[VCS_HW] = {
					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VR,  .mbox_reg = GEN6_RVSYNC },
					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VB,  .mbox_reg = GEN6_BVSYNC },
					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC },
2529
				},
2530 2531 2532 2533
				[BCS_HW] = {
					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BR,  .mbox_reg = GEN6_RBSYNC },
					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BV,  .mbox_reg = GEN6_VBSYNC },
					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC },
2534
				},
2535 2536 2537 2538
				[VECS_HW] = {
					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC },
					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC },
					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC },
2539 2540 2541 2542 2543
				},
			};
			u32 wait_mbox;
			i915_reg_t mbox_reg;

2544
			if (i == engine->hw_id) {
2545 2546 2547
				wait_mbox = MI_SEMAPHORE_SYNC_INVALID;
				mbox_reg = GEN6_NOSYNC;
			} else {
2548 2549
				wait_mbox = sem_data[engine->hw_id][i].wait_mbox;
				mbox_reg = sem_data[engine->hw_id][i].mbox_reg;
2550 2551 2552 2553 2554
			}

			engine->semaphore.mbox.wait[i] = wait_mbox;
			engine->semaphore.mbox.signal[i] = mbox_reg;
		}
2555
	}
2556 2557 2558 2559 2560 2561 2562 2563

	return;

err_obj:
	i915_gem_object_put(obj);
err:
	DRM_DEBUG_DRIVER("Failed to allocate space for semaphores, disabling\n");
	i915.semaphores = 0;
2564 2565
}

2566 2567 2568
static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
				struct intel_engine_cs *engine)
{
2569 2570
	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << engine->irq_shift;

2571
	if (INTEL_GEN(dev_priv) >= 8) {
2572 2573
		engine->irq_enable = gen8_irq_enable;
		engine->irq_disable = gen8_irq_disable;
2574 2575
		engine->irq_seqno_barrier = gen6_seqno_barrier;
	} else if (INTEL_GEN(dev_priv) >= 6) {
2576 2577
		engine->irq_enable = gen6_irq_enable;
		engine->irq_disable = gen6_irq_disable;
2578 2579
		engine->irq_seqno_barrier = gen6_seqno_barrier;
	} else if (INTEL_GEN(dev_priv) >= 5) {
2580 2581
		engine->irq_enable = gen5_irq_enable;
		engine->irq_disable = gen5_irq_disable;
2582
		engine->irq_seqno_barrier = gen5_seqno_barrier;
2583
	} else if (INTEL_GEN(dev_priv) >= 3) {
2584 2585
		engine->irq_enable = i9xx_irq_enable;
		engine->irq_disable = i9xx_irq_disable;
2586
	} else {
2587 2588
		engine->irq_enable = i8xx_irq_enable;
		engine->irq_disable = i8xx_irq_disable;
2589 2590 2591
	}
}

2592 2593 2594
static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
				      struct intel_engine_cs *engine)
{
2595 2596 2597
	intel_ring_init_irq(dev_priv, engine);
	intel_ring_init_semaphores(dev_priv, engine);

2598
	engine->init_hw = init_ring_common;
2599
	engine->reset_hw = reset_ring_common;
2600

2601 2602 2603
	engine->context_pin = intel_ring_context_pin;
	engine->context_unpin = intel_ring_context_unpin;

2604 2605
	engine->request_alloc = ring_request_alloc;

2606
	engine->emit_breadcrumb = i9xx_emit_breadcrumb;
2607 2608 2609 2610
	engine->emit_breadcrumb_sz = i9xx_emit_breadcrumb_sz;
	if (i915.semaphores) {
		int num_rings;

2611
		engine->emit_breadcrumb = gen6_sema_emit_breadcrumb;
2612 2613 2614 2615 2616 2617 2618 2619 2620 2621

		num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1;
		if (INTEL_GEN(dev_priv) >= 8) {
			engine->emit_breadcrumb_sz += num_rings * 6;
		} else {
			engine->emit_breadcrumb_sz += num_rings * 3;
			if (num_rings & 1)
				engine->emit_breadcrumb_sz++;
		}
	}
2622
	engine->submit_request = i9xx_submit_request;
2623 2624

	if (INTEL_GEN(dev_priv) >= 8)
2625
		engine->emit_bb_start = gen8_emit_bb_start;
2626
	else if (INTEL_GEN(dev_priv) >= 6)
2627
		engine->emit_bb_start = gen6_emit_bb_start;
2628
	else if (INTEL_GEN(dev_priv) >= 4)
2629
		engine->emit_bb_start = i965_emit_bb_start;
2630
	else if (IS_I830(dev_priv) || IS_I845G(dev_priv))
2631
		engine->emit_bb_start = i830_emit_bb_start;
2632
	else
2633
		engine->emit_bb_start = i915_emit_bb_start;
2634 2635
}

2636
int intel_init_render_ring_buffer(struct intel_engine_cs *engine)
2637
{
2638
	struct drm_i915_private *dev_priv = engine->i915;
2639
	int ret;
2640

2641 2642
	intel_ring_default_vfuncs(dev_priv, engine);

2643 2644
	if (HAS_L3_DPF(dev_priv))
		engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2645

2646
	if (INTEL_GEN(dev_priv) >= 8) {
2647
		engine->init_context = intel_rcs_ctx_init;
2648
		engine->emit_breadcrumb = gen8_render_emit_breadcrumb;
2649
		engine->emit_breadcrumb_sz = gen8_render_emit_breadcrumb_sz;
2650
		engine->emit_flush = gen8_render_ring_flush;
2651 2652 2653
		if (i915.semaphores) {
			int num_rings;

2654
			engine->semaphore.signal = gen8_rcs_signal;
2655 2656 2657 2658 2659

			num_rings =
				hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1;
			engine->emit_breadcrumb_sz += num_rings * 6;
		}
2660
	} else if (INTEL_GEN(dev_priv) >= 6) {
2661
		engine->init_context = intel_rcs_ctx_init;
2662
		engine->emit_flush = gen7_render_ring_flush;
2663
		if (IS_GEN6(dev_priv))
2664
			engine->emit_flush = gen6_render_ring_flush;
2665
	} else if (IS_GEN5(dev_priv)) {
2666
		engine->emit_flush = gen4_render_ring_flush;
2667
	} else {
2668
		if (INTEL_GEN(dev_priv) < 4)
2669
			engine->emit_flush = gen2_render_ring_flush;
2670
		else
2671
			engine->emit_flush = gen4_render_ring_flush;
2672
		engine->irq_enable_mask = I915_USER_INTERRUPT;
2673
	}
B
Ben Widawsky 已提交
2674

2675
	if (IS_HASWELL(dev_priv))
2676
		engine->emit_bb_start = hsw_emit_bb_start;
2677

2678 2679
	engine->init_hw = init_render_ring;
	engine->cleanup = render_ring_cleanup;
2680

2681
	ret = intel_init_ring_buffer(engine);
2682 2683 2684
	if (ret)
		return ret;

2685
	if (INTEL_GEN(dev_priv) >= 6) {
2686
		ret = intel_engine_create_scratch(engine, 4096);
2687 2688 2689
		if (ret)
			return ret;
	} else if (HAS_BROKEN_CS_TLB(dev_priv)) {
2690
		ret = intel_engine_create_scratch(engine, I830_WA_SIZE);
2691 2692 2693 2694 2695
		if (ret)
			return ret;
	}

	return 0;
2696 2697
}

2698
int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine)
2699
{
2700
	struct drm_i915_private *dev_priv = engine->i915;
2701

2702 2703
	intel_ring_default_vfuncs(dev_priv, engine);

2704
	if (INTEL_GEN(dev_priv) >= 6) {
2705
		/* gen6 bsd needs a special wa for tail updates */
2706
		if (IS_GEN6(dev_priv))
2707
			engine->submit_request = gen6_bsd_submit_request;
2708
		engine->emit_flush = gen6_bsd_ring_flush;
2709
		if (INTEL_GEN(dev_priv) < 8)
2710
			engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
2711
	} else {
2712
		engine->mmio_base = BSD_RING_BASE;
2713
		engine->emit_flush = bsd_ring_flush;
2714
		if (IS_GEN5(dev_priv))
2715
			engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
2716
		else
2717
			engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
2718 2719
	}

2720
	return intel_init_ring_buffer(engine);
2721
}
2722

2723
/**
2724
 * Initialize the second BSD ring (eg. Broadwell GT3, Skylake GT3)
2725
 */
2726
int intel_init_bsd2_ring_buffer(struct intel_engine_cs *engine)
2727
{
2728
	struct drm_i915_private *dev_priv = engine->i915;
2729 2730 2731

	intel_ring_default_vfuncs(dev_priv, engine);

2732
	engine->emit_flush = gen6_bsd_ring_flush;
2733

2734
	return intel_init_ring_buffer(engine);
2735 2736
}

2737
int intel_init_blt_ring_buffer(struct intel_engine_cs *engine)
2738
{
2739
	struct drm_i915_private *dev_priv = engine->i915;
2740 2741 2742

	intel_ring_default_vfuncs(dev_priv, engine);

2743
	engine->emit_flush = gen6_ring_flush;
2744
	if (INTEL_GEN(dev_priv) < 8)
2745
		engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
2746

2747
	return intel_init_ring_buffer(engine);
2748
}
2749

2750
int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine)
B
Ben Widawsky 已提交
2751
{
2752
	struct drm_i915_private *dev_priv = engine->i915;
2753 2754 2755

	intel_ring_default_vfuncs(dev_priv, engine);

2756
	engine->emit_flush = gen6_ring_flush;
2757

2758
	if (INTEL_GEN(dev_priv) < 8) {
2759
		engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
2760 2761
		engine->irq_enable = hsw_vebox_irq_enable;
		engine->irq_disable = hsw_vebox_irq_disable;
2762
	}
B
Ben Widawsky 已提交
2763

2764
	return intel_init_ring_buffer(engine);
B
Ben Widawsky 已提交
2765
}