intel_ringbuffer.c 44.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 * Copyright © 2008-2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *    Zou Nan hai <nanhai.zou@intel.com>
 *    Xiang Hai hao<haihao.xiang@intel.com>
 *
 */

30
#include <drm/drmP.h>
31
#include "i915_drv.h"
32
#include <drm/i915_drm.h>
33
#include "i915_trace.h"
34
#include "intel_drv.h"
35

36 37 38 39 40 41 42 43 44 45
/*
 * 965+ support PIPE_CONTROL commands, which provide finer grained control
 * over cache flushing.
 */
struct pipe_control {
	struct drm_i915_gem_object *obj;
	volatile u32 *cpu_page;
	u32 gtt_offset;
};

46 47 48 49 50 51 52 53
static inline int ring_space(struct intel_ring_buffer *ring)
{
	int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
	if (space < 0)
		space += ring->size;
	return space;
}

54
static int
55 56 57 58 59 60 61 62
gen2_render_ring_flush(struct intel_ring_buffer *ring,
		       u32	invalidate_domains,
		       u32	flush_domains)
{
	u32 cmd;
	int ret;

	cmd = MI_FLUSH;
63
	if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
		cmd |= MI_NO_WRITE_FLUSH;

	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
		cmd |= MI_READ_FLUSH;

	ret = intel_ring_begin(ring, 2);
	if (ret)
		return ret;

	intel_ring_emit(ring, cmd);
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);

	return 0;
}

static int
gen4_render_ring_flush(struct intel_ring_buffer *ring,
		       u32	invalidate_domains,
		       u32	flush_domains)
84
{
85
	struct drm_device *dev = ring->dev;
86
	u32 cmd;
87
	int ret;
88

89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
	/*
	 * read/write caches:
	 *
	 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
	 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
	 * also flushed at 2d versus 3d pipeline switches.
	 *
	 * read-only caches:
	 *
	 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
	 * MI_READ_FLUSH is set, and is always flushed on 965.
	 *
	 * I915_GEM_DOMAIN_COMMAND may not exist?
	 *
	 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
	 * invalidated when MI_EXE_FLUSH is set.
	 *
	 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
	 * invalidated with every MI_FLUSH.
	 *
	 * TLBs:
	 *
	 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
	 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
	 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
	 * are flushed at any MI_FLUSH.
	 */

	cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
118
	if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
119 120 121
		cmd &= ~MI_NO_WRITE_FLUSH;
	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
		cmd |= MI_EXE_FLUSH;
122

123 124 125
	if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
	    (IS_G4X(dev) || IS_GEN5(dev)))
		cmd |= MI_INVALIDATE_ISP;
126

127 128 129
	ret = intel_ring_begin(ring, 2);
	if (ret)
		return ret;
130

131 132 133
	intel_ring_emit(ring, cmd);
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
134 135

	return 0;
136 137
}

138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
/**
 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 * implementing two workarounds on gen6.  From section 1.4.7.1
 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 *
 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 * produced by non-pipelined state commands), software needs to first
 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 * 0.
 *
 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 *
 * And the workaround for these two requires this workaround first:
 *
 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 * BEFORE the pipe-control with a post-sync op and no write-cache
 * flushes.
 *
 * And this last workaround is tricky because of the requirements on
 * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 * volume 2 part 1:
 *
 *     "1 of the following must also be set:
 *      - Render Target Cache Flush Enable ([12] of DW1)
 *      - Depth Cache Flush Enable ([0] of DW1)
 *      - Stall at Pixel Scoreboard ([1] of DW1)
 *      - Depth Stall ([13] of DW1)
 *      - Post-Sync Operation ([13] of DW1)
 *      - Notify Enable ([8] of DW1)"
 *
 * The cache flushes require the workaround flush that triggered this
 * one, so we can't use it.  Depth stall would trigger the same.
 * Post-sync nonzero is what triggered this second workaround, so we
 * can't use that one either.  Notify enable is IRQs, which aren't
 * really our business.  That leaves only stall at scoreboard.
 */
static int
intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
{
	struct pipe_control *pc = ring->private;
	u32 scratch_addr = pc->gtt_offset + 128;
	int ret;


	ret = intel_ring_begin(ring, 6);
	if (ret)
		return ret;

	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
			PIPE_CONTROL_STALL_AT_SCOREBOARD);
	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
	intel_ring_emit(ring, 0); /* low dword */
	intel_ring_emit(ring, 0); /* high dword */
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);

	ret = intel_ring_begin(ring, 6);
	if (ret)
		return ret;

	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
	intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
	intel_ring_emit(ring, 0);
	intel_ring_emit(ring, 0);
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);

	return 0;
}

static int
gen6_render_ring_flush(struct intel_ring_buffer *ring,
                         u32 invalidate_domains, u32 flush_domains)
{
	u32 flags = 0;
	struct pipe_control *pc = ring->private;
	u32 scratch_addr = pc->gtt_offset + 128;
	int ret;

220 221 222 223 224
	/* Force SNB workarounds for PIPE_CONTROL flushes */
	ret = intel_emit_post_sync_nonzero_flush(ring);
	if (ret)
		return ret;

225 226 227 228
	/* Just flush everything.  Experiments have shown that reducing the
	 * number of bits based on the write domains has little performance
	 * impact.
	 */
229 230 231 232 233 234 235
	if (flush_domains) {
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
		/*
		 * Ensure that any following seqno writes only happen
		 * when the render cache is indeed flushed.
		 */
236
		flags |= PIPE_CONTROL_CS_STALL;
237 238 239 240 241 242 243 244 245 246 247
	}
	if (invalidate_domains) {
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		/*
		 * TLB invalidate requires a post-sync write.
		 */
248
		flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
249
	}
250

251
	ret = intel_ring_begin(ring, 4);
252 253 254
	if (ret)
		return ret;

255
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
256 257
	intel_ring_emit(ring, flags);
	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
258
	intel_ring_emit(ring, 0);
259 260 261 262 263
	intel_ring_advance(ring);

	return 0;
}

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
static int
gen7_render_ring_cs_stall_wa(struct intel_ring_buffer *ring)
{
	int ret;

	ret = intel_ring_begin(ring, 4);
	if (ret)
		return ret;

	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
	intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
			      PIPE_CONTROL_STALL_AT_SCOREBOARD);
	intel_ring_emit(ring, 0);
	intel_ring_emit(ring, 0);
	intel_ring_advance(ring);

	return 0;
}

283 284 285 286 287 288 289 290 291
static int
gen7_render_ring_flush(struct intel_ring_buffer *ring,
		       u32 invalidate_domains, u32 flush_domains)
{
	u32 flags = 0;
	struct pipe_control *pc = ring->private;
	u32 scratch_addr = pc->gtt_offset + 128;
	int ret;

292 293 294 295 296 297 298 299 300 301
	/*
	 * Ensure that any following seqno writes only happen when the render
	 * cache is indeed flushed.
	 *
	 * Workaround: 4th PIPE_CONTROL command (except the ones with only
	 * read-cache invalidate bits set) must have the CS_STALL bit set. We
	 * don't try to be clever and just set it unconditionally.
	 */
	flags |= PIPE_CONTROL_CS_STALL;

302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
	/* Just flush everything.  Experiments have shown that reducing the
	 * number of bits based on the write domains has little performance
	 * impact.
	 */
	if (flush_domains) {
		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
	}
	if (invalidate_domains) {
		flags |= PIPE_CONTROL_TLB_INVALIDATE;
		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
		/*
		 * TLB invalidate requires a post-sync write.
		 */
		flags |= PIPE_CONTROL_QW_WRITE;
321 322 323 324 325

		/* Workaround: we must issue a pipe_control with CS-stall bit
		 * set before a pipe_control command that has the state cache
		 * invalidate bit set. */
		gen7_render_ring_cs_stall_wa(ring);
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340
	}

	ret = intel_ring_begin(ring, 4);
	if (ret)
		return ret;

	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
	intel_ring_emit(ring, flags);
	intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
	intel_ring_emit(ring, 0);
	intel_ring_advance(ring);

	return 0;
}

341
static void ring_write_tail(struct intel_ring_buffer *ring,
342
			    u32 value)
343
{
344
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
345
	I915_WRITE_TAIL(ring, value);
346 347
}

348
u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
349
{
350 351
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
	u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
D
Daniel Vetter 已提交
352
			RING_ACTHD(ring->mmio_base) : ACTHD;
353 354 355 356

	return I915_READ(acthd_reg);
}

357
static int init_ring_common(struct intel_ring_buffer *ring)
358
{
359 360
	struct drm_device *dev = ring->dev;
	drm_i915_private_t *dev_priv = dev->dev_private;
361
	struct drm_i915_gem_object *obj = ring->obj;
362
	int ret = 0;
363 364
	u32 head;

365 366 367
	if (HAS_FORCE_WAKE(dev))
		gen6_gt_force_wake_get(dev_priv);

368
	/* Stop the ring if it's running. */
369
	I915_WRITE_CTL(ring, 0);
370
	I915_WRITE_HEAD(ring, 0);
371
	ring->write_tail(ring, 0);
372

373
	head = I915_READ_HEAD(ring) & HEAD_ADDR;
374 375 376

	/* G45 ring initialization fails to reset head to zero */
	if (head != 0) {
377 378 379 380 381 382 383
		DRM_DEBUG_KMS("%s head not reset to zero "
			      "ctl %08x head %08x tail %08x start %08x\n",
			      ring->name,
			      I915_READ_CTL(ring),
			      I915_READ_HEAD(ring),
			      I915_READ_TAIL(ring),
			      I915_READ_START(ring));
384

385
		I915_WRITE_HEAD(ring, 0);
386

387 388 389 390 391 392 393 394 395
		if (I915_READ_HEAD(ring) & HEAD_ADDR) {
			DRM_ERROR("failed to set %s head to zero "
				  "ctl %08x head %08x tail %08x start %08x\n",
				  ring->name,
				  I915_READ_CTL(ring),
				  I915_READ_HEAD(ring),
				  I915_READ_TAIL(ring),
				  I915_READ_START(ring));
		}
396 397
	}

398 399 400 401 402
	/* Initialize the ring. This must happen _after_ we've cleared the ring
	 * registers with the above sequence (the readback of the HEAD registers
	 * also enforces ordering), otherwise the hw might lose the new ring
	 * register values. */
	I915_WRITE_START(ring, obj->gtt_offset);
403
	I915_WRITE_CTL(ring,
404
			((ring->size - PAGE_SIZE) & RING_NR_PAGES)
405
			| RING_VALID);
406 407

	/* If the head is still not zero, the ring is dead */
408 409 410
	if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
		     I915_READ_START(ring) == obj->gtt_offset &&
		     (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
411 412 413 414 415 416 417
		DRM_ERROR("%s initialization failed "
				"ctl %08x head %08x tail %08x start %08x\n",
				ring->name,
				I915_READ_CTL(ring),
				I915_READ_HEAD(ring),
				I915_READ_TAIL(ring),
				I915_READ_START(ring));
418 419
		ret = -EIO;
		goto out;
420 421
	}

422 423
	if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
		i915_kernel_lost_context(ring->dev);
424
	else {
425
		ring->head = I915_READ_HEAD(ring);
426
		ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
427
		ring->space = ring_space(ring);
428
		ring->last_retired_head = -1;
429
	}
430

431 432 433 434 435
out:
	if (HAS_FORCE_WAKE(dev))
		gen6_gt_force_wake_put(dev_priv);

	return ret;
436 437
}

438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457
static int
init_pipe_control(struct intel_ring_buffer *ring)
{
	struct pipe_control *pc;
	struct drm_i915_gem_object *obj;
	int ret;

	if (ring->private)
		return 0;

	pc = kmalloc(sizeof(*pc), GFP_KERNEL);
	if (!pc)
		return -ENOMEM;

	obj = i915_gem_alloc_object(ring->dev, 4096);
	if (obj == NULL) {
		DRM_ERROR("Failed to allocate seqno page\n");
		ret = -ENOMEM;
		goto err;
	}
458 459

	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
460

461
	ret = i915_gem_object_pin(obj, 4096, true, false);
462 463 464 465
	if (ret)
		goto err_unref;

	pc->gtt_offset = obj->gtt_offset;
466
	pc->cpu_page =  kmap(sg_page(obj->pages->sgl));
467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
	if (pc->cpu_page == NULL)
		goto err_unpin;

	pc->obj = obj;
	ring->private = pc;
	return 0;

err_unpin:
	i915_gem_object_unpin(obj);
err_unref:
	drm_gem_object_unreference(&obj->base);
err:
	kfree(pc);
	return ret;
}

static void
cleanup_pipe_control(struct intel_ring_buffer *ring)
{
	struct pipe_control *pc = ring->private;
	struct drm_i915_gem_object *obj;

	if (!ring->private)
		return;

	obj = pc->obj;
493 494

	kunmap(sg_page(obj->pages->sgl));
495 496 497 498 499 500 501
	i915_gem_object_unpin(obj);
	drm_gem_object_unreference(&obj->base);

	kfree(pc);
	ring->private = NULL;
}

502
static int init_render_ring(struct intel_ring_buffer *ring)
503
{
504
	struct drm_device *dev = ring->dev;
505
	struct drm_i915_private *dev_priv = dev->dev_private;
506
	int ret = init_ring_common(ring);
507

508
	if (INTEL_INFO(dev)->gen > 3) {
509
		I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
510 511
		if (IS_GEN7(dev))
			I915_WRITE(GFX_MODE_GEN7,
512 513
				   _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
				   _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
514
	}
515

516
	if (INTEL_INFO(dev)->gen >= 5) {
517 518 519 520 521
		ret = init_pipe_control(ring);
		if (ret)
			return ret;
	}

522
	if (IS_GEN6(dev)) {
523 524 525 526 527 528
		/* From the Sandybridge PRM, volume 1 part 3, page 24:
		 * "If this bit is set, STCunit will have LRA as replacement
		 *  policy. [...] This bit must be reset.  LRA replacement
		 *  policy is not supported."
		 */
		I915_WRITE(CACHE_MODE_0,
529
			   _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
530 531 532 533 534 535 536

		/* This is not explicitly set for GEN6, so read the register.
		 * see intel_ring_mi_set_context() for why we care.
		 * TODO: consider explicitly setting the bit for GEN5
		 */
		ring->itlb_before_ctx_switch =
			!!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
537 538
	}

539 540
	if (INTEL_INFO(dev)->gen >= 6)
		I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
541

542
	if (HAS_L3_GPU_CACHE(dev))
543 544
		I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);

545 546 547
	return ret;
}

548 549 550 551 552 553 554 555
static void render_ring_cleanup(struct intel_ring_buffer *ring)
{
	if (!ring->private)
		return;

	cleanup_pipe_control(ring);
}

556
static void
557 558 559
update_mboxes(struct intel_ring_buffer *ring,
	    u32 seqno,
	    u32 mmio_offset)
560
{
561 562 563 564
	intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
			      MI_SEMAPHORE_GLOBAL_GTT |
			      MI_SEMAPHORE_REGISTER |
			      MI_SEMAPHORE_UPDATE);
565
	intel_ring_emit(ring, seqno);
566
	intel_ring_emit(ring, mmio_offset);
567 568
}

569 570 571 572 573 574 575 576 577
/**
 * gen6_add_request - Update the semaphore mailbox registers
 * 
 * @ring - ring that is adding a request
 * @seqno - return seqno stuck into the ring
 *
 * Update the mailbox registers in the *other* rings with the current seqno.
 * This acts like a signal in the canonical semaphore.
 */
578 579
static int
gen6_add_request(struct intel_ring_buffer *ring,
580
		 u32 *seqno)
581
{
582 583
	u32 mbox1_reg;
	u32 mbox2_reg;
584 585 586 587 588 589
	int ret;

	ret = intel_ring_begin(ring, 10);
	if (ret)
		return ret;

590 591
	mbox1_reg = ring->signal_mbox[0];
	mbox2_reg = ring->signal_mbox[1];
592

593
	*seqno = i915_gem_next_request_seqno(ring);
594 595 596

	update_mboxes(ring, *seqno, mbox1_reg);
	update_mboxes(ring, *seqno, mbox2_reg);
597 598
	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
599
	intel_ring_emit(ring, *seqno);
600 601 602 603 604 605
	intel_ring_emit(ring, MI_USER_INTERRUPT);
	intel_ring_advance(ring);

	return 0;
}

606 607 608 609 610 611 612 613
/**
 * intel_ring_sync - sync the waiter to the signaller on seqno
 *
 * @waiter - ring that is waiting
 * @signaller - ring which has, or will signal
 * @seqno - seqno which the waiter will block on
 */
static int
614 615 616
gen6_ring_sync(struct intel_ring_buffer *waiter,
	       struct intel_ring_buffer *signaller,
	       u32 seqno)
617 618
{
	int ret;
619 620 621
	u32 dw1 = MI_SEMAPHORE_MBOX |
		  MI_SEMAPHORE_COMPARE |
		  MI_SEMAPHORE_REGISTER;
622

623 624 625 626 627 628
	/* Throughout all of the GEM code, seqno passed implies our current
	 * seqno is >= the last seqno executed. However for hardware the
	 * comparison is strictly greater than.
	 */
	seqno -= 1;

629 630 631
	WARN_ON(signaller->semaphore_register[waiter->id] ==
		MI_SEMAPHORE_SYNC_INVALID);

632
	ret = intel_ring_begin(waiter, 4);
633 634 635
	if (ret)
		return ret;

636 637
	intel_ring_emit(waiter,
			dw1 | signaller->semaphore_register[waiter->id]);
638 639 640 641
	intel_ring_emit(waiter, seqno);
	intel_ring_emit(waiter, 0);
	intel_ring_emit(waiter, MI_NOOP);
	intel_ring_advance(waiter);
642 643 644 645

	return 0;
}

646 647
#define PIPE_CONTROL_FLUSH(ring__, addr__)					\
do {									\
648 649
	intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |		\
		 PIPE_CONTROL_DEPTH_STALL);				\
650 651 652 653 654 655 656 657 658
	intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);			\
	intel_ring_emit(ring__, 0);							\
	intel_ring_emit(ring__, 0);							\
} while (0)

static int
pc_render_add_request(struct intel_ring_buffer *ring,
		      u32 *result)
{
659
	u32 seqno = i915_gem_next_request_seqno(ring);
660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675
	struct pipe_control *pc = ring->private;
	u32 scratch_addr = pc->gtt_offset + 128;
	int ret;

	/* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
	 * incoherent with writes to memory, i.e. completely fubar,
	 * so we need to use PIPE_NOTIFY instead.
	 *
	 * However, we also need to workaround the qword write
	 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
	 * memory before requesting an interrupt.
	 */
	ret = intel_ring_begin(ring, 32);
	if (ret)
		return ret;

676
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
677 678
			PIPE_CONTROL_WRITE_FLUSH |
			PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
679 680 681 682 683 684 685 686 687 688 689 690 691 692
	intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
	intel_ring_emit(ring, seqno);
	intel_ring_emit(ring, 0);
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
	scratch_addr += 128; /* write to separate cachelines */
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
	scratch_addr += 128;
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
	scratch_addr += 128;
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
	scratch_addr += 128;
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
	scratch_addr += 128;
	PIPE_CONTROL_FLUSH(ring, scratch_addr);
693

694
	intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
695 696
			PIPE_CONTROL_WRITE_FLUSH |
			PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
697 698 699 700 701 702 703 704 705 706
			PIPE_CONTROL_NOTIFY);
	intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
	intel_ring_emit(ring, seqno);
	intel_ring_emit(ring, 0);
	intel_ring_advance(ring);

	*result = seqno;
	return 0;
}

707
static u32
708
gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
709 710 711 712
{
	/* Workaround to force correct ordering between irq and seqno writes on
	 * ivb (and maybe also on snb) by reading from a CS register (like
	 * ACTHD) before reading the status page. */
713
	if (!lazy_coherency)
714 715 716 717
		intel_ring_get_active_head(ring);
	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
}

718
static u32
719
ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
720
{
721 722 723
	return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
}

724
static u32
725
pc_render_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
726 727 728 729 730
{
	struct pipe_control *pc = ring->private;
	return pc->cpu_page[0];
}

731 732 733 734 735
static bool
gen5_ring_get_irq(struct intel_ring_buffer *ring)
{
	struct drm_device *dev = ring->dev;
	drm_i915_private_t *dev_priv = dev->dev_private;
736
	unsigned long flags;
737 738 739 740

	if (!dev->irq_enabled)
		return false;

741
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
742 743 744 745 746
	if (ring->irq_refcount++ == 0) {
		dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
		I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		POSTING_READ(GTIMR);
	}
747
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
748 749 750 751 752 753 754 755 756

	return true;
}

static void
gen5_ring_put_irq(struct intel_ring_buffer *ring)
{
	struct drm_device *dev = ring->dev;
	drm_i915_private_t *dev_priv = dev->dev_private;
757
	unsigned long flags;
758

759
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
760 761 762 763 764
	if (--ring->irq_refcount == 0) {
		dev_priv->gt_irq_mask |= ring->irq_enable_mask;
		I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		POSTING_READ(GTIMR);
	}
765
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
766 767
}

768
static bool
769
i9xx_ring_get_irq(struct intel_ring_buffer *ring)
770
{
771
	struct drm_device *dev = ring->dev;
772
	drm_i915_private_t *dev_priv = dev->dev_private;
773
	unsigned long flags;
774

775 776 777
	if (!dev->irq_enabled)
		return false;

778
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
779 780 781 782 783
	if (ring->irq_refcount++ == 0) {
		dev_priv->irq_mask &= ~ring->irq_enable_mask;
		I915_WRITE(IMR, dev_priv->irq_mask);
		POSTING_READ(IMR);
	}
784
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
785 786

	return true;
787 788
}

789
static void
790
i9xx_ring_put_irq(struct intel_ring_buffer *ring)
791
{
792
	struct drm_device *dev = ring->dev;
793
	drm_i915_private_t *dev_priv = dev->dev_private;
794
	unsigned long flags;
795

796
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
797 798 799 800 801
	if (--ring->irq_refcount == 0) {
		dev_priv->irq_mask |= ring->irq_enable_mask;
		I915_WRITE(IMR, dev_priv->irq_mask);
		POSTING_READ(IMR);
	}
802
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
803 804
}

C
Chris Wilson 已提交
805 806 807 808 809
static bool
i8xx_ring_get_irq(struct intel_ring_buffer *ring)
{
	struct drm_device *dev = ring->dev;
	drm_i915_private_t *dev_priv = dev->dev_private;
810
	unsigned long flags;
C
Chris Wilson 已提交
811 812 813 814

	if (!dev->irq_enabled)
		return false;

815
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
C
Chris Wilson 已提交
816 817 818 819 820
	if (ring->irq_refcount++ == 0) {
		dev_priv->irq_mask &= ~ring->irq_enable_mask;
		I915_WRITE16(IMR, dev_priv->irq_mask);
		POSTING_READ16(IMR);
	}
821
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
C
Chris Wilson 已提交
822 823 824 825 826 827 828 829 830

	return true;
}

static void
i8xx_ring_put_irq(struct intel_ring_buffer *ring)
{
	struct drm_device *dev = ring->dev;
	drm_i915_private_t *dev_priv = dev->dev_private;
831
	unsigned long flags;
C
Chris Wilson 已提交
832

833
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
C
Chris Wilson 已提交
834 835 836 837 838
	if (--ring->irq_refcount == 0) {
		dev_priv->irq_mask |= ring->irq_enable_mask;
		I915_WRITE16(IMR, dev_priv->irq_mask);
		POSTING_READ16(IMR);
	}
839
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
C
Chris Wilson 已提交
840 841
}

842
void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
843
{
844
	struct drm_device *dev = ring->dev;
845
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
846 847 848 849 850 851 852
	u32 mmio = 0;

	/* The ring status page addresses are no longer next to the rest of
	 * the ring registers as of gen7.
	 */
	if (IS_GEN7(dev)) {
		switch (ring->id) {
853
		case RCS:
854 855
			mmio = RENDER_HWS_PGA_GEN7;
			break;
856
		case BCS:
857 858
			mmio = BLT_HWS_PGA_GEN7;
			break;
859
		case VCS:
860 861 862 863 864 865 866 867 868
			mmio = BSD_HWS_PGA_GEN7;
			break;
		}
	} else if (IS_GEN6(ring->dev)) {
		mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
	} else {
		mmio = RING_HWS_PGA(ring->mmio_base);
	}

869 870
	I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
	POSTING_READ(mmio);
871 872
}

873
static int
874 875 876
bsd_ring_flush(struct intel_ring_buffer *ring,
	       u32     invalidate_domains,
	       u32     flush_domains)
877
{
878 879 880 881 882 883 884 885 886 887
	int ret;

	ret = intel_ring_begin(ring, 2);
	if (ret)
		return ret;

	intel_ring_emit(ring, MI_FLUSH);
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);
	return 0;
888 889
}

890
static int
891
i9xx_add_request(struct intel_ring_buffer *ring,
892
		 u32 *result)
893 894
{
	u32 seqno;
895 896 897 898 899
	int ret;

	ret = intel_ring_begin(ring, 4);
	if (ret)
		return ret;
900

901
	seqno = i915_gem_next_request_seqno(ring);
902

903 904 905 906 907
	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
	intel_ring_emit(ring, seqno);
	intel_ring_emit(ring, MI_USER_INTERRUPT);
	intel_ring_advance(ring);
908

909 910
	*result = seqno;
	return 0;
911 912
}

913
static bool
914
gen6_ring_get_irq(struct intel_ring_buffer *ring)
915 916
{
	struct drm_device *dev = ring->dev;
917
	drm_i915_private_t *dev_priv = dev->dev_private;
918
	unsigned long flags;
919 920 921 922

	if (!dev->irq_enabled)
	       return false;

923 924 925
	/* It looks like we need to prevent the gt from suspending while waiting
	 * for an notifiy irq, otherwise irqs seem to get lost on at least the
	 * blt/bsd rings on ivb. */
926
	gen6_gt_force_wake_get(dev_priv);
927

928
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
929
	if (ring->irq_refcount++ == 0) {
930
		if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
931 932 933 934
			I915_WRITE_IMR(ring, ~(ring->irq_enable_mask |
						GEN6_RENDER_L3_PARITY_ERROR));
		else
			I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
935 936 937
		dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
		I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		POSTING_READ(GTIMR);
938
	}
939
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
940 941 942 943 944

	return true;
}

static void
945
gen6_ring_put_irq(struct intel_ring_buffer *ring)
946 947
{
	struct drm_device *dev = ring->dev;
948
	drm_i915_private_t *dev_priv = dev->dev_private;
949
	unsigned long flags;
950

951
	spin_lock_irqsave(&dev_priv->irq_lock, flags);
952
	if (--ring->irq_refcount == 0) {
953
		if (HAS_L3_GPU_CACHE(dev) && ring->id == RCS)
954 955 956
			I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
		else
			I915_WRITE_IMR(ring, ~0);
957 958 959
		dev_priv->gt_irq_mask |= ring->irq_enable_mask;
		I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
		POSTING_READ(GTIMR);
960
	}
961
	spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
962

963
	gen6_gt_force_wake_put(dev_priv);
964 965 966
}

static int
967 968 969
i965_dispatch_execbuffer(struct intel_ring_buffer *ring,
			 u32 offset, u32 length,
			 unsigned flags)
970
{
971
	int ret;
972

973 974 975 976
	ret = intel_ring_begin(ring, 2);
	if (ret)
		return ret;

977
	intel_ring_emit(ring,
978 979
			MI_BATCH_BUFFER_START |
			MI_BATCH_GTT |
980
			(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
981
	intel_ring_emit(ring, offset);
982 983
	intel_ring_advance(ring);

984 985 986
	return 0;
}

987
static int
988
i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
989 990
				u32 offset, u32 len,
				unsigned flags)
991
{
992
	int ret;
993

994 995 996
	ret = intel_ring_begin(ring, 4);
	if (ret)
		return ret;
997

998
	intel_ring_emit(ring, MI_BATCH_BUFFER);
999
	intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1000 1001 1002
	intel_ring_emit(ring, offset + len - 8);
	intel_ring_emit(ring, 0);
	intel_ring_advance(ring);
1003

1004 1005 1006 1007 1008
	return 0;
}

static int
i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
1009 1010
			 u32 offset, u32 len,
			 unsigned flags)
1011 1012 1013 1014 1015 1016 1017
{
	int ret;

	ret = intel_ring_begin(ring, 2);
	if (ret)
		return ret;

1018
	intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
1019
	intel_ring_emit(ring, offset | (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE));
1020
	intel_ring_advance(ring);
1021 1022 1023 1024

	return 0;
}

1025
static void cleanup_status_page(struct intel_ring_buffer *ring)
1026
{
1027
	struct drm_i915_gem_object *obj;
1028

1029 1030
	obj = ring->status_page.obj;
	if (obj == NULL)
1031 1032
		return;

1033
	kunmap(sg_page(obj->pages->sgl));
1034
	i915_gem_object_unpin(obj);
1035
	drm_gem_object_unreference(&obj->base);
1036
	ring->status_page.obj = NULL;
1037 1038
}

1039
static int init_status_page(struct intel_ring_buffer *ring)
1040
{
1041
	struct drm_device *dev = ring->dev;
1042
	struct drm_i915_gem_object *obj;
1043 1044 1045 1046 1047 1048 1049 1050
	int ret;

	obj = i915_gem_alloc_object(dev, 4096);
	if (obj == NULL) {
		DRM_ERROR("Failed to allocate status page\n");
		ret = -ENOMEM;
		goto err;
	}
1051 1052

	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
1053

1054
	ret = i915_gem_object_pin(obj, 4096, true, false);
1055 1056 1057 1058
	if (ret != 0) {
		goto err_unref;
	}

1059
	ring->status_page.gfx_addr = obj->gtt_offset;
1060
	ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
1061
	if (ring->status_page.page_addr == NULL) {
1062
		ret = -ENOMEM;
1063 1064
		goto err_unpin;
	}
1065 1066
	ring->status_page.obj = obj;
	memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1067

1068
	intel_ring_setup_status_page(ring);
1069 1070
	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
			ring->name, ring->status_page.gfx_addr);
1071 1072 1073 1074 1075 1076

	return 0;

err_unpin:
	i915_gem_object_unpin(obj);
err_unref:
1077
	drm_gem_object_unreference(&obj->base);
1078
err:
1079
	return ret;
1080 1081
}

1082 1083
static int intel_init_ring_buffer(struct drm_device *dev,
				  struct intel_ring_buffer *ring)
1084
{
1085
	struct drm_i915_gem_object *obj;
1086
	struct drm_i915_private *dev_priv = dev->dev_private;
1087 1088
	int ret;

1089
	ring->dev = dev;
1090 1091
	INIT_LIST_HEAD(&ring->active_list);
	INIT_LIST_HEAD(&ring->request_list);
1092
	ring->size = 32 * PAGE_SIZE;
1093

1094
	init_waitqueue_head(&ring->irq_queue);
1095

1096
	if (I915_NEED_GFX_HWS(dev)) {
1097
		ret = init_status_page(ring);
1098 1099 1100
		if (ret)
			return ret;
	}
1101

1102
	obj = i915_gem_alloc_object(dev, ring->size);
1103 1104
	if (obj == NULL) {
		DRM_ERROR("Failed to allocate ringbuffer\n");
1105
		ret = -ENOMEM;
1106
		goto err_hws;
1107 1108
	}

1109
	ring->obj = obj;
1110

1111
	ret = i915_gem_object_pin(obj, PAGE_SIZE, true, false);
1112 1113
	if (ret)
		goto err_unref;
1114

1115 1116 1117 1118
	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		goto err_unpin;

1119 1120 1121
	ring->virtual_start =
		ioremap_wc(dev_priv->mm.gtt->gma_bus_addr + obj->gtt_offset,
			   ring->size);
1122
	if (ring->virtual_start == NULL) {
1123
		DRM_ERROR("Failed to map ringbuffer.\n");
1124
		ret = -EINVAL;
1125
		goto err_unpin;
1126 1127
	}

1128
	ret = ring->init(ring);
1129 1130
	if (ret)
		goto err_unmap;
1131

1132 1133 1134 1135 1136
	/* Workaround an erratum on the i830 which causes a hang if
	 * the TAIL pointer points to within the last 2 cachelines
	 * of the buffer.
	 */
	ring->effective_size = ring->size;
1137
	if (IS_I830(ring->dev) || IS_845G(ring->dev))
1138 1139
		ring->effective_size -= 128;

1140
	return 0;
1141 1142

err_unmap:
1143
	iounmap(ring->virtual_start);
1144 1145 1146
err_unpin:
	i915_gem_object_unpin(obj);
err_unref:
1147 1148
	drm_gem_object_unreference(&obj->base);
	ring->obj = NULL;
1149
err_hws:
1150
	cleanup_status_page(ring);
1151
	return ret;
1152 1153
}

1154
void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1155
{
1156 1157 1158
	struct drm_i915_private *dev_priv;
	int ret;

1159
	if (ring->obj == NULL)
1160 1161
		return;

1162 1163
	/* Disable the ring buffer. The ring must be idle at this point */
	dev_priv = ring->dev->dev_private;
1164
	ret = intel_wait_ring_idle(ring);
1165 1166 1167 1168
	if (ret)
		DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
			  ring->name, ret);

1169 1170
	I915_WRITE_CTL(ring, 0);

1171
	iounmap(ring->virtual_start);
1172

1173 1174 1175
	i915_gem_object_unpin(ring->obj);
	drm_gem_object_unreference(&ring->obj->base);
	ring->obj = NULL;
1176

Z
Zou Nan hai 已提交
1177 1178 1179
	if (ring->cleanup)
		ring->cleanup(ring);

1180
	cleanup_status_page(ring);
1181 1182
}

1183
static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1184
{
1185
	uint32_t __iomem *virt;
1186
	int rem = ring->size - ring->tail;
1187

1188
	if (ring->space < rem) {
1189
		int ret = intel_wait_ring_buffer(ring, rem);
1190 1191 1192 1193
		if (ret)
			return ret;
	}

1194 1195 1196 1197
	virt = ring->virtual_start + ring->tail;
	rem /= 4;
	while (rem--)
		iowrite32(MI_NOOP, virt++);
1198

1199
	ring->tail = 0;
1200
	ring->space = ring_space(ring);
1201 1202 1203 1204

	return 0;
}

1205 1206 1207 1208
static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
{
	int ret;

1209
	ret = i915_wait_seqno(ring, seqno);
1210 1211
	if (!ret)
		i915_gem_retire_requests_ring(ring);
1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272

	return ret;
}

static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
{
	struct drm_i915_gem_request *request;
	u32 seqno = 0;
	int ret;

	i915_gem_retire_requests_ring(ring);

	if (ring->last_retired_head != -1) {
		ring->head = ring->last_retired_head;
		ring->last_retired_head = -1;
		ring->space = ring_space(ring);
		if (ring->space >= n)
			return 0;
	}

	list_for_each_entry(request, &ring->request_list, list) {
		int space;

		if (request->tail == -1)
			continue;

		space = request->tail - (ring->tail + 8);
		if (space < 0)
			space += ring->size;
		if (space >= n) {
			seqno = request->seqno;
			break;
		}

		/* Consume this request in case we need more space than
		 * is available and so need to prevent a race between
		 * updating last_retired_head and direct reads of
		 * I915_RING_HEAD. It also provides a nice sanity check.
		 */
		request->tail = -1;
	}

	if (seqno == 0)
		return -ENOSPC;

	ret = intel_ring_wait_seqno(ring, seqno);
	if (ret)
		return ret;

	if (WARN_ON(ring->last_retired_head == -1))
		return -ENOSPC;

	ring->head = ring->last_retired_head;
	ring->last_retired_head = -1;
	ring->space = ring_space(ring);
	if (WARN_ON(ring->space < n))
		return -ENOSPC;

	return 0;
}

1273
int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
1274
{
1275
	struct drm_device *dev = ring->dev;
1276
	struct drm_i915_private *dev_priv = dev->dev_private;
1277
	unsigned long end;
1278
	int ret;
1279

1280 1281 1282 1283
	ret = intel_ring_wait_request(ring, n);
	if (ret != -ENOSPC)
		return ret;

C
Chris Wilson 已提交
1284
	trace_i915_ring_wait_begin(ring);
1285 1286 1287 1288 1289 1290
	/* With GEM the hangcheck timer should kick us out of the loop,
	 * leaving it early runs the risk of corrupting GEM state (due
	 * to running on almost untested codepaths). But on resume
	 * timers don't work yet, so prevent a complete hang in that
	 * case by choosing an insanely large timeout. */
	end = jiffies + 60 * HZ;
1291

1292
	do {
1293 1294
		ring->head = I915_READ_HEAD(ring);
		ring->space = ring_space(ring);
1295
		if (ring->space >= n) {
C
Chris Wilson 已提交
1296
			trace_i915_ring_wait_end(ring);
1297 1298 1299 1300 1301 1302 1303 1304
			return 0;
		}

		if (dev->primary->master) {
			struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
			if (master_priv->sarea_priv)
				master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
		}
1305

1306
		msleep(1);
1307 1308 1309 1310

		ret = i915_gem_check_wedge(dev_priv, dev_priv->mm.interruptible);
		if (ret)
			return ret;
1311
	} while (!time_after(jiffies, end));
C
Chris Wilson 已提交
1312
	trace_i915_ring_wait_end(ring);
1313 1314
	return -EBUSY;
}
1315

1316 1317
int intel_ring_begin(struct intel_ring_buffer *ring,
		     int num_dwords)
1318
{
1319
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
1320
	int n = 4*num_dwords;
1321
	int ret;
1322

1323 1324 1325
	ret = i915_gem_check_wedge(dev_priv, dev_priv->mm.interruptible);
	if (ret)
		return ret;
1326

1327
	if (unlikely(ring->tail + n > ring->effective_size)) {
1328 1329 1330 1331
		ret = intel_wrap_ring_buffer(ring);
		if (unlikely(ret))
			return ret;
	}
1332

1333 1334 1335 1336 1337
	if (unlikely(ring->space < n)) {
		ret = intel_wait_ring_buffer(ring, n);
		if (unlikely(ret))
			return ret;
	}
1338 1339

	ring->space -= n;
1340
	return 0;
1341
}
1342

1343
void intel_ring_advance(struct intel_ring_buffer *ring)
1344
{
1345 1346
	struct drm_i915_private *dev_priv = ring->dev->dev_private;

1347
	ring->tail &= ring->size - 1;
1348 1349
	if (dev_priv->stop_rings & intel_ring_flag(ring))
		return;
1350
	ring->write_tail(ring, ring->tail);
1351
}
1352

1353

1354
static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1355
				     u32 value)
1356
{
1357
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
1358 1359

       /* Every tail move must follow the sequence below */
1360 1361 1362 1363

	/* Disable notification that the ring is IDLE. The GT
	 * will then assume that it is busy and bring it out of rc6.
	 */
1364
	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1365 1366 1367 1368
		   _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));

	/* Clear the context id. Here be magic! */
	I915_WRITE64(GEN6_BSD_RNCID, 0x0);
1369

1370
	/* Wait for the ring not to be idle, i.e. for it to wake up. */
1371
	if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1372 1373 1374
		      GEN6_BSD_SLEEP_INDICATOR) == 0,
		     50))
		DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
1375

1376
	/* Now that the ring is fully powered up, update the tail */
1377
	I915_WRITE_TAIL(ring, value);
1378 1379 1380 1381 1382
	POSTING_READ(RING_TAIL(ring->mmio_base));

	/* Let the ring send IDLE messages to the GT again,
	 * and so let it sleep to conserve power when idle.
	 */
1383
	I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1384
		   _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1385 1386
}

1387
static int gen6_ring_flush(struct intel_ring_buffer *ring,
1388
			   u32 invalidate, u32 flush)
1389
{
1390
	uint32_t cmd;
1391 1392 1393 1394 1395 1396
	int ret;

	ret = intel_ring_begin(ring, 4);
	if (ret)
		return ret;

1397
	cmd = MI_FLUSH_DW;
1398 1399 1400 1401 1402 1403
	/*
	 * Bspec vol 1c.5 - video engine command streamer:
	 * "If ENABLED, all TLBs will be invalidated once the flush
	 * operation is complete. This bit is only valid when the
	 * Post-Sync Operation field is a value of 1h or 3h."
	 */
1404
	if (invalidate & I915_GEM_GPU_DOMAINS)
1405 1406
		cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
			MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
1407
	intel_ring_emit(ring, cmd);
1408
	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
1409
	intel_ring_emit(ring, 0);
1410
	intel_ring_emit(ring, MI_NOOP);
1411 1412
	intel_ring_advance(ring);
	return 0;
1413 1414
}

1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435
static int
hsw_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
			      u32 offset, u32 len,
			      unsigned flags)
{
	int ret;

	ret = intel_ring_begin(ring, 2);
	if (ret)
		return ret;

	intel_ring_emit(ring,
			MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
			(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
	/* bit0-7 is the length on GEN6+ */
	intel_ring_emit(ring, offset);
	intel_ring_advance(ring);

	return 0;
}

1436
static int
1437
gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1438 1439
			      u32 offset, u32 len,
			      unsigned flags)
1440
{
1441
	int ret;
1442

1443 1444 1445
	ret = intel_ring_begin(ring, 2);
	if (ret)
		return ret;
1446

1447 1448 1449
	intel_ring_emit(ring,
			MI_BATCH_BUFFER_START |
			(flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965));
1450 1451 1452
	/* bit0-7 is the length on GEN6+ */
	intel_ring_emit(ring, offset);
	intel_ring_advance(ring);
1453

1454
	return 0;
1455 1456
}

1457 1458
/* Blitter support (SandyBridge+) */

1459
static int blt_ring_flush(struct intel_ring_buffer *ring,
1460
			  u32 invalidate, u32 flush)
Z
Zou Nan hai 已提交
1461
{
1462
	uint32_t cmd;
1463 1464
	int ret;

1465
	ret = intel_ring_begin(ring, 4);
1466 1467 1468
	if (ret)
		return ret;

1469
	cmd = MI_FLUSH_DW;
1470 1471 1472 1473 1474 1475
	/*
	 * Bspec vol 1c.3 - blitter engine command streamer:
	 * "If ENABLED, all TLBs will be invalidated once the flush
	 * operation is complete. This bit is only valid when the
	 * Post-Sync Operation field is a value of 1h or 3h."
	 */
1476
	if (invalidate & I915_GEM_DOMAIN_RENDER)
1477 1478
		cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
			MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_OP_STOREDW;
1479
	intel_ring_emit(ring, cmd);
1480
	intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
1481
	intel_ring_emit(ring, 0);
1482
	intel_ring_emit(ring, MI_NOOP);
1483 1484
	intel_ring_advance(ring);
	return 0;
Z
Zou Nan hai 已提交
1485 1486
}

1487 1488 1489
int intel_init_render_ring_buffer(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
1490
	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1491

1492 1493 1494 1495
	ring->name = "render ring";
	ring->id = RCS;
	ring->mmio_base = RENDER_RING_BASE;

1496 1497
	if (INTEL_INFO(dev)->gen >= 6) {
		ring->add_request = gen6_add_request;
1498
		ring->flush = gen7_render_ring_flush;
1499
		if (INTEL_INFO(dev)->gen == 6)
1500
			ring->flush = gen6_render_ring_flush;
1501 1502
		ring->irq_get = gen6_ring_get_irq;
		ring->irq_put = gen6_ring_put_irq;
D
Daniel Vetter 已提交
1503
		ring->irq_enable_mask = GT_USER_INTERRUPT;
1504
		ring->get_seqno = gen6_ring_get_seqno;
1505
		ring->sync_to = gen6_ring_sync;
1506 1507 1508 1509 1510
		ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
		ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
		ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
		ring->signal_mbox[0] = GEN6_VRSYNC;
		ring->signal_mbox[1] = GEN6_BRSYNC;
1511 1512
	} else if (IS_GEN5(dev)) {
		ring->add_request = pc_render_add_request;
1513
		ring->flush = gen4_render_ring_flush;
1514
		ring->get_seqno = pc_render_get_seqno;
1515 1516
		ring->irq_get = gen5_ring_get_irq;
		ring->irq_put = gen5_ring_put_irq;
1517
		ring->irq_enable_mask = GT_USER_INTERRUPT | GT_PIPE_NOTIFY;
1518
	} else {
1519
		ring->add_request = i9xx_add_request;
1520 1521 1522 1523
		if (INTEL_INFO(dev)->gen < 4)
			ring->flush = gen2_render_ring_flush;
		else
			ring->flush = gen4_render_ring_flush;
1524
		ring->get_seqno = ring_get_seqno;
C
Chris Wilson 已提交
1525 1526 1527 1528 1529 1530 1531
		if (IS_GEN2(dev)) {
			ring->irq_get = i8xx_ring_get_irq;
			ring->irq_put = i8xx_ring_put_irq;
		} else {
			ring->irq_get = i9xx_ring_get_irq;
			ring->irq_put = i9xx_ring_put_irq;
		}
1532
		ring->irq_enable_mask = I915_USER_INTERRUPT;
1533
	}
1534
	ring->write_tail = ring_write_tail;
1535 1536 1537
	if (IS_HASWELL(dev))
		ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
	else if (INTEL_INFO(dev)->gen >= 6)
1538 1539 1540 1541 1542 1543 1544
		ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
	else if (INTEL_INFO(dev)->gen >= 4)
		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
	else if (IS_I830(dev) || IS_845G(dev))
		ring->dispatch_execbuffer = i830_dispatch_execbuffer;
	else
		ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1545 1546 1547
	ring->init = init_render_ring;
	ring->cleanup = render_ring_cleanup;

1548 1549

	if (!I915_NEED_GFX_HWS(dev)) {
1550 1551
		ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
		memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1552 1553
	}

1554
	return intel_init_ring_buffer(dev, ring);
1555 1556
}

1557 1558 1559 1560 1561
int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct intel_ring_buffer *ring = &dev_priv->ring[RCS];

1562 1563 1564 1565
	ring->name = "render ring";
	ring->id = RCS;
	ring->mmio_base = RENDER_RING_BASE;

1566
	if (INTEL_INFO(dev)->gen >= 6) {
1567 1568
		/* non-kms not supported on gen6+ */
		return -ENODEV;
1569
	}
1570 1571 1572 1573 1574

	/* Note: gem is not supported on gen5/ilk without kms (the corresponding
	 * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
	 * the special gen5 functions. */
	ring->add_request = i9xx_add_request;
1575 1576 1577 1578
	if (INTEL_INFO(dev)->gen < 4)
		ring->flush = gen2_render_ring_flush;
	else
		ring->flush = gen4_render_ring_flush;
1579
	ring->get_seqno = ring_get_seqno;
C
Chris Wilson 已提交
1580 1581 1582 1583 1584 1585 1586
	if (IS_GEN2(dev)) {
		ring->irq_get = i8xx_ring_get_irq;
		ring->irq_put = i8xx_ring_put_irq;
	} else {
		ring->irq_get = i9xx_ring_get_irq;
		ring->irq_put = i9xx_ring_put_irq;
	}
1587
	ring->irq_enable_mask = I915_USER_INTERRUPT;
1588
	ring->write_tail = ring_write_tail;
1589 1590 1591 1592 1593 1594
	if (INTEL_INFO(dev)->gen >= 4)
		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
	else if (IS_I830(dev) || IS_845G(dev))
		ring->dispatch_execbuffer = i830_dispatch_execbuffer;
	else
		ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1595 1596
	ring->init = init_render_ring;
	ring->cleanup = render_ring_cleanup;
1597

1598 1599 1600
	if (!I915_NEED_GFX_HWS(dev))
		ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;

1601 1602 1603 1604 1605 1606
	ring->dev = dev;
	INIT_LIST_HEAD(&ring->active_list);
	INIT_LIST_HEAD(&ring->request_list);

	ring->size = size;
	ring->effective_size = ring->size;
1607
	if (IS_I830(ring->dev) || IS_845G(ring->dev))
1608 1609
		ring->effective_size -= 128;

1610 1611
	ring->virtual_start = ioremap_wc(start, size);
	if (ring->virtual_start == NULL) {
1612 1613 1614 1615 1616 1617 1618 1619
		DRM_ERROR("can not ioremap virtual address for"
			  " ring buffer\n");
		return -ENOMEM;
	}

	return 0;
}

1620 1621 1622
int intel_init_bsd_ring_buffer(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
1623
	struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1624

1625 1626 1627
	ring->name = "bsd ring";
	ring->id = VCS;

1628
	ring->write_tail = ring_write_tail;
1629 1630
	if (IS_GEN6(dev) || IS_GEN7(dev)) {
		ring->mmio_base = GEN6_BSD_RING_BASE;
1631 1632 1633
		/* gen6 bsd needs a special wa for tail updates */
		if (IS_GEN6(dev))
			ring->write_tail = gen6_bsd_ring_write_tail;
1634 1635 1636 1637 1638 1639 1640
		ring->flush = gen6_ring_flush;
		ring->add_request = gen6_add_request;
		ring->get_seqno = gen6_ring_get_seqno;
		ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
		ring->irq_get = gen6_ring_get_irq;
		ring->irq_put = gen6_ring_put_irq;
		ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1641
		ring->sync_to = gen6_ring_sync;
1642 1643 1644 1645 1646 1647 1648 1649
		ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
		ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
		ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
		ring->signal_mbox[0] = GEN6_RVSYNC;
		ring->signal_mbox[1] = GEN6_BVSYNC;
	} else {
		ring->mmio_base = BSD_RING_BASE;
		ring->flush = bsd_ring_flush;
1650
		ring->add_request = i9xx_add_request;
1651
		ring->get_seqno = ring_get_seqno;
1652
		if (IS_GEN5(dev)) {
1653
			ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1654 1655 1656
			ring->irq_get = gen5_ring_get_irq;
			ring->irq_put = gen5_ring_put_irq;
		} else {
1657
			ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1658 1659 1660
			ring->irq_get = i9xx_ring_get_irq;
			ring->irq_put = i9xx_ring_put_irq;
		}
1661
		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1662 1663 1664
	}
	ring->init = init_ring_common;

1665

1666
	return intel_init_ring_buffer(dev, ring);
1667
}
1668 1669 1670 1671

int intel_init_blt_ring_buffer(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
1672
	struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1673

1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685
	ring->name = "blitter ring";
	ring->id = BCS;

	ring->mmio_base = BLT_RING_BASE;
	ring->write_tail = ring_write_tail;
	ring->flush = blt_ring_flush;
	ring->add_request = gen6_add_request;
	ring->get_seqno = gen6_ring_get_seqno;
	ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
	ring->irq_get = gen6_ring_get_irq;
	ring->irq_put = gen6_ring_put_irq;
	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1686
	ring->sync_to = gen6_ring_sync;
1687 1688 1689 1690 1691 1692
	ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
	ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
	ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
	ring->signal_mbox[0] = GEN6_RBSYNC;
	ring->signal_mbox[1] = GEN6_VBSYNC;
	ring->init = init_ring_common;
1693

1694
	return intel_init_ring_buffer(dev, ring);
1695
}
1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733

int
intel_ring_flush_all_caches(struct intel_ring_buffer *ring)
{
	int ret;

	if (!ring->gpu_caches_dirty)
		return 0;

	ret = ring->flush(ring, 0, I915_GEM_GPU_DOMAINS);
	if (ret)
		return ret;

	trace_i915_gem_ring_flush(ring, 0, I915_GEM_GPU_DOMAINS);

	ring->gpu_caches_dirty = false;
	return 0;
}

int
intel_ring_invalidate_all_caches(struct intel_ring_buffer *ring)
{
	uint32_t flush_domains;
	int ret;

	flush_domains = 0;
	if (ring->gpu_caches_dirty)
		flush_domains = I915_GEM_GPU_DOMAINS;

	ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);
	if (ret)
		return ret;

	trace_i915_gem_ring_flush(ring, I915_GEM_GPU_DOMAINS, flush_domains);

	ring->gpu_caches_dirty = false;
	return 0;
}