intel_ringbuffer.h 17.2 KB
Newer Older
1 2 3
#ifndef _INTEL_RINGBUFFER_H_
#define _INTEL_RINGBUFFER_H_

4
#include <linux/hashtable.h>
5
#include "i915_gem_batch_pool.h"
6 7 8

#define I915_CMD_HASH_ORDER 9

9 10 11 12 13 14
/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
 * but keeps the logic simple. Indeed, the whole purpose of this macro is just
 * to give some inclination as to some of the magic values used in the various
 * workarounds!
 */
#define CACHELINE_BYTES 64
15
#define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(uint32_t))
16

17 18 19 20 21 22 23 24 25 26 27
/*
 * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use"
 * Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use"
 * Gen4+ BSpec "vol1c Memory Interface and Command Stream" / 5.3.4.5 "Ring Buffer Use"
 *
 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the same
 * cacheline, the Head Pointer must not be greater than the Tail
 * Pointer."
 */
#define I915_RING_FREE_SPACE 64

28
struct  intel_hw_status_page {
29
	u32		*page_addr;
30
	unsigned int	gfx_addr;
31
	struct		drm_i915_gem_object *obj;
32 33
};

B
Ben Widawsky 已提交
34 35
#define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base))
#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val)
36

B
Ben Widawsky 已提交
37 38
#define I915_READ_START(ring) I915_READ(RING_START((ring)->mmio_base))
#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val)
39

B
Ben Widawsky 已提交
40 41
#define I915_READ_HEAD(ring)  I915_READ(RING_HEAD((ring)->mmio_base))
#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val)
42

B
Ben Widawsky 已提交
43 44
#define I915_READ_CTL(ring) I915_READ(RING_CTL((ring)->mmio_base))
#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val)
45

B
Ben Widawsky 已提交
46 47
#define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base))
#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
48

49
#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base))
50
#define I915_WRITE_MODE(ring, val) I915_WRITE(RING_MI_MODE((ring)->mmio_base), val)
51

52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
/* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
 * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
 */
#define i915_semaphore_seqno_size sizeof(uint64_t)
#define GEN8_SIGNAL_OFFSET(__ring, to)			     \
	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
	((__ring)->id * I915_NUM_RINGS * i915_semaphore_seqno_size) +	\
	(i915_semaphore_seqno_size * (to)))

#define GEN8_WAIT_OFFSET(__ring, from)			     \
	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
	((from) * I915_NUM_RINGS * i915_semaphore_seqno_size) + \
	(i915_semaphore_seqno_size * (__ring)->id))

#define GEN8_RING_SEMAPHORE_INIT do { \
	if (!dev_priv->semaphore_obj) { \
		break; \
	} \
	ring->semaphore.signal_ggtt[RCS] = GEN8_SIGNAL_OFFSET(ring, RCS); \
	ring->semaphore.signal_ggtt[VCS] = GEN8_SIGNAL_OFFSET(ring, VCS); \
	ring->semaphore.signal_ggtt[BCS] = GEN8_SIGNAL_OFFSET(ring, BCS); \
	ring->semaphore.signal_ggtt[VECS] = GEN8_SIGNAL_OFFSET(ring, VECS); \
	ring->semaphore.signal_ggtt[VCS2] = GEN8_SIGNAL_OFFSET(ring, VCS2); \
	ring->semaphore.signal_ggtt[ring->id] = MI_SEMAPHORE_SYNC_INVALID; \
	} while(0)

78
enum intel_ring_hangcheck_action {
79
	HANGCHECK_IDLE = 0,
80 81
	HANGCHECK_WAIT,
	HANGCHECK_ACTIVE,
82
	HANGCHECK_ACTIVE_LOOP,
83 84 85
	HANGCHECK_KICK,
	HANGCHECK_HUNG,
};
86

87 88
#define HANGCHECK_SCORE_RING_HUNG 31

89
struct intel_ring_hangcheck {
90
	u64 acthd;
91
	u64 max_acthd;
92
	u32 seqno;
93
	int score;
94
	enum intel_ring_hangcheck_action action;
95
	int deadlock;
96
	u32 instdone[I915_NUM_INSTDONE_REG];
97 98
};

99 100 101
struct intel_ringbuffer {
	struct drm_i915_gem_object *obj;
	void __iomem *virtual_start;
102
	struct i915_vma *vma;
103

104
	struct intel_engine_cs *ring;
105
	struct list_head link;
106

107 108 109 110 111
	u32 head;
	u32 tail;
	int space;
	int size;
	int effective_size;
112 113 114
	int reserved_size;
	int reserved_tail;
	bool reserved_in_use;
115 116 117 118 119 120 121 122 123 124 125 126

	/** We track the position of the requests in the ring buffer, and
	 * when each is retired we increment last_retired_head as the GPU
	 * must have finished processing the request and so we know we
	 * can advance the ringbuffer up to that position.
	 *
	 * last_retired_head is set to -1 after the value is consumed so
	 * we can detect new retirements.
	 */
	u32 last_retired_head;
};

127
struct	intel_context;
128
struct drm_i915_reg_descriptor;
129

130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
/*
 * we use a single page to load ctx workarounds so all of these
 * values are referred in terms of dwords
 *
 * struct i915_wa_ctx_bb:
 *  offset: specifies batch starting position, also helpful in case
 *    if we want to have multiple batches at different offsets based on
 *    some criteria. It is not a requirement at the moment but provides
 *    an option for future use.
 *  size: size of the batch in DWORDS
 */
struct  i915_ctx_workarounds {
	struct i915_wa_ctx_bb {
		u32 offset;
		u32 size;
	} indirect_ctx, per_ctx;
	struct drm_i915_gem_object *obj;
};

149
struct  intel_engine_cs {
150
	const char	*name;
151
	enum intel_ring_id {
152 153 154
		RCS = 0x0,
		VCS,
		BCS,
155
		VECS,
156
		VCS2
157
	} id;
158
#define I915_NUM_RINGS 5
159
#define LAST_USER_RING (VECS + 1)
160
	u32		mmio_base;
161
	struct		drm_device *dev;
162
	struct intel_ringbuffer *buffer;
163
	struct list_head buffers;
164

165 166 167 168 169 170 171
	/*
	 * A pool of objects to use as shadow copies of client batch buffers
	 * when the command parser is enabled. Prevents the client from
	 * modifying the batch contents after software parsing.
	 */
	struct i915_gem_batch_pool batch_pool;

172
	struct intel_hw_status_page status_page;
173
	struct i915_ctx_workarounds wa_ctx;
174

175
	unsigned irq_refcount; /* protected by dev_priv->irq_lock */
D
Daniel Vetter 已提交
176
	u32		irq_enable_mask;	/* bitmask to enable ring interrupt */
177
	struct drm_i915_gem_request *trace_irq_req;
178 179
	bool __must_check (*irq_get)(struct intel_engine_cs *ring);
	void		(*irq_put)(struct intel_engine_cs *ring);
180

181
	int		(*init_hw)(struct intel_engine_cs *ring);
182

183
	int		(*init_context)(struct drm_i915_gem_request *req);
184

185
	void		(*write_tail)(struct intel_engine_cs *ring,
186
				      u32 value);
187
	int __must_check (*flush)(struct drm_i915_gem_request *req,
188 189
				  u32	invalidate_domains,
				  u32	flush_domains);
190
	int		(*add_request)(struct drm_i915_gem_request *req);
191 192 193 194 195 196
	/* Some chipsets are not quite as coherent as advertised and need
	 * an expensive kick to force a true read of the up-to-date seqno.
	 * However, the up-to-date seqno is not always required and the last
	 * seen value is good enough. Note that the seqno will always be
	 * monotonic, even if not coherent.
	 */
197
	u32		(*get_seqno)(struct intel_engine_cs *ring,
198
				     bool lazy_coherency);
199
	void		(*set_seqno)(struct intel_engine_cs *ring,
M
Mika Kuoppala 已提交
200
				     u32 seqno);
201
	int		(*dispatch_execbuffer)(struct drm_i915_gem_request *req,
B
Ben Widawsky 已提交
202
					       u64 offset, u32 length,
203
					       unsigned dispatch_flags);
204
#define I915_DISPATCH_SECURE 0x1
205
#define I915_DISPATCH_PINNED 0x2
206
#define I915_DISPATCH_RS     0x4
207
	void		(*cleanup)(struct intel_engine_cs *ring);
208

209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
	/* GEN8 signal/wait table - never trust comments!
	 *	  signal to	signal to    signal to   signal to      signal to
	 *	    RCS		   VCS          BCS        VECS		 VCS2
	 *      --------------------------------------------------------------------
	 *  RCS | NOP (0x00) | VCS (0x08) | BCS (0x10) | VECS (0x18) | VCS2 (0x20) |
	 *	|-------------------------------------------------------------------
	 *  VCS | RCS (0x28) | NOP (0x30) | BCS (0x38) | VECS (0x40) | VCS2 (0x48) |
	 *	|-------------------------------------------------------------------
	 *  BCS | RCS (0x50) | VCS (0x58) | NOP (0x60) | VECS (0x68) | VCS2 (0x70) |
	 *	|-------------------------------------------------------------------
	 * VECS | RCS (0x78) | VCS (0x80) | BCS (0x88) |  NOP (0x90) | VCS2 (0x98) |
	 *	|-------------------------------------------------------------------
	 * VCS2 | RCS (0xa0) | VCS (0xa8) | BCS (0xb0) | VECS (0xb8) | NOP  (0xc0) |
	 *	|-------------------------------------------------------------------
	 *
	 * Generalization:
	 *  f(x, y) := (x->id * NUM_RINGS * seqno_size) + (seqno_size * y->id)
	 *  ie. transpose of g(x, y)
	 *
	 *	 sync from	sync from    sync from    sync from	sync from
	 *	    RCS		   VCS          BCS        VECS		 VCS2
	 *      --------------------------------------------------------------------
	 *  RCS | NOP (0x00) | VCS (0x28) | BCS (0x50) | VECS (0x78) | VCS2 (0xa0) |
	 *	|-------------------------------------------------------------------
	 *  VCS | RCS (0x08) | NOP (0x30) | BCS (0x58) | VECS (0x80) | VCS2 (0xa8) |
	 *	|-------------------------------------------------------------------
	 *  BCS | RCS (0x10) | VCS (0x38) | NOP (0x60) | VECS (0x88) | VCS2 (0xb0) |
	 *	|-------------------------------------------------------------------
	 * VECS | RCS (0x18) | VCS (0x40) | BCS (0x68) |  NOP (0x90) | VCS2 (0xb8) |
	 *	|-------------------------------------------------------------------
	 * VCS2 | RCS (0x20) | VCS (0x48) | BCS (0x70) | VECS (0x98) |  NOP (0xc0) |
	 *	|-------------------------------------------------------------------
	 *
	 * Generalization:
	 *  g(x, y) := (y->id * NUM_RINGS * seqno_size) + (seqno_size * x->id)
	 *  ie. transpose of f(x, y)
	 */
246 247
	struct {
		u32	sync_seqno[I915_NUM_RINGS-1];
248

249 250 251 252 253
		union {
			struct {
				/* our mbox written by others */
				u32		wait[I915_NUM_RINGS];
				/* mboxes this ring signals to */
254
				i915_reg_t	signal[I915_NUM_RINGS];
255 256 257
			} mbox;
			u64		signal_ggtt[I915_NUM_RINGS];
		};
258 259

		/* AKA wait() */
260 261
		int	(*sync_to)(struct drm_i915_gem_request *to_req,
				   struct intel_engine_cs *from,
262
				   u32 seqno);
263
		int	(*signal)(struct drm_i915_gem_request *signaller_req,
264 265
				  /* num_dwords needed by caller */
				  unsigned int num_dwords);
266
	} semaphore;
267

268
	/* Execlists */
269 270
	spinlock_t execlist_lock;
	struct list_head execlist_queue;
271
	struct list_head execlist_retired_req_list;
272
	u8 next_context_status_buffer;
273 274
	bool disable_lite_restore_wa;
	u32 ctx_desc_template;
275
	u32             irq_keep_mask; /* bitmask for interrupts that should not be masked */
276
	int		(*emit_request)(struct drm_i915_gem_request *request);
277
	int		(*emit_flush)(struct drm_i915_gem_request *request,
278 279
				      u32 invalidate_domains,
				      u32 flush_domains);
280
	int		(*emit_bb_start)(struct drm_i915_gem_request *req,
281
					 u64 offset, unsigned dispatch_flags);
282

283 284 285 286 287
	/**
	 * List of objects currently involved in rendering from the
	 * ringbuffer.
	 *
	 * Includes buffers having the contents of their GPU caches
288
	 * flushed, not necessarily primitives.  last_read_req
289 290 291 292 293 294 295 296 297 298 299 300
	 * represents when the rendering involved will be completed.
	 *
	 * A reference is held on the buffer while on this list.
	 */
	struct list_head active_list;

	/**
	 * List of breadcrumbs associated with GPU requests currently
	 * outstanding.
	 */
	struct list_head request_list;

301 302 303 304 305 306 307
	/**
	 * Seqno of request most recently submitted to request_list.
	 * Used exclusively by hang checker to avoid grabbing lock while
	 * inspecting request list.
	 */
	u32 last_submitted_seqno;

308
	bool gpu_caches_dirty;
309

310
	wait_queue_head_t irq_queue;
Z
Zou Nan hai 已提交
311

312
	struct intel_context *last_context;
313

314 315
	struct intel_ring_hangcheck hangcheck;

316 317 318 319 320
	struct {
		struct drm_i915_gem_object *obj;
		u32 gtt_offset;
		volatile u32 *cpu_page;
	} scratch;
321

322 323
	bool needs_cmd_parser;

324
	/*
325
	 * Table of commands the command parser needs to know about
326 327
	 * for this ring.
	 */
328
	DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
329 330 331 332

	/*
	 * Table of registers allowed in commands that read/write registers.
	 */
333
	const struct drm_i915_reg_descriptor *reg_table;
334 335 336 337 338 339
	int reg_count;

	/*
	 * Table of registers allowed in commands that read/write registers, but
	 * only from the DRM master.
	 */
340
	const struct drm_i915_reg_descriptor *master_reg_table;
341 342 343 344 345 346 347 348 349 350 351 352 353
	int master_reg_count;

	/*
	 * Returns the bitmask for the length field of the specified command.
	 * Return 0 for an unrecognized/invalid command.
	 *
	 * If the command parser finds an entry for a command in the ring's
	 * cmd_tables, it gets the command's length based on the table entry.
	 * If not, it calls this function to determine the per-ring length field
	 * encoding for the command (i.e. certain opcode ranges use certain bits
	 * to encode the command length in the header).
	 */
	u32 (*get_cmd_length_mask)(u32 cmd_header);
354 355
};

356 357 358 359 360
static inline bool
intel_ring_initialized(struct intel_engine_cs *ring)
{
	return ring->dev != NULL;
}
361

362
static inline unsigned
363
intel_ring_flag(struct intel_engine_cs *ring)
364 365 366 367
{
	return 1 << ring->id;
}

368
static inline u32
369 370
intel_ring_sync_index(struct intel_engine_cs *ring,
		      struct intel_engine_cs *other)
371 372 373 374
{
	int idx;

	/*
R
Rodrigo Vivi 已提交
375 376 377 378 379
	 * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
	 * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
	 * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
	 * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
	 * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
380 381 382 383 384 385 386 387 388
	 */

	idx = (other - ring) - 1;
	if (idx < 0)
		idx += I915_NUM_RINGS;

	return idx;
}

389 390 391 392 393 394 395
static inline void
intel_flush_status_page(struct intel_engine_cs *ring, int reg)
{
	drm_clflush_virt_range(&ring->status_page.page_addr[reg],
			       sizeof(uint32_t));
}

396
static inline u32
397
intel_read_status_page(struct intel_engine_cs *ring,
398
		       int reg)
399
{
400 401 402
	/* Ensure that the compiler doesn't optimize away the load. */
	barrier();
	return ring->status_page.page_addr[reg];
403 404
}

M
Mika Kuoppala 已提交
405
static inline void
406
intel_write_status_page(struct intel_engine_cs *ring,
M
Mika Kuoppala 已提交
407 408 409 410 411
			int reg, u32 value)
{
	ring->status_page.page_addr[reg] = value;
}

412
/*
C
Chris Wilson 已提交
413 414 415 416 417 418 419 420 421 422 423
 * Reads a dword out of the status page, which is written to from the command
 * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
 * MI_STORE_DATA_IMM.
 *
 * The following dwords have a reserved meaning:
 * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
 * 0x04: ring 0 head pointer
 * 0x05: ring 1 head pointer (915-class)
 * 0x06: ring 2 head pointer (915-class)
 * 0x10-0x1b: Context status DWords (GM45)
 * 0x1f: Last written status offset. (GM45)
424
 * 0x20-0x2f: Reserved (Gen6+)
C
Chris Wilson 已提交
425
 *
426
 * The area from dword 0x30 to 0x3ff is available for driver usage.
C
Chris Wilson 已提交
427
 */
428 429
#define I915_GEM_HWS_INDEX		0x30
#define I915_GEM_HWS_SCRATCH_INDEX	0x40
430
#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
C
Chris Wilson 已提交
431

432 433
struct intel_ringbuffer *
intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size);
434 435
int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
				     struct intel_ringbuffer *ringbuf);
436 437
void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf);
void intel_ringbuffer_free(struct intel_ringbuffer *ring);
438

439 440
void intel_stop_ring_buffer(struct intel_engine_cs *ring);
void intel_cleanup_ring_buffer(struct intel_engine_cs *ring);
441

442 443
int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request);

444
int __must_check intel_ring_begin(struct drm_i915_gem_request *req, int n);
445
int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
446
static inline void intel_ring_emit(struct intel_engine_cs *ring,
447
				   u32 data)
448
{
449 450 451
	struct intel_ringbuffer *ringbuf = ring->buffer;
	iowrite32(data, ringbuf->virtual_start + ringbuf->tail);
	ringbuf->tail += 4;
452
}
453
static inline void intel_ring_emit_reg(struct intel_engine_cs *ring,
454
				       i915_reg_t reg)
455
{
456
	intel_ring_emit(ring, i915_mmio_reg_offset(reg));
457
}
458
static inline void intel_ring_advance(struct intel_engine_cs *ring)
459
{
460 461
	struct intel_ringbuffer *ringbuf = ring->buffer;
	ringbuf->tail &= ringbuf->size - 1;
462
}
463
int __intel_ring_space(int head, int tail, int size);
464
void intel_ring_update_space(struct intel_ringbuffer *ringbuf);
465 466
int intel_ring_space(struct intel_ringbuffer *ringbuf);
bool intel_ring_stopped(struct intel_engine_cs *ring);
467

468 469
int __must_check intel_ring_idle(struct intel_engine_cs *ring);
void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno);
470
int intel_ring_flush_all_caches(struct drm_i915_gem_request *req);
471
int intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req);
472

473 474 475
void intel_fini_pipe_control(struct intel_engine_cs *ring);
int intel_init_pipe_control(struct intel_engine_cs *ring);

476 477
int intel_init_render_ring_buffer(struct drm_device *dev);
int intel_init_bsd_ring_buffer(struct drm_device *dev);
478
int intel_init_bsd2_ring_buffer(struct drm_device *dev);
479
int intel_init_blt_ring_buffer(struct drm_device *dev);
B
Ben Widawsky 已提交
480
int intel_init_vebox_ring_buffer(struct drm_device *dev);
481

482
u64 intel_ring_get_active_head(struct intel_engine_cs *ring);
483

484 485
int init_workarounds_ring(struct intel_engine_cs *ring);

486
static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf)
487
{
488
	return ringbuf->tail;
489 490
}

491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512
/*
 * Arbitrary size for largest possible 'add request' sequence. The code paths
 * are complex and variable. Empirical measurement shows that the worst case
 * is ILK at 136 words. Reserving too much is better than reserving too little
 * as that allows for corner cases that might have been missed. So the figure
 * has been rounded up to 160 words.
 */
#define MIN_SPACE_FOR_ADD_REQUEST	160

/*
 * Reserve space in the ring to guarantee that the i915_add_request() call
 * will always have sufficient room to do its stuff. The request creation
 * code calls this automatically.
 */
void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size);
/* Cancel the reservation, e.g. because the request is being discarded. */
void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf);
/* Use the reserved space - for use by i915_add_request() only. */
void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf);
/* Finish with the reserved space - for use by i915_add_request() only. */
void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf);

513 514 515
/* Legacy ringbuffer specific portion of reservation code: */
int intel_ring_reserve_space(struct drm_i915_gem_request *request);

516
#endif /* _INTEL_RINGBUFFER_H_ */