intel_ringbuffer.h 17.2 KB
Newer Older
1 2 3
#ifndef _INTEL_RINGBUFFER_H_
#define _INTEL_RINGBUFFER_H_

4
#include <linux/hashtable.h>
5
#include "i915_gem_batch_pool.h"
6 7 8

#define I915_CMD_HASH_ORDER 9

9 10 11 12 13 14
/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
 * but keeps the logic simple. Indeed, the whole purpose of this macro is just
 * to give some inclination as to some of the magic values used in the various
 * workarounds!
 */
#define CACHELINE_BYTES 64
15
#define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(uint32_t))
16

17 18 19 20 21 22 23 24 25 26 27
/*
 * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use"
 * Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use"
 * Gen4+ BSpec "vol1c Memory Interface and Command Stream" / 5.3.4.5 "Ring Buffer Use"
 *
 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the same
 * cacheline, the Head Pointer must not be greater than the Tail
 * Pointer."
 */
#define I915_RING_FREE_SPACE 64

28
struct  intel_hw_status_page {
29
	u32		*page_addr;
30
	unsigned int	gfx_addr;
31
	struct		drm_i915_gem_object *obj;
32 33
};

B
Ben Widawsky 已提交
34 35
#define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base))
#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val)
36

B
Ben Widawsky 已提交
37 38
#define I915_READ_START(ring) I915_READ(RING_START((ring)->mmio_base))
#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val)
39

B
Ben Widawsky 已提交
40 41
#define I915_READ_HEAD(ring)  I915_READ(RING_HEAD((ring)->mmio_base))
#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val)
42

B
Ben Widawsky 已提交
43 44
#define I915_READ_CTL(ring) I915_READ(RING_CTL((ring)->mmio_base))
#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val)
45

B
Ben Widawsky 已提交
46 47
#define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base))
#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
48

49
#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base))
50
#define I915_WRITE_MODE(ring, val) I915_WRITE(RING_MI_MODE((ring)->mmio_base), val)
51

52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
/* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
 * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
 */
#define i915_semaphore_seqno_size sizeof(uint64_t)
#define GEN8_SIGNAL_OFFSET(__ring, to)			     \
	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
	((__ring)->id * I915_NUM_RINGS * i915_semaphore_seqno_size) +	\
	(i915_semaphore_seqno_size * (to)))

#define GEN8_WAIT_OFFSET(__ring, from)			     \
	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
	((from) * I915_NUM_RINGS * i915_semaphore_seqno_size) + \
	(i915_semaphore_seqno_size * (__ring)->id))

#define GEN8_RING_SEMAPHORE_INIT do { \
	if (!dev_priv->semaphore_obj) { \
		break; \
	} \
	ring->semaphore.signal_ggtt[RCS] = GEN8_SIGNAL_OFFSET(ring, RCS); \
	ring->semaphore.signal_ggtt[VCS] = GEN8_SIGNAL_OFFSET(ring, VCS); \
	ring->semaphore.signal_ggtt[BCS] = GEN8_SIGNAL_OFFSET(ring, BCS); \
	ring->semaphore.signal_ggtt[VECS] = GEN8_SIGNAL_OFFSET(ring, VECS); \
	ring->semaphore.signal_ggtt[VCS2] = GEN8_SIGNAL_OFFSET(ring, VCS2); \
	ring->semaphore.signal_ggtt[ring->id] = MI_SEMAPHORE_SYNC_INVALID; \
	} while(0)

78
enum intel_ring_hangcheck_action {
79
	HANGCHECK_IDLE = 0,
80 81
	HANGCHECK_WAIT,
	HANGCHECK_ACTIVE,
82
	HANGCHECK_ACTIVE_LOOP,
83 84 85
	HANGCHECK_KICK,
	HANGCHECK_HUNG,
};
86

87 88
#define HANGCHECK_SCORE_RING_HUNG 31

89
struct intel_ring_hangcheck {
90
	u64 acthd;
91
	u64 max_acthd;
92
	u32 seqno;
93
	int score;
94
	enum intel_ring_hangcheck_action action;
95
	int deadlock;
96
	u32 instdone[I915_NUM_INSTDONE_REG];
97 98
};

99 100 101 102
struct intel_ringbuffer {
	struct drm_i915_gem_object *obj;
	void __iomem *virtual_start;

103
	struct intel_engine_cs *ring;
104
	struct list_head link;
105

106 107 108 109 110
	u32 head;
	u32 tail;
	int space;
	int size;
	int effective_size;
111 112 113
	int reserved_size;
	int reserved_tail;
	bool reserved_in_use;
114 115 116 117 118 119 120 121 122 123 124 125

	/** We track the position of the requests in the ring buffer, and
	 * when each is retired we increment last_retired_head as the GPU
	 * must have finished processing the request and so we know we
	 * can advance the ringbuffer up to that position.
	 *
	 * last_retired_head is set to -1 after the value is consumed so
	 * we can detect new retirements.
	 */
	u32 last_retired_head;
};

126
struct	intel_context;
127
struct drm_i915_reg_descriptor;
128

129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
/*
 * we use a single page to load ctx workarounds so all of these
 * values are referred in terms of dwords
 *
 * struct i915_wa_ctx_bb:
 *  offset: specifies batch starting position, also helpful in case
 *    if we want to have multiple batches at different offsets based on
 *    some criteria. It is not a requirement at the moment but provides
 *    an option for future use.
 *  size: size of the batch in DWORDS
 */
struct  i915_ctx_workarounds {
	struct i915_wa_ctx_bb {
		u32 offset;
		u32 size;
	} indirect_ctx, per_ctx;
	struct drm_i915_gem_object *obj;
};

148
struct  intel_engine_cs {
149
	const char	*name;
150
	enum intel_ring_id {
151 152 153
		RCS = 0x0,
		VCS,
		BCS,
154
		VECS,
155
		VCS2
156
	} id;
157
#define I915_NUM_RINGS 5
158
#define LAST_USER_RING (VECS + 1)
159
	u32		mmio_base;
160
	struct		drm_device *dev;
161
	struct intel_ringbuffer *buffer;
162
	struct list_head buffers;
163

164 165 166 167 168 169 170
	/*
	 * A pool of objects to use as shadow copies of client batch buffers
	 * when the command parser is enabled. Prevents the client from
	 * modifying the batch contents after software parsing.
	 */
	struct i915_gem_batch_pool batch_pool;

171
	struct intel_hw_status_page status_page;
172
	struct i915_ctx_workarounds wa_ctx;
173

174
	unsigned irq_refcount; /* protected by dev_priv->irq_lock */
D
Daniel Vetter 已提交
175
	u32		irq_enable_mask;	/* bitmask to enable ring interrupt */
176
	struct drm_i915_gem_request *trace_irq_req;
177 178
	bool __must_check (*irq_get)(struct intel_engine_cs *ring);
	void		(*irq_put)(struct intel_engine_cs *ring);
179

180
	int		(*init_hw)(struct intel_engine_cs *ring);
181

182
	int		(*init_context)(struct drm_i915_gem_request *req);
183

184
	void		(*write_tail)(struct intel_engine_cs *ring,
185
				      u32 value);
186
	int __must_check (*flush)(struct drm_i915_gem_request *req,
187 188
				  u32	invalidate_domains,
				  u32	flush_domains);
189
	int		(*add_request)(struct drm_i915_gem_request *req);
190 191 192 193 194 195
	/* Some chipsets are not quite as coherent as advertised and need
	 * an expensive kick to force a true read of the up-to-date seqno.
	 * However, the up-to-date seqno is not always required and the last
	 * seen value is good enough. Note that the seqno will always be
	 * monotonic, even if not coherent.
	 */
196
	u32		(*get_seqno)(struct intel_engine_cs *ring,
197
				     bool lazy_coherency);
198
	void		(*set_seqno)(struct intel_engine_cs *ring,
M
Mika Kuoppala 已提交
199
				     u32 seqno);
200
	int		(*dispatch_execbuffer)(struct drm_i915_gem_request *req,
B
Ben Widawsky 已提交
201
					       u64 offset, u32 length,
202
					       unsigned dispatch_flags);
203
#define I915_DISPATCH_SECURE 0x1
204
#define I915_DISPATCH_PINNED 0x2
205
#define I915_DISPATCH_RS     0x4
206
	void		(*cleanup)(struct intel_engine_cs *ring);
207

208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
	/* GEN8 signal/wait table - never trust comments!
	 *	  signal to	signal to    signal to   signal to      signal to
	 *	    RCS		   VCS          BCS        VECS		 VCS2
	 *      --------------------------------------------------------------------
	 *  RCS | NOP (0x00) | VCS (0x08) | BCS (0x10) | VECS (0x18) | VCS2 (0x20) |
	 *	|-------------------------------------------------------------------
	 *  VCS | RCS (0x28) | NOP (0x30) | BCS (0x38) | VECS (0x40) | VCS2 (0x48) |
	 *	|-------------------------------------------------------------------
	 *  BCS | RCS (0x50) | VCS (0x58) | NOP (0x60) | VECS (0x68) | VCS2 (0x70) |
	 *	|-------------------------------------------------------------------
	 * VECS | RCS (0x78) | VCS (0x80) | BCS (0x88) |  NOP (0x90) | VCS2 (0x98) |
	 *	|-------------------------------------------------------------------
	 * VCS2 | RCS (0xa0) | VCS (0xa8) | BCS (0xb0) | VECS (0xb8) | NOP  (0xc0) |
	 *	|-------------------------------------------------------------------
	 *
	 * Generalization:
	 *  f(x, y) := (x->id * NUM_RINGS * seqno_size) + (seqno_size * y->id)
	 *  ie. transpose of g(x, y)
	 *
	 *	 sync from	sync from    sync from    sync from	sync from
	 *	    RCS		   VCS          BCS        VECS		 VCS2
	 *      --------------------------------------------------------------------
	 *  RCS | NOP (0x00) | VCS (0x28) | BCS (0x50) | VECS (0x78) | VCS2 (0xa0) |
	 *	|-------------------------------------------------------------------
	 *  VCS | RCS (0x08) | NOP (0x30) | BCS (0x58) | VECS (0x80) | VCS2 (0xa8) |
	 *	|-------------------------------------------------------------------
	 *  BCS | RCS (0x10) | VCS (0x38) | NOP (0x60) | VECS (0x88) | VCS2 (0xb0) |
	 *	|-------------------------------------------------------------------
	 * VECS | RCS (0x18) | VCS (0x40) | BCS (0x68) |  NOP (0x90) | VCS2 (0xb8) |
	 *	|-------------------------------------------------------------------
	 * VCS2 | RCS (0x20) | VCS (0x48) | BCS (0x70) | VECS (0x98) |  NOP (0xc0) |
	 *	|-------------------------------------------------------------------
	 *
	 * Generalization:
	 *  g(x, y) := (y->id * NUM_RINGS * seqno_size) + (seqno_size * x->id)
	 *  ie. transpose of f(x, y)
	 */
245 246
	struct {
		u32	sync_seqno[I915_NUM_RINGS-1];
247

248 249 250 251 252
		union {
			struct {
				/* our mbox written by others */
				u32		wait[I915_NUM_RINGS];
				/* mboxes this ring signals to */
253
				i915_reg_t	signal[I915_NUM_RINGS];
254 255 256
			} mbox;
			u64		signal_ggtt[I915_NUM_RINGS];
		};
257 258

		/* AKA wait() */
259 260
		int	(*sync_to)(struct drm_i915_gem_request *to_req,
				   struct intel_engine_cs *from,
261
				   u32 seqno);
262
		int	(*signal)(struct drm_i915_gem_request *signaller_req,
263 264
				  /* num_dwords needed by caller */
				  unsigned int num_dwords);
265
	} semaphore;
266

267
	/* Execlists */
268 269
	spinlock_t execlist_lock;
	struct list_head execlist_queue;
270
	struct list_head execlist_retired_req_list;
271
	u8 next_context_status_buffer;
272
	u32             irq_keep_mask; /* bitmask for interrupts that should not be masked */
273
	int		(*emit_request)(struct drm_i915_gem_request *request);
274
	int		(*emit_flush)(struct drm_i915_gem_request *request,
275 276
				      u32 invalidate_domains,
				      u32 flush_domains);
277
	int		(*emit_bb_start)(struct drm_i915_gem_request *req,
278
					 u64 offset, unsigned dispatch_flags);
279

280 281 282 283 284
	/**
	 * List of objects currently involved in rendering from the
	 * ringbuffer.
	 *
	 * Includes buffers having the contents of their GPU caches
285
	 * flushed, not necessarily primitives.  last_read_req
286 287 288 289 290 291 292 293 294 295 296 297
	 * represents when the rendering involved will be completed.
	 *
	 * A reference is held on the buffer while on this list.
	 */
	struct list_head active_list;

	/**
	 * List of breadcrumbs associated with GPU requests currently
	 * outstanding.
	 */
	struct list_head request_list;

298 299 300 301 302 303 304
	/**
	 * Seqno of request most recently submitted to request_list.
	 * Used exclusively by hang checker to avoid grabbing lock while
	 * inspecting request list.
	 */
	u32 last_submitted_seqno;

305
	bool gpu_caches_dirty;
306

307
	wait_queue_head_t irq_queue;
Z
Zou Nan hai 已提交
308

309 310
	struct intel_context *default_context;
	struct intel_context *last_context;
311

312 313
	struct intel_ring_hangcheck hangcheck;

314 315 316 317 318
	struct {
		struct drm_i915_gem_object *obj;
		u32 gtt_offset;
		volatile u32 *cpu_page;
	} scratch;
319

320 321
	bool needs_cmd_parser;

322
	/*
323
	 * Table of commands the command parser needs to know about
324 325
	 * for this ring.
	 */
326
	DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
327 328 329 330

	/*
	 * Table of registers allowed in commands that read/write registers.
	 */
331
	const struct drm_i915_reg_descriptor *reg_table;
332 333 334 335 336 337
	int reg_count;

	/*
	 * Table of registers allowed in commands that read/write registers, but
	 * only from the DRM master.
	 */
338
	const struct drm_i915_reg_descriptor *master_reg_table;
339 340 341 342 343 344 345 346 347 348 349 350 351
	int master_reg_count;

	/*
	 * Returns the bitmask for the length field of the specified command.
	 * Return 0 for an unrecognized/invalid command.
	 *
	 * If the command parser finds an entry for a command in the ring's
	 * cmd_tables, it gets the command's length based on the table entry.
	 * If not, it calls this function to determine the per-ring length field
	 * encoding for the command (i.e. certain opcode ranges use certain bits
	 * to encode the command length in the header).
	 */
	u32 (*get_cmd_length_mask)(u32 cmd_header);
352 353
};

354 355 356 357 358
static inline bool
intel_ring_initialized(struct intel_engine_cs *ring)
{
	return ring->dev != NULL;
}
359

360
static inline unsigned
361
intel_ring_flag(struct intel_engine_cs *ring)
362 363 364 365
{
	return 1 << ring->id;
}

366
static inline u32
367 368
intel_ring_sync_index(struct intel_engine_cs *ring,
		      struct intel_engine_cs *other)
369 370 371 372
{
	int idx;

	/*
R
Rodrigo Vivi 已提交
373 374 375 376 377
	 * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
	 * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
	 * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
	 * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
	 * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
378 379 380 381 382 383 384 385 386
	 */

	idx = (other - ring) - 1;
	if (idx < 0)
		idx += I915_NUM_RINGS;

	return idx;
}

387 388 389 390 391 392 393
static inline void
intel_flush_status_page(struct intel_engine_cs *ring, int reg)
{
	drm_clflush_virt_range(&ring->status_page.page_addr[reg],
			       sizeof(uint32_t));
}

394
static inline u32
395
intel_read_status_page(struct intel_engine_cs *ring,
396
		       int reg)
397
{
398 399 400
	/* Ensure that the compiler doesn't optimize away the load. */
	barrier();
	return ring->status_page.page_addr[reg];
401 402
}

M
Mika Kuoppala 已提交
403
static inline void
404
intel_write_status_page(struct intel_engine_cs *ring,
M
Mika Kuoppala 已提交
405 406 407 408 409
			int reg, u32 value)
{
	ring->status_page.page_addr[reg] = value;
}

C
Chris Wilson 已提交
410 411 412 413 414 415 416 417 418 419 420 421
/**
 * Reads a dword out of the status page, which is written to from the command
 * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
 * MI_STORE_DATA_IMM.
 *
 * The following dwords have a reserved meaning:
 * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
 * 0x04: ring 0 head pointer
 * 0x05: ring 1 head pointer (915-class)
 * 0x06: ring 2 head pointer (915-class)
 * 0x10-0x1b: Context status DWords (GM45)
 * 0x1f: Last written status offset. (GM45)
422
 * 0x20-0x2f: Reserved (Gen6+)
C
Chris Wilson 已提交
423
 *
424
 * The area from dword 0x30 to 0x3ff is available for driver usage.
C
Chris Wilson 已提交
425
 */
426 427
#define I915_GEM_HWS_INDEX		0x30
#define I915_GEM_HWS_SCRATCH_INDEX	0x40
428
#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
C
Chris Wilson 已提交
429

430 431
struct intel_ringbuffer *
intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size);
432 433
int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
				     struct intel_ringbuffer *ringbuf);
434 435
void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf);
void intel_ringbuffer_free(struct intel_ringbuffer *ring);
436

437 438
void intel_stop_ring_buffer(struct intel_engine_cs *ring);
void intel_cleanup_ring_buffer(struct intel_engine_cs *ring);
439

440 441
int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request);

442
int __must_check intel_ring_begin(struct drm_i915_gem_request *req, int n);
443
int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
444
static inline void intel_ring_emit(struct intel_engine_cs *ring,
445
				   u32 data)
446
{
447 448 449
	struct intel_ringbuffer *ringbuf = ring->buffer;
	iowrite32(data, ringbuf->virtual_start + ringbuf->tail);
	ringbuf->tail += 4;
450
}
451
static inline void intel_ring_emit_reg(struct intel_engine_cs *ring,
452
				       i915_reg_t reg)
453
{
454
	intel_ring_emit(ring, i915_mmio_reg_offset(reg));
455
}
456
static inline void intel_ring_advance(struct intel_engine_cs *ring)
457
{
458 459
	struct intel_ringbuffer *ringbuf = ring->buffer;
	ringbuf->tail &= ringbuf->size - 1;
460
}
461
int __intel_ring_space(int head, int tail, int size);
462
void intel_ring_update_space(struct intel_ringbuffer *ringbuf);
463 464
int intel_ring_space(struct intel_ringbuffer *ringbuf);
bool intel_ring_stopped(struct intel_engine_cs *ring);
465

466 467
int __must_check intel_ring_idle(struct intel_engine_cs *ring);
void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno);
468
int intel_ring_flush_all_caches(struct drm_i915_gem_request *req);
469
int intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req);
470

471 472 473
void intel_fini_pipe_control(struct intel_engine_cs *ring);
int intel_init_pipe_control(struct intel_engine_cs *ring);

474 475
int intel_init_render_ring_buffer(struct drm_device *dev);
int intel_init_bsd_ring_buffer(struct drm_device *dev);
476
int intel_init_bsd2_ring_buffer(struct drm_device *dev);
477
int intel_init_blt_ring_buffer(struct drm_device *dev);
B
Ben Widawsky 已提交
478
int intel_init_vebox_ring_buffer(struct drm_device *dev);
479

480
u64 intel_ring_get_active_head(struct intel_engine_cs *ring);
481

482 483
int init_workarounds_ring(struct intel_engine_cs *ring);

484
static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf)
485
{
486
	return ringbuf->tail;
487 488
}

489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510
/*
 * Arbitrary size for largest possible 'add request' sequence. The code paths
 * are complex and variable. Empirical measurement shows that the worst case
 * is ILK at 136 words. Reserving too much is better than reserving too little
 * as that allows for corner cases that might have been missed. So the figure
 * has been rounded up to 160 words.
 */
#define MIN_SPACE_FOR_ADD_REQUEST	160

/*
 * Reserve space in the ring to guarantee that the i915_add_request() call
 * will always have sufficient room to do its stuff. The request creation
 * code calls this automatically.
 */
void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size);
/* Cancel the reservation, e.g. because the request is being discarded. */
void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf);
/* Use the reserved space - for use by i915_add_request() only. */
void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf);
/* Finish with the reserved space - for use by i915_add_request() only. */
void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf);

511 512 513
/* Legacy ringbuffer specific portion of reservation code: */
int intel_ring_reserve_space(struct drm_i915_gem_request *request);

514
#endif /* _INTEL_RINGBUFFER_H_ */