intel_ringbuffer.h 13.2 KB
Newer Older
1 2 3
#ifndef _INTEL_RINGBUFFER_H_
#define _INTEL_RINGBUFFER_H_

4 5 6 7
#include <linux/hashtable.h>

#define I915_CMD_HASH_ORDER 9

8 9 10 11 12 13 14 15 16 17 18
/*
 * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use"
 * Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use"
 * Gen4+ BSpec "vol1c Memory Interface and Command Stream" / 5.3.4.5 "Ring Buffer Use"
 *
 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the same
 * cacheline, the Head Pointer must not be greater than the Tail
 * Pointer."
 */
#define I915_RING_FREE_SPACE 64

19
struct  intel_hw_status_page {
20
	u32		*page_addr;
21
	unsigned int	gfx_addr;
22
	struct		drm_i915_gem_object *obj;
23 24
};

B
Ben Widawsky 已提交
25 26
#define I915_READ_TAIL(ring) I915_READ(RING_TAIL((ring)->mmio_base))
#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val)
27

B
Ben Widawsky 已提交
28 29
#define I915_READ_START(ring) I915_READ(RING_START((ring)->mmio_base))
#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val)
30

B
Ben Widawsky 已提交
31 32
#define I915_READ_HEAD(ring)  I915_READ(RING_HEAD((ring)->mmio_base))
#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val)
33

B
Ben Widawsky 已提交
34 35
#define I915_READ_CTL(ring) I915_READ(RING_CTL((ring)->mmio_base))
#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val)
36

B
Ben Widawsky 已提交
37 38
#define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base))
#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
39

40
#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base))
41
#define I915_WRITE_MODE(ring, val) I915_WRITE(RING_MI_MODE((ring)->mmio_base), val)
42

43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
/* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
 * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
 */
#define i915_semaphore_seqno_size sizeof(uint64_t)
#define GEN8_SIGNAL_OFFSET(__ring, to)			     \
	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
	((__ring)->id * I915_NUM_RINGS * i915_semaphore_seqno_size) +	\
	(i915_semaphore_seqno_size * (to)))

#define GEN8_WAIT_OFFSET(__ring, from)			     \
	(i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
	((from) * I915_NUM_RINGS * i915_semaphore_seqno_size) + \
	(i915_semaphore_seqno_size * (__ring)->id))

#define GEN8_RING_SEMAPHORE_INIT do { \
	if (!dev_priv->semaphore_obj) { \
		break; \
	} \
	ring->semaphore.signal_ggtt[RCS] = GEN8_SIGNAL_OFFSET(ring, RCS); \
	ring->semaphore.signal_ggtt[VCS] = GEN8_SIGNAL_OFFSET(ring, VCS); \
	ring->semaphore.signal_ggtt[BCS] = GEN8_SIGNAL_OFFSET(ring, BCS); \
	ring->semaphore.signal_ggtt[VECS] = GEN8_SIGNAL_OFFSET(ring, VECS); \
	ring->semaphore.signal_ggtt[VCS2] = GEN8_SIGNAL_OFFSET(ring, VCS2); \
	ring->semaphore.signal_ggtt[ring->id] = MI_SEMAPHORE_SYNC_INVALID; \
	} while(0)

69
enum intel_ring_hangcheck_action {
70
	HANGCHECK_IDLE = 0,
71 72
	HANGCHECK_WAIT,
	HANGCHECK_ACTIVE,
73
	HANGCHECK_ACTIVE_LOOP,
74 75 76
	HANGCHECK_KICK,
	HANGCHECK_HUNG,
};
77

78 79
#define HANGCHECK_SCORE_RING_HUNG 31

80
struct intel_ring_hangcheck {
81
	u64 acthd;
82
	u64 max_acthd;
83
	u32 seqno;
84
	int score;
85
	enum intel_ring_hangcheck_action action;
86
	int deadlock;
87 88
};

89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
struct intel_ringbuffer {
	struct drm_i915_gem_object *obj;
	void __iomem *virtual_start;

	u32 head;
	u32 tail;
	int space;
	int size;
	int effective_size;

	/** We track the position of the requests in the ring buffer, and
	 * when each is retired we increment last_retired_head as the GPU
	 * must have finished processing the request and so we know we
	 * can advance the ringbuffer up to that position.
	 *
	 * last_retired_head is set to -1 after the value is consumed so
	 * we can detect new retirements.
	 */
	u32 last_retired_head;
};

110
struct  intel_engine_cs {
111
	const char	*name;
112
	enum intel_ring_id {
113 114 115
		RCS = 0x0,
		VCS,
		BCS,
116
		VECS,
117
		VCS2
118
	} id;
119
#define I915_NUM_RINGS 5
120
#define LAST_USER_RING (VECS + 1)
121
	u32		mmio_base;
122
	struct		drm_device *dev;
123
	struct intel_ringbuffer *buffer;
124 125 126

	struct intel_hw_status_page status_page;

127
	unsigned irq_refcount; /* protected by dev_priv->irq_lock */
D
Daniel Vetter 已提交
128
	u32		irq_enable_mask;	/* bitmask to enable ring interrupt */
C
Chris Wilson 已提交
129
	u32		trace_irq_seqno;
130 131
	bool __must_check (*irq_get)(struct intel_engine_cs *ring);
	void		(*irq_put)(struct intel_engine_cs *ring);
132

133
	int		(*init)(struct intel_engine_cs *ring);
134

135
	void		(*write_tail)(struct intel_engine_cs *ring,
136
				      u32 value);
137
	int __must_check (*flush)(struct intel_engine_cs *ring,
138 139
				  u32	invalidate_domains,
				  u32	flush_domains);
140
	int		(*add_request)(struct intel_engine_cs *ring);
141 142 143 144 145 146
	/* Some chipsets are not quite as coherent as advertised and need
	 * an expensive kick to force a true read of the up-to-date seqno.
	 * However, the up-to-date seqno is not always required and the last
	 * seen value is good enough. Note that the seqno will always be
	 * monotonic, even if not coherent.
	 */
147
	u32		(*get_seqno)(struct intel_engine_cs *ring,
148
				     bool lazy_coherency);
149
	void		(*set_seqno)(struct intel_engine_cs *ring,
M
Mika Kuoppala 已提交
150
				     u32 seqno);
151
	int		(*dispatch_execbuffer)(struct intel_engine_cs *ring,
B
Ben Widawsky 已提交
152
					       u64 offset, u32 length,
153 154
					       unsigned flags);
#define I915_DISPATCH_SECURE 0x1
155
#define I915_DISPATCH_PINNED 0x2
156
	void		(*cleanup)(struct intel_engine_cs *ring);
157

158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194
	/* GEN8 signal/wait table - never trust comments!
	 *	  signal to	signal to    signal to   signal to      signal to
	 *	    RCS		   VCS          BCS        VECS		 VCS2
	 *      --------------------------------------------------------------------
	 *  RCS | NOP (0x00) | VCS (0x08) | BCS (0x10) | VECS (0x18) | VCS2 (0x20) |
	 *	|-------------------------------------------------------------------
	 *  VCS | RCS (0x28) | NOP (0x30) | BCS (0x38) | VECS (0x40) | VCS2 (0x48) |
	 *	|-------------------------------------------------------------------
	 *  BCS | RCS (0x50) | VCS (0x58) | NOP (0x60) | VECS (0x68) | VCS2 (0x70) |
	 *	|-------------------------------------------------------------------
	 * VECS | RCS (0x78) | VCS (0x80) | BCS (0x88) |  NOP (0x90) | VCS2 (0x98) |
	 *	|-------------------------------------------------------------------
	 * VCS2 | RCS (0xa0) | VCS (0xa8) | BCS (0xb0) | VECS (0xb8) | NOP  (0xc0) |
	 *	|-------------------------------------------------------------------
	 *
	 * Generalization:
	 *  f(x, y) := (x->id * NUM_RINGS * seqno_size) + (seqno_size * y->id)
	 *  ie. transpose of g(x, y)
	 *
	 *	 sync from	sync from    sync from    sync from	sync from
	 *	    RCS		   VCS          BCS        VECS		 VCS2
	 *      --------------------------------------------------------------------
	 *  RCS | NOP (0x00) | VCS (0x28) | BCS (0x50) | VECS (0x78) | VCS2 (0xa0) |
	 *	|-------------------------------------------------------------------
	 *  VCS | RCS (0x08) | NOP (0x30) | BCS (0x58) | VECS (0x80) | VCS2 (0xa8) |
	 *	|-------------------------------------------------------------------
	 *  BCS | RCS (0x10) | VCS (0x38) | NOP (0x60) | VECS (0x88) | VCS2 (0xb0) |
	 *	|-------------------------------------------------------------------
	 * VECS | RCS (0x18) | VCS (0x40) | BCS (0x68) |  NOP (0x90) | VCS2 (0xb8) |
	 *	|-------------------------------------------------------------------
	 * VCS2 | RCS (0x20) | VCS (0x48) | BCS (0x70) | VECS (0x98) |  NOP (0xc0) |
	 *	|-------------------------------------------------------------------
	 *
	 * Generalization:
	 *  g(x, y) := (y->id * NUM_RINGS * seqno_size) + (seqno_size * x->id)
	 *  ie. transpose of f(x, y)
	 */
195 196
	struct {
		u32	sync_seqno[I915_NUM_RINGS-1];
197

198 199 200 201 202 203 204 205 206
		union {
			struct {
				/* our mbox written by others */
				u32		wait[I915_NUM_RINGS];
				/* mboxes this ring signals to */
				u32		signal[I915_NUM_RINGS];
			} mbox;
			u64		signal_ggtt[I915_NUM_RINGS];
		};
207 208

		/* AKA wait() */
209 210
		int	(*sync_to)(struct intel_engine_cs *ring,
				   struct intel_engine_cs *to,
211
				   u32 seqno);
212
		int	(*signal)(struct intel_engine_cs *signaller,
213 214
				  /* num_dwords needed by caller */
				  unsigned int num_dwords);
215
	} semaphore;
216

217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
	/**
	 * List of objects currently involved in rendering from the
	 * ringbuffer.
	 *
	 * Includes buffers having the contents of their GPU caches
	 * flushed, not necessarily primitives.  last_rendering_seqno
	 * represents when the rendering involved will be completed.
	 *
	 * A reference is held on the buffer while on this list.
	 */
	struct list_head active_list;

	/**
	 * List of breadcrumbs associated with GPU requests currently
	 * outstanding.
	 */
	struct list_head request_list;

235 236 237
	/**
	 * Do we have some not yet emitted requests outstanding?
	 */
238
	struct drm_i915_gem_request *preallocated_lazy_request;
239
	u32 outstanding_lazy_seqno;
240
	bool gpu_caches_dirty;
241
	bool fbc_dirty;
242

243
	wait_queue_head_t irq_queue;
Z
Zou Nan hai 已提交
244

245 246
	struct intel_context *default_context;
	struct intel_context *last_context;
247

248 249
	struct intel_ring_hangcheck hangcheck;

250 251 252 253 254
	struct {
		struct drm_i915_gem_object *obj;
		u32 gtt_offset;
		volatile u32 *cpu_page;
	} scratch;
255

256 257
	bool needs_cmd_parser;

258
	/*
259
	 * Table of commands the command parser needs to know about
260 261
	 * for this ring.
	 */
262
	DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287

	/*
	 * Table of registers allowed in commands that read/write registers.
	 */
	const u32 *reg_table;
	int reg_count;

	/*
	 * Table of registers allowed in commands that read/write registers, but
	 * only from the DRM master.
	 */
	const u32 *master_reg_table;
	int master_reg_count;

	/*
	 * Returns the bitmask for the length field of the specified command.
	 * Return 0 for an unrecognized/invalid command.
	 *
	 * If the command parser finds an entry for a command in the ring's
	 * cmd_tables, it gets the command's length based on the table entry.
	 * If not, it calls this function to determine the per-ring length field
	 * encoding for the command (i.e. certain opcode ranges use certain bits
	 * to encode the command length in the header).
	 */
	u32 (*get_cmd_length_mask)(u32 cmd_header);
288 289
};

290
static inline bool
291
intel_ring_initialized(struct intel_engine_cs *ring)
292
{
293
	return ring->buffer && ring->buffer->obj;
294 295
}

296
static inline unsigned
297
intel_ring_flag(struct intel_engine_cs *ring)
298 299 300 301
{
	return 1 << ring->id;
}

302
static inline u32
303 304
intel_ring_sync_index(struct intel_engine_cs *ring,
		      struct intel_engine_cs *other)
305 306 307 308
{
	int idx;

	/*
R
Rodrigo Vivi 已提交
309 310 311 312 313
	 * rcs -> 0 = vcs, 1 = bcs, 2 = vecs, 3 = vcs2;
	 * vcs -> 0 = bcs, 1 = vecs, 2 = vcs2, 3 = rcs;
	 * bcs -> 0 = vecs, 1 = vcs2. 2 = rcs, 3 = vcs;
	 * vecs -> 0 = vcs2, 1 = rcs, 2 = vcs, 3 = bcs;
	 * vcs2 -> 0 = rcs, 1 = vcs, 2 = bcs, 3 = vecs;
314 315 316 317 318 319 320 321 322
	 */

	idx = (other - ring) - 1;
	if (idx < 0)
		idx += I915_NUM_RINGS;

	return idx;
}

323
static inline u32
324
intel_read_status_page(struct intel_engine_cs *ring,
325
		       int reg)
326
{
327 328 329
	/* Ensure that the compiler doesn't optimize away the load. */
	barrier();
	return ring->status_page.page_addr[reg];
330 331
}

M
Mika Kuoppala 已提交
332
static inline void
333
intel_write_status_page(struct intel_engine_cs *ring,
M
Mika Kuoppala 已提交
334 335 336 337 338
			int reg, u32 value)
{
	ring->status_page.page_addr[reg] = value;
}

C
Chris Wilson 已提交
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
/**
 * Reads a dword out of the status page, which is written to from the command
 * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
 * MI_STORE_DATA_IMM.
 *
 * The following dwords have a reserved meaning:
 * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
 * 0x04: ring 0 head pointer
 * 0x05: ring 1 head pointer (915-class)
 * 0x06: ring 2 head pointer (915-class)
 * 0x10-0x1b: Context status DWords (GM45)
 * 0x1f: Last written status offset. (GM45)
 *
 * The area from dword 0x20 to 0x3ff is available for driver usage.
 */
#define I915_GEM_HWS_INDEX		0x20
355 356
#define I915_GEM_HWS_SCRATCH_INDEX	0x30
#define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
C
Chris Wilson 已提交
357

358 359
void intel_stop_ring_buffer(struct intel_engine_cs *ring);
void intel_cleanup_ring_buffer(struct intel_engine_cs *ring);
360

361 362 363
int __must_check intel_ring_begin(struct intel_engine_cs *ring, int n);
int __must_check intel_ring_cacheline_align(struct intel_engine_cs *ring);
static inline void intel_ring_emit(struct intel_engine_cs *ring,
364
				   u32 data)
365
{
366 367 368
	struct intel_ringbuffer *ringbuf = ring->buffer;
	iowrite32(data, ringbuf->virtual_start + ringbuf->tail);
	ringbuf->tail += 4;
369
}
370
static inline void intel_ring_advance(struct intel_engine_cs *ring)
371
{
372 373
	struct intel_ringbuffer *ringbuf = ring->buffer;
	ringbuf->tail &= ringbuf->size - 1;
374
}
375
void __intel_ring_advance(struct intel_engine_cs *ring);
376

377 378 379 380
int __must_check intel_ring_idle(struct intel_engine_cs *ring);
void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno);
int intel_ring_flush_all_caches(struct intel_engine_cs *ring);
int intel_ring_invalidate_all_caches(struct intel_engine_cs *ring);
381

382 383
int intel_init_render_ring_buffer(struct drm_device *dev);
int intel_init_bsd_ring_buffer(struct drm_device *dev);
384
int intel_init_bsd2_ring_buffer(struct drm_device *dev);
385
int intel_init_blt_ring_buffer(struct drm_device *dev);
B
Ben Widawsky 已提交
386
int intel_init_vebox_ring_buffer(struct drm_device *dev);
387

388 389
u64 intel_ring_get_active_head(struct intel_engine_cs *ring);
void intel_ring_setup_status_page(struct intel_engine_cs *ring);
390

391
static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf)
392
{
393
	return ringbuf->tail;
394 395
}

396
static inline u32 intel_ring_get_seqno(struct intel_engine_cs *ring)
397
{
398 399
	BUG_ON(ring->outstanding_lazy_seqno == 0);
	return ring->outstanding_lazy_seqno;
400 401
}

402
static inline void i915_trace_irq_get(struct intel_engine_cs *ring, u32 seqno)
C
Chris Wilson 已提交
403 404 405 406 407
{
	if (ring->trace_irq_seqno == 0 && ring->irq_get(ring))
		ring->trace_irq_seqno = seqno;
}

408 409 410
/* DRI warts */
int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size);

411
#endif /* _INTEL_RINGBUFFER_H_ */