radeon_ring.c 23.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
/*
 * Copyright 2008 Advanced Micro Devices, Inc.
 * Copyright 2008 Red Hat Inc.
 * Copyright 2009 Jerome Glisse.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Dave Airlie
 *          Alex Deucher
 *          Jerome Glisse
J
Jerome Glisse 已提交
27
 *          Christian König
28 29
 */
#include <linux/seq_file.h>
30
#include <linux/slab.h>
31 32
#include <drm/drmP.h>
#include <drm/radeon_drm.h>
33 34 35 36
#include "radeon_reg.h"
#include "radeon.h"
#include "atom.h"

J
Jerome Glisse 已提交
37
/*
38 39 40 41 42 43 44
 * IB
 * IBs (Indirect Buffers) and areas of GPU accessible memory where
 * commands are stored.  You can put a pointer to the IB in the
 * command ring and the hw will fetch the commands from the IB
 * and execute them.  Generally userspace acceleration drivers
 * produce command buffers which are send to the kernel and
 * put in IBs for execution by the requested ring.
J
Jerome Glisse 已提交
45
 */
46
static int radeon_debugfs_sa_init(struct radeon_device *rdev);
47

48 49 50 51 52 53 54 55 56 57 58 59
/**
 * radeon_ib_get - request an IB (Indirect Buffer)
 *
 * @rdev: radeon_device pointer
 * @ring: ring index the IB is associated with
 * @ib: IB object returned
 * @size: requested IB size
 *
 * Request an IB (all asics).  IBs are allocated using the
 * suballocator.
 * Returns 0 on success, error on failure.
 */
60
int radeon_ib_get(struct radeon_device *rdev, int ring,
61 62
		  struct radeon_ib *ib, struct radeon_vm *vm,
		  unsigned size)
63
{
64
	int i, r;
65

66
	r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &ib->sa_bo, size, 256, true);
J
Jerome Glisse 已提交
67 68 69
	if (r) {
		dev_err(rdev->dev, "failed to get a new IB (%d)\n", r);
		return r;
70
	}
J
Jerome Glisse 已提交
71

72 73 74 75 76
	r = radeon_semaphore_create(rdev, &ib->semaphore);
	if (r) {
		return r;
	}

77 78
	ib->ring = ring;
	ib->fence = NULL;
79
	ib->ptr = radeon_sa_bo_cpu_addr(ib->sa_bo);
80 81
	ib->vm = vm;
	if (vm) {
82 83
		/* ib pool is bound at RADEON_VA_IB_OFFSET in virtual address
		 * space and soffset is the offset inside the pool bo
84
		 */
85
		ib->gpu_addr = ib->sa_bo->soffset + RADEON_VA_IB_OFFSET;
86 87 88
	} else {
		ib->gpu_addr = radeon_sa_bo_gpu_addr(ib->sa_bo);
	}
89
	ib->is_const_ib = false;
90 91
	for (i = 0; i < RADEON_NUM_RINGS; ++i)
		ib->sync_to[i] = NULL;
J
Jerome Glisse 已提交
92 93

	return 0;
94 95
}

96 97 98 99 100 101 102 103
/**
 * radeon_ib_free - free an IB (Indirect Buffer)
 *
 * @rdev: radeon_device pointer
 * @ib: IB object to free
 *
 * Free an IB (all asics).
 */
104
void radeon_ib_free(struct radeon_device *rdev, struct radeon_ib *ib)
105
{
106
	radeon_semaphore_free(rdev, &ib->semaphore, ib->fence);
107 108
	radeon_sa_bo_free(rdev, &ib->sa_bo, ib->fence);
	radeon_fence_unref(&ib->fence);
109 110
}

111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
/**
 * radeon_ib_schedule - schedule an IB (Indirect Buffer) on the ring
 *
 * @rdev: radeon_device pointer
 * @ib: IB object to schedule
 * @const_ib: Const IB to schedule (SI only)
 *
 * Schedule an IB on the associated ring (all asics).
 * Returns 0 on success, error on failure.
 *
 * On SI, there are two parallel engines fed from the primary ring,
 * the CE (Constant Engine) and the DE (Drawing Engine).  Since
 * resource descriptors have moved to memory, the CE allows you to
 * prime the caches while the DE is updating register state so that
 * the resource descriptors will be already in cache when the draw is
 * processed.  To accomplish this, the userspace driver submits two
 * IBs, one for the CE and one for the DE.  If there is a CE IB (called
 * a CONST_IB), it will be put on the ring prior to the DE IB.  Prior
 * to SI there was just a DE IB.
 */
131 132
int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
		       struct radeon_ib *const_ib)
133
{
134
	struct radeon_ring *ring = &rdev->ring[ib->ring];
135 136
	bool need_sync = false;
	int i, r = 0;
137

138
	if (!ib->length_dw || !ring->ready) {
139
		/* TODO: Nothings in the ib we should report. */
J
Jerome Glisse 已提交
140
		dev_err(rdev->dev, "couldn't schedule ib\n");
141 142
		return -EINVAL;
	}
143

144
	/* 64 dwords should be enough for fence too */
145
	r = radeon_ring_lock(rdev, ring, 64 + RADEON_NUM_RINGS * 8);
146
	if (r) {
J
Jerome Glisse 已提交
147
		dev_err(rdev->dev, "scheduling IB failed (%d).\n", r);
148 149
		return r;
	}
150 151 152 153 154 155 156 157 158 159 160 161 162
	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
		struct radeon_fence *fence = ib->sync_to[i];
		if (radeon_fence_need_sync(fence, ib->ring)) {
			need_sync = true;
			radeon_semaphore_sync_rings(rdev, ib->semaphore,
						    fence->ring, ib->ring);
			radeon_fence_note_sync(fence, ib->ring);
		}
	}
	/* immediately free semaphore when we don't need to sync */
	if (!need_sync) {
		radeon_semaphore_free(rdev, &ib->semaphore, NULL);
	}
163 164
	/* if we can't remember our last VM flush then flush now! */
	if (ib->vm && !ib->vm->last_flush) {
165
		radeon_ring_vm_flush(rdev, ib->ring, ib->vm);
166
	}
167 168 169 170
	if (const_ib) {
		radeon_ring_ib_execute(rdev, const_ib->ring, const_ib);
		radeon_semaphore_free(rdev, &const_ib->semaphore, NULL);
	}
171 172 173 174 175 176 177
	radeon_ring_ib_execute(rdev, ib->ring, ib);
	r = radeon_fence_emit(rdev, &ib->fence, ib->ring);
	if (r) {
		dev_err(rdev->dev, "failed to emit fence for new IB (%d)\n", r);
		radeon_ring_unlock_undo(rdev, ring);
		return r;
	}
178 179 180
	if (const_ib) {
		const_ib->fence = radeon_fence_ref(ib->fence);
	}
181 182 183 184
	/* we just flushed the VM, remember that */
	if (ib->vm && !ib->vm->last_flush) {
		ib->vm->last_flush = radeon_fence_ref(ib->fence);
	}
185
	radeon_ring_unlock_commit(rdev, ring);
186 187 188
	return 0;
}

189 190 191 192 193 194 195 196 197
/**
 * radeon_ib_pool_init - Init the IB (Indirect Buffer) pool
 *
 * @rdev: radeon_device pointer
 *
 * Initialize the suballocator to manage a pool of memory
 * for use as IBs (all asics).
 * Returns 0 on success, error on failure.
 */
198 199
int radeon_ib_pool_init(struct radeon_device *rdev)
{
J
Jerome Glisse 已提交
200
	int r;
201

J
Jerome Glisse 已提交
202
	if (rdev->ib_pool_ready) {
203 204
		return 0;
	}
J
Jerome Glisse 已提交
205
	r = radeon_sa_bo_manager_init(rdev, &rdev->ring_tmp_bo,
206 207 208 209 210
				      RADEON_IB_POOL_SIZE*64*1024,
				      RADEON_GEM_DOMAIN_GTT);
	if (r) {
		return r;
	}
211 212 213 214 215 216

	r = radeon_sa_bo_manager_start(rdev, &rdev->ring_tmp_bo);
	if (r) {
		return r;
	}

J
Jerome Glisse 已提交
217 218 219
	rdev->ib_pool_ready = true;
	if (radeon_debugfs_sa_init(rdev)) {
		dev_err(rdev->dev, "failed to register debugfs file for SA\n");
220
	}
221
	return 0;
222 223
}

224 225 226 227 228 229 230 231
/**
 * radeon_ib_pool_fini - Free the IB (Indirect Buffer) pool
 *
 * @rdev: radeon_device pointer
 *
 * Tear down the suballocator managing the pool of memory
 * for use as IBs (all asics).
 */
232 233
void radeon_ib_pool_fini(struct radeon_device *rdev)
{
J
Jerome Glisse 已提交
234
	if (rdev->ib_pool_ready) {
235
		radeon_sa_bo_manager_suspend(rdev, &rdev->ring_tmp_bo);
J
Jerome Glisse 已提交
236 237
		radeon_sa_bo_manager_fini(rdev, &rdev->ring_tmp_bo);
		rdev->ib_pool_ready = false;
238 239 240
	}
}

241 242 243 244 245 246 247 248 249 250
/**
 * radeon_ib_ring_tests - test IBs on the rings
 *
 * @rdev: radeon_device pointer
 *
 * Test an IB (Indirect Buffer) on each ring.
 * If the test fails, disable the ring.
 * Returns 0 on success, error if the primary GFX ring
 * IB test fails.
 */
251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
int radeon_ib_ring_tests(struct radeon_device *rdev)
{
	unsigned i;
	int r;

	for (i = 0; i < RADEON_NUM_RINGS; ++i) {
		struct radeon_ring *ring = &rdev->ring[i];

		if (!ring->ready)
			continue;

		r = radeon_ib_test(rdev, i, ring);
		if (r) {
			ring->ready = false;

			if (i == RADEON_RING_TYPE_GFX_INDEX) {
				/* oh, oh, that's really bad */
				DRM_ERROR("radeon: failed testing IB on GFX ring (%d).\n", r);
		                rdev->accel_working = false;
				return r;

			} else {
				/* still not good, but we can live with it */
				DRM_ERROR("radeon: failed testing IB on ring %d (%d).\n", i, r);
			}
		}
	}
	return 0;
}

281
/*
282 283 284 285 286 287 288 289 290 291 292
 * Rings
 * Most engines on the GPU are fed via ring buffers.  Ring
 * buffers are areas of GPU accessible memory that the host
 * writes commands into and the GPU reads commands out of.
 * There is a rptr (read pointer) that determines where the
 * GPU is currently reading, and a wptr (write pointer)
 * which determines where the host has written.  When the
 * pointers are equal, the ring is idle.  When the host
 * writes commands to the ring buffer, it increments the
 * wptr.  The GPU then starts fetching commands and executes
 * them until the pointers are equal again.
293
 */
294
static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring);
J
Jerome Glisse 已提交
295

296 297 298 299 300 301 302 303
/**
 * radeon_ring_write - write a value to the ring
 *
 * @ring: radeon_ring structure holding ring information
 * @v: dword (dw) value to write
 *
 * Write a value to the requested ring buffer (all asics).
 */
J
Jerome Glisse 已提交
304 305 306 307
void radeon_ring_write(struct radeon_ring *ring, uint32_t v)
{
#if DRM_DEBUG_CODE
	if (ring->count_dw <= 0) {
308
		DRM_ERROR("radeon: writing more dwords to the ring than expected!\n");
J
Jerome Glisse 已提交
309 310 311 312 313 314 315 316
	}
#endif
	ring->ring[ring->wptr++] = v;
	ring->wptr &= ring->ptr_mask;
	ring->count_dw--;
	ring->ring_free_dw--;
}

317 318 319 320 321 322 323 324 325 326
/**
 * radeon_ring_supports_scratch_reg - check if the ring supports
 * writing to scratch registers
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 *
 * Check if a specific ring supports writing to scratch registers (all asics).
 * Returns true if the ring supports writing to scratch regs, false if not.
 */
327 328 329 330 331 332 333 334 335 336 337 338 339
bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev,
				      struct radeon_ring *ring)
{
	switch (ring->idx) {
	case RADEON_RING_TYPE_GFX_INDEX:
	case CAYMAN_RING_TYPE_CP1_INDEX:
	case CAYMAN_RING_TYPE_CP2_INDEX:
		return true;
	default:
		return false;
	}
}

340 341 342 343 344 345 346 347
/**
 * radeon_ring_free_size - update the free size
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 *
 * Update the free dw slots in the ring buffer (all asics).
 */
348
void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)
349
{
350 351
	u32 rptr;

352
	if (rdev->wb.enabled)
353
		rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
354
	else
355 356
		rptr = RREG32(ring->rptr_reg);
	ring->rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
357
	/* This works because ring_size is a power of 2 */
358 359 360 361 362
	ring->ring_free_dw = (ring->rptr + (ring->ring_size / 4));
	ring->ring_free_dw -= ring->wptr;
	ring->ring_free_dw &= ring->ptr_mask;
	if (!ring->ring_free_dw) {
		ring->ring_free_dw = ring->ring_size / 4;
363 364 365
	}
}

366 367 368 369 370 371 372 373 374 375
/**
 * radeon_ring_alloc - allocate space on the ring buffer
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 * @ndw: number of dwords to allocate in the ring buffer
 *
 * Allocate @ndw dwords in the ring buffer (all asics).
 * Returns 0 on success, error on failure.
 */
376
int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
377 378 379 380 381
{
	int r;

	/* Align requested size with padding so unlock_commit can
	 * pad safely */
382 383 384 385
	ndw = (ndw + ring->align_mask) & ~ring->align_mask;
	while (ndw > (ring->ring_free_dw - 1)) {
		radeon_ring_free_size(rdev, ring);
		if (ndw < ring->ring_free_dw) {
386 387
			break;
		}
388
		r = radeon_fence_wait_next_locked(rdev, ring->idx);
389
		if (r)
390 391
			return r;
	}
392 393
	ring->count_dw = ndw;
	ring->wptr_old = ring->wptr;
394 395 396
	return 0;
}

397 398 399 400 401 402 403 404 405 406 407
/**
 * radeon_ring_lock - lock the ring and allocate space on it
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 * @ndw: number of dwords to allocate in the ring buffer
 *
 * Lock the ring and allocate @ndw dwords in the ring buffer
 * (all asics).
 * Returns 0 on success, error on failure.
 */
408
int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)
409 410 411
{
	int r;

412
	mutex_lock(&rdev->ring_lock);
413
	r = radeon_ring_alloc(rdev, ring, ndw);
414
	if (r) {
415
		mutex_unlock(&rdev->ring_lock);
416 417 418 419 420
		return r;
	}
	return 0;
}

421 422 423 424 425 426 427 428 429 430
/**
 * radeon_ring_commit - tell the GPU to execute the new
 * commands on the ring buffer
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 *
 * Update the wptr (write pointer) to tell the GPU to
 * execute new commands on the ring buffer (all asics).
 */
431
void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring)
432 433
{
	/* We pad to match fetch size */
434
	while (ring->wptr & ring->align_mask) {
435
		radeon_ring_write(ring, ring->nop);
436 437
	}
	DRM_MEMORYBARRIER();
438
	WREG32(ring->wptr_reg, (ring->wptr << ring->ptr_reg_shift) & ring->ptr_reg_mask);
439
	(void)RREG32(ring->wptr_reg);
440 441
}

442 443 444 445 446 447 448 449 450
/**
 * radeon_ring_unlock_commit - tell the GPU to execute the new
 * commands on the ring buffer and unlock it
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 *
 * Call radeon_ring_commit() then unlock the ring (all asics).
 */
451
void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring)
452
{
453
	radeon_ring_commit(rdev, ring);
454
	mutex_unlock(&rdev->ring_lock);
455 456
}

457 458 459 460 461 462 463
/**
 * radeon_ring_undo - reset the wptr
 *
 * @ring: radeon_ring structure holding ring information
 *
 * Reset the driver's copy of the wtpr (all asics).
 */
464
void radeon_ring_undo(struct radeon_ring *ring)
465
{
466
	ring->wptr = ring->wptr_old;
467 468
}

469 470 471 472 473 474 475
/**
 * radeon_ring_unlock_undo - reset the wptr and unlock the ring
 *
 * @ring: radeon_ring structure holding ring information
 *
 * Call radeon_ring_undo() then unlock the ring (all asics).
 */
476 477 478 479
void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring)
{
	radeon_ring_undo(ring);
	mutex_unlock(&rdev->ring_lock);
480 481
}

482 483 484 485 486 487 488 489 490
/**
 * radeon_ring_force_activity - add some nop packets to the ring
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 *
 * Add some nop packets to the ring to force activity (all asics).
 * Used for lockup detection to see if the rptr is advancing.
 */
491 492 493 494 495 496 497 498 499 500 501 502 503 504
void radeon_ring_force_activity(struct radeon_device *rdev, struct radeon_ring *ring)
{
	int r;

	radeon_ring_free_size(rdev, ring);
	if (ring->rptr == ring->wptr) {
		r = radeon_ring_alloc(rdev, ring, 1);
		if (!r) {
			radeon_ring_write(ring, ring->nop);
			radeon_ring_commit(rdev, ring);
		}
	}
}

505 506 507 508 509 510 511
/**
 * radeon_ring_force_activity - update lockup variables
 *
 * @ring: radeon_ring structure holding ring information
 *
 * Update the last rptr value and timestamp (all asics).
 */
512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556
void radeon_ring_lockup_update(struct radeon_ring *ring)
{
	ring->last_rptr = ring->rptr;
	ring->last_activity = jiffies;
}

/**
 * radeon_ring_test_lockup() - check if ring is lockedup by recording information
 * @rdev:       radeon device structure
 * @ring:       radeon_ring structure holding ring information
 *
 * We don't need to initialize the lockup tracking information as we will either
 * have CP rptr to a different value of jiffies wrap around which will force
 * initialization of the lockup tracking informations.
 *
 * A possible false positivie is if we get call after while and last_cp_rptr ==
 * the current CP rptr, even if it's unlikely it might happen. To avoid this
 * if the elapsed time since last call is bigger than 2 second than we return
 * false and update the tracking information. Due to this the caller must call
 * radeon_ring_test_lockup several time in less than 2sec for lockup to be reported
 * the fencing code should be cautious about that.
 *
 * Caller should write to the ring to force CP to do something so we don't get
 * false positive when CP is just gived nothing to do.
 *
 **/
bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
{
	unsigned long cjiffies, elapsed;
	uint32_t rptr;

	cjiffies = jiffies;
	if (!time_after(cjiffies, ring->last_activity)) {
		/* likely a wrap around */
		radeon_ring_lockup_update(ring);
		return false;
	}
	rptr = RREG32(ring->rptr_reg);
	ring->rptr = (rptr & ring->ptr_reg_mask) >> ring->ptr_reg_shift;
	if (ring->rptr != ring->last_rptr) {
		/* CP is still working no lockup */
		radeon_ring_lockup_update(ring);
		return false;
	}
	elapsed = jiffies_to_msecs(cjiffies - ring->last_activity);
557
	if (radeon_lockup_timeout && elapsed >= radeon_lockup_timeout) {
558 559 560 561 562 563 564
		dev_err(rdev->dev, "GPU lockup CP stall for more than %lumsec\n", elapsed);
		return true;
	}
	/* give a chance to the GPU ... */
	return false;
}

565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581
/**
 * radeon_ring_backup - Back up the content of a ring
 *
 * @rdev: radeon_device pointer
 * @ring: the ring we want to back up
 *
 * Saves all unprocessed commits from a ring, returns the number of dwords saved.
 */
unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,
			    uint32_t **data)
{
	unsigned size, ptr, i;

	/* just in case lock the ring */
	mutex_lock(&rdev->ring_lock);
	*data = NULL;

582
	if (ring->ring_obj == NULL) {
583 584 585 586 587
		mutex_unlock(&rdev->ring_lock);
		return 0;
	}

	/* it doesn't make sense to save anything if all fences are signaled */
588
	if (!radeon_fence_count_emitted(rdev, ring->idx)) {
589 590 591 592 593
		mutex_unlock(&rdev->ring_lock);
		return 0;
	}

	/* calculate the number of dw on the ring */
594 595 596 597 598 599 600 601 602 603
	if (ring->rptr_save_reg)
		ptr = RREG32(ring->rptr_save_reg);
	else if (rdev->wb.enabled)
		ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);
	else {
		/* no way to read back the next rptr */
		mutex_unlock(&rdev->ring_lock);
		return 0;
	}

604 605 606 607 608 609 610 611 612
	size = ring->wptr + (ring->ring_size / 4);
	size -= ptr;
	size &= ring->ptr_mask;
	if (size == 0) {
		mutex_unlock(&rdev->ring_lock);
		return 0;
	}

	/* and then save the content of the ring */
613 614 615 616 617
	*data = kmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
	if (!*data) {
		mutex_unlock(&rdev->ring_lock);
		return 0;
	}
618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658
	for (i = 0; i < size; ++i) {
		(*data)[i] = ring->ring[ptr++];
		ptr &= ring->ptr_mask;
	}

	mutex_unlock(&rdev->ring_lock);
	return size;
}

/**
 * radeon_ring_restore - append saved commands to the ring again
 *
 * @rdev: radeon_device pointer
 * @ring: ring to append commands to
 * @size: number of dwords we want to write
 * @data: saved commands
 *
 * Allocates space on the ring and restore the previously saved commands.
 */
int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,
			unsigned size, uint32_t *data)
{
	int i, r;

	if (!size || !data)
		return 0;

	/* restore the saved ring content */
	r = radeon_ring_lock(rdev, ring, size);
	if (r)
		return r;

	for (i = 0; i < size; ++i) {
		radeon_ring_write(ring, data[i]);
	}

	radeon_ring_unlock_commit(rdev, ring);
	kfree(data);
	return 0;
}

659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
/**
 * radeon_ring_init - init driver ring struct.
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 * @ring_size: size of the ring
 * @rptr_offs: offset of the rptr writeback location in the WB buffer
 * @rptr_reg: MMIO offset of the rptr register
 * @wptr_reg: MMIO offset of the wptr register
 * @ptr_reg_shift: bit offset of the rptr/wptr values
 * @ptr_reg_mask: bit mask of the rptr/wptr values
 * @nop: nop packet for this ring
 *
 * Initialize the driver information for the selected ring (all asics).
 * Returns 0 on success, error on failure.
 */
675
int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size,
676 677
		     unsigned rptr_offs, unsigned rptr_reg, unsigned wptr_reg,
		     u32 ptr_reg_shift, u32 ptr_reg_mask, u32 nop)
678 679 680
{
	int r;

681 682 683 684
	ring->ring_size = ring_size;
	ring->rptr_offs = rptr_offs;
	ring->rptr_reg = rptr_reg;
	ring->wptr_reg = wptr_reg;
685 686 687
	ring->ptr_reg_shift = ptr_reg_shift;
	ring->ptr_reg_mask = ptr_reg_mask;
	ring->nop = nop;
688
	/* Allocate ring buffer */
689 690
	if (ring->ring_obj == NULL) {
		r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true,
691 692
				     RADEON_GEM_DOMAIN_GTT,
				     NULL, &ring->ring_obj);
693
		if (r) {
694
			dev_err(rdev->dev, "(%d) ring create failed\n", r);
695 696
			return r;
		}
697
		r = radeon_bo_reserve(ring->ring_obj, false);
698 699
		if (unlikely(r != 0))
			return r;
700 701
		r = radeon_bo_pin(ring->ring_obj, RADEON_GEM_DOMAIN_GTT,
					&ring->gpu_addr);
702
		if (r) {
703
			radeon_bo_unreserve(ring->ring_obj);
704
			dev_err(rdev->dev, "(%d) ring pin failed\n", r);
705 706
			return r;
		}
707 708 709
		r = radeon_bo_kmap(ring->ring_obj,
				       (void **)&ring->ring);
		radeon_bo_unreserve(ring->ring_obj);
710
		if (r) {
711
			dev_err(rdev->dev, "(%d) ring map failed\n", r);
712 713 714
			return r;
		}
	}
715 716
	ring->ptr_mask = (ring->ring_size / 4) - 1;
	ring->ring_free_dw = ring->ring_size / 4;
717 718 719 720 721
	if (rdev->wb.enabled) {
		u32 index = RADEON_WB_RING0_NEXT_RPTR + (ring->idx * 4);
		ring->next_rptr_gpu_addr = rdev->wb.gpu_addr + index;
		ring->next_rptr_cpu_addr = &rdev->wb.wb[index/4];
	}
722 723 724
	if (radeon_debugfs_ring_init(rdev, ring)) {
		DRM_ERROR("Failed to register debugfs file for rings !\n");
	}
725
	radeon_ring_lockup_update(ring);
726 727 728
	return 0;
}

729 730 731 732 733 734 735 736
/**
 * radeon_ring_fini - tear down the driver ring struct.
 *
 * @rdev: radeon_device pointer
 * @ring: radeon_ring structure holding ring information
 *
 * Tear down the driver information for the selected ring (all asics).
 */
737
void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)
738
{
739
	int r;
740
	struct radeon_bo *ring_obj;
741

742
	mutex_lock(&rdev->ring_lock);
743
	ring_obj = ring->ring_obj;
744
	ring->ready = false;
745 746
	ring->ring = NULL;
	ring->ring_obj = NULL;
747
	mutex_unlock(&rdev->ring_lock);
748 749 750

	if (ring_obj) {
		r = radeon_bo_reserve(ring_obj, false);
751
		if (likely(r == 0)) {
752 753 754
			radeon_bo_kunmap(ring_obj);
			radeon_bo_unpin(ring_obj);
			radeon_bo_unreserve(ring_obj);
755
		}
756
		radeon_bo_unref(&ring_obj);
757 758 759 760 761 762 763
	}
}

/*
 * Debugfs info
 */
#if defined(CONFIG_DEBUG_FS)
764 765 766 767 768 769 770 771 772 773 774 775 776 777

static int radeon_debugfs_ring_info(struct seq_file *m, void *data)
{
	struct drm_info_node *node = (struct drm_info_node *) m->private;
	struct drm_device *dev = node->minor->dev;
	struct radeon_device *rdev = dev->dev_private;
	int ridx = *(int*)node->info_ent->data;
	struct radeon_ring *ring = &rdev->ring[ridx];
	unsigned count, i, j;

	radeon_ring_free_size(rdev, ring);
	count = (ring->ring_size / 4) - ring->ring_free_dw;
	seq_printf(m, "wptr(0x%04x): 0x%08x\n", ring->wptr_reg, RREG32(ring->wptr_reg));
	seq_printf(m, "rptr(0x%04x): 0x%08x\n", ring->rptr_reg, RREG32(ring->rptr_reg));
778 779 780 781
	if (ring->rptr_save_reg) {
		seq_printf(m, "rptr next(0x%04x): 0x%08x\n", ring->rptr_save_reg,
			   RREG32(ring->rptr_save_reg));
	}
782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803
	seq_printf(m, "driver's copy of the wptr: 0x%08x\n", ring->wptr);
	seq_printf(m, "driver's copy of the rptr: 0x%08x\n", ring->rptr);
	seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);
	seq_printf(m, "%u dwords in ring\n", count);
	i = ring->rptr;
	for (j = 0; j <= count; j++) {
		seq_printf(m, "r[%04d]=0x%08x\n", i, ring->ring[i]);
		i = (i + 1) & ring->ptr_mask;
	}
	return 0;
}

static int radeon_ring_type_gfx_index = RADEON_RING_TYPE_GFX_INDEX;
static int cayman_ring_type_cp1_index = CAYMAN_RING_TYPE_CP1_INDEX;
static int cayman_ring_type_cp2_index = CAYMAN_RING_TYPE_CP2_INDEX;

static struct drm_info_list radeon_debugfs_ring_info_list[] = {
	{"radeon_ring_gfx", radeon_debugfs_ring_info, 0, &radeon_ring_type_gfx_index},
	{"radeon_ring_cp1", radeon_debugfs_ring_info, 0, &cayman_ring_type_cp1_index},
	{"radeon_ring_cp2", radeon_debugfs_ring_info, 0, &cayman_ring_type_cp2_index},
};

804 805 806 807 808 809
static int radeon_debugfs_sa_info(struct seq_file *m, void *data)
{
	struct drm_info_node *node = (struct drm_info_node *) m->private;
	struct drm_device *dev = node->minor->dev;
	struct radeon_device *rdev = dev->dev_private;

J
Jerome Glisse 已提交
810
	radeon_sa_bo_dump_debug_info(&rdev->ring_tmp_bo, m);
811 812 813 814 815 816 817 818 819

	return 0;

}

static struct drm_info_list radeon_debugfs_sa_list[] = {
        {"radeon_sa_info", &radeon_debugfs_sa_info, 0, NULL},
};

820 821
#endif

822
static int radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring)
823 824
{
#if defined(CONFIG_DEBUG_FS)
825 826 827 828 829 830 831 832 833 834 835 836 837
	unsigned i;
	for (i = 0; i < ARRAY_SIZE(radeon_debugfs_ring_info_list); ++i) {
		struct drm_info_list *info = &radeon_debugfs_ring_info_list[i];
		int ridx = *(int*)radeon_debugfs_ring_info_list[i].data;
		unsigned r;

		if (&rdev->ring[ridx] != ring)
			continue;

		r = radeon_debugfs_add_files(rdev, info, 1);
		if (r)
			return r;
	}
838
#endif
839
	return 0;
840 841
}

842
static int radeon_debugfs_sa_init(struct radeon_device *rdev)
843 844
{
#if defined(CONFIG_DEBUG_FS)
J
Jerome Glisse 已提交
845
	return radeon_debugfs_add_files(rdev, radeon_debugfs_sa_list, 1);
846 847 848 849
#else
	return 0;
#endif
}