r300.c 38.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
/*
 * Copyright 2008 Advanced Micro Devices, Inc.
 * Copyright 2008 Red Hat Inc.
 * Copyright 2009 Jerome Glisse.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Dave Airlie
 *          Alex Deucher
 *          Jerome Glisse
 */
#include <linux/seq_file.h>
29
#include <linux/slab.h>
30 31 32
#include <drm/drmP.h>
#include <drm/drm.h>
#include <drm/drm_crtc_helper.h>
33 34
#include "radeon_reg.h"
#include "radeon.h"
35
#include "radeon_asic.h"
36
#include "radeon_drm.h"
37
#include "r100_track.h"
38
#include "r300d.h"
39
#include "rv350d.h"
40 41
#include "r300_reg_safe.h"

42 43 44 45 46 47 48 49 50
/* This files gather functions specifics to: r300,r350,rv350,rv370,rv380
 *
 * GPU Errata:
 * - HOST_PATH_CNTL: r300 family seems to dislike write to HOST_PATH_CNTL
 *   using MMIO to flush host path read cache, this lead to HARDLOCKUP.
 *   However, scheduling such write to the ring seems harmless, i suspect
 *   the CP read collide with the flush somehow, or maybe the MC, hard to
 *   tell. (Jerome Glisse)
 */
51 52 53 54

/*
 * rv370,rv380 PCIE GART
 */
55 56
static int rv370_debugfs_pcie_gart_info_init(struct radeon_device *rdev);

57 58 59 60 61 62 63 64 65 66 67 68
void rv370_pcie_gart_tlb_flush(struct radeon_device *rdev)
{
	uint32_t tmp;
	int i;

	/* Workaround HW bug do flush 2 times */
	for (i = 0; i < 2; i++) {
		tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_CNTL);
		WREG32_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp | RADEON_PCIE_TX_GART_INVALIDATE_TLB);
		(void)RREG32_PCIE(RADEON_PCIE_TX_GART_CNTL);
		WREG32_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp);
	}
69
	mb();
70 71
}

72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
int rv370_pcie_gart_set_page(struct radeon_device *rdev, int i, uint64_t addr)
{
	void __iomem *ptr = (void *)rdev->gart.table.vram.ptr;

	if (i < 0 || i > rdev->gart.num_gpu_pages) {
		return -EINVAL;
	}
	addr = (lower_32_bits(addr) >> 8) |
	       ((upper_32_bits(addr) & 0xff) << 24) |
	       0xc;
	/* on x86 we want this to be CPU endian, on powerpc
	 * on powerpc without HW swappers, it'll get swapped on way
	 * into VRAM - so no need for cpu_to_le32 on VRAM tables */
	writel(addr, ((void __iomem *)ptr) + (i * 4));
	return 0;
}

int rv370_pcie_gart_init(struct radeon_device *rdev)
90 91 92
{
	int r;

93 94 95 96
	if (rdev->gart.table.vram.robj) {
		WARN(1, "RV370 PCIE GART already initialized.\n");
		return 0;
	}
97 98
	/* Initialize common gart structure */
	r = radeon_gart_init(rdev);
99
	if (r)
100 101
		return r;
	r = rv370_debugfs_pcie_gart_info_init(rdev);
102
	if (r)
103 104
		DRM_ERROR("Failed to register debugfs file for PCIE gart !\n");
	rdev->gart.table_size = rdev->gart.num_gpu_pages * 4;
105 106 107 108 109 110 111 112 113 114 115 116 117 118
	rdev->asic->gart_tlb_flush = &rv370_pcie_gart_tlb_flush;
	rdev->asic->gart_set_page = &rv370_pcie_gart_set_page;
	return radeon_gart_table_vram_alloc(rdev);
}

int rv370_pcie_gart_enable(struct radeon_device *rdev)
{
	uint32_t table_addr;
	uint32_t tmp;
	int r;

	if (rdev->gart.table.vram.robj == NULL) {
		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
		return -EINVAL;
119
	}
120 121 122
	r = radeon_gart_table_vram_pin(rdev);
	if (r)
		return r;
123
	radeon_gart_restore(rdev);
124 125 126
	/* discard memory request outside of configured range */
	tmp = RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_DISCARD;
	WREG32_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp);
127 128
	WREG32_PCIE(RADEON_PCIE_TX_GART_START_LO, rdev->mc.gtt_start);
	tmp = rdev->mc.gtt_end & ~RADEON_GPU_PAGE_MASK;
129 130 131 132 133 134
	WREG32_PCIE(RADEON_PCIE_TX_GART_END_LO, tmp);
	WREG32_PCIE(RADEON_PCIE_TX_GART_START_HI, 0);
	WREG32_PCIE(RADEON_PCIE_TX_GART_END_HI, 0);
	table_addr = rdev->gart.table_addr;
	WREG32_PCIE(RADEON_PCIE_TX_GART_BASE, table_addr);
	/* FIXME: setup default page */
135
	WREG32_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_LO, rdev->mc.vram_start);
136 137 138 139 140 141 142 143 144
	WREG32_PCIE(RADEON_PCIE_TX_DISCARD_RD_ADDR_HI, 0);
	/* Clear error */
	WREG32_PCIE(0x18, 0);
	tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_CNTL);
	tmp |= RADEON_PCIE_TX_GART_EN;
	tmp |= RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_DISCARD;
	WREG32_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp);
	rv370_pcie_gart_tlb_flush(rdev);
	DRM_INFO("PCIE GART of %uM enabled (table at 0x%08X).\n",
145
		 (unsigned)(rdev->mc.gtt_size >> 20), table_addr);
146 147 148 149 150 151
	rdev->gart.ready = true;
	return 0;
}

void rv370_pcie_gart_disable(struct radeon_device *rdev)
{
152 153
	u32 tmp;
	int r;
154

155 156 157 158
	WREG32_PCIE(RADEON_PCIE_TX_GART_START_LO, 0);
	WREG32_PCIE(RADEON_PCIE_TX_GART_END_LO, 0);
	WREG32_PCIE(RADEON_PCIE_TX_GART_START_HI, 0);
	WREG32_PCIE(RADEON_PCIE_TX_GART_END_HI, 0);
159 160 161 162
	tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_CNTL);
	tmp |= RADEON_PCIE_TX_GART_UNMAPPED_ACCESS_DISCARD;
	WREG32_PCIE(RADEON_PCIE_TX_GART_CNTL, tmp & ~RADEON_PCIE_TX_GART_EN);
	if (rdev->gart.table.vram.robj) {
163 164 165 166 167 168
		r = radeon_bo_reserve(rdev->gart.table.vram.robj, false);
		if (likely(r == 0)) {
			radeon_bo_kunmap(rdev->gart.table.vram.robj);
			radeon_bo_unpin(rdev->gart.table.vram.robj);
			radeon_bo_unreserve(rdev->gart.table.vram.robj);
		}
169 170 171
	}
}

172
void rv370_pcie_gart_fini(struct radeon_device *rdev)
173
{
174
	radeon_gart_fini(rdev);
175 176
	rv370_pcie_gart_disable(rdev);
	radeon_gart_table_vram_free(rdev);
177 178 179 180 181 182 183 184
}

void r300_fence_ring_emit(struct radeon_device *rdev,
			  struct radeon_fence *fence)
{
	/* Who ever call radeon_fence_emit should call ring_lock and ask
	 * for enough space (today caller are ib schedule and buffer move) */
	/* Write SC register so SC & US assert idle */
185
	radeon_ring_write(rdev, PACKET0(R300_RE_SCISSORS_TL, 0));
186
	radeon_ring_write(rdev, 0);
187
	radeon_ring_write(rdev, PACKET0(R300_RE_SCISSORS_BR, 0));
188 189
	radeon_ring_write(rdev, 0);
	/* Flush 3D cache */
190 191 192 193
	radeon_ring_write(rdev, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
	radeon_ring_write(rdev, R300_RB3D_DC_FLUSH);
	radeon_ring_write(rdev, PACKET0(R300_RB3D_ZCACHE_CTLSTAT, 0));
	radeon_ring_write(rdev, R300_ZC_FLUSH);
194
	/* Wait until IDLE & CLEAN */
195 196 197 198
	radeon_ring_write(rdev, PACKET0(RADEON_WAIT_UNTIL, 0));
	radeon_ring_write(rdev, (RADEON_WAIT_3D_IDLECLEAN |
				 RADEON_WAIT_2D_IDLECLEAN |
				 RADEON_WAIT_DMA_GUI_IDLE));
199 200 201 202 203
	radeon_ring_write(rdev, PACKET0(RADEON_HOST_PATH_CNTL, 0));
	radeon_ring_write(rdev, rdev->config.r300.hdp_cntl |
				RADEON_HDP_READ_BUFFER_INVALIDATE);
	radeon_ring_write(rdev, PACKET0(RADEON_HOST_PATH_CNTL, 0));
	radeon_ring_write(rdev, rdev->config.r300.hdp_cntl);
204 205 206 207 208 209 210 211 212 213 214 215 216 217
	/* Emit fence sequence & fire IRQ */
	radeon_ring_write(rdev, PACKET0(rdev->fence_drv.scratch_reg, 0));
	radeon_ring_write(rdev, fence->seq);
	radeon_ring_write(rdev, PACKET0(RADEON_GEN_INT_STATUS, 0));
	radeon_ring_write(rdev, RADEON_SW_INT_FIRE);
}

void r300_ring_start(struct radeon_device *rdev)
{
	unsigned gb_tile_config;
	int r;

	/* Sub pixel 1/12 so we can have 4K rendering according to doc */
	gb_tile_config = (R300_ENABLE_TILING | R300_TILE_SIZE_16);
218
	switch(rdev->num_gb_pipes) {
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
	case 2:
		gb_tile_config |= R300_PIPE_COUNT_R300;
		break;
	case 3:
		gb_tile_config |= R300_PIPE_COUNT_R420_3P;
		break;
	case 4:
		gb_tile_config |= R300_PIPE_COUNT_R420;
		break;
	case 1:
	default:
		gb_tile_config |= R300_PIPE_COUNT_RV350;
		break;
	}

	r = radeon_ring_lock(rdev, 64);
	if (r) {
		return;
	}
	radeon_ring_write(rdev, PACKET0(RADEON_ISYNC_CNTL, 0));
	radeon_ring_write(rdev,
			  RADEON_ISYNC_ANY2D_IDLE3D |
			  RADEON_ISYNC_ANY3D_IDLE2D |
			  RADEON_ISYNC_WAIT_IDLEGUI |
			  RADEON_ISYNC_CPSCRATCH_IDLEGUI);
	radeon_ring_write(rdev, PACKET0(R300_GB_TILE_CONFIG, 0));
	radeon_ring_write(rdev, gb_tile_config);
	radeon_ring_write(rdev, PACKET0(RADEON_WAIT_UNTIL, 0));
	radeon_ring_write(rdev,
			  RADEON_WAIT_2D_IDLECLEAN |
			  RADEON_WAIT_3D_IDLECLEAN);
250 251
	radeon_ring_write(rdev, PACKET0(R300_DST_PIPE_CONFIG, 0));
	radeon_ring_write(rdev, R300_PIPE_AUTO_CONFIG);
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
	radeon_ring_write(rdev, PACKET0(R300_GB_SELECT, 0));
	radeon_ring_write(rdev, 0);
	radeon_ring_write(rdev, PACKET0(R300_GB_ENABLE, 0));
	radeon_ring_write(rdev, 0);
	radeon_ring_write(rdev, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
	radeon_ring_write(rdev, R300_RB3D_DC_FLUSH | R300_RB3D_DC_FREE);
	radeon_ring_write(rdev, PACKET0(R300_RB3D_ZCACHE_CTLSTAT, 0));
	radeon_ring_write(rdev, R300_ZC_FLUSH | R300_ZC_FREE);
	radeon_ring_write(rdev, PACKET0(RADEON_WAIT_UNTIL, 0));
	radeon_ring_write(rdev,
			  RADEON_WAIT_2D_IDLECLEAN |
			  RADEON_WAIT_3D_IDLECLEAN);
	radeon_ring_write(rdev, PACKET0(R300_GB_AA_CONFIG, 0));
	radeon_ring_write(rdev, 0);
	radeon_ring_write(rdev, PACKET0(R300_RB3D_DSTCACHE_CTLSTAT, 0));
	radeon_ring_write(rdev, R300_RB3D_DC_FLUSH | R300_RB3D_DC_FREE);
	radeon_ring_write(rdev, PACKET0(R300_RB3D_ZCACHE_CTLSTAT, 0));
	radeon_ring_write(rdev, R300_ZC_FLUSH | R300_ZC_FREE);
	radeon_ring_write(rdev, PACKET0(R300_GB_MSPOS0, 0));
	radeon_ring_write(rdev,
			  ((6 << R300_MS_X0_SHIFT) |
			   (6 << R300_MS_Y0_SHIFT) |
			   (6 << R300_MS_X1_SHIFT) |
			   (6 << R300_MS_Y1_SHIFT) |
			   (6 << R300_MS_X2_SHIFT) |
			   (6 << R300_MS_Y2_SHIFT) |
			   (6 << R300_MSBD0_Y_SHIFT) |
			   (6 << R300_MSBD0_X_SHIFT)));
	radeon_ring_write(rdev, PACKET0(R300_GB_MSPOS1, 0));
	radeon_ring_write(rdev,
			  ((6 << R300_MS_X3_SHIFT) |
			   (6 << R300_MS_Y3_SHIFT) |
			   (6 << R300_MS_X4_SHIFT) |
			   (6 << R300_MS_Y4_SHIFT) |
			   (6 << R300_MS_X5_SHIFT) |
			   (6 << R300_MS_Y5_SHIFT) |
			   (6 << R300_MSBD1_SHIFT)));
	radeon_ring_write(rdev, PACKET0(R300_GA_ENHANCE, 0));
	radeon_ring_write(rdev, R300_GA_DEADLOCK_CNTL | R300_GA_FASTSYNC_CNTL);
	radeon_ring_write(rdev, PACKET0(R300_GA_POLY_MODE, 0));
	radeon_ring_write(rdev,
			  R300_FRONT_PTYPE_TRIANGE | R300_BACK_PTYPE_TRIANGE);
	radeon_ring_write(rdev, PACKET0(R300_GA_ROUND_MODE, 0));
	radeon_ring_write(rdev,
			  R300_GEOMETRY_ROUND_NEAREST |
			  R300_COLOR_ROUND_NEAREST);
	radeon_ring_unlock_commit(rdev);
}

void r300_errata(struct radeon_device *rdev)
{
	rdev->pll_errata = 0;

	if (rdev->family == CHIP_R300 &&
	    (RREG32(RADEON_CONFIG_CNTL) & RADEON_CFG_ATI_REV_ID_MASK) == RADEON_CFG_ATI_REV_A11) {
		rdev->pll_errata |= CHIP_ERRATA_R300_CG;
	}
}

int r300_mc_wait_for_idle(struct radeon_device *rdev)
{
	unsigned i;
	uint32_t tmp;

	for (i = 0; i < rdev->usec_timeout; i++) {
		/* read MC_STATUS */
318 319
		tmp = RREG32(RADEON_MC_STATUS);
		if (tmp & R300_MC_IDLE) {
320 321 322 323 324 325 326 327 328 329 330
			return 0;
		}
		DRM_UDELAY(1);
	}
	return -1;
}

void r300_gpu_init(struct radeon_device *rdev)
{
	uint32_t gb_tile_config, tmp;

331
	if ((rdev->family == CHIP_R300 && rdev->pdev->device != 0x4144) ||
332
	    (rdev->family == CHIP_R350 && rdev->pdev->device != 0x4148)) {
333 334 335
		/* r300,r350 */
		rdev->num_gb_pipes = 2;
	} else {
336
		/* rv350,rv370,rv380,r300 AD, r350 AH */
337 338
		rdev->num_gb_pipes = 1;
	}
339
	rdev->num_z_pipes = 1;
340 341 342 343 344 345 346 347 348 349 350 351
	gb_tile_config = (R300_ENABLE_TILING | R300_TILE_SIZE_16);
	switch (rdev->num_gb_pipes) {
	case 2:
		gb_tile_config |= R300_PIPE_COUNT_R300;
		break;
	case 3:
		gb_tile_config |= R300_PIPE_COUNT_R420_3P;
		break;
	case 4:
		gb_tile_config |= R300_PIPE_COUNT_R420;
		break;
	default:
352
	case 1:
353 354 355 356 357 358 359 360 361 362
		gb_tile_config |= R300_PIPE_COUNT_RV350;
		break;
	}
	WREG32(R300_GB_TILE_CONFIG, gb_tile_config);

	if (r100_gui_wait_for_idle(rdev)) {
		printk(KERN_WARNING "Failed to wait GUI idle while "
		       "programming pipes. Bad things might happen.\n");
	}

363 364
	tmp = RREG32(R300_DST_PIPE_CONFIG);
	WREG32(R300_DST_PIPE_CONFIG, tmp | R300_PIPE_AUTO_CONFIG);
365 366 367 368 369 370 371 372 373 374 375 376 377

	WREG32(R300_RB2D_DSTCACHE_MODE,
	       R300_DC_AUTOFLUSH_ENABLE |
	       R300_DC_DC_DISABLE_IGNORE_PE);

	if (r100_gui_wait_for_idle(rdev)) {
		printk(KERN_WARNING "Failed to wait GUI idle while "
		       "programming pipes. Bad things might happen.\n");
	}
	if (r300_mc_wait_for_idle(rdev)) {
		printk(KERN_WARNING "Failed to wait MC idle while "
		       "programming pipes. Bad things might happen.\n");
	}
378 379
	DRM_INFO("radeon: %d quad pipes, %d Z pipes initialized.\n",
		 rdev->num_gb_pipes, rdev->num_z_pipes);
380 381
}

382
bool r300_gpu_is_lockup(struct radeon_device *rdev)
383
{
384 385
	u32 rbbm_status;
	int r;
386

387 388 389 390
	rbbm_status = RREG32(R_000E40_RBBM_STATUS);
	if (!G_000E40_GUI_ACTIVE(rbbm_status)) {
		r100_gpu_lockup_update(&rdev->config.r300.lockup, &rdev->cp);
		return false;
391
	}
392 393 394 395 396 397 398
	/* force CP activities */
	r = radeon_ring_lock(rdev, 2);
	if (!r) {
		/* PACKET2 NOP */
		radeon_ring_write(rdev, 0x80000000);
		radeon_ring_write(rdev, 0x80000000);
		radeon_ring_unlock_commit(rdev);
399
	}
400 401
	rdev->cp.rptr = RREG32(RADEON_CP_RB_RPTR);
	return r100_gpu_cp_is_lockup(rdev, &rdev->config.r300.lockup, &rdev->cp);
402 403
}

404
int r300_asic_reset(struct radeon_device *rdev)
405
{
406 407
	struct r100_mc_save save;
	u32 status, tmp;
408

409 410 411 412
	r100_mc_stop(rdev, &save);
	status = RREG32(R_000E40_RBBM_STATUS);
	if (!G_000E40_GUI_ACTIVE(status)) {
		return 0;
413
	}
414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449
	status = RREG32(R_000E40_RBBM_STATUS);
	dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status);
	/* stop CP */
	WREG32(RADEON_CP_CSQ_CNTL, 0);
	tmp = RREG32(RADEON_CP_RB_CNTL);
	WREG32(RADEON_CP_RB_CNTL, tmp | RADEON_RB_RPTR_WR_ENA);
	WREG32(RADEON_CP_RB_RPTR_WR, 0);
	WREG32(RADEON_CP_RB_WPTR, 0);
	WREG32(RADEON_CP_RB_CNTL, tmp);
	/* save PCI state */
	pci_save_state(rdev->pdev);
	/* disable bus mastering */
	r100_bm_disable(rdev);
	WREG32(R_0000F0_RBBM_SOFT_RESET, S_0000F0_SOFT_RESET_VAP(1) |
					S_0000F0_SOFT_RESET_GA(1));
	RREG32(R_0000F0_RBBM_SOFT_RESET);
	mdelay(500);
	WREG32(R_0000F0_RBBM_SOFT_RESET, 0);
	mdelay(1);
	status = RREG32(R_000E40_RBBM_STATUS);
	dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status);
	/* resetting the CP seems to be problematic sometimes it end up
	 * hard locking the computer, but it's necessary for successfull
	 * reset more test & playing is needed on R3XX/R4XX to find a
	 * reliable (if any solution)
	 */
	WREG32(R_0000F0_RBBM_SOFT_RESET, S_0000F0_SOFT_RESET_CP(1));
	RREG32(R_0000F0_RBBM_SOFT_RESET);
	mdelay(500);
	WREG32(R_0000F0_RBBM_SOFT_RESET, 0);
	mdelay(1);
	status = RREG32(R_000E40_RBBM_STATUS);
	dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status);
	/* restore PCI & busmastering */
	pci_restore_state(rdev->pdev);
	r100_enable_bm(rdev);
450
	/* Check if GPU is idle */
451 452 453
	if (G_000E40_GA_BUSY(status) || G_000E40_VAP_BUSY(status)) {
		dev_err(rdev->dev, "failed to reset GPU\n");
		rdev->gpu_lockup = true;
454 455
		return -1;
	}
456 457
	r100_mc_resume(rdev, &save);
	dev_info(rdev->dev, "GPU reset succeed\n");
458 459 460 461 462 463
	return 0;
}

/*
 * r300,r350,rv350,rv380 VRAM info
 */
464
void r300_mc_init(struct radeon_device *rdev)
465
{
466 467
	u64 base;
	u32 tmp;
468 469 470 471

	/* DDR for all card after R300 & IGP */
	rdev->mc.vram_is_ddr = true;
	tmp = RREG32(RADEON_MEM_CNTL);
472 473 474 475 476 477
	tmp &= R300_MEM_NUM_CHANNELS_MASK;
	switch (tmp) {
	case 0: rdev->mc.vram_width = 64; break;
	case 1: rdev->mc.vram_width = 128; break;
	case 2: rdev->mc.vram_width = 256; break;
	default:  rdev->mc.vram_width = 128; break;
478
	}
479
	r100_vram_init_sizes(rdev);
480 481 482 483
	base = rdev->mc.aper_base;
	if (rdev->flags & RADEON_IS_IGP)
		base = (RREG32(RADEON_NB_TOM) & 0xffff) << 16;
	radeon_vram_location(rdev, &rdev->mc, base);
484
	rdev->mc.gtt_base_align = 0;
485 486
	if (!(rdev->flags & RADEON_IS_AGP))
		radeon_gtt_location(rdev, &rdev->mc);
487
	radeon_update_bandwidth_info(rdev);
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548
}

void rv370_set_pcie_lanes(struct radeon_device *rdev, int lanes)
{
	uint32_t link_width_cntl, mask;

	if (rdev->flags & RADEON_IS_IGP)
		return;

	if (!(rdev->flags & RADEON_IS_PCIE))
		return;

	/* FIXME wait for idle */

	switch (lanes) {
	case 0:
		mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
		break;
	case 1:
		mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
		break;
	case 2:
		mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
		break;
	case 4:
		mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
		break;
	case 8:
		mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
		break;
	case 12:
		mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
		break;
	case 16:
	default:
		mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
		break;
	}

	link_width_cntl = RREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL);

	if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
	    (mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
		return;

	link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK |
			     RADEON_PCIE_LC_RECONFIG_NOW |
			     RADEON_PCIE_LC_RECONFIG_LATER |
			     RADEON_PCIE_LC_SHORT_RECONFIG_EN);
	link_width_cntl |= mask;
	WREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
	WREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl |
						     RADEON_PCIE_LC_RECONFIG_NOW));

	/* wait for lane set to complete */
	link_width_cntl = RREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
	while (link_width_cntl == 0xffffffff)
		link_width_cntl = RREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL);

}

549 550 551 552 553 554 555 556 557 558 559 560
int rv370_get_pcie_lanes(struct radeon_device *rdev)
{
	u32 link_width_cntl;

	if (rdev->flags & RADEON_IS_IGP)
		return 0;

	if (!(rdev->flags & RADEON_IS_PCIE))
		return 0;

	/* FIXME wait for idle */

561 562 563 564
	if (rdev->family < CHIP_R600)
		link_width_cntl = RREG32_PCIE(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
	else
		link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582

	switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
	case RADEON_PCIE_LC_LINK_WIDTH_X0:
		return 0;
	case RADEON_PCIE_LC_LINK_WIDTH_X1:
		return 1;
	case RADEON_PCIE_LC_LINK_WIDTH_X2:
		return 2;
	case RADEON_PCIE_LC_LINK_WIDTH_X4:
		return 4;
	case RADEON_PCIE_LC_LINK_WIDTH_X8:
		return 8;
	case RADEON_PCIE_LC_LINK_WIDTH_X16:
	default:
		return 16;
	}
}

583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612
#if defined(CONFIG_DEBUG_FS)
static int rv370_debugfs_pcie_gart_info(struct seq_file *m, void *data)
{
	struct drm_info_node *node = (struct drm_info_node *) m->private;
	struct drm_device *dev = node->minor->dev;
	struct radeon_device *rdev = dev->dev_private;
	uint32_t tmp;

	tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_CNTL);
	seq_printf(m, "PCIE_TX_GART_CNTL 0x%08x\n", tmp);
	tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_BASE);
	seq_printf(m, "PCIE_TX_GART_BASE 0x%08x\n", tmp);
	tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_START_LO);
	seq_printf(m, "PCIE_TX_GART_START_LO 0x%08x\n", tmp);
	tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_START_HI);
	seq_printf(m, "PCIE_TX_GART_START_HI 0x%08x\n", tmp);
	tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_END_LO);
	seq_printf(m, "PCIE_TX_GART_END_LO 0x%08x\n", tmp);
	tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_END_HI);
	seq_printf(m, "PCIE_TX_GART_END_HI 0x%08x\n", tmp);
	tmp = RREG32_PCIE(RADEON_PCIE_TX_GART_ERROR);
	seq_printf(m, "PCIE_TX_GART_ERROR 0x%08x\n", tmp);
	return 0;
}

static struct drm_info_list rv370_pcie_gart_info_list[] = {
	{"rv370_pcie_gart_info", rv370_debugfs_pcie_gart_info, 0, NULL},
};
#endif

613
static int rv370_debugfs_pcie_gart_info_init(struct radeon_device *rdev)
614 615 616 617 618 619 620 621 622 623 624 625 626
{
#if defined(CONFIG_DEBUG_FS)
	return radeon_debugfs_add_files(rdev, rv370_pcie_gart_info_list, 1);
#else
	return 0;
#endif
}

static int r300_packet0_check(struct radeon_cs_parser *p,
		struct radeon_cs_packet *pkt,
		unsigned idx, unsigned reg)
{
	struct radeon_cs_reloc *reloc;
627
	struct r100_cs_track *track;
628
	volatile uint32_t *ib;
629
	uint32_t tmp, tile_flags = 0;
630 631
	unsigned i;
	int r;
632
	u32 idx_value;
633 634

	ib = p->ib->ptr;
635
	track = (struct r100_cs_track *)p->track;
636 637
	idx_value = radeon_get_ib_value(p, idx);

638
	switch(reg) {
639 640 641 642 643 644 645 646 647 648
	case AVIVO_D1MODE_VLINE_START_END:
	case RADEON_CRTC_GUI_TRIG_VLINE:
		r = r100_cs_packet_parse_vline(p);
		if (r) {
			DRM_ERROR("No reloc for ib[%d]=0x%04X\n",
					idx, reg);
			r100_cs_dump_packet(p, pkt);
			return r;
		}
		break;
649 650
	case RADEON_DST_PITCH_OFFSET:
	case RADEON_SRC_PITCH_OFFSET:
651 652
		r = r100_reloc_pitch_offset(p, pkt, idx, reg);
		if (r)
653 654 655 656 657 658 659 660 661 662 663 664 665 666 667
			return r;
		break;
	case R300_RB3D_COLOROFFSET0:
	case R300_RB3D_COLOROFFSET1:
	case R300_RB3D_COLOROFFSET2:
	case R300_RB3D_COLOROFFSET3:
		i = (reg - R300_RB3D_COLOROFFSET0) >> 2;
		r = r100_cs_packet_next_reloc(p, &reloc);
		if (r) {
			DRM_ERROR("No reloc for ib[%d]=0x%04X\n",
					idx, reg);
			r100_cs_dump_packet(p, pkt);
			return r;
		}
		track->cb[i].robj = reloc->robj;
668 669
		track->cb[i].offset = idx_value;
		ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
670 671 672 673 674 675 676 677 678 679
		break;
	case R300_ZB_DEPTHOFFSET:
		r = r100_cs_packet_next_reloc(p, &reloc);
		if (r) {
			DRM_ERROR("No reloc for ib[%d]=0x%04X\n",
					idx, reg);
			r100_cs_dump_packet(p, pkt);
			return r;
		}
		track->zb.robj = reloc->robj;
680 681
		track->zb.offset = idx_value;
		ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698
		break;
	case R300_TX_OFFSET_0:
	case R300_TX_OFFSET_0+4:
	case R300_TX_OFFSET_0+8:
	case R300_TX_OFFSET_0+12:
	case R300_TX_OFFSET_0+16:
	case R300_TX_OFFSET_0+20:
	case R300_TX_OFFSET_0+24:
	case R300_TX_OFFSET_0+28:
	case R300_TX_OFFSET_0+32:
	case R300_TX_OFFSET_0+36:
	case R300_TX_OFFSET_0+40:
	case R300_TX_OFFSET_0+44:
	case R300_TX_OFFSET_0+48:
	case R300_TX_OFFSET_0+52:
	case R300_TX_OFFSET_0+56:
	case R300_TX_OFFSET_0+60:
699
		i = (reg - R300_TX_OFFSET_0) >> 2;
700 701 702 703 704 705 706
		r = r100_cs_packet_next_reloc(p, &reloc);
		if (r) {
			DRM_ERROR("No reloc for ib[%d]=0x%04X\n",
					idx, reg);
			r100_cs_dump_packet(p, pkt);
			return r;
		}
707 708 709 710 711

		if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
			tile_flags |= R300_TXO_MACRO_TILE;
		if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO)
			tile_flags |= R300_TXO_MICRO_TILE;
712 713
		else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO_SQUARE)
			tile_flags |= R300_TXO_MICRO_TILE_SQUARE;
714 715 716 717

		tmp = idx_value + ((u32)reloc->lobj.gpu_offset);
		tmp |= tile_flags;
		ib[idx] = tmp;
718
		track->textures[i].robj = reloc->robj;
719 720
		break;
	/* Tracked registers */
721 722
	case 0x2084:
		/* VAP_VF_CNTL */
723
		track->vap_vf_cntl = idx_value;
724 725 726
		break;
	case 0x20B4:
		/* VAP_VTX_SIZE */
727
		track->vtx_size = idx_value & 0x7F;
728 729 730
		break;
	case 0x2134:
		/* VAP_VF_MAX_VTX_INDX */
731
		track->max_indx = idx_value & 0x00FFFFFFUL;
732
		break;
733 734 735 736 737 738
	case 0x2088:
		/* VAP_ALT_NUM_VERTICES - only valid on r500 */
		if (p->rdev->family < CHIP_RV515)
			goto fail;
		track->vap_alt_nverts = idx_value & 0xFFFFFF;
		break;
739 740
	case 0x43E4:
		/* SC_SCISSOR1 */
741
		track->maxy = ((idx_value >> 13) & 0x1FFF) + 1;
742 743 744 745 746 747
		if (p->rdev->family < CHIP_RV515) {
			track->maxy -= 1440;
		}
		break;
	case 0x4E00:
		/* RB3D_CCTL */
748
		track->num_cb = ((idx_value >> 5) & 0x3) + 1;
749 750 751 752 753 754 755 756 757
		break;
	case 0x4E38:
	case 0x4E3C:
	case 0x4E40:
	case 0x4E44:
		/* RB3D_COLORPITCH0 */
		/* RB3D_COLORPITCH1 */
		/* RB3D_COLORPITCH2 */
		/* RB3D_COLORPITCH3 */
758 759 760 761 762 763 764 765 766 767 768 769
		r = r100_cs_packet_next_reloc(p, &reloc);
		if (r) {
			DRM_ERROR("No reloc for ib[%d]=0x%04X\n",
				  idx, reg);
			r100_cs_dump_packet(p, pkt);
			return r;
		}

		if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
			tile_flags |= R300_COLOR_TILE_ENABLE;
		if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO)
			tile_flags |= R300_COLOR_MICROTILE_ENABLE;
770 771
		else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO_SQUARE)
			tile_flags |= R300_COLOR_MICROTILE_SQUARE_ENABLE;
772

773
		tmp = idx_value & ~(0x7 << 16);
774 775
		tmp |= tile_flags;
		ib[idx] = tmp;
776
		i = (reg - 0x4E38) >> 2;
777 778
		track->cb[i].pitch = idx_value & 0x3FFE;
		switch (((idx_value >> 21) & 0xF)) {
779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800
		case 9:
		case 11:
		case 12:
			track->cb[i].cpp = 1;
			break;
		case 3:
		case 4:
		case 13:
		case 15:
			track->cb[i].cpp = 2;
			break;
		case 6:
			track->cb[i].cpp = 4;
			break;
		case 10:
			track->cb[i].cpp = 8;
			break;
		case 7:
			track->cb[i].cpp = 16;
			break;
		default:
			DRM_ERROR("Invalid color buffer format (%d) !\n",
801
				  ((idx_value >> 21) & 0xF));
802 803 804 805 806
			return -EINVAL;
		}
		break;
	case 0x4F00:
		/* ZB_CNTL */
807
		if (idx_value & 2) {
808 809 810 811 812 813 814
			track->z_enabled = true;
		} else {
			track->z_enabled = false;
		}
		break;
	case 0x4F10:
		/* ZB_FORMAT */
815
		switch ((idx_value & 0xF)) {
816 817 818 819 820 821 822 823 824
		case 0:
		case 1:
			track->zb.cpp = 2;
			break;
		case 2:
			track->zb.cpp = 4;
			break;
		default:
			DRM_ERROR("Invalid z buffer format (%d) !\n",
825
				  (idx_value & 0xF));
826 827 828 829 830
			return -EINVAL;
		}
		break;
	case 0x4F24:
		/* ZB_DEPTHPITCH */
831 832 833 834 835 836 837 838 839 840 841
		r = r100_cs_packet_next_reloc(p, &reloc);
		if (r) {
			DRM_ERROR("No reloc for ib[%d]=0x%04X\n",
				  idx, reg);
			r100_cs_dump_packet(p, pkt);
			return r;
		}

		if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
			tile_flags |= R300_DEPTHMACROTILE_ENABLE;
		if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO)
842 843 844
			tile_flags |= R300_DEPTHMICROTILE_TILED;
		else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO_SQUARE)
			tile_flags |= R300_DEPTHMICROTILE_TILED_SQUARE;
845

846
		tmp = idx_value & ~(0x7 << 16);
847 848 849
		tmp |= tile_flags;
		ib[idx] = tmp;

850
		track->zb.pitch = idx_value & 0x3FFC;
851
		break;
852 853 854 855
	case 0x4104:
		for (i = 0; i < 16; i++) {
			bool enabled;

856
			enabled = !!(idx_value & (1 << i));
857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877
			track->textures[i].enabled = enabled;
		}
		break;
	case 0x44C0:
	case 0x44C4:
	case 0x44C8:
	case 0x44CC:
	case 0x44D0:
	case 0x44D4:
	case 0x44D8:
	case 0x44DC:
	case 0x44E0:
	case 0x44E4:
	case 0x44E8:
	case 0x44EC:
	case 0x44F0:
	case 0x44F4:
	case 0x44F8:
	case 0x44FC:
		/* TX_FORMAT1_[0-15] */
		i = (reg - 0x44C0) >> 2;
878
		tmp = (idx_value >> 25) & 0x3;
879
		track->textures[i].tex_coord_type = tmp;
880
		switch ((idx_value & 0x1F)) {
881 882 883
		case R300_TX_FORMAT_X8:
		case R300_TX_FORMAT_Y4X4:
		case R300_TX_FORMAT_Z3Y3X2:
884
			track->textures[i].cpp = 1;
885
			track->textures[i].compress_format = R100_TRACK_COMP_NONE;
886
			break;
887 888 889 890 891 892 893 894 895
		case R300_TX_FORMAT_X16:
		case R300_TX_FORMAT_Y8X8:
		case R300_TX_FORMAT_Z5Y6X5:
		case R300_TX_FORMAT_Z6Y5X5:
		case R300_TX_FORMAT_W4Z4Y4X4:
		case R300_TX_FORMAT_W1Z5Y5X5:
		case R300_TX_FORMAT_D3DMFT_CxV8U8:
		case R300_TX_FORMAT_B8G8_B8G8:
		case R300_TX_FORMAT_G8R8_G8B8:
896
			track->textures[i].cpp = 2;
897
			track->textures[i].compress_format = R100_TRACK_COMP_NONE;
898
			break;
899 900 901 902 903 904 905 906
		case R300_TX_FORMAT_Y16X16:
		case R300_TX_FORMAT_Z11Y11X10:
		case R300_TX_FORMAT_Z10Y11X11:
		case R300_TX_FORMAT_W8Z8Y8X8:
		case R300_TX_FORMAT_W2Z10Y10X10:
		case 0x17:
		case R300_TX_FORMAT_FL_I32:
		case 0x1e:
907
			track->textures[i].cpp = 4;
908
			track->textures[i].compress_format = R100_TRACK_COMP_NONE;
909
			break;
910 911 912
		case R300_TX_FORMAT_W16Z16Y16X16:
		case R300_TX_FORMAT_FL_R16G16B16A16:
		case R300_TX_FORMAT_FL_I32A32:
913
			track->textures[i].cpp = 8;
914
			track->textures[i].compress_format = R100_TRACK_COMP_NONE;
915
			break;
916
		case R300_TX_FORMAT_FL_R32G32B32A32:
917
			track->textures[i].cpp = 16;
918
			track->textures[i].compress_format = R100_TRACK_COMP_NONE;
919
			break;
920 921 922 923
		case R300_TX_FORMAT_DXT1:
			track->textures[i].cpp = 1;
			track->textures[i].compress_format = R100_TRACK_COMP_DXT1;
			break;
924 925 926 927 928 929 930 931
		case R300_TX_FORMAT_ATI2N:
			if (p->rdev->family < CHIP_R420) {
				DRM_ERROR("Invalid texture format %u\n",
					  (idx_value & 0x1F));
				return -EINVAL;
			}
			/* The same rules apply as for DXT3/5. */
			/* Pass through. */
932 933 934 935 936
		case R300_TX_FORMAT_DXT3:
		case R300_TX_FORMAT_DXT5:
			track->textures[i].cpp = 1;
			track->textures[i].compress_format = R100_TRACK_COMP_DXT35;
			break;
937 938
		default:
			DRM_ERROR("Invalid texture format %u\n",
939
				  (idx_value & 0x1F));
940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961
			return -EINVAL;
			break;
		}
		break;
	case 0x4400:
	case 0x4404:
	case 0x4408:
	case 0x440C:
	case 0x4410:
	case 0x4414:
	case 0x4418:
	case 0x441C:
	case 0x4420:
	case 0x4424:
	case 0x4428:
	case 0x442C:
	case 0x4430:
	case 0x4434:
	case 0x4438:
	case 0x443C:
		/* TX_FILTER0_[0-15] */
		i = (reg - 0x4400) >> 2;
962
		tmp = idx_value & 0x7;
963 964 965
		if (tmp == 2 || tmp == 4 || tmp == 6) {
			track->textures[i].roundup_w = false;
		}
966
		tmp = (idx_value >> 3) & 0x7;
967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988
		if (tmp == 2 || tmp == 4 || tmp == 6) {
			track->textures[i].roundup_h = false;
		}
		break;
	case 0x4500:
	case 0x4504:
	case 0x4508:
	case 0x450C:
	case 0x4510:
	case 0x4514:
	case 0x4518:
	case 0x451C:
	case 0x4520:
	case 0x4524:
	case 0x4528:
	case 0x452C:
	case 0x4530:
	case 0x4534:
	case 0x4538:
	case 0x453C:
		/* TX_FORMAT2_[0-15] */
		i = (reg - 0x4500) >> 2;
989
		tmp = idx_value & 0x3FFF;
990 991
		track->textures[i].pitch = tmp + 1;
		if (p->rdev->family >= CHIP_RV515) {
992
			tmp = ((idx_value >> 15) & 1) << 11;
993
			track->textures[i].width_11 = tmp;
994
			tmp = ((idx_value >> 16) & 1) << 11;
995
			track->textures[i].height_11 = tmp;
996 997 998 999 1000 1001 1002 1003 1004 1005

			/* ATI1N */
			if (idx_value & (1 << 14)) {
				/* The same rules apply as for DXT1. */
				track->textures[i].compress_format =
					R100_TRACK_COMP_DXT1;
			}
		} else if (idx_value & (1 << 14)) {
			DRM_ERROR("Forbidden bit TXFORMAT_MSB\n");
			return -EINVAL;
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
		}
		break;
	case 0x4480:
	case 0x4484:
	case 0x4488:
	case 0x448C:
	case 0x4490:
	case 0x4494:
	case 0x4498:
	case 0x449C:
	case 0x44A0:
	case 0x44A4:
	case 0x44A8:
	case 0x44AC:
	case 0x44B0:
	case 0x44B4:
	case 0x44B8:
	case 0x44BC:
		/* TX_FORMAT0_[0-15] */
		i = (reg - 0x4480) >> 2;
1026
		tmp = idx_value & 0x7FF;
1027
		track->textures[i].width = tmp + 1;
1028
		tmp = (idx_value >> 11) & 0x7FF;
1029
		track->textures[i].height = tmp + 1;
1030
		tmp = (idx_value >> 26) & 0xF;
1031
		track->textures[i].num_levels = tmp;
1032
		tmp = idx_value & (1 << 31);
1033
		track->textures[i].use_pitch = !!tmp;
1034
		tmp = (idx_value >> 22) & 0xF;
1035 1036
		track->textures[i].txdepth = tmp;
		break;
1037 1038 1039 1040 1041 1042 1043 1044
	case R300_ZB_ZPASS_ADDR:
		r = r100_cs_packet_next_reloc(p, &reloc);
		if (r) {
			DRM_ERROR("No reloc for ib[%d]=0x%04X\n",
					idx, reg);
			r100_cs_dump_packet(p, pkt);
			return r;
		}
1045
		ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
1046
		break;
1047 1048 1049 1050 1051 1052
	case 0x4e0c:
		/* RB3D_COLOR_CHANNEL_MASK */
		track->color_channel_mask = idx_value;
		break;
	case 0x4d1c:
		/* ZB_BW_CNTL */
1053
		track->zb_cb_clear = !!(idx_value & (1 << 5));
1054 1055 1056 1057 1058
		break;
	case 0x4e04:
		/* RB3D_BLENDCNTL */
		track->blend_read_enable = !!(idx_value & (1 << 2));
		break;
1059 1060 1061 1062 1063
	case 0x4be8:
		/* valid register only on RV530 */
		if (p->rdev->family == CHIP_RV530)
			break;
		/* fallthrough do not move */
1064
	default:
1065
		goto fail;
1066 1067
	}
	return 0;
1068 1069 1070 1071
fail:
	printk(KERN_ERR "Forbidden register 0x%04X in cs at %d\n",
	       reg, idx);
	return -EINVAL;
1072 1073 1074 1075 1076 1077
}

static int r300_packet3_check(struct radeon_cs_parser *p,
			      struct radeon_cs_packet *pkt)
{
	struct radeon_cs_reloc *reloc;
1078
	struct r100_cs_track *track;
1079 1080 1081 1082 1083 1084
	volatile uint32_t *ib;
	unsigned idx;
	int r;

	ib = p->ib->ptr;
	idx = pkt->idx + 1;
1085
	track = (struct r100_cs_track *)p->track;
1086
	switch(pkt->opcode) {
1087
	case PACKET3_3D_LOAD_VBPNTR:
1088 1089 1090
		r = r100_packet3_load_vbpntr(p, pkt, idx);
		if (r)
			return r;
1091 1092 1093 1094 1095 1096 1097 1098
		break;
	case PACKET3_INDX_BUFFER:
		r = r100_cs_packet_next_reloc(p, &reloc);
		if (r) {
			DRM_ERROR("No reloc for packet3 %d\n", pkt->opcode);
			r100_cs_dump_packet(p, pkt);
			return r;
		}
1099
		ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->lobj.gpu_offset);
1100 1101 1102 1103
		r = r100_cs_track_check_pkt3_indx_buffer(p, pkt, reloc->robj);
		if (r) {
			return r;
		}
1104 1105 1106
		break;
	/* Draw packet */
	case PACKET3_3D_DRAW_IMMD:
1107 1108 1109
		/* Number of dwords is vtx_size * (num_vertices - 1)
		 * PRIM_WALK must be equal to 3 vertex data in embedded
		 * in cmd stream */
1110
		if (((radeon_get_ib_value(p, idx + 1) >> 4) & 0x3) != 3) {
1111 1112 1113
			DRM_ERROR("PRIM_WALK must be 3 for IMMD draw\n");
			return -EINVAL;
		}
1114
		track->vap_vf_cntl = radeon_get_ib_value(p, idx + 1);
1115
		track->immd_dwords = pkt->count - 1;
1116
		r = r100_cs_track_check(p->rdev, track);
1117 1118 1119 1120
		if (r) {
			return r;
		}
		break;
1121
	case PACKET3_3D_DRAW_IMMD_2:
1122 1123 1124
		/* Number of dwords is vtx_size * (num_vertices - 1)
		 * PRIM_WALK must be equal to 3 vertex data in embedded
		 * in cmd stream */
1125
		if (((radeon_get_ib_value(p, idx) >> 4) & 0x3) != 3) {
1126 1127 1128
			DRM_ERROR("PRIM_WALK must be 3 for IMMD draw\n");
			return -EINVAL;
		}
1129
		track->vap_vf_cntl = radeon_get_ib_value(p, idx);
1130
		track->immd_dwords = pkt->count;
1131
		r = r100_cs_track_check(p->rdev, track);
1132 1133 1134 1135 1136
		if (r) {
			return r;
		}
		break;
	case PACKET3_3D_DRAW_VBUF:
1137
		track->vap_vf_cntl = radeon_get_ib_value(p, idx + 1);
1138
		r = r100_cs_track_check(p->rdev, track);
1139 1140 1141 1142 1143
		if (r) {
			return r;
		}
		break;
	case PACKET3_3D_DRAW_VBUF_2:
1144
		track->vap_vf_cntl = radeon_get_ib_value(p, idx);
1145
		r = r100_cs_track_check(p->rdev, track);
1146 1147 1148 1149 1150
		if (r) {
			return r;
		}
		break;
	case PACKET3_3D_DRAW_INDX:
1151
		track->vap_vf_cntl = radeon_get_ib_value(p, idx + 1);
1152
		r = r100_cs_track_check(p->rdev, track);
1153 1154 1155 1156
		if (r) {
			return r;
		}
		break;
1157
	case PACKET3_3D_DRAW_INDX_2:
1158
		track->vap_vf_cntl = radeon_get_ib_value(p, idx);
1159
		r = r100_cs_track_check(p->rdev, track);
1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
		if (r) {
			return r;
		}
		break;
	case PACKET3_NOP:
		break;
	default:
		DRM_ERROR("Packet3 opcode %x not supported\n", pkt->opcode);
		return -EINVAL;
	}
	return 0;
}

int r300_cs_parse(struct radeon_cs_parser *p)
{
	struct radeon_cs_packet pkt;
1176
	struct r100_cs_track *track;
1177 1178
	int r;

1179
	track = kzalloc(sizeof(*track), GFP_KERNEL);
1180 1181
	if (track == NULL)
		return -ENOMEM;
1182 1183
	r100_cs_track_clear(p->rdev, track);
	p->track = track;
1184 1185 1186 1187 1188 1189 1190 1191 1192
	do {
		r = r100_cs_packet_parse(p, &pkt, p->idx);
		if (r) {
			return r;
		}
		p->idx += pkt.count + 2;
		switch (pkt.type) {
		case PACKET_TYPE0:
			r = r100_cs_parse_packet0(p, &pkt,
1193 1194
						  p->rdev->config.r300.reg_safe_bm,
						  p->rdev->config.r300.reg_safe_bm_size,
1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
						  &r300_packet0_check);
			break;
		case PACKET_TYPE2:
			break;
		case PACKET_TYPE3:
			r = r300_packet3_check(p, &pkt);
			break;
		default:
			DRM_ERROR("Unknown packet type %d !\n", pkt.type);
			return -EINVAL;
		}
		if (r) {
			return r;
		}
	} while (p->idx < p->chunks[p->chunk_ib_idx].length_dw);
	return 0;
}
1212

1213
void r300_set_reg_safe(struct radeon_device *rdev)
1214 1215 1216
{
	rdev->config.r300.reg_safe_bm = r300_reg_safe_bm;
	rdev->config.r300.reg_safe_bm_size = ARRAY_SIZE(r300_reg_safe_bm);
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251
}

void r300_mc_program(struct radeon_device *rdev)
{
	struct r100_mc_save save;
	int r;

	r = r100_debugfs_mc_info_init(rdev);
	if (r) {
		dev_err(rdev->dev, "Failed to create r100_mc debugfs file.\n");
	}

	/* Stops all mc clients */
	r100_mc_stop(rdev, &save);
	if (rdev->flags & RADEON_IS_AGP) {
		WREG32(R_00014C_MC_AGP_LOCATION,
			S_00014C_MC_AGP_START(rdev->mc.gtt_start >> 16) |
			S_00014C_MC_AGP_TOP(rdev->mc.gtt_end >> 16));
		WREG32(R_000170_AGP_BASE, lower_32_bits(rdev->mc.agp_base));
		WREG32(R_00015C_AGP_BASE_2,
			upper_32_bits(rdev->mc.agp_base) & 0xff);
	} else {
		WREG32(R_00014C_MC_AGP_LOCATION, 0x0FFFFFFF);
		WREG32(R_000170_AGP_BASE, 0);
		WREG32(R_00015C_AGP_BASE_2, 0);
	}
	/* Wait for mc idle */
	if (r300_mc_wait_for_idle(rdev))
		DRM_INFO("Failed to wait MC idle before programming MC.\n");
	/* Program MC, should be a 32bits limited address space */
	WREG32(R_000148_MC_FB_LOCATION,
		S_000148_MC_FB_START(rdev->mc.vram_start >> 16) |
		S_000148_MC_FB_TOP(rdev->mc.vram_end >> 16));
	r100_mc_resume(rdev, &save);
}
1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265

void r300_clock_startup(struct radeon_device *rdev)
{
	u32 tmp;

	if (radeon_dynclks != -1 && radeon_dynclks)
		radeon_legacy_set_clock_gating(rdev, 1);
	/* We need to force on some of the block */
	tmp = RREG32_PLL(R_00000D_SCLK_CNTL);
	tmp |= S_00000D_FORCE_CP(1) | S_00000D_FORCE_VIP(1);
	if ((rdev->family == CHIP_RV350) || (rdev->family == CHIP_RV380))
		tmp |= S_00000D_FORCE_VAP(1);
	WREG32_PLL(R_00000D_SCLK_CNTL, tmp);
}
1266 1267 1268 1269 1270

static int r300_startup(struct radeon_device *rdev)
{
	int r;

1271 1272 1273
	/* set common regs */
	r100_set_common_regs(rdev);
	/* program mc */
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285
	r300_mc_program(rdev);
	/* Resume clock */
	r300_clock_startup(rdev);
	/* Initialize GPU configuration (# pipes, ...) */
	r300_gpu_init(rdev);
	/* Initialize GART (initialize after TTM so we can allocate
	 * memory through TTM but finalize after TTM) */
	if (rdev->flags & RADEON_IS_PCIE) {
		r = rv370_pcie_gart_enable(rdev);
		if (r)
			return r;
	}
1286 1287 1288 1289 1290 1291

	if (rdev->family == CHIP_R300 ||
	    rdev->family == CHIP_R350 ||
	    rdev->family == CHIP_RV350)
		r100_enable_bm(rdev);

1292 1293 1294 1295 1296 1297 1298
	if (rdev->flags & RADEON_IS_PCI) {
		r = r100_pci_gart_enable(rdev);
		if (r)
			return r;
	}
	/* Enable IRQ */
	r100_irq_set(rdev);
1299
	rdev->config.r300.hdp_cntl = RREG32(RADEON_HOST_PATH_CNTL);
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326
	/* 1M ring buffer */
	r = r100_cp_init(rdev, 1024 * 1024);
	if (r) {
		dev_err(rdev->dev, "failled initializing CP (%d).\n", r);
		return r;
	}
	r = r100_wb_init(rdev);
	if (r)
		dev_err(rdev->dev, "failled initializing WB (%d).\n", r);
	r = r100_ib_init(rdev);
	if (r) {
		dev_err(rdev->dev, "failled initializing IB (%d).\n", r);
		return r;
	}
	return 0;
}

int r300_resume(struct radeon_device *rdev)
{
	/* Make sur GART are not working */
	if (rdev->flags & RADEON_IS_PCIE)
		rv370_pcie_gart_disable(rdev);
	if (rdev->flags & RADEON_IS_PCI)
		r100_pci_gart_disable(rdev);
	/* Resume clock before doing reset */
	r300_clock_startup(rdev);
	/* Reset gpu before posting otherwise ATOM will enter infinite loop */
1327
	if (radeon_asic_reset(rdev)) {
1328 1329 1330 1331 1332 1333 1334 1335
		dev_warn(rdev->dev, "GPU reset failed ! (0xE40=0x%08X, 0x7C0=0x%08X)\n",
			RREG32(R_000E40_RBBM_STATUS),
			RREG32(R_0007C0_CP_STAT));
	}
	/* post */
	radeon_combios_asic_init(rdev->ddev);
	/* Resume clock after posting */
	r300_clock_startup(rdev);
1336 1337
	/* Initialize surface registers */
	radeon_surface_init(rdev);
1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362
	return r300_startup(rdev);
}

int r300_suspend(struct radeon_device *rdev)
{
	r100_cp_disable(rdev);
	r100_wb_disable(rdev);
	r100_irq_disable(rdev);
	if (rdev->flags & RADEON_IS_PCIE)
		rv370_pcie_gart_disable(rdev);
	if (rdev->flags & RADEON_IS_PCI)
		r100_pci_gart_disable(rdev);
	return 0;
}

void r300_fini(struct radeon_device *rdev)
{
	r100_cp_fini(rdev);
	r100_wb_fini(rdev);
	r100_ib_fini(rdev);
	radeon_gem_fini(rdev);
	if (rdev->flags & RADEON_IS_PCIE)
		rv370_pcie_gart_fini(rdev);
	if (rdev->flags & RADEON_IS_PCI)
		r100_pci_gart_fini(rdev);
1363
	radeon_agp_fini(rdev);
1364 1365
	radeon_irq_kms_fini(rdev);
	radeon_fence_driver_fini(rdev);
1366
	radeon_bo_fini(rdev);
1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
	radeon_atombios_fini(rdev);
	kfree(rdev->bios);
	rdev->bios = NULL;
}

int r300_init(struct radeon_device *rdev)
{
	int r;

	/* Disable VGA */
	r100_vga_render_disable(rdev);
	/* Initialize scratch registers */
	radeon_scratch_init(rdev);
	/* Initialize surface registers */
	radeon_surface_init(rdev);
	/* TODO: disable VGA need to use VGA request */
1383 1384
	/* restore some register to sane defaults */
	r100_restore_sanity(rdev);
1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
	/* BIOS*/
	if (!radeon_get_bios(rdev)) {
		if (ASIC_IS_AVIVO(rdev))
			return -EINVAL;
	}
	if (rdev->is_atom_bios) {
		dev_err(rdev->dev, "Expecting combios for RS400/RS480 GPU\n");
		return -EINVAL;
	} else {
		r = radeon_combios_init(rdev);
		if (r)
			return r;
	}
	/* Reset gpu before posting otherwise ATOM will enter infinite loop */
1399
	if (radeon_asic_reset(rdev)) {
1400 1401 1402 1403 1404 1405
		dev_warn(rdev->dev,
			"GPU reset failed ! (0xE40=0x%08X, 0x7C0=0x%08X)\n",
			RREG32(R_000E40_RBBM_STATUS),
			RREG32(R_0007C0_CP_STAT));
	}
	/* check if cards are posted or not */
1406 1407
	if (radeon_boot_test_post_card(rdev) == false)
		return -EINVAL;
1408 1409 1410 1411
	/* Set asic errata */
	r300_errata(rdev);
	/* Initialize clocks */
	radeon_get_clock_info(rdev->ddev);
1412 1413 1414 1415 1416 1417 1418 1419 1420
	/* initialize AGP */
	if (rdev->flags & RADEON_IS_AGP) {
		r = radeon_agp_init(rdev);
		if (r) {
			radeon_agp_disable(rdev);
		}
	}
	/* initialize memory controller */
	r300_mc_init(rdev);
1421 1422 1423 1424 1425 1426 1427 1428
	/* Fence driver */
	r = radeon_fence_driver_init(rdev);
	if (r)
		return r;
	r = radeon_irq_kms_init(rdev);
	if (r)
		return r;
	/* Memory manager */
1429
	r = radeon_bo_init(rdev);
1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450
	if (r)
		return r;
	if (rdev->flags & RADEON_IS_PCIE) {
		r = rv370_pcie_gart_init(rdev);
		if (r)
			return r;
	}
	if (rdev->flags & RADEON_IS_PCI) {
		r = r100_pci_gart_init(rdev);
		if (r)
			return r;
	}
	r300_set_reg_safe(rdev);
	rdev->accel_working = true;
	r = r300_startup(rdev);
	if (r) {
		/* Somethings want wront with the accel init stop accel */
		dev_err(rdev->dev, "Disabling GPU acceleration\n");
		r100_cp_fini(rdev);
		r100_wb_fini(rdev);
		r100_ib_fini(rdev);
1451
		radeon_irq_kms_fini(rdev);
1452 1453 1454 1455
		if (rdev->flags & RADEON_IS_PCIE)
			rv370_pcie_gart_fini(rdev);
		if (rdev->flags & RADEON_IS_PCI)
			r100_pci_gart_fini(rdev);
1456
		radeon_agp_fini(rdev);
1457 1458 1459 1460
		rdev->accel_working = false;
	}
	return 0;
}