amdgpu_device.c 90.0 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright 2008 Advanced Micro Devices, Inc.
 * Copyright 2008 Red Hat Inc.
 * Copyright 2009 Jerome Glisse.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Dave Airlie
 *          Alex Deucher
 *          Jerome Glisse
 */
28
#include <linux/kthread.h>
A
Alex Deucher 已提交
29 30 31 32
#include <linux/console.h>
#include <linux/slab.h>
#include <drm/drmP.h>
#include <drm/drm_crtc_helper.h>
33
#include <drm/drm_atomic_helper.h>
A
Alex Deucher 已提交
34 35 36 37 38
#include <drm/amdgpu_drm.h>
#include <linux/vgaarb.h>
#include <linux/vga_switcheroo.h>
#include <linux/efi.h>
#include "amdgpu.h"
39
#include "amdgpu_trace.h"
A
Alex Deucher 已提交
40 41 42
#include "amdgpu_i2c.h"
#include "atom.h"
#include "amdgpu_atombios.h"
43
#include "amdgpu_atomfirmware.h"
44
#include "amd_pcie.h"
K
Ken Wang 已提交
45 46 47
#ifdef CONFIG_DRM_AMDGPU_SI
#include "si.h"
#endif
48 49 50
#ifdef CONFIG_DRM_AMDGPU_CIK
#include "cik.h"
#endif
51
#include "vi.h"
52
#include "soc15.h"
A
Alex Deucher 已提交
53
#include "bif/bif_4_1_d.h"
54
#include <linux/pci.h>
55
#include <linux/firmware.h>
56
#include "amdgpu_vf_error.h"
A
Alex Deucher 已提交
57

58
#include "amdgpu_amdkfd.h"
59
#include "amdgpu_pm.h"
A
Alex Deucher 已提交
60

61
MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
62
MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
63
MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
64

65 66
#define AMDGPU_RESUME_MS		2000

A
Alex Deucher 已提交
67
static const char *amdgpu_asic_name[] = {
68 69 70 71 72
	"TAHITI",
	"PITCAIRN",
	"VERDE",
	"OLAND",
	"HAINAN",
A
Alex Deucher 已提交
73 74 75 76 77 78 79
	"BONAIRE",
	"KAVERI",
	"KABINI",
	"HAWAII",
	"MULLINS",
	"TOPAZ",
	"TONGA",
80
	"FIJI",
A
Alex Deucher 已提交
81
	"CARRIZO",
S
Samuel Li 已提交
82
	"STONEY",
83 84
	"POLARIS10",
	"POLARIS11",
85
	"POLARIS12",
L
Leo Liu 已提交
86
	"VEGAM",
K
Ken Wang 已提交
87
	"VEGA10",
88
	"VEGA12",
89
	"VEGA20",
90
	"RAVEN",
A
Alex Deucher 已提交
91 92 93
	"LAST",
};

94 95
static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);

96 97 98 99 100 101 102 103
/**
 * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control
 *
 * @dev: drm_device pointer
 *
 * Returns true if the device is a dGPU with HG/PX power control,
 * otherwise return false.
 */
A
Alex Deucher 已提交
104 105 106 107
bool amdgpu_device_is_px(struct drm_device *dev)
{
	struct amdgpu_device *adev = dev->dev_private;

108
	if (adev->flags & AMD_IS_PX)
A
Alex Deucher 已提交
109 110 111 112 113 114 115
		return true;
	return false;
}

/*
 * MMIO register access helper functions.
 */
116 117 118 119 120 121 122 123 124
/**
 * amdgpu_mm_rreg - read a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 * @acc_flags: access flags which require special behavior
 *
 * Returns the 32 bit value from the offset specified.
 */
A
Alex Deucher 已提交
125
uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
M
Monk Liu 已提交
126
			uint32_t acc_flags)
A
Alex Deucher 已提交
127
{
128 129
	uint32_t ret;

130
	if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
131 132
		return amdgpu_virt_kiq_rreg(adev, reg);

M
Monk Liu 已提交
133
	if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
134
		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
A
Alex Deucher 已提交
135 136 137 138 139 140 141 142
	else {
		unsigned long flags;

		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
		writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
		ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
	}
143 144
	trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
	return ret;
A
Alex Deucher 已提交
145 146
}

147 148 149 150 151 152
/*
 * MMIO register read with bytes helper functions
 * @offset:bytes offset from MMIO start
 *
*/

153 154 155 156 157 158 159 160
/**
 * amdgpu_mm_rreg8 - read a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @offset: byte aligned register offset
 *
 * Returns the 8 bit value from the offset specified.
 */
161 162 163 164 165 166 167 168 169 170 171 172
uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) {
	if (offset < adev->rmmio_size)
		return (readb(adev->rmmio + offset));
	BUG();
}

/*
 * MMIO register write with bytes helper functions
 * @offset:bytes offset from MMIO start
 * @value: the value want to be written to the register
 *
*/
173 174 175 176 177 178 179 180 181
/**
 * amdgpu_mm_wreg8 - read a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @offset: byte aligned register offset
 * @value: 8 bit value to write
 *
 * Writes the value specified to the offset specified.
 */
182 183 184 185 186 187 188
void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) {
	if (offset < adev->rmmio_size)
		writeb(value, adev->rmmio + offset);
	else
		BUG();
}

189 190 191 192 193 194 195 196 197 198
/**
 * amdgpu_mm_wreg - write to a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 * @v: 32 bit value to write to the register
 * @acc_flags: access flags which require special behavior
 *
 * Writes the value specified to the offset specified.
 */
A
Alex Deucher 已提交
199
void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
M
Monk Liu 已提交
200
		    uint32_t acc_flags)
A
Alex Deucher 已提交
201
{
202
	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
203

204 205 206 207
	if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
		adev->last_mm_index = v;
	}

208
	if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
209 210
		return amdgpu_virt_kiq_wreg(adev, reg, v);

M
Monk Liu 已提交
211
	if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
A
Alex Deucher 已提交
212 213 214 215 216 217 218 219 220
		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
	else {
		unsigned long flags;

		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
		writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
		writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
	}
221 222 223 224

	if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
		udelay(500);
	}
A
Alex Deucher 已提交
225 226
}

227 228 229 230 231 232 233 234
/**
 * amdgpu_io_rreg - read an IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 *
 * Returns the 32 bit value from the offset specified.
 */
A
Alex Deucher 已提交
235 236 237 238 239 240 241 242 243 244
u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
{
	if ((reg * 4) < adev->rio_mem_size)
		return ioread32(adev->rio_mem + (reg * 4));
	else {
		iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
		return ioread32(adev->rio_mem + (mmMM_DATA * 4));
	}
}

245 246 247 248 249 250 251 252 253
/**
 * amdgpu_io_wreg - write to an IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 * @v: 32 bit value to write to the register
 *
 * Writes the value specified to the offset specified.
 */
A
Alex Deucher 已提交
254 255
void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
{
256 257 258
	if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
		adev->last_mm_index = v;
	}
A
Alex Deucher 已提交
259 260 261 262 263 264 265

	if ((reg * 4) < adev->rio_mem_size)
		iowrite32(v, adev->rio_mem + (reg * 4));
	else {
		iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
		iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
	}
266 267 268 269

	if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
		udelay(500);
	}
A
Alex Deucher 已提交
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
}

/**
 * amdgpu_mm_rdoorbell - read a doorbell dword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 *
 * Returns the value in the doorbell aperture at the
 * requested doorbell index (CIK).
 */
u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
{
	if (index < adev->doorbell.num_doorbells) {
		return readl(adev->doorbell.ptr + index);
	} else {
		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
		return 0;
	}
}

/**
 * amdgpu_mm_wdoorbell - write a doorbell dword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 * @v: value to write
 *
 * Writes @v to the doorbell aperture at the
 * requested doorbell index (CIK).
 */
void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
{
	if (index < adev->doorbell.num_doorbells) {
		writel(v, adev->doorbell.ptr + index);
	} else {
		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
	}
}

310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
/**
 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 *
 * Returns the value in the doorbell aperture at the
 * requested doorbell index (VEGA10+).
 */
u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
{
	if (index < adev->doorbell.num_doorbells) {
		return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
	} else {
		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
		return 0;
	}
}

/**
 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 * @v: value to write
 *
 * Writes @v to the doorbell aperture at the
 * requested doorbell index (VEGA10+).
 */
void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
{
	if (index < adev->doorbell.num_doorbells) {
		atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
	} else {
		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
	}
}

A
Alex Deucher 已提交
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
/**
 * amdgpu_invalid_rreg - dummy reg read function
 *
 * @adev: amdgpu device pointer
 * @reg: offset of register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 * Returns the value in the register.
 */
static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
{
	DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
	BUG();
	return 0;
}

/**
 * amdgpu_invalid_wreg - dummy reg write function
 *
 * @adev: amdgpu device pointer
 * @reg: offset of register
 * @v: value to write to the register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 */
static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
{
	DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
		  reg, v);
	BUG();
}

/**
 * amdgpu_block_invalid_rreg - dummy reg read function
 *
 * @adev: amdgpu device pointer
 * @block: offset of instance
 * @reg: offset of register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 * Returns the value in the register.
 */
static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
					  uint32_t block, uint32_t reg)
{
	DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
		  reg, block);
	BUG();
	return 0;
}

/**
 * amdgpu_block_invalid_wreg - dummy reg write function
 *
 * @adev: amdgpu device pointer
 * @block: offset of instance
 * @reg: offset of register
 * @v: value to write to the register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 */
static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
				      uint32_t block,
				      uint32_t reg, uint32_t v)
{
	DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
		  reg, block, v);
	BUG();
}

422 423 424 425 426 427 428 429
/**
 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
 *
 * @adev: amdgpu device pointer
 *
 * Allocates a scratch page of VRAM for use by various things in the
 * driver.
 */
430
static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
431
{
432 433 434 435 436
	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
				       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
				       &adev->vram_scratch.robj,
				       &adev->vram_scratch.gpu_addr,
				       (void **)&adev->vram_scratch.ptr);
A
Alex Deucher 已提交
437 438
}

439 440 441 442 443 444 445
/**
 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
 *
 * @adev: amdgpu device pointer
 *
 * Frees the VRAM scratch page.
 */
446
static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
A
Alex Deucher 已提交
447
{
448
	amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
A
Alex Deucher 已提交
449 450 451
}

/**
452
 * amdgpu_device_program_register_sequence - program an array of registers.
A
Alex Deucher 已提交
453 454 455 456 457 458 459 460
 *
 * @adev: amdgpu_device pointer
 * @registers: pointer to the register array
 * @array_size: size of the register array
 *
 * Programs an array or registers with and and or masks.
 * This is a helper for setting golden registers.
 */
461 462 463
void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
					     const u32 *registers,
					     const u32 array_size)
A
Alex Deucher 已提交
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
{
	u32 tmp, reg, and_mask, or_mask;
	int i;

	if (array_size % 3)
		return;

	for (i = 0; i < array_size; i +=3) {
		reg = registers[i + 0];
		and_mask = registers[i + 1];
		or_mask = registers[i + 2];

		if (and_mask == 0xffffffff) {
			tmp = or_mask;
		} else {
			tmp = RREG32(reg);
			tmp &= ~and_mask;
			tmp |= or_mask;
		}
		WREG32(reg, tmp);
	}
}

487 488 489 490 491 492 493 494
/**
 * amdgpu_device_pci_config_reset - reset the GPU
 *
 * @adev: amdgpu_device pointer
 *
 * Resets the GPU using the pci config reset sequence.
 * Only applicable to asics prior to vega10.
 */
495
void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
A
Alex Deucher 已提交
496 497 498 499 500 501 502 503
{
	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
}

/*
 * GPU doorbell aperture helpers function.
 */
/**
504
 * amdgpu_device_doorbell_init - Init doorbell driver information.
A
Alex Deucher 已提交
505 506 507 508 509 510
 *
 * @adev: amdgpu_device pointer
 *
 * Init doorbell driver information (CIK)
 * Returns 0 on success, error on failure.
 */
511
static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
512
{
513 514 515 516 517 518 519 520 521
	/* No doorbell on SI hardware generation */
	if (adev->asic_type < CHIP_BONAIRE) {
		adev->doorbell.base = 0;
		adev->doorbell.size = 0;
		adev->doorbell.num_doorbells = 0;
		adev->doorbell.ptr = NULL;
		return 0;
	}

522 523 524
	if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
		return -EINVAL;

A
Alex Deucher 已提交
525 526 527 528
	/* doorbell bar mapping */
	adev->doorbell.base = pci_resource_start(adev->pdev, 2);
	adev->doorbell.size = pci_resource_len(adev->pdev, 2);

529
	adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
A
Alex Deucher 已提交
530 531 532 533
					     AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
	if (adev->doorbell.num_doorbells == 0)
		return -EINVAL;

534 535 536 537
	adev->doorbell.ptr = ioremap(adev->doorbell.base,
				     adev->doorbell.num_doorbells *
				     sizeof(u32));
	if (adev->doorbell.ptr == NULL)
A
Alex Deucher 已提交
538 539 540 541 542 543
		return -ENOMEM;

	return 0;
}

/**
544
 * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
A
Alex Deucher 已提交
545 546 547 548 549
 *
 * @adev: amdgpu_device pointer
 *
 * Tear down doorbell driver information (CIK)
 */
550
static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
A
Alex Deucher 已提交
551 552 553 554 555
{
	iounmap(adev->doorbell.ptr);
	adev->doorbell.ptr = NULL;
}

556

A
Alex Deucher 已提交
557 558

/*
559
 * amdgpu_device_wb_*()
560
 * Writeback is the method by which the GPU updates special pages in memory
A
Alex Xie 已提交
561
 * with the status of certain GPU events (fences, ring pointers,etc.).
A
Alex Deucher 已提交
562 563 564
 */

/**
565
 * amdgpu_device_wb_fini - Disable Writeback and free memory
A
Alex Deucher 已提交
566 567 568 569 570 571
 *
 * @adev: amdgpu_device pointer
 *
 * Disables Writeback and frees the Writeback memory (all asics).
 * Used at driver shutdown.
 */
572
static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
A
Alex Deucher 已提交
573 574
{
	if (adev->wb.wb_obj) {
575 576 577
		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
				      &adev->wb.gpu_addr,
				      (void **)&adev->wb.wb);
A
Alex Deucher 已提交
578 579 580 581 582
		adev->wb.wb_obj = NULL;
	}
}

/**
583
 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
A
Alex Deucher 已提交
584 585 586
 *
 * @adev: amdgpu_device pointer
 *
587
 * Initializes writeback and allocates writeback memory (all asics).
A
Alex Deucher 已提交
588 589 590
 * Used at driver startup.
 * Returns 0 on success or an -error on failure.
 */
591
static int amdgpu_device_wb_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
592 593 594 595
{
	int r;

	if (adev->wb.wb_obj == NULL) {
596 597
		/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
598 599 600
					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
					    (void **)&adev->wb.wb);
A
Alex Deucher 已提交
601 602 603 604 605 606 607 608 609
		if (r) {
			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
			return r;
		}

		adev->wb.num_wb = AMDGPU_MAX_WB;
		memset(&adev->wb.used, 0, sizeof(adev->wb.used));

		/* clear wb memory */
M
Monk Liu 已提交
610
		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
A
Alex Deucher 已提交
611 612 613 614 615 616
	}

	return 0;
}

/**
617
 * amdgpu_device_wb_get - Allocate a wb entry
A
Alex Deucher 已提交
618 619 620 621 622 623 624
 *
 * @adev: amdgpu_device pointer
 * @wb: wb index
 *
 * Allocate a wb slot for use by the driver (all asics).
 * Returns 0 on success or -EINVAL on failure.
 */
625
int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
A
Alex Deucher 已提交
626 627 628
{
	unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);

629
	if (offset < adev->wb.num_wb) {
K
Ken Wang 已提交
630
		__set_bit(offset, adev->wb.used);
M
Monk Liu 已提交
631
		*wb = offset << 3; /* convert to dw offset */
632 633 634 635 636 637
		return 0;
	} else {
		return -EINVAL;
	}
}

A
Alex Deucher 已提交
638
/**
639
 * amdgpu_device_wb_free - Free a wb entry
A
Alex Deucher 已提交
640 641 642 643 644 645
 *
 * @adev: amdgpu_device pointer
 * @wb: wb index
 *
 * Free a wb slot allocated for use by the driver (all asics)
 */
646
void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
A
Alex Deucher 已提交
647
{
M
Monk Liu 已提交
648
	wb >>= 3;
A
Alex Deucher 已提交
649
	if (wb < adev->wb.num_wb)
M
Monk Liu 已提交
650
		__clear_bit(wb, adev->wb.used);
A
Alex Deucher 已提交
651 652 653
}

/**
654
 * amdgpu_device_vram_location - try to find VRAM location
655
 *
A
Alex Deucher 已提交
656 657 658 659
 * @adev: amdgpu device structure holding all necessary informations
 * @mc: memory controller structure holding memory informations
 * @base: base address at which to put VRAM
 *
660
 * Function will try to place VRAM at base address provided
661
 * as parameter.
A
Alex Deucher 已提交
662
 */
663
void amdgpu_device_vram_location(struct amdgpu_device *adev,
664
				 struct amdgpu_gmc *mc, u64 base)
A
Alex Deucher 已提交
665 666 667 668 669 670 671 672 673 674 675 676 677
{
	uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;

	mc->vram_start = base;
	mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
	if (limit && limit < mc->real_vram_size)
		mc->real_vram_size = limit;
	dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
			mc->mc_vram_size >> 20, mc->vram_start,
			mc->vram_end, mc->real_vram_size >> 20);
}

/**
678
 * amdgpu_device_gart_location - try to find GTT location
679
 *
A
Alex Deucher 已提交
680 681 682 683 684 685 686 687 688 689
 * @adev: amdgpu device structure holding all necessary informations
 * @mc: memory controller structure holding memory informations
 *
 * Function will place try to place GTT before or after VRAM.
 *
 * If GTT size is bigger than space left then we ajust GTT size.
 * Thus function will never fails.
 *
 * FIXME: when reducing GTT size align new size on power of 2.
 */
690
void amdgpu_device_gart_location(struct amdgpu_device *adev,
691
				 struct amdgpu_gmc *mc)
A
Alex Deucher 已提交
692 693 694
{
	u64 size_af, size_bf;

695 696
	mc->gart_size += adev->pm.smu_prv_buffer_size;

697
	size_af = adev->gmc.mc_mask - mc->vram_end;
698
	size_bf = mc->vram_start;
A
Alex Deucher 已提交
699
	if (size_bf > size_af) {
700
		if (mc->gart_size > size_bf) {
A
Alex Deucher 已提交
701
			dev_warn(adev->dev, "limiting GTT\n");
702
			mc->gart_size = size_bf;
A
Alex Deucher 已提交
703
		}
704
		mc->gart_start = 0;
A
Alex Deucher 已提交
705
	} else {
706
		if (mc->gart_size > size_af) {
A
Alex Deucher 已提交
707
			dev_warn(adev->dev, "limiting GTT\n");
708
			mc->gart_size = size_af;
A
Alex Deucher 已提交
709
		}
710 711 712 713
		/* VCE doesn't like it when BOs cross a 4GB segment, so align
		 * the GART base on a 4GB boundary as well.
		 */
		mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL);
A
Alex Deucher 已提交
714
	}
715
	mc->gart_end = mc->gart_start + mc->gart_size - 1;
A
Alex Deucher 已提交
716
	dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
717
			mc->gart_size >> 20, mc->gart_start, mc->gart_end);
A
Alex Deucher 已提交
718 719
}

720 721 722 723 724 725 726 727 728 729 730
/**
 * amdgpu_device_resize_fb_bar - try to resize FB BAR
 *
 * @adev: amdgpu_device pointer
 *
 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
 * to fail, but if any of the BARs is not accessible after the size we abort
 * driver loading by returning -ENODEV.
 */
int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
{
731
	u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
732
	u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
733 734 735
	struct pci_bus *root;
	struct resource *res;
	unsigned i;
736 737 738
	u16 cmd;
	int r;

739 740 741 742
	/* Bypass for VF */
	if (amdgpu_sriov_vf(adev))
		return 0;

743 744 745 746 747 748
	/* Check if the root BUS has 64bit memory resources */
	root = adev->pdev->bus;
	while (root->parent)
		root = root->parent;

	pci_bus_for_each_resource(root, res, i) {
749
		if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
750 751 752 753 754 755 756 757
		    res->start > 0x100000000ull)
			break;
	}

	/* Trying to resize is pointless without a root hub window above 4GB */
	if (!res)
		return 0;

758 759 760 761 762 763
	/* Disable memory decoding while we change the BAR addresses and size */
	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
	pci_write_config_word(adev->pdev, PCI_COMMAND,
			      cmd & ~PCI_COMMAND_MEMORY);

	/* Free the VRAM and doorbell BAR, we most likely need to move both. */
764
	amdgpu_device_doorbell_fini(adev);
765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780
	if (adev->asic_type >= CHIP_BONAIRE)
		pci_release_resource(adev->pdev, 2);

	pci_release_resource(adev->pdev, 0);

	r = pci_resize_resource(adev->pdev, 0, rbar_size);
	if (r == -ENOSPC)
		DRM_INFO("Not enough PCI address space for a large BAR.");
	else if (r && r != -ENOTSUPP)
		DRM_ERROR("Problem resizing BAR0 (%d).", r);

	pci_assign_unassigned_bus_resources(adev->pdev->bus);

	/* When the doorbell or fb BAR isn't available we have no chance of
	 * using the device.
	 */
781
	r = amdgpu_device_doorbell_init(adev);
782 783 784 785 786 787 788
	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
		return -ENODEV;

	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);

	return 0;
}
789

A
Alex Deucher 已提交
790 791 792 793
/*
 * GPU helpers function.
 */
/**
A
Alex Deucher 已提交
794
 * amdgpu_device_need_post - check if the hw need post or not
A
Alex Deucher 已提交
795 796 797
 *
 * @adev: amdgpu_device pointer
 *
798 799 800
 * Check if the asic has been initialized (all asics) at driver startup
 * or post is needed if  hw reset is performed.
 * Returns true if need or false if not.
A
Alex Deucher 已提交
801
 */
A
Alex Deucher 已提交
802
bool amdgpu_device_need_post(struct amdgpu_device *adev)
A
Alex Deucher 已提交
803 804 805
{
	uint32_t reg;

806 807 808 809
	if (amdgpu_sriov_vf(adev))
		return false;

	if (amdgpu_passthrough(adev)) {
M
Monk Liu 已提交
810 811 812 813
		/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
		 * some old smc fw still need driver do vPost otherwise gpu hang, while
		 * those smc fw version above 22.15 doesn't have this flaw, so we force
		 * vpost executed for smc version below 22.15
814 815 816 817 818 819 820 821 822 823
		 */
		if (adev->asic_type == CHIP_FIJI) {
			int err;
			uint32_t fw_ver;
			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
			/* force vPost if error occured */
			if (err)
				return true;

			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
M
Monk Liu 已提交
824 825
			if (fw_ver < 0x00160e00)
				return true;
826 827
		}
	}
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844

	if (adev->has_hw_reset) {
		adev->has_hw_reset = false;
		return true;
	}

	/* bios scratch used on CIK+ */
	if (adev->asic_type >= CHIP_BONAIRE)
		return amdgpu_atombios_scratch_need_asic_init(adev);

	/* check MEM_SIZE for older asics */
	reg = amdgpu_asic_get_config_memsize(adev);

	if ((reg != 0) && (reg != 0xffffffff))
		return false;

	return true;
845 846
}

A
Alex Deucher 已提交
847 848
/* if we get transitioned to only one device, take VGA back */
/**
849
 * amdgpu_device_vga_set_decode - enable/disable vga decode
A
Alex Deucher 已提交
850 851 852 853 854 855 856
 *
 * @cookie: amdgpu_device pointer
 * @state: enable/disable vga decode
 *
 * Enable/disable vga decode (all asics).
 * Returns VGA resource flags.
 */
857
static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
A
Alex Deucher 已提交
858 859 860 861 862 863 864 865 866 867
{
	struct amdgpu_device *adev = cookie;
	amdgpu_asic_set_vga_state(adev, state);
	if (state)
		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
	else
		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
}

868 869 870 871 872 873 874 875 876 877
/**
 * amdgpu_device_check_block_size - validate the vm block size
 *
 * @adev: amdgpu_device pointer
 *
 * Validates the vm block size specified via module parameter.
 * The vm block size defines number of bits in page table versus page directory,
 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
 * page table and the remaining bits are in the page directory.
 */
878
static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
879 880 881 882
{
	/* defines number of bits in page table versus page directory,
	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
	 * page table and the remaining bits are in the page directory */
883 884
	if (amdgpu_vm_block_size == -1)
		return;
885

886
	if (amdgpu_vm_block_size < 9) {
887 888
		dev_warn(adev->dev, "VM page table size (%d) too small\n",
			 amdgpu_vm_block_size);
889
		amdgpu_vm_block_size = -1;
890 891 892
	}
}

893 894 895 896 897 898 899 900
/**
 * amdgpu_device_check_vm_size - validate the vm size
 *
 * @adev: amdgpu_device pointer
 *
 * Validates the vm size in GB specified via module parameter.
 * The VM size is the size of the GPU virtual memory space in GB.
 */
901
static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
902
{
903 904 905 906
	/* no need to check the default value */
	if (amdgpu_vm_size == -1)
		return;

907 908 909
	if (amdgpu_vm_size < 1) {
		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
			 amdgpu_vm_size);
910
		amdgpu_vm_size = -1;
911 912 913
	}
}

914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953
static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
{
	struct sysinfo si;
	bool is_os_64 = (sizeof(void *) == 8) ? true : false;
	uint64_t total_memory;
	uint64_t dram_size_seven_GB = 0x1B8000000;
	uint64_t dram_size_three_GB = 0xB8000000;

	if (amdgpu_smu_memory_pool_size == 0)
		return;

	if (!is_os_64) {
		DRM_WARN("Not 64-bit OS, feature not supported\n");
		goto def_value;
	}
	si_meminfo(&si);
	total_memory = (uint64_t)si.totalram * si.mem_unit;

	if ((amdgpu_smu_memory_pool_size == 1) ||
		(amdgpu_smu_memory_pool_size == 2)) {
		if (total_memory < dram_size_three_GB)
			goto def_value1;
	} else if ((amdgpu_smu_memory_pool_size == 4) ||
		(amdgpu_smu_memory_pool_size == 8)) {
		if (total_memory < dram_size_seven_GB)
			goto def_value1;
	} else {
		DRM_WARN("Smu memory pool size not supported\n");
		goto def_value;
	}
	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;

	return;

def_value1:
	DRM_WARN("No enough system memory\n");
def_value:
	adev->pm.smu_prv_buffer_size = 0;
}

A
Alex Deucher 已提交
954
/**
955
 * amdgpu_device_check_arguments - validate module params
A
Alex Deucher 已提交
956 957 958 959 960 961
 *
 * @adev: amdgpu_device pointer
 *
 * Validates certain module parameters and updates
 * the associated values used by the driver (all asics).
 */
962
static void amdgpu_device_check_arguments(struct amdgpu_device *adev)
A
Alex Deucher 已提交
963
{
964 965 966 967
	if (amdgpu_sched_jobs < 4) {
		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
			 amdgpu_sched_jobs);
		amdgpu_sched_jobs = 4;
968
	} else if (!is_power_of_2(amdgpu_sched_jobs)){
969 970 971 972
		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
			 amdgpu_sched_jobs);
		amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
	}
A
Alex Deucher 已提交
973

974
	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
975 976 977
		/* gart size must be greater or equal to 32M */
		dev_warn(adev->dev, "gart size (%d) too small\n",
			 amdgpu_gart_size);
978
		amdgpu_gart_size = -1;
A
Alex Deucher 已提交
979 980
	}

981
	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
982
		/* gtt size must be greater or equal to 32M */
983 984 985
		dev_warn(adev->dev, "gtt size (%d) too small\n",
				 amdgpu_gtt_size);
		amdgpu_gtt_size = -1;
A
Alex Deucher 已提交
986 987
	}

988 989 990 991 992 993 994
	/* valid range is between 4 and 9 inclusive */
	if (amdgpu_vm_fragment_size != -1 &&
	    (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
		dev_warn(adev->dev, "valid range is between 4 and 9\n");
		amdgpu_vm_fragment_size = -1;
	}

995 996
	amdgpu_device_check_smu_prv_buffer_size(adev);

997
	amdgpu_device_check_vm_size(adev);
A
Alex Deucher 已提交
998

999
	amdgpu_device_check_block_size(adev);
C
Christian König 已提交
1000

1001
	if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
1002
	    !is_power_of_2(amdgpu_vram_page_split))) {
C
Christian König 已提交
1003 1004 1005 1006
		dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
			 amdgpu_vram_page_split);
		amdgpu_vram_page_split = 1024;
	}
1007 1008 1009 1010 1011

	if (amdgpu_lockup_timeout == 0) {
		dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n");
		amdgpu_lockup_timeout = 10000;
	}
1012 1013

	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
A
Alex Deucher 已提交
1014 1015 1016 1017 1018 1019
}

/**
 * amdgpu_switcheroo_set_state - set switcheroo state
 *
 * @pdev: pci dev pointer
1020
 * @state: vga_switcheroo state
A
Alex Deucher 已提交
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032
 *
 * Callback for the switcheroo driver.  Suspends or resumes the
 * the asics before or after it is powered up using ACPI methods.
 */
static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
{
	struct drm_device *dev = pci_get_drvdata(pdev);

	if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
		return;

	if (state == VGA_SWITCHEROO_ON) {
1033
		pr_info("amdgpu: switched on\n");
A
Alex Deucher 已提交
1034 1035 1036
		/* don't suspend or resume card normally */
		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;

1037
		amdgpu_device_resume(dev, true, true);
A
Alex Deucher 已提交
1038 1039 1040 1041

		dev->switch_power_state = DRM_SWITCH_POWER_ON;
		drm_kms_helper_poll_enable(dev);
	} else {
1042
		pr_info("amdgpu: switched off\n");
A
Alex Deucher 已提交
1043 1044
		drm_kms_helper_poll_disable(dev);
		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1045
		amdgpu_device_suspend(dev, true, true);
A
Alex Deucher 已提交
1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
	}
}

/**
 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
 *
 * @pdev: pci dev pointer
 *
 * Callback for the switcheroo driver.  Check of the switcheroo
 * state can be changed.
 * Returns true if the state can be changed, false if not.
 */
static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
{
	struct drm_device *dev = pci_get_drvdata(pdev);

	/*
	* FIXME: open_count is protected by drm_global_mutex but that would lead to
	* locking inversion with the driver load path. And the access here is
	* completely racy anyway. So don't bother with locking for now.
	*/
	return dev->open_count == 0;
}

static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
	.set_gpu_state = amdgpu_switcheroo_set_state,
	.reprobe = NULL,
	.can_switch = amdgpu_switcheroo_can_switch,
};

1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
/**
 * amdgpu_device_ip_set_clockgating_state - set the CG state
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 * @state: clockgating state (gate or ungate)
 *
 * Sets the requested clockgating state for all instances of
 * the hardware IP specified.
 * Returns the error code from the last instance.
 */
1088
int amdgpu_device_ip_set_clockgating_state(void *dev,
1089 1090
					   enum amd_ip_block_type block_type,
					   enum amd_clockgating_state state)
A
Alex Deucher 已提交
1091
{
1092
	struct amdgpu_device *adev = dev;
A
Alex Deucher 已提交
1093 1094 1095
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1096
		if (!adev->ip_blocks[i].status.valid)
1097
			continue;
1098 1099 1100 1101 1102 1103 1104 1105 1106
		if (adev->ip_blocks[i].version->type != block_type)
			continue;
		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
			continue;
		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
			(void *)adev, state);
		if (r)
			DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
A
Alex Deucher 已提交
1107 1108 1109 1110
	}
	return r;
}

1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
/**
 * amdgpu_device_ip_set_powergating_state - set the PG state
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 * @state: powergating state (gate or ungate)
 *
 * Sets the requested powergating state for all instances of
 * the hardware IP specified.
 * Returns the error code from the last instance.
 */
1122
int amdgpu_device_ip_set_powergating_state(void *dev,
1123 1124
					   enum amd_ip_block_type block_type,
					   enum amd_powergating_state state)
A
Alex Deucher 已提交
1125
{
1126
	struct amdgpu_device *adev = dev;
A
Alex Deucher 已提交
1127 1128 1129
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1130
		if (!adev->ip_blocks[i].status.valid)
1131
			continue;
1132 1133 1134 1135 1136 1137 1138 1139 1140
		if (adev->ip_blocks[i].version->type != block_type)
			continue;
		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
			continue;
		r = adev->ip_blocks[i].version->funcs->set_powergating_state(
			(void *)adev, state);
		if (r)
			DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
A
Alex Deucher 已提交
1141 1142 1143 1144
	}
	return r;
}

1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155
/**
 * amdgpu_device_ip_get_clockgating_state - get the CG state
 *
 * @adev: amdgpu_device pointer
 * @flags: clockgating feature flags
 *
 * Walks the list of IPs on the device and updates the clockgating
 * flags for each IP.
 * Updates @flags with the feature flags for each hardware IP where
 * clockgating is enabled.
 */
1156 1157
void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
					    u32 *flags)
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
{
	int i;

	for (i = 0; i < adev->num_ip_blocks; i++) {
		if (!adev->ip_blocks[i].status.valid)
			continue;
		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
			adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
	}
}

1169 1170 1171 1172 1173 1174 1175 1176 1177
/**
 * amdgpu_device_ip_wait_for_idle - wait for idle
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 *
 * Waits for the request hardware IP to be idle.
 * Returns 0 for success or a negative error code on failure.
 */
1178 1179
int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
				   enum amd_ip_block_type block_type)
1180 1181 1182 1183
{
	int i, r;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1184
		if (!adev->ip_blocks[i].status.valid)
1185
			continue;
1186 1187
		if (adev->ip_blocks[i].version->type == block_type) {
			r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1188 1189 1190 1191 1192 1193 1194 1195 1196
			if (r)
				return r;
			break;
		}
	}
	return 0;

}

1197 1198 1199 1200 1201 1202 1203 1204 1205
/**
 * amdgpu_device_ip_is_idle - is the hardware IP idle
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 *
 * Check if the hardware IP is idle or not.
 * Returns true if it the IP is idle, false if not.
 */
1206 1207
bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
			      enum amd_ip_block_type block_type)
1208 1209 1210 1211
{
	int i;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1212
		if (!adev->ip_blocks[i].status.valid)
1213
			continue;
1214 1215
		if (adev->ip_blocks[i].version->type == block_type)
			return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1216 1217 1218 1219 1220
	}
	return true;

}

1221 1222 1223 1224 1225 1226 1227 1228 1229
/**
 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 *
 * Returns a pointer to the hardware IP block structure
 * if it exists for the asic, otherwise NULL.
 */
1230 1231 1232
struct amdgpu_ip_block *
amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
			      enum amd_ip_block_type type)
A
Alex Deucher 已提交
1233 1234 1235 1236
{
	int i;

	for (i = 0; i < adev->num_ip_blocks; i++)
1237
		if (adev->ip_blocks[i].version->type == type)
A
Alex Deucher 已提交
1238 1239 1240 1241 1242 1243
			return &adev->ip_blocks[i];

	return NULL;
}

/**
1244
 * amdgpu_device_ip_block_version_cmp
A
Alex Deucher 已提交
1245 1246
 *
 * @adev: amdgpu_device pointer
1247
 * @type: enum amd_ip_block_type
A
Alex Deucher 已提交
1248 1249 1250 1251 1252 1253
 * @major: major version
 * @minor: minor version
 *
 * return 0 if equal or greater
 * return 1 if smaller or the ip_block doesn't exist
 */
1254 1255 1256
int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
				       enum amd_ip_block_type type,
				       u32 major, u32 minor)
A
Alex Deucher 已提交
1257
{
1258
	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
A
Alex Deucher 已提交
1259

1260 1261 1262
	if (ip_block && ((ip_block->version->major > major) ||
			((ip_block->version->major == major) &&
			(ip_block->version->minor >= minor))))
A
Alex Deucher 已提交
1263 1264 1265 1266 1267
		return 0;

	return 1;
}

1268
/**
1269
 * amdgpu_device_ip_block_add
1270 1271 1272 1273 1274 1275 1276
 *
 * @adev: amdgpu_device pointer
 * @ip_block_version: pointer to the IP to add
 *
 * Adds the IP block driver information to the collection of IPs
 * on the asic.
 */
1277 1278
int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
			       const struct amdgpu_ip_block_version *ip_block_version)
1279 1280 1281 1282
{
	if (!ip_block_version)
		return -EINVAL;

1283
	DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1284 1285
		  ip_block_version->funcs->name);

1286 1287 1288 1289 1290
	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;

	return 0;
}

1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302
/**
 * amdgpu_device_enable_virtual_display - enable virtual display feature
 *
 * @adev: amdgpu_device pointer
 *
 * Enabled the virtual display feature if the user has enabled it via
 * the module parameter virtual_display.  This feature provides a virtual
 * display hardware on headless boards or in virtualized environments.
 * This function parses and validates the configuration string specified by
 * the user and configues the virtual display configuration (number of
 * virtual connectors, crtcs, etc.) specified.
 */
1303
static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1304 1305 1306 1307 1308 1309
{
	adev->enable_virtual_display = false;

	if (amdgpu_virtual_display) {
		struct drm_device *ddev = adev->ddev;
		const char *pci_address_name = pci_name(ddev->pdev);
1310
		char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1311 1312 1313

		pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
		pciaddstr_tmp = pciaddstr;
1314 1315
		while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
			pciaddname = strsep(&pciaddname_tmp, ",");
1316 1317
			if (!strcmp("all", pciaddname)
			    || !strcmp(pci_address_name, pciaddname)) {
1318 1319 1320
				long num_crtc;
				int res = -1;

1321
				adev->enable_virtual_display = true;
1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335

				if (pciaddname_tmp)
					res = kstrtol(pciaddname_tmp, 10,
						      &num_crtc);

				if (!res) {
					if (num_crtc < 1)
						num_crtc = 1;
					if (num_crtc > 6)
						num_crtc = 6;
					adev->mode_info.num_crtc = num_crtc;
				} else {
					adev->mode_info.num_crtc = 1;
				}
1336 1337 1338 1339
				break;
			}
		}

1340 1341 1342
		DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
			 amdgpu_virtual_display, pci_address_name,
			 adev->enable_virtual_display, adev->mode_info.num_crtc);
1343 1344 1345 1346 1347

		kfree(pciaddstr);
	}
}

1348 1349 1350 1351 1352 1353 1354 1355 1356 1357
/**
 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
 *
 * @adev: amdgpu_device pointer
 *
 * Parses the asic configuration parameters specified in the gpu info
 * firmware and makes them availale to the driver for use in configuring
 * the asic.
 * Returns 0 on success, -EINVAL on failure.
 */
1358 1359 1360 1361 1362 1363 1364
static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
{
	const char *chip_name;
	char fw_name[30];
	int err;
	const struct gpu_info_firmware_header_v1_0 *hdr;

1365 1366
	adev->firmware.gpu_info_fw = NULL;

1367 1368 1369 1370 1371
	switch (adev->asic_type) {
	case CHIP_TOPAZ:
	case CHIP_TONGA:
	case CHIP_FIJI:
	case CHIP_POLARIS10:
1372
	case CHIP_POLARIS11:
1373
	case CHIP_POLARIS12:
1374
	case CHIP_VEGAM:
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390
	case CHIP_CARRIZO:
	case CHIP_STONEY:
#ifdef CONFIG_DRM_AMDGPU_SI
	case CHIP_VERDE:
	case CHIP_TAHITI:
	case CHIP_PITCAIRN:
	case CHIP_OLAND:
	case CHIP_HAINAN:
#endif
#ifdef CONFIG_DRM_AMDGPU_CIK
	case CHIP_BONAIRE:
	case CHIP_HAWAII:
	case CHIP_KAVERI:
	case CHIP_KABINI:
	case CHIP_MULLINS:
#endif
1391
	case CHIP_VEGA20:
1392 1393 1394 1395 1396
	default:
		return 0;
	case CHIP_VEGA10:
		chip_name = "vega10";
		break;
1397 1398 1399
	case CHIP_VEGA12:
		chip_name = "vega12";
		break;
1400 1401 1402
	case CHIP_RAVEN:
		chip_name = "raven";
		break;
1403 1404 1405
	}

	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1406
	err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1407 1408 1409 1410 1411 1412
	if (err) {
		dev_err(adev->dev,
			"Failed to load gpu_info firmware \"%s\"\n",
			fw_name);
		goto out;
	}
1413
	err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1414 1415 1416 1417 1418 1419 1420
	if (err) {
		dev_err(adev->dev,
			"Failed to validate gpu_info firmware \"%s\"\n",
			fw_name);
		goto out;
	}

1421
	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1422 1423 1424 1425 1426 1427
	amdgpu_ucode_print_gpu_info_hdr(&hdr->header);

	switch (hdr->version_major) {
	case 1:
	{
		const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1428
			(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1429 1430
								le32_to_cpu(hdr->header.ucode_array_offset_bytes));

1431 1432 1433 1434
		adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
		adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
		adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
		adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1435
		adev->gfx.config.max_texture_channel_caches =
1436 1437 1438 1439 1440
			le32_to_cpu(gpu_info_fw->gc_num_tccs);
		adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
		adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
		adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
		adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1441
		adev->gfx.config.double_offchip_lds_buf =
1442 1443
			le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
		adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1444 1445 1446 1447 1448
		adev->gfx.cu_info.max_waves_per_simd =
			le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
		adev->gfx.cu_info.max_scratch_slots_per_cu =
			le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
		adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460
		break;
	}
	default:
		dev_err(adev->dev,
			"Unsupported gpu_info table %d\n", hdr->header.ucode_version);
		err = -EINVAL;
		goto out;
	}
out:
	return err;
}

1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
/**
 * amdgpu_device_ip_early_init - run early init for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Early initialization pass for hardware IPs.  The hardware IPs that make
 * up each asic are discovered each IP's early_init callback is run.  This
 * is the first stage in initializing the asic.
 * Returns 0 on success, negative error code on failure.
 */
1471
static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
1472
{
1473
	int i, r;
A
Alex Deucher 已提交
1474

1475
	amdgpu_device_enable_virtual_display(adev);
1476

A
Alex Deucher 已提交
1477
	switch (adev->asic_type) {
1478 1479
	case CHIP_TOPAZ:
	case CHIP_TONGA:
1480
	case CHIP_FIJI:
1481
	case CHIP_POLARIS10:
1482
	case CHIP_POLARIS11:
1483
	case CHIP_POLARIS12:
1484
	case CHIP_VEGAM:
1485
	case CHIP_CARRIZO:
1486 1487
	case CHIP_STONEY:
		if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1488 1489 1490 1491 1492 1493 1494 1495
			adev->family = AMDGPU_FAMILY_CZ;
		else
			adev->family = AMDGPU_FAMILY_VI;

		r = vi_set_ip_blocks(adev);
		if (r)
			return r;
		break;
K
Ken Wang 已提交
1496 1497 1498 1499 1500 1501
#ifdef CONFIG_DRM_AMDGPU_SI
	case CHIP_VERDE:
	case CHIP_TAHITI:
	case CHIP_PITCAIRN:
	case CHIP_OLAND:
	case CHIP_HAINAN:
K
Ken Wang 已提交
1502
		adev->family = AMDGPU_FAMILY_SI;
K
Ken Wang 已提交
1503 1504 1505 1506 1507
		r = si_set_ip_blocks(adev);
		if (r)
			return r;
		break;
#endif
1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
#ifdef CONFIG_DRM_AMDGPU_CIK
	case CHIP_BONAIRE:
	case CHIP_HAWAII:
	case CHIP_KAVERI:
	case CHIP_KABINI:
	case CHIP_MULLINS:
		if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
			adev->family = AMDGPU_FAMILY_CI;
		else
			adev->family = AMDGPU_FAMILY_KV;

		r = cik_set_ip_blocks(adev);
		if (r)
			return r;
		break;
#endif
1524 1525
	case CHIP_VEGA10:
	case CHIP_VEGA12:
1526
	case CHIP_VEGA20:
1527
	case CHIP_RAVEN:
1528 1529 1530 1531
		if (adev->asic_type == CHIP_RAVEN)
			adev->family = AMDGPU_FAMILY_RV;
		else
			adev->family = AMDGPU_FAMILY_AI;
1532 1533 1534 1535 1536

		r = soc15_set_ip_blocks(adev);
		if (r)
			return r;
		break;
A
Alex Deucher 已提交
1537 1538 1539 1540 1541
	default:
		/* FIXME: not supported yet */
		return -EINVAL;
	}

1542 1543 1544 1545
	r = amdgpu_device_parse_gpu_info_fw(adev);
	if (r)
		return r;

1546 1547
	amdgpu_amdkfd_device_probe(adev);

1548 1549 1550
	if (amdgpu_sriov_vf(adev)) {
		r = amdgpu_virt_request_full_gpu(adev, true);
		if (r)
1551
			return -EAGAIN;
1552 1553
	}

1554 1555
	adev->powerplay.pp_feature = amdgpu_pp_feature_mask;

A
Alex Deucher 已提交
1556 1557
	for (i = 0; i < adev->num_ip_blocks; i++) {
		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1558 1559
			DRM_ERROR("disabled ip block: %d <%s>\n",
				  i, adev->ip_blocks[i].version->funcs->name);
1560
			adev->ip_blocks[i].status.valid = false;
A
Alex Deucher 已提交
1561
		} else {
1562 1563
			if (adev->ip_blocks[i].version->funcs->early_init) {
				r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1564
				if (r == -ENOENT) {
1565
					adev->ip_blocks[i].status.valid = false;
1566
				} else if (r) {
1567 1568
					DRM_ERROR("early_init of IP block <%s> failed %d\n",
						  adev->ip_blocks[i].version->funcs->name, r);
A
Alex Deucher 已提交
1569
					return r;
1570
				} else {
1571
					adev->ip_blocks[i].status.valid = true;
1572
				}
1573
			} else {
1574
				adev->ip_blocks[i].status.valid = true;
A
Alex Deucher 已提交
1575 1576 1577 1578
			}
		}
	}

1579 1580 1581
	adev->cg_flags &= amdgpu_cg_mask;
	adev->pg_flags &= amdgpu_pg_mask;

A
Alex Deucher 已提交
1582 1583 1584
	return 0;
}

1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595
/**
 * amdgpu_device_ip_init - run init for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Main initialization pass for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
 * are run.  sw_init initializes the software state associated with each IP
 * and hw_init initializes the hardware associated with each IP.
 * Returns 0 on success, negative error code on failure.
 */
1596
static int amdgpu_device_ip_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
1597 1598 1599 1600
{
	int i, r;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1601
		if (!adev->ip_blocks[i].status.valid)
A
Alex Deucher 已提交
1602
			continue;
1603
		r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1604
		if (r) {
1605 1606
			DRM_ERROR("sw_init of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
A
Alex Deucher 已提交
1607
			return r;
1608
		}
1609
		adev->ip_blocks[i].status.sw = true;
1610

A
Alex Deucher 已提交
1611
		/* need to do gmc hw init early so we can allocate gpu mem */
1612
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1613
			r = amdgpu_device_vram_scratch_init(adev);
1614 1615
			if (r) {
				DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
A
Alex Deucher 已提交
1616
				return r;
1617
			}
1618
			r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1619 1620
			if (r) {
				DRM_ERROR("hw_init %d failed %d\n", i, r);
A
Alex Deucher 已提交
1621
				return r;
1622
			}
1623
			r = amdgpu_device_wb_init(adev);
1624
			if (r) {
1625
				DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
A
Alex Deucher 已提交
1626
				return r;
1627
			}
1628
			adev->ip_blocks[i].status.hw = true;
M
Monk Liu 已提交
1629 1630 1631 1632 1633 1634 1635 1636 1637

			/* right after GMC hw init, we create CSA */
			if (amdgpu_sriov_vf(adev)) {
				r = amdgpu_allocate_static_csa(adev);
				if (r) {
					DRM_ERROR("allocate CSA failed %d\n", r);
					return r;
				}
			}
A
Alex Deucher 已提交
1638 1639 1640 1641
		}
	}

	for (i = 0; i < adev->num_ip_blocks; i++) {
1642
		if (!adev->ip_blocks[i].status.sw)
A
Alex Deucher 已提交
1643
			continue;
1644
		if (adev->ip_blocks[i].status.hw)
A
Alex Deucher 已提交
1645
			continue;
1646
		r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1647
		if (r) {
1648 1649
			DRM_ERROR("hw_init of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
A
Alex Deucher 已提交
1650
			return r;
1651
		}
1652
		adev->ip_blocks[i].status.hw = true;
A
Alex Deucher 已提交
1653 1654
	}

1655
	amdgpu_amdkfd_device_init(adev);
1656 1657 1658 1659

	if (amdgpu_sriov_vf(adev))
		amdgpu_virt_release_full_gpu(adev, true);

A
Alex Deucher 已提交
1660 1661 1662
	return 0;
}

1663 1664 1665 1666 1667 1668 1669 1670 1671
/**
 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
 *
 * @adev: amdgpu_device pointer
 *
 * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
 * this function before a GPU reset.  If the value is retained after a
 * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
 */
1672
static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
1673 1674 1675 1676
{
	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
}

1677 1678 1679 1680 1681 1682 1683 1684 1685 1686
/**
 * amdgpu_device_check_vram_lost - check if vram is valid
 *
 * @adev: amdgpu_device pointer
 *
 * Checks the reset magic value written to the gart pointer in VRAM.
 * The driver calls this after a GPU reset to see if the contents of
 * VRAM is lost or now.
 * returns true if vram is lost, false if not.
 */
1687
static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
1688 1689 1690 1691 1692
{
	return !!memcmp(adev->gart.ptr, adev->reset_magic,
			AMDGPU_RESET_MAGIC_NUM);
}

1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703
/**
 * amdgpu_device_ip_late_set_cg_state - late init for clockgating
 *
 * @adev: amdgpu_device pointer
 *
 * Late initialization pass enabling clockgating for hardware IPs.
 * The list of all the hardware IPs that make up the asic is walked and the
 * set_clockgating_state callbacks are run.  This stage is run late
 * in the init process.
 * Returns 0 on success, negative error code on failure.
 */
1704
static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev)
A
Alex Deucher 已提交
1705 1706 1707
{
	int i = 0, r;

1708 1709 1710
	if (amdgpu_emu_mode == 1)
		return 0;

1711 1712 1713 1714
	r = amdgpu_ib_ring_tests(adev);
	if (r)
		DRM_ERROR("ib ring test failed (%d).\n", r);

A
Alex Deucher 已提交
1715
	for (i = 0; i < adev->num_ip_blocks; i++) {
1716
		if (!adev->ip_blocks[i].status.valid)
A
Alex Deucher 已提交
1717
			continue;
1718
		/* skip CG for VCE/UVD, it's handled specially */
1719
		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1720 1721
		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
		    adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1722
			/* enable clockgating to save power */
1723 1724
			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
										     AMD_CG_STATE_GATE);
1725 1726
			if (r) {
				DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1727
					  adev->ip_blocks[i].version->funcs->name, r);
1728 1729
				return r;
			}
1730
		}
A
Alex Deucher 已提交
1731
	}
1732 1733 1734
	return 0;
}

1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746
/**
 * amdgpu_device_ip_late_init - run late init for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Late initialization pass for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked and the late_init callbacks are run.
 * late_init covers any special initialization that an IP requires
 * after all of the have been initialized or something that needs to happen
 * late in the init process.
 * Returns 0 on success, negative error code on failure.
 */
1747
static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764
{
	int i = 0, r;

	for (i = 0; i < adev->num_ip_blocks; i++) {
		if (!adev->ip_blocks[i].status.valid)
			continue;
		if (adev->ip_blocks[i].version->funcs->late_init) {
			r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
			if (r) {
				DRM_ERROR("late_init of IP block <%s> failed %d\n",
					  adev->ip_blocks[i].version->funcs->name, r);
				return r;
			}
			adev->ip_blocks[i].status.late_initialized = true;
		}
	}

1765 1766
	queue_delayed_work(system_wq, &adev->late_init_work,
			   msecs_to_jiffies(AMDGPU_RESUME_MS));
A
Alex Deucher 已提交
1767

1768
	amdgpu_device_fill_reset_magic(adev);
A
Alex Deucher 已提交
1769 1770 1771 1772

	return 0;
}

1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783
/**
 * amdgpu_device_ip_fini - run fini for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Main teardown pass for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
 * are run.  hw_fini tears down the hardware associated with each IP
 * and sw_fini tears down any software state associated with each IP.
 * Returns 0 on success, negative error code on failure.
 */
1784
static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
A
Alex Deucher 已提交
1785 1786 1787
{
	int i, r;

1788
	amdgpu_amdkfd_device_fini(adev);
1789 1790
	/* need to disable SMC first */
	for (i = 0; i < adev->num_ip_blocks; i++) {
1791
		if (!adev->ip_blocks[i].status.hw)
1792
			continue;
1793 1794
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC &&
			adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1795
			/* ungate blocks before hw fini so that we can shutdown the blocks safely */
1796 1797
			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
										     AMD_CG_STATE_UNGATE);
1798 1799
			if (r) {
				DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1800
					  adev->ip_blocks[i].version->funcs->name, r);
1801 1802
				return r;
			}
1803
			r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1804 1805 1806
			/* XXX handle errors */
			if (r) {
				DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1807
					  adev->ip_blocks[i].version->funcs->name, r);
1808
			}
1809
			adev->ip_blocks[i].status.hw = false;
1810 1811 1812 1813
			break;
		}
	}

A
Alex Deucher 已提交
1814
	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1815
		if (!adev->ip_blocks[i].status.hw)
A
Alex Deucher 已提交
1816
			continue;
1817 1818

		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1819 1820
			adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
			adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1821 1822 1823 1824 1825 1826 1827 1828
			/* ungate blocks before hw fini so that we can shutdown the blocks safely */
			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
										     AMD_CG_STATE_UNGATE);
			if (r) {
				DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
					  adev->ip_blocks[i].version->funcs->name, r);
				return r;
			}
1829
		}
1830

1831
		r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
A
Alex Deucher 已提交
1832
		/* XXX handle errors */
1833
		if (r) {
1834 1835
			DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
1836
		}
1837

1838
		adev->ip_blocks[i].status.hw = false;
A
Alex Deucher 已提交
1839 1840
	}

1841

A
Alex Deucher 已提交
1842
	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1843
		if (!adev->ip_blocks[i].status.sw)
A
Alex Deucher 已提交
1844
			continue;
1845 1846 1847 1848 1849 1850 1851

		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
			amdgpu_free_static_csa(adev);
			amdgpu_device_wb_fini(adev);
			amdgpu_device_vram_scratch_fini(adev);
		}

1852
		r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
A
Alex Deucher 已提交
1853
		/* XXX handle errors */
1854
		if (r) {
1855 1856
			DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
1857
		}
1858 1859
		adev->ip_blocks[i].status.sw = false;
		adev->ip_blocks[i].status.valid = false;
A
Alex Deucher 已提交
1860 1861
	}

M
Monk Liu 已提交
1862
	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1863
		if (!adev->ip_blocks[i].status.late_initialized)
1864
			continue;
1865 1866 1867
		if (adev->ip_blocks[i].version->funcs->late_fini)
			adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
		adev->ip_blocks[i].status.late_initialized = false;
M
Monk Liu 已提交
1868 1869
	}

1870
	if (amdgpu_sriov_vf(adev))
1871 1872
		if (amdgpu_virt_release_full_gpu(adev, false))
			DRM_ERROR("failed to release exclusive mode on fini\n");
M
Monk Liu 已提交
1873

A
Alex Deucher 已提交
1874 1875 1876
	return 0;
}

1877 1878 1879 1880 1881 1882 1883 1884 1885
/**
 * amdgpu_device_ip_late_init_func_handler - work handler for clockgating
 *
 * @work: work_struct
 *
 * Work handler for amdgpu_device_ip_late_set_cg_state.  We put the
 * clockgating setup into a worker thread to speed up driver init and
 * resume from suspend.
 */
1886
static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work)
1887 1888 1889
{
	struct amdgpu_device *adev =
		container_of(work, struct amdgpu_device, late_init_work.work);
1890
	amdgpu_device_ip_late_set_cg_state(adev);
1891 1892
}

1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903
/**
 * amdgpu_device_ip_suspend - run suspend for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Main suspend function for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked, clockgating is disabled and the
 * suspend callbacks are run.  suspend puts the hardware and software state
 * in each IP into a state suitable for suspend.
 * Returns 0 on success, negative error code on failure.
 */
1904
int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
A
Alex Deucher 已提交
1905 1906 1907
{
	int i, r;

1908 1909 1910
	if (amdgpu_sriov_vf(adev))
		amdgpu_virt_request_full_gpu(adev, false);

1911 1912 1913 1914 1915 1916
	/* ungate SMC block powergating */
	if (adev->powerplay.pp_feature & PP_GFXOFF_MASK)
		amdgpu_device_ip_set_powergating_state(adev,
						       AMD_IP_BLOCK_TYPE_SMC,
						       AMD_CG_STATE_UNGATE);

1917
	/* ungate SMC block first */
1918 1919
	r = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
						   AMD_CG_STATE_UNGATE);
1920
	if (r) {
1921
		DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n", r);
1922 1923
	}

A
Alex Deucher 已提交
1924
	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1925
		if (!adev->ip_blocks[i].status.valid)
A
Alex Deucher 已提交
1926 1927
			continue;
		/* ungate blocks so that suspend can properly shut them down */
1928
		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_SMC &&
1929
			adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1930 1931
			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
										     AMD_CG_STATE_UNGATE);
1932
			if (r) {
1933 1934
				DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
					  adev->ip_blocks[i].version->funcs->name, r);
1935
			}
1936
		}
A
Alex Deucher 已提交
1937
		/* XXX handle errors */
1938
		r = adev->ip_blocks[i].version->funcs->suspend(adev);
A
Alex Deucher 已提交
1939
		/* XXX handle errors */
1940
		if (r) {
1941 1942
			DRM_ERROR("suspend of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
1943
		}
A
Alex Deucher 已提交
1944 1945
	}

1946 1947 1948
	if (amdgpu_sriov_vf(adev))
		amdgpu_virt_release_full_gpu(adev, false);

A
Alex Deucher 已提交
1949 1950 1951
	return 0;
}

1952
static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
1953 1954 1955
{
	int i, r;

1956 1957 1958 1959 1960
	static enum amd_ip_block_type ip_order[] = {
		AMD_IP_BLOCK_TYPE_GMC,
		AMD_IP_BLOCK_TYPE_COMMON,
		AMD_IP_BLOCK_TYPE_IH,
	};
1961

1962 1963 1964
	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
		int j;
		struct amdgpu_ip_block *block;
1965

1966 1967 1968 1969 1970 1971 1972 1973 1974
		for (j = 0; j < adev->num_ip_blocks; j++) {
			block = &adev->ip_blocks[j];

			if (block->version->type != ip_order[i] ||
				!block->status.valid)
				continue;

			r = block->version->funcs->hw_init(adev);
			DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1975 1976
			if (r)
				return r;
1977 1978 1979 1980 1981 1982
		}
	}

	return 0;
}

1983
static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
1984 1985 1986
{
	int i, r;

1987 1988
	static enum amd_ip_block_type ip_order[] = {
		AMD_IP_BLOCK_TYPE_SMC,
M
Monk Liu 已提交
1989
		AMD_IP_BLOCK_TYPE_PSP,
1990 1991 1992
		AMD_IP_BLOCK_TYPE_DCE,
		AMD_IP_BLOCK_TYPE_GFX,
		AMD_IP_BLOCK_TYPE_SDMA,
1993 1994
		AMD_IP_BLOCK_TYPE_UVD,
		AMD_IP_BLOCK_TYPE_VCE
1995
	};
1996

1997 1998 1999
	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
		int j;
		struct amdgpu_ip_block *block;
2000

2001 2002 2003 2004 2005 2006 2007 2008 2009
		for (j = 0; j < adev->num_ip_blocks; j++) {
			block = &adev->ip_blocks[j];

			if (block->version->type != ip_order[i] ||
				!block->status.valid)
				continue;

			r = block->version->funcs->hw_init(adev);
			DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
2010 2011
			if (r)
				return r;
2012 2013 2014 2015 2016 2017
		}
	}

	return 0;
}

2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029
/**
 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * First resume function for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked and the resume callbacks are run for
 * COMMON, GMC, and IH.  resume puts the hardware into a functional state
 * after a suspend and updates the software state as necessary.  This
 * function is also used for restoring the GPU after a GPU reset.
 * Returns 0 on success, negative error code on failure.
 */
2030
static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
A
Alex Deucher 已提交
2031 2032 2033
{
	int i, r;

2034 2035 2036 2037
	for (i = 0; i < adev->num_ip_blocks; i++) {
		if (!adev->ip_blocks[i].status.valid)
			continue;
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2038 2039
		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2040 2041 2042 2043 2044 2045
			r = adev->ip_blocks[i].version->funcs->resume(adev);
			if (r) {
				DRM_ERROR("resume of IP block <%s> failed %d\n",
					  adev->ip_blocks[i].version->funcs->name, r);
				return r;
			}
2046 2047 2048 2049 2050 2051
		}
	}

	return 0;
}

2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064
/**
 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * First resume function for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked and the resume callbacks are run for
 * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
 * functional state after a suspend and updates the software state as
 * necessary.  This function is also used for restoring the GPU after a GPU
 * reset.
 * Returns 0 on success, negative error code on failure.
 */
2065
static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
A
Alex Deucher 已提交
2066 2067 2068 2069
{
	int i, r;

	for (i = 0; i < adev->num_ip_blocks; i++) {
2070
		if (!adev->ip_blocks[i].status.valid)
A
Alex Deucher 已提交
2071
			continue;
2072
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2073 2074
		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)
2075
			continue;
2076
		r = adev->ip_blocks[i].version->funcs->resume(adev);
2077
		if (r) {
2078 2079
			DRM_ERROR("resume of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
A
Alex Deucher 已提交
2080
			return r;
2081
		}
A
Alex Deucher 已提交
2082 2083 2084 2085 2086
	}

	return 0;
}

2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098
/**
 * amdgpu_device_ip_resume - run resume for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Main resume function for hardware IPs.  The hardware IPs
 * are split into two resume functions because they are
 * are also used in in recovering from a GPU reset and some additional
 * steps need to be take between them.  In this case (S3/S4) they are
 * run sequentially.
 * Returns 0 on success, negative error code on failure.
 */
2099
static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
2100 2101 2102
{
	int r;

2103
	r = amdgpu_device_ip_resume_phase1(adev);
2104 2105
	if (r)
		return r;
2106
	r = amdgpu_device_ip_resume_phase2(adev);
2107 2108 2109 2110

	return r;
}

2111 2112 2113 2114 2115 2116 2117
/**
 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
 *
 * @adev: amdgpu_device pointer
 *
 * Query the VBIOS data tables to determine if the board supports SR-IOV.
 */
2118
static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2119
{
M
Monk Liu 已提交
2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130
	if (amdgpu_sriov_vf(adev)) {
		if (adev->is_atom_fw) {
			if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
		} else {
			if (amdgpu_atombios_has_gpu_virtualization_table(adev))
				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
		}

		if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2131
	}
2132 2133
}

2134 2135 2136 2137 2138 2139 2140 2141
/**
 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
 *
 * @asic_type: AMD asic type
 *
 * Check if there is DC (new modesetting infrastructre) support for an asic.
 * returns true if DC has support, false if not.
 */
2142 2143 2144 2145 2146 2147
bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
{
	switch (asic_type) {
#if defined(CONFIG_DRM_AMD_DC)
	case CHIP_BONAIRE:
	case CHIP_HAWAII:
2148
	case CHIP_KAVERI:
2149 2150
	case CHIP_KABINI:
	case CHIP_MULLINS:
2151 2152 2153
	case CHIP_CARRIZO:
	case CHIP_STONEY:
	case CHIP_POLARIS10:
L
Leo Liu 已提交
2154
	case CHIP_POLARIS11:
2155
	case CHIP_POLARIS12:
L
Leo Liu 已提交
2156
	case CHIP_VEGAM:
2157 2158
	case CHIP_TONGA:
	case CHIP_FIJI:
2159
	case CHIP_VEGA10:
2160
	case CHIP_VEGA12:
2161
	case CHIP_VEGA20:
2162
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
2163
	case CHIP_RAVEN:
2164
#endif
2165
		return amdgpu_dc != 0;
2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180
#endif
	default:
		return false;
	}
}

/**
 * amdgpu_device_has_dc_support - check if dc is supported
 *
 * @adev: amdgpu_device_pointer
 *
 * Returns true for supported, false for not supported
 */
bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
{
X
Xiangliang Yu 已提交
2181 2182 2183
	if (amdgpu_sriov_vf(adev))
		return false;

2184 2185 2186
	return amdgpu_device_asic_has_dc_support(adev->asic_type);
}

A
Alex Deucher 已提交
2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205
/**
 * amdgpu_device_init - initialize the driver
 *
 * @adev: amdgpu_device pointer
 * @pdev: drm dev pointer
 * @pdev: pci dev pointer
 * @flags: driver flags
 *
 * Initializes the driver info and hw (all asics).
 * Returns 0 for success or an error on failure.
 * Called at driver startup.
 */
int amdgpu_device_init(struct amdgpu_device *adev,
		       struct drm_device *ddev,
		       struct pci_dev *pdev,
		       uint32_t flags)
{
	int r, i;
	bool runtime = false;
2206
	u32 max_MBps;
A
Alex Deucher 已提交
2207 2208 2209 2210 2211 2212

	adev->shutdown = false;
	adev->dev = &pdev->dev;
	adev->ddev = ddev;
	adev->pdev = pdev;
	adev->flags = flags;
2213
	adev->asic_type = flags & AMD_ASIC_MASK;
A
Alex Deucher 已提交
2214
	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2215 2216
	if (amdgpu_emu_mode == 1)
		adev->usec_timeout *= 2;
2217
	adev->gmc.gart_size = 512 * 1024 * 1024;
A
Alex Deucher 已提交
2218 2219 2220 2221 2222
	adev->accel_working = false;
	adev->num_rings = 0;
	adev->mman.buffer_funcs = NULL;
	adev->mman.buffer_funcs_ring = NULL;
	adev->vm_manager.vm_pte_funcs = NULL;
2223
	adev->vm_manager.vm_pte_num_rings = 0;
2224
	adev->gmc.gmc_funcs = NULL;
2225
	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2226
	bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
A
Alex Deucher 已提交
2227 2228 2229 2230 2231

	adev->smc_rreg = &amdgpu_invalid_rreg;
	adev->smc_wreg = &amdgpu_invalid_wreg;
	adev->pcie_rreg = &amdgpu_invalid_rreg;
	adev->pcie_wreg = &amdgpu_invalid_wreg;
2232 2233
	adev->pciep_rreg = &amdgpu_invalid_rreg;
	adev->pciep_wreg = &amdgpu_invalid_wreg;
A
Alex Deucher 已提交
2234 2235 2236 2237
	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
	adev->didt_rreg = &amdgpu_invalid_rreg;
	adev->didt_wreg = &amdgpu_invalid_wreg;
2238 2239
	adev->gc_cac_rreg = &amdgpu_invalid_rreg;
	adev->gc_cac_wreg = &amdgpu_invalid_wreg;
A
Alex Deucher 已提交
2240 2241 2242
	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;

2243 2244 2245
	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
		 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
		 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
A
Alex Deucher 已提交
2246 2247 2248 2249

	/* mutex initialization are all done here so we
	 * can recall function without having locking issues */
	atomic_set(&adev->irq.ih.lock, 0);
2250
	mutex_init(&adev->firmware.mutex);
A
Alex Deucher 已提交
2251 2252 2253
	mutex_init(&adev->pm.mutex);
	mutex_init(&adev->gfx.gpu_clock_mutex);
	mutex_init(&adev->srbm_mutex);
2254
	mutex_init(&adev->gfx.pipe_reserve_mutex);
A
Alex Deucher 已提交
2255 2256
	mutex_init(&adev->grbm_idx_mutex);
	mutex_init(&adev->mn_lock);
A
Alex Deucher 已提交
2257
	mutex_init(&adev->virt.vf_errors.lock);
A
Alex Deucher 已提交
2258
	hash_init(adev->mn_hash);
2259
	mutex_init(&adev->lock_reset);
A
Alex Deucher 已提交
2260

2261
	amdgpu_device_check_arguments(adev);
A
Alex Deucher 已提交
2262 2263 2264 2265 2266 2267

	spin_lock_init(&adev->mmio_idx_lock);
	spin_lock_init(&adev->smc_idx_lock);
	spin_lock_init(&adev->pcie_idx_lock);
	spin_lock_init(&adev->uvd_ctx_idx_lock);
	spin_lock_init(&adev->didt_idx_lock);
2268
	spin_lock_init(&adev->gc_cac_idx_lock);
2269
	spin_lock_init(&adev->se_cac_idx_lock);
A
Alex Deucher 已提交
2270
	spin_lock_init(&adev->audio_endpt_idx_lock);
2271
	spin_lock_init(&adev->mm_stats.lock);
A
Alex Deucher 已提交
2272

2273 2274 2275
	INIT_LIST_HEAD(&adev->shadow_list);
	mutex_init(&adev->shadow_list_lock);

2276 2277 2278
	INIT_LIST_HEAD(&adev->ring_lru_list);
	spin_lock_init(&adev->ring_lru_list_lock);

2279 2280
	INIT_DELAYED_WORK(&adev->late_init_work,
			  amdgpu_device_ip_late_init_func_handler);
2281

2282 2283
	/* Registers mapping */
	/* TODO: block userspace mapping of io register */
2284 2285 2286 2287 2288 2289 2290
	if (adev->asic_type >= CHIP_BONAIRE) {
		adev->rmmio_base = pci_resource_start(adev->pdev, 5);
		adev->rmmio_size = pci_resource_len(adev->pdev, 5);
	} else {
		adev->rmmio_base = pci_resource_start(adev->pdev, 2);
		adev->rmmio_size = pci_resource_len(adev->pdev, 2);
	}
A
Alex Deucher 已提交
2291 2292 2293 2294 2295 2296 2297 2298

	adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
	if (adev->rmmio == NULL) {
		return -ENOMEM;
	}
	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
	DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);

2299
	/* doorbell bar mapping */
2300
	amdgpu_device_doorbell_init(adev);
A
Alex Deucher 已提交
2301 2302 2303 2304 2305 2306 2307 2308 2309 2310

	/* io port mapping */
	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
		if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
			adev->rio_mem_size = pci_resource_len(adev->pdev, i);
			adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
			break;
		}
	}
	if (adev->rio_mem == NULL)
2311
		DRM_INFO("PCI I/O BAR is not found.\n");
A
Alex Deucher 已提交
2312

2313 2314
	amdgpu_device_get_pcie_info(adev);

A
Alex Deucher 已提交
2315
	/* early init functions */
2316
	r = amdgpu_device_ip_early_init(adev);
A
Alex Deucher 已提交
2317 2318 2319 2320 2321 2322
	if (r)
		return r;

	/* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
	/* this will fail for cards that aren't VGA class devices, just
	 * ignore it */
2323
	vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
A
Alex Deucher 已提交
2324

2325
	if (amdgpu_device_is_px(ddev))
A
Alex Deucher 已提交
2326
		runtime = true;
2327 2328 2329
	if (!pci_is_thunderbolt_attached(adev->pdev))
		vga_switcheroo_register_client(adev->pdev,
					       &amdgpu_switcheroo_ops, runtime);
A
Alex Deucher 已提交
2330 2331 2332
	if (runtime)
		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);

2333 2334 2335
	if (amdgpu_emu_mode == 1) {
		/* post the asic on emulation mode */
		emu_soc_asic_init(adev);
2336
		goto fence_driver_init;
2337
	}
2338

A
Alex Deucher 已提交
2339
	/* Read BIOS */
2340 2341 2342 2343
	if (!amdgpu_get_bios(adev)) {
		r = -EINVAL;
		goto failed;
	}
2344

A
Alex Deucher 已提交
2345
	r = amdgpu_atombios_init(adev);
2346 2347
	if (r) {
		dev_err(adev->dev, "amdgpu_atombios_init failed\n");
A
Alex Deucher 已提交
2348
		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2349
		goto failed;
2350
	}
A
Alex Deucher 已提交
2351

2352 2353
	/* detect if we are with an SRIOV vbios */
	amdgpu_device_detect_sriov_bios(adev);
2354

A
Alex Deucher 已提交
2355
	/* Post card if necessary */
A
Alex Deucher 已提交
2356
	if (amdgpu_device_need_post(adev)) {
A
Alex Deucher 已提交
2357
		if (!adev->bios) {
2358
			dev_err(adev->dev, "no vBIOS found\n");
2359 2360
			r = -EINVAL;
			goto failed;
A
Alex Deucher 已提交
2361
		}
2362
		DRM_INFO("GPU posting now...\n");
2363 2364 2365 2366 2367
		r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
		if (r) {
			dev_err(adev->dev, "gpu post error!\n");
			goto failed;
		}
A
Alex Deucher 已提交
2368 2369
	}

2370 2371 2372 2373 2374
	if (adev->is_atom_fw) {
		/* Initialize clocks */
		r = amdgpu_atomfirmware_get_clock_info(adev);
		if (r) {
			dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
A
Alex Deucher 已提交
2375
			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2376 2377 2378
			goto failed;
		}
	} else {
2379 2380 2381 2382
		/* Initialize clocks */
		r = amdgpu_atombios_get_clock_info(adev);
		if (r) {
			dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
A
Alex Deucher 已提交
2383
			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2384
			goto failed;
2385 2386
		}
		/* init i2c buses */
2387 2388
		if (!amdgpu_device_has_dc_support(adev))
			amdgpu_atombios_i2c_init(adev);
2389
	}
A
Alex Deucher 已提交
2390

2391
fence_driver_init:
A
Alex Deucher 已提交
2392 2393
	/* Fence driver */
	r = amdgpu_fence_driver_init(adev);
2394 2395
	if (r) {
		dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
A
Alex Deucher 已提交
2396
		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2397
		goto failed;
2398
	}
A
Alex Deucher 已提交
2399 2400 2401 2402

	/* init the mode config */
	drm_mode_config_init(adev->ddev);

2403
	r = amdgpu_device_ip_init(adev);
A
Alex Deucher 已提交
2404
	if (r) {
2405 2406 2407 2408 2409 2410
		/* failed in exclusive mode due to timeout */
		if (amdgpu_sriov_vf(adev) &&
		    !amdgpu_sriov_runtime(adev) &&
		    amdgpu_virt_mmio_blocked(adev) &&
		    !amdgpu_virt_wait_reset(adev)) {
			dev_err(adev->dev, "VF exclusive mode timeout\n");
2411 2412 2413
			/* Don't send request since VF is inactive. */
			adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
			adev->virt.ops = NULL;
2414 2415 2416
			r = -EAGAIN;
			goto failed;
		}
2417
		dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
A
Alex Deucher 已提交
2418
		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2419
		goto failed;
A
Alex Deucher 已提交
2420 2421 2422 2423
	}

	adev->accel_working = true;

2424 2425
	amdgpu_vm_check_compute_bug(adev);

2426 2427 2428 2429 2430 2431 2432 2433
	/* Initialize the buffer migration limit. */
	if (amdgpu_moverate >= 0)
		max_MBps = amdgpu_moverate;
	else
		max_MBps = 8; /* Allow 8 MB/s. */
	/* Get a log2 for easy divisions. */
	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));

A
Alex Deucher 已提交
2434 2435 2436
	r = amdgpu_ib_pool_init(adev);
	if (r) {
		dev_err(adev->dev, "IB initialization failed (%d).\n", r);
A
Alex Deucher 已提交
2437
		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2438
		goto failed;
A
Alex Deucher 已提交
2439 2440
	}

2441 2442 2443
	if (amdgpu_sriov_vf(adev))
		amdgpu_virt_init_data_exchange(adev);

2444 2445
	amdgpu_fbdev_init(adev);

2446 2447 2448 2449
	r = amdgpu_pm_sysfs_init(adev);
	if (r)
		DRM_ERROR("registering pm debugfs failed (%d).\n", r);

2450
	r = amdgpu_debugfs_gem_init(adev);
M
Monk Liu 已提交
2451
	if (r)
A
Alex Deucher 已提交
2452 2453 2454
		DRM_ERROR("registering gem debugfs failed (%d).\n", r);

	r = amdgpu_debugfs_regs_init(adev);
M
Monk Liu 已提交
2455
	if (r)
A
Alex Deucher 已提交
2456 2457
		DRM_ERROR("registering register debugfs failed (%d).\n", r);

2458
	r = amdgpu_debugfs_firmware_init(adev);
M
Monk Liu 已提交
2459
	if (r)
2460 2461
		DRM_ERROR("registering firmware debugfs failed (%d).\n", r);

2462
	r = amdgpu_debugfs_init(adev);
2463
	if (r)
2464
		DRM_ERROR("Creating debugfs files failed (%d).\n", r);
2465

A
Alex Deucher 已提交
2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481
	if ((amdgpu_testing & 1)) {
		if (adev->accel_working)
			amdgpu_test_moves(adev);
		else
			DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
	}
	if (amdgpu_benchmarking) {
		if (adev->accel_working)
			amdgpu_benchmark(adev, amdgpu_benchmarking);
		else
			DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
	}

	/* enable clockgating, etc. after ib tests, etc. since some blocks require
	 * explicit gating rather than handling it automatically.
	 */
2482
	r = amdgpu_device_ip_late_init(adev);
2483
	if (r) {
2484
		dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
A
Alex Deucher 已提交
2485
		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2486
		goto failed;
2487
	}
A
Alex Deucher 已提交
2488 2489

	return 0;
2490 2491

failed:
2492
	amdgpu_vf_error_trans_all(adev);
2493 2494
	if (runtime)
		vga_switcheroo_fini_domain_pm_ops(adev->dev);
2495

2496
	return r;
A
Alex Deucher 已提交
2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512
}

/**
 * amdgpu_device_fini - tear down the driver
 *
 * @adev: amdgpu_device pointer
 *
 * Tear down the driver info (all asics).
 * Called at driver shutdown.
 */
void amdgpu_device_fini(struct amdgpu_device *adev)
{
	int r;

	DRM_INFO("amdgpu: finishing device.\n");
	adev->shutdown = true;
2513 2514
	/* disable all interrupts */
	amdgpu_irq_disable_all(adev);
2515 2516 2517 2518 2519 2520
	if (adev->mode_info.mode_config_initialized){
		if (!amdgpu_device_has_dc_support(adev))
			drm_crtc_force_disable_all(adev->ddev);
		else
			drm_atomic_helper_shutdown(adev->ddev);
	}
A
Alex Deucher 已提交
2521 2522
	amdgpu_ib_pool_fini(adev);
	amdgpu_fence_driver_fini(adev);
2523
	amdgpu_pm_sysfs_fini(adev);
A
Alex Deucher 已提交
2524
	amdgpu_fbdev_fini(adev);
2525
	r = amdgpu_device_ip_fini(adev);
2526 2527 2528 2529
	if (adev->firmware.gpu_info_fw) {
		release_firmware(adev->firmware.gpu_info_fw);
		adev->firmware.gpu_info_fw = NULL;
	}
A
Alex Deucher 已提交
2530
	adev->accel_working = false;
2531
	cancel_delayed_work_sync(&adev->late_init_work);
A
Alex Deucher 已提交
2532
	/* free i2c buses */
2533 2534
	if (!amdgpu_device_has_dc_support(adev))
		amdgpu_i2c_fini(adev);
2535 2536 2537 2538

	if (amdgpu_emu_mode != 1)
		amdgpu_atombios_fini(adev);

A
Alex Deucher 已提交
2539 2540
	kfree(adev->bios);
	adev->bios = NULL;
2541 2542
	if (!pci_is_thunderbolt_attached(adev->pdev))
		vga_switcheroo_unregister_client(adev->pdev);
2543 2544
	if (adev->flags & AMD_IS_PX)
		vga_switcheroo_fini_domain_pm_ops(adev->dev);
A
Alex Deucher 已提交
2545 2546 2547 2548 2549 2550
	vga_client_register(adev->pdev, NULL, NULL, NULL);
	if (adev->rio_mem)
		pci_iounmap(adev->pdev, adev->rio_mem);
	adev->rio_mem = NULL;
	iounmap(adev->rmmio);
	adev->rmmio = NULL;
2551
	amdgpu_device_doorbell_fini(adev);
A
Alex Deucher 已提交
2552 2553 2554 2555 2556 2557 2558 2559
	amdgpu_debugfs_regs_cleanup(adev);
}


/*
 * Suspend & resume.
 */
/**
2560
 * amdgpu_device_suspend - initiate device suspend
A
Alex Deucher 已提交
2561 2562 2563 2564 2565 2566 2567 2568
 *
 * @pdev: drm dev pointer
 * @state: suspend state
 *
 * Puts the hw in the suspend state (all asics).
 * Returns 0 for success or an error on failure.
 * Called at driver suspend.
 */
2569
int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
A
Alex Deucher 已提交
2570 2571 2572 2573
{
	struct amdgpu_device *adev;
	struct drm_crtc *crtc;
	struct drm_connector *connector;
2574
	int r;
A
Alex Deucher 已提交
2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586

	if (dev == NULL || dev->dev_private == NULL) {
		return -ENODEV;
	}

	adev = dev->dev_private;

	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
		return 0;

	drm_kms_helper_poll_disable(dev);

2587 2588 2589 2590 2591 2592 2593
	if (!amdgpu_device_has_dc_support(adev)) {
		/* turn off display hw */
		drm_modeset_lock_all(dev);
		list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
		}
		drm_modeset_unlock_all(dev);
A
Alex Deucher 已提交
2594 2595
	}

2596 2597
	amdgpu_amdkfd_suspend(adev);

2598
	/* unpin the front buffers and cursors */
A
Alex Deucher 已提交
2599
	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2600
		struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2601
		struct drm_framebuffer *fb = crtc->primary->fb;
A
Alex Deucher 已提交
2602 2603
		struct amdgpu_bo *robj;

2604 2605
		if (amdgpu_crtc->cursor_bo) {
			struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2606
			r = amdgpu_bo_reserve(aobj, true);
2607 2608 2609 2610 2611 2612
			if (r == 0) {
				amdgpu_bo_unpin(aobj);
				amdgpu_bo_unreserve(aobj);
			}
		}

2613
		if (fb == NULL || fb->obj[0] == NULL) {
A
Alex Deucher 已提交
2614 2615
			continue;
		}
2616
		robj = gem_to_amdgpu_bo(fb->obj[0]);
A
Alex Deucher 已提交
2617 2618
		/* don't unpin kernel fb objects */
		if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2619
			r = amdgpu_bo_reserve(robj, true);
A
Alex Deucher 已提交
2620 2621 2622 2623 2624 2625 2626 2627 2628
			if (r == 0) {
				amdgpu_bo_unpin(robj);
				amdgpu_bo_unreserve(robj);
			}
		}
	}
	/* evict vram memory */
	amdgpu_bo_evict_vram(adev);

2629
	amdgpu_fence_driver_suspend(adev);
A
Alex Deucher 已提交
2630

2631
	r = amdgpu_device_ip_suspend(adev);
A
Alex Deucher 已提交
2632

2633 2634 2635 2636
	/* evict remaining vram memory
	 * This second call to evict vram is to evict the gart page table
	 * using the CPU.
	 */
A
Alex Deucher 已提交
2637 2638 2639 2640 2641 2642 2643
	amdgpu_bo_evict_vram(adev);

	pci_save_state(dev->pdev);
	if (suspend) {
		/* Shut down the device */
		pci_disable_device(dev->pdev);
		pci_set_power_state(dev->pdev, PCI_D3hot);
J
jimqu 已提交
2644 2645 2646 2647
	} else {
		r = amdgpu_asic_reset(adev);
		if (r)
			DRM_ERROR("amdgpu asic reset failed\n");
A
Alex Deucher 已提交
2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658
	}

	if (fbcon) {
		console_lock();
		amdgpu_fbdev_set_suspend(adev, 1);
		console_unlock();
	}
	return 0;
}

/**
2659
 * amdgpu_device_resume - initiate device resume
A
Alex Deucher 已提交
2660 2661 2662 2663 2664 2665 2666
 *
 * @pdev: drm dev pointer
 *
 * Bring the hw back to operating state (all asics).
 * Returns 0 for success or an error on failure.
 * Called at driver resume.
 */
2667
int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
A
Alex Deucher 已提交
2668 2669 2670
{
	struct drm_connector *connector;
	struct amdgpu_device *adev = dev->dev_private;
2671
	struct drm_crtc *crtc;
2672
	int r = 0;
A
Alex Deucher 已提交
2673 2674 2675 2676

	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
		return 0;

J
jimqu 已提交
2677
	if (fbcon)
A
Alex Deucher 已提交
2678
		console_lock();
J
jimqu 已提交
2679

A
Alex Deucher 已提交
2680 2681 2682
	if (resume) {
		pci_set_power_state(dev->pdev, PCI_D0);
		pci_restore_state(dev->pdev);
J
jimqu 已提交
2683
		r = pci_enable_device(dev->pdev);
2684 2685
		if (r)
			goto unlock;
A
Alex Deucher 已提交
2686 2687 2688
	}

	/* post card */
A
Alex Deucher 已提交
2689
	if (amdgpu_device_need_post(adev)) {
J
jimqu 已提交
2690 2691 2692 2693
		r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
		if (r)
			DRM_ERROR("amdgpu asic init failed\n");
	}
A
Alex Deucher 已提交
2694

2695
	r = amdgpu_device_ip_resume(adev);
2696
	if (r) {
2697
		DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
2698
		goto unlock;
2699
	}
2700 2701
	amdgpu_fence_driver_resume(adev);

A
Alex Deucher 已提交
2702

2703
	r = amdgpu_device_ip_late_init(adev);
2704 2705
	if (r)
		goto unlock;
A
Alex Deucher 已提交
2706

2707 2708 2709 2710 2711 2712
	/* pin cursors */
	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
		struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);

		if (amdgpu_crtc->cursor_bo) {
			struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2713
			r = amdgpu_bo_reserve(aobj, true);
2714 2715 2716 2717 2718 2719 2720 2721 2722 2723
			if (r == 0) {
				r = amdgpu_bo_pin(aobj,
						  AMDGPU_GEM_DOMAIN_VRAM,
						  &amdgpu_crtc->cursor_addr);
				if (r != 0)
					DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
				amdgpu_bo_unreserve(aobj);
			}
		}
	}
2724 2725 2726
	r = amdgpu_amdkfd_resume(adev);
	if (r)
		return r;
2727

A
Alex Deucher 已提交
2728 2729
	/* blat the mode back in */
	if (fbcon) {
2730 2731 2732 2733 2734 2735 2736 2737 2738 2739
		if (!amdgpu_device_has_dc_support(adev)) {
			/* pre DCE11 */
			drm_helper_resume_force_mode(dev);

			/* turn on display hw */
			drm_modeset_lock_all(dev);
			list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
				drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
			}
			drm_modeset_unlock_all(dev);
A
Alex Deucher 已提交
2740 2741 2742 2743
		}
	}

	drm_kms_helper_poll_enable(dev);
2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756

	/*
	 * Most of the connector probing functions try to acquire runtime pm
	 * refs to ensure that the GPU is powered on when connector polling is
	 * performed. Since we're calling this from a runtime PM callback,
	 * trying to acquire rpm refs will cause us to deadlock.
	 *
	 * Since we're guaranteed to be holding the rpm lock, it's safe to
	 * temporarily disable the rpm helpers so this doesn't deadlock us.
	 */
#ifdef CONFIG_PM
	dev->dev->power.disable_depth++;
#endif
2757 2758 2759 2760
	if (!amdgpu_device_has_dc_support(adev))
		drm_helper_hpd_irq_event(dev);
	else
		drm_kms_helper_hotplug_event(dev);
2761 2762 2763
#ifdef CONFIG_PM
	dev->dev->power.disable_depth--;
#endif
A
Alex Deucher 已提交
2764

2765
	if (fbcon)
A
Alex Deucher 已提交
2766
		amdgpu_fbdev_set_suspend(adev, 0);
2767 2768 2769

unlock:
	if (fbcon)
A
Alex Deucher 已提交
2770 2771
		console_unlock();

2772
	return r;
A
Alex Deucher 已提交
2773 2774
}

2775 2776 2777 2778 2779 2780 2781 2782 2783 2784
/**
 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
 *
 * @adev: amdgpu_device pointer
 *
 * The list of all the hardware IPs that make up the asic is walked and
 * the check_soft_reset callbacks are run.  check_soft_reset determines
 * if the asic is still hung or not.
 * Returns true if any of the IPs are still in a hung state, false if not.
 */
2785
static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
2786 2787 2788 2789
{
	int i;
	bool asic_hang = false;

2790 2791 2792
	if (amdgpu_sriov_vf(adev))
		return true;

2793 2794 2795
	if (amdgpu_asic_need_full_reset(adev))
		return true;

2796
	for (i = 0; i < adev->num_ip_blocks; i++) {
2797
		if (!adev->ip_blocks[i].status.valid)
2798
			continue;
2799 2800 2801 2802 2803
		if (adev->ip_blocks[i].version->funcs->check_soft_reset)
			adev->ip_blocks[i].status.hang =
				adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
		if (adev->ip_blocks[i].status.hang) {
			DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2804 2805 2806 2807 2808 2809
			asic_hang = true;
		}
	}
	return asic_hang;
}

2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820
/**
 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
 *
 * @adev: amdgpu_device pointer
 *
 * The list of all the hardware IPs that make up the asic is walked and the
 * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
 * handles any IP specific hardware or software state changes that are
 * necessary for a soft reset to succeed.
 * Returns 0 on success, negative error code on failure.
 */
2821
static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
2822 2823 2824 2825
{
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
2826
		if (!adev->ip_blocks[i].status.valid)
2827
			continue;
2828 2829 2830
		if (adev->ip_blocks[i].status.hang &&
		    adev->ip_blocks[i].version->funcs->pre_soft_reset) {
			r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
2831 2832 2833 2834 2835 2836 2837 2838
			if (r)
				return r;
		}
	}

	return 0;
}

2839 2840 2841 2842 2843 2844 2845 2846 2847
/**
 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
 *
 * @adev: amdgpu_device pointer
 *
 * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
 * reset is necessary to recover.
 * Returns true if a full asic reset is required, false if not.
 */
2848
static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
2849
{
2850 2851
	int i;

2852 2853 2854
	if (amdgpu_asic_need_full_reset(adev))
		return true;

2855
	for (i = 0; i < adev->num_ip_blocks; i++) {
2856
		if (!adev->ip_blocks[i].status.valid)
2857
			continue;
2858 2859 2860
		if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2861 2862
		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2863
			if (adev->ip_blocks[i].status.hang) {
2864 2865 2866 2867
				DRM_INFO("Some block need full reset!\n");
				return true;
			}
		}
2868 2869 2870 2871
	}
	return false;
}

2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882
/**
 * amdgpu_device_ip_soft_reset - do a soft reset
 *
 * @adev: amdgpu_device pointer
 *
 * The list of all the hardware IPs that make up the asic is walked and the
 * soft_reset callbacks are run if the block is hung.  soft_reset handles any
 * IP specific hardware or software state changes that are necessary to soft
 * reset the IP.
 * Returns 0 on success, negative error code on failure.
 */
2883
static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
2884 2885 2886 2887
{
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
2888
		if (!adev->ip_blocks[i].status.valid)
2889
			continue;
2890 2891 2892
		if (adev->ip_blocks[i].status.hang &&
		    adev->ip_blocks[i].version->funcs->soft_reset) {
			r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
2893 2894 2895 2896 2897 2898 2899 2900
			if (r)
				return r;
		}
	}

	return 0;
}

2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911
/**
 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
 *
 * @adev: amdgpu_device pointer
 *
 * The list of all the hardware IPs that make up the asic is walked and the
 * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
 * handles any IP specific hardware or software state changes that are
 * necessary after the IP has been soft reset.
 * Returns 0 on success, negative error code on failure.
 */
2912
static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
2913 2914 2915 2916
{
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
2917
		if (!adev->ip_blocks[i].status.valid)
2918
			continue;
2919 2920 2921
		if (adev->ip_blocks[i].status.hang &&
		    adev->ip_blocks[i].version->funcs->post_soft_reset)
			r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
2922 2923 2924 2925 2926 2927 2928
		if (r)
			return r;
	}

	return 0;
}

2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941
/**
 * amdgpu_device_recover_vram_from_shadow - restore shadowed VRAM buffers
 *
 * @adev: amdgpu_device pointer
 * @ring: amdgpu_ring for the engine handling the buffer operations
 * @bo: amdgpu_bo buffer whose shadow is being restored
 * @fence: dma_fence associated with the operation
 *
 * Restores the VRAM buffer contents from the shadow in GTT.  Used to
 * restore things like GPUVM page tables after a GPU reset where
 * the contents of VRAM might be lost.
 * Returns 0 on success, negative error code on failure.
 */
2942 2943 2944 2945
static int amdgpu_device_recover_vram_from_shadow(struct amdgpu_device *adev,
						  struct amdgpu_ring *ring,
						  struct amdgpu_bo *bo,
						  struct dma_fence **fence)
2946 2947 2948 2949
{
	uint32_t domain;
	int r;

R
Roger.He 已提交
2950 2951 2952
	if (!bo->shadow)
		return 0;

2953
	r = amdgpu_bo_reserve(bo, true);
R
Roger.He 已提交
2954 2955 2956 2957 2958
	if (r)
		return r;
	domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
	/* if bo has been evicted, then no need to recover */
	if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2959 2960 2961 2962 2963 2964
		r = amdgpu_bo_validate(bo->shadow);
		if (r) {
			DRM_ERROR("bo validate failed!\n");
			goto err;
		}

R
Roger.He 已提交
2965
		r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2966
						 NULL, fence, true);
R
Roger.He 已提交
2967 2968 2969 2970 2971
		if (r) {
			DRM_ERROR("recover page table failed!\n");
			goto err;
		}
	}
2972
err:
R
Roger.He 已提交
2973 2974
	amdgpu_bo_unreserve(bo);
	return r;
2975 2976
}

2977 2978 2979 2980 2981 2982 2983 2984 2985 2986
/**
 * amdgpu_device_handle_vram_lost - Handle the loss of VRAM contents
 *
 * @adev: amdgpu_device pointer
 *
 * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
 * restore things like GPUVM page tables after a GPU reset where
 * the contents of VRAM might be lost.
 * Returns 0 on success, 1 on failure.
 */
2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039
static int amdgpu_device_handle_vram_lost(struct amdgpu_device *adev)
{
	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
	struct amdgpu_bo *bo, *tmp;
	struct dma_fence *fence = NULL, *next = NULL;
	long r = 1;
	int i = 0;
	long tmo;

	if (amdgpu_sriov_runtime(adev))
		tmo = msecs_to_jiffies(amdgpu_lockup_timeout);
	else
		tmo = msecs_to_jiffies(100);

	DRM_INFO("recover vram bo from shadow start\n");
	mutex_lock(&adev->shadow_list_lock);
	list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
		next = NULL;
		amdgpu_device_recover_vram_from_shadow(adev, ring, bo, &next);
		if (fence) {
			r = dma_fence_wait_timeout(fence, false, tmo);
			if (r == 0)
				pr_err("wait fence %p[%d] timeout\n", fence, i);
			else if (r < 0)
				pr_err("wait fence %p[%d] interrupted\n", fence, i);
			if (r < 1) {
				dma_fence_put(fence);
				fence = next;
				break;
			}
			i++;
		}

		dma_fence_put(fence);
		fence = next;
	}
	mutex_unlock(&adev->shadow_list_lock);

	if (fence) {
		r = dma_fence_wait_timeout(fence, false, tmo);
		if (r == 0)
			pr_err("wait fence %p[%d] timeout\n", fence, i);
		else if (r < 0)
			pr_err("wait fence %p[%d] interrupted\n", fence, i);

	}
	dma_fence_put(fence);

	if (r > 0)
		DRM_INFO("recover vram bo from shadow done\n");
	else
		DRM_ERROR("recover vram bo from shadow failed\n");

3040
	return (r > 0) ? 0 : 1;
3041 3042
}

3043
/**
3044
 * amdgpu_device_reset - reset ASIC/GPU for bare-metal or passthrough
3045 3046 3047
 *
 * @adev: amdgpu device pointer
 *
3048 3049
 * attempt to do soft-reset or full-reset and reinitialize Asic
 * return 0 means successed otherwise failed
3050
 */
3051
static int amdgpu_device_reset(struct amdgpu_device *adev)
3052
{
3053 3054
	bool need_full_reset, vram_lost = 0;
	int r;
3055

3056
	need_full_reset = amdgpu_device_ip_need_full_reset(adev);
3057

3058
	if (!need_full_reset) {
3059 3060 3061 3062
		amdgpu_device_ip_pre_soft_reset(adev);
		r = amdgpu_device_ip_soft_reset(adev);
		amdgpu_device_ip_post_soft_reset(adev);
		if (r || amdgpu_device_ip_check_soft_reset(adev)) {
3063 3064 3065 3066
			DRM_INFO("soft reset failed, will fallback to full reset!\n");
			need_full_reset = true;
		}
	}
3067

3068
	if (need_full_reset) {
3069
		r = amdgpu_device_ip_suspend(adev);
3070

3071 3072 3073 3074
retry:
		r = amdgpu_asic_reset(adev);
		/* post card */
		amdgpu_atom_asic_init(adev->mode_info.atom_context);
3075

3076 3077
		if (!r) {
			dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
3078
			r = amdgpu_device_ip_resume_phase1(adev);
3079 3080
			if (r)
				goto out;
3081

3082
			vram_lost = amdgpu_device_check_vram_lost(adev);
3083 3084 3085 3086 3087
			if (vram_lost) {
				DRM_ERROR("VRAM is lost!\n");
				atomic_inc(&adev->vram_lost_counter);
			}

3088 3089
			r = amdgpu_gtt_mgr_recover(
				&adev->mman.bdev.man[TTM_PL_TT]);
3090 3091 3092
			if (r)
				goto out;

3093
			r = amdgpu_device_ip_resume_phase2(adev);
3094 3095 3096 3097
			if (r)
				goto out;

			if (vram_lost)
3098
				amdgpu_device_fill_reset_magic(adev);
3099
		}
3100
	}
3101

3102 3103 3104 3105 3106 3107
out:
	if (!r) {
		amdgpu_irq_gpu_reset_resume_helper(adev);
		r = amdgpu_ib_ring_tests(adev);
		if (r) {
			dev_err(adev->dev, "ib ring test failed (%d).\n", r);
3108
			r = amdgpu_device_ip_suspend(adev);
3109 3110 3111 3112
			need_full_reset = true;
			goto retry;
		}
	}
3113

3114 3115
	if (!r && ((need_full_reset && !(adev->flags & AMD_IS_APU)) || vram_lost))
		r = amdgpu_device_handle_vram_lost(adev);
3116

3117 3118
	return r;
}
3119

3120
/**
3121
 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
3122 3123 3124 3125 3126
 *
 * @adev: amdgpu device pointer
 *
 * do VF FLR and reinitialize Asic
 * return 0 means successed otherwise failed
3127 3128 3129
 */
static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
				     bool from_hypervisor)
3130 3131 3132 3133 3134 3135 3136 3137 3138
{
	int r;

	if (from_hypervisor)
		r = amdgpu_virt_request_full_gpu(adev, true);
	else
		r = amdgpu_virt_reset_gpu(adev);
	if (r)
		return r;
3139 3140

	/* Resume IP prior to SMC */
3141
	r = amdgpu_device_ip_reinit_early_sriov(adev);
3142 3143
	if (r)
		goto error;
3144 3145

	/* we need recover gart prior to run SMC/CP/SDMA resume */
3146
	amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
3147 3148

	/* now we are okay to resume SMC/CP/SDMA */
3149
	r = amdgpu_device_ip_reinit_late_sriov(adev);
3150 3151
	if (r)
		goto error;
3152 3153

	amdgpu_irq_gpu_reset_resume_helper(adev);
3154
	r = amdgpu_ib_ring_tests(adev);
3155

3156 3157
error:
	amdgpu_virt_release_full_gpu(adev, true);
3158 3159 3160
	if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
		atomic_inc(&adev->vram_lost_counter);
		r = amdgpu_device_handle_vram_lost(adev);
3161 3162 3163 3164 3165
	}

	return r;
}

A
Alex Deucher 已提交
3166
/**
3167
 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
A
Alex Deucher 已提交
3168 3169
 *
 * @adev: amdgpu device pointer
3170
 * @job: which job trigger hang
3171
 * @force forces reset regardless of amdgpu_gpu_recovery
A
Alex Deucher 已提交
3172
 *
3173
 * Attempt to reset the GPU if it has hung (all asics).
A
Alex Deucher 已提交
3174 3175
 * Returns 0 for success or an error on failure.
 */
3176 3177
int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
			      struct amdgpu_job *job, bool force)
A
Alex Deucher 已提交
3178
{
3179
	struct drm_atomic_state *state = NULL;
3180
	int i, r, resched;
3181

3182
	if (!force && !amdgpu_device_ip_check_soft_reset(adev)) {
3183 3184 3185
		DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
		return 0;
	}
A
Alex Deucher 已提交
3186

3187 3188 3189 3190 3191 3192
	if (!force && (amdgpu_gpu_recovery == 0 ||
			(amdgpu_gpu_recovery == -1  && !amdgpu_sriov_vf(adev)))) {
		DRM_INFO("GPU recovery disabled.\n");
		return 0;
	}

3193 3194
	dev_info(adev->dev, "GPU reset begin!\n");

3195
	mutex_lock(&adev->lock_reset);
3196
	atomic_inc(&adev->gpu_reset_counter);
3197
	adev->in_gpu_reset = 1;
A
Alex Deucher 已提交
3198

3199 3200
	/* block TTM */
	resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3201

3202 3203 3204
	/* store modesetting */
	if (amdgpu_device_has_dc_support(adev))
		state = drm_atomic_helper_suspend(adev->ddev);
3205

3206
	/* block all schedulers and reset given job's ring */
3207 3208 3209
	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
		struct amdgpu_ring *ring = adev->rings[i];

C
Chunming Zhou 已提交
3210
		if (!ring || !ring->sched.thread)
3211
			continue;
3212

3213 3214
		kthread_park(ring->sched.thread);

3215 3216 3217
		if (job && job->ring->idx != i)
			continue;

3218
		drm_sched_hw_job_reset(&ring->sched, &job->base);
3219

M
Monk Liu 已提交
3220 3221
		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
		amdgpu_fence_driver_force_completion(ring);
3222
	}
A
Alex Deucher 已提交
3223

3224
	if (amdgpu_sriov_vf(adev))
3225
		r = amdgpu_device_reset_sriov(adev, job ? false : true);
3226
	else
3227
		r = amdgpu_device_reset(adev);
3228

3229 3230
	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
		struct amdgpu_ring *ring = adev->rings[i];
3231

3232 3233
		if (!ring || !ring->sched.thread)
			continue;
3234

3235 3236 3237 3238 3239
		/* only need recovery sched of the given job's ring
		 * or all rings (in the case @job is NULL)
		 * after above amdgpu_reset accomplished
		 */
		if ((!job || job->ring->idx == i) && !r)
3240
			drm_sched_job_recovery(&ring->sched);
3241

3242
		kthread_unpark(ring->sched.thread);
A
Alex Deucher 已提交
3243 3244
	}

3245
	if (amdgpu_device_has_dc_support(adev)) {
3246 3247 3248
		if (drm_atomic_helper_resume(adev->ddev, state))
			dev_info(adev->dev, "drm resume failed:%d\n", r);
	} else {
3249
		drm_helper_resume_force_mode(adev->ddev);
3250
	}
A
Alex Deucher 已提交
3251 3252

	ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
3253

3254
	if (r) {
A
Alex Deucher 已提交
3255
		/* bad news, how to tell it to userspace ? */
3256 3257 3258 3259
		dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
	} else {
		dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
3260
	}
A
Alex Deucher 已提交
3261

3262
	amdgpu_vf_error_trans_all(adev);
3263 3264
	adev->in_gpu_reset = 0;
	mutex_unlock(&adev->lock_reset);
A
Alex Deucher 已提交
3265 3266 3267
	return r;
}

3268 3269 3270 3271 3272 3273 3274 3275 3276
/**
 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
 *
 * @adev: amdgpu_device pointer
 *
 * Fetchs and stores in the driver the PCIE capabilities (gen speed
 * and lanes) of the slot the device is in. Handles APUs and
 * virtualized environments where PCIE config space may not be available.
 */
3277
static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
3278 3279 3280 3281
{
	u32 mask;
	int ret;

3282 3283
	if (amdgpu_pcie_gen_cap)
		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
3284

3285 3286
	if (amdgpu_pcie_lane_cap)
		adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
3287

3288 3289 3290 3291 3292 3293
	/* covers APUs as well */
	if (pci_is_root_bus(adev->pdev->bus)) {
		if (adev->pm.pcie_gen_mask == 0)
			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
		if (adev->pm.pcie_mlw_mask == 0)
			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3294
		return;
3295
	}
3296

3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364
	if (adev->pm.pcie_gen_mask == 0) {
		ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
		if (!ret) {
			adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);

			if (mask & DRM_PCIE_SPEED_25)
				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
			if (mask & DRM_PCIE_SPEED_50)
				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
			if (mask & DRM_PCIE_SPEED_80)
				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
		} else {
			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
		}
	}
	if (adev->pm.pcie_mlw_mask == 0) {
		ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
		if (!ret) {
			switch (mask) {
			case 32:
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
			case 16:
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
			case 12:
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
			case 8:
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
			case 4:
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
			case 2:
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
			case 1:
				adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
				break;
			default:
				break;
			}
		} else {
			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3365 3366 3367
		}
	}
}
A
Alex Deucher 已提交
3368