amdgpu_device.c 109.2 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright 2008 Advanced Micro Devices, Inc.
 * Copyright 2008 Red Hat Inc.
 * Copyright 2009 Jerome Glisse.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Dave Airlie
 *          Alex Deucher
 *          Jerome Glisse
 */
28
#include <linux/power_supply.h>
29
#include <linux/kthread.h>
30
#include <linux/module.h>
A
Alex Deucher 已提交
31 32
#include <linux/console.h>
#include <linux/slab.h>
33

34
#include <drm/drm_atomic_helper.h>
35
#include <drm/drm_probe_helper.h>
A
Alex Deucher 已提交
36 37 38 39 40
#include <drm/amdgpu_drm.h>
#include <linux/vgaarb.h>
#include <linux/vga_switcheroo.h>
#include <linux/efi.h>
#include "amdgpu.h"
41
#include "amdgpu_trace.h"
A
Alex Deucher 已提交
42 43 44
#include "amdgpu_i2c.h"
#include "atom.h"
#include "amdgpu_atombios.h"
45
#include "amdgpu_atomfirmware.h"
46
#include "amd_pcie.h"
K
Ken Wang 已提交
47 48 49
#ifdef CONFIG_DRM_AMDGPU_SI
#include "si.h"
#endif
50 51 52
#ifdef CONFIG_DRM_AMDGPU_CIK
#include "cik.h"
#endif
53
#include "vi.h"
54
#include "soc15.h"
55
#include "nv.h"
A
Alex Deucher 已提交
56
#include "bif/bif_4_1_d.h"
57
#include <linux/pci.h>
58
#include <linux/firmware.h>
59
#include "amdgpu_vf_error.h"
A
Alex Deucher 已提交
60

61
#include "amdgpu_amdkfd.h"
62
#include "amdgpu_pm.h"
A
Alex Deucher 已提交
63

64
#include "amdgpu_xgmi.h"
65
#include "amdgpu_ras.h"
J
Jonathan Kim 已提交
66
#include "amdgpu_pmu.h"
67

68
MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
69
MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
70
MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
71
MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
72
MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
73
MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
74
MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin");
75
MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
76
MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
77
MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
78

79 80
#define AMDGPU_RESUME_MS		2000

A
Alex Deucher 已提交
81
static const char *amdgpu_asic_name[] = {
82 83 84 85 86
	"TAHITI",
	"PITCAIRN",
	"VERDE",
	"OLAND",
	"HAINAN",
A
Alex Deucher 已提交
87 88 89 90 91 92 93
	"BONAIRE",
	"KAVERI",
	"KABINI",
	"HAWAII",
	"MULLINS",
	"TOPAZ",
	"TONGA",
94
	"FIJI",
A
Alex Deucher 已提交
95
	"CARRIZO",
S
Samuel Li 已提交
96
	"STONEY",
97 98
	"POLARIS10",
	"POLARIS11",
99
	"POLARIS12",
L
Leo Liu 已提交
100
	"VEGAM",
K
Ken Wang 已提交
101
	"VEGA10",
102
	"VEGA12",
103
	"VEGA20",
104
	"RAVEN",
L
Le Ma 已提交
105
	"ARCTURUS",
106
	"RENOIR",
H
Huang Rui 已提交
107
	"NAVI10",
X
Xiaojie Yuan 已提交
108
	"NAVI14",
X
Xiaojie Yuan 已提交
109
	"NAVI12",
A
Alex Deucher 已提交
110 111 112
	"LAST",
};

113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
/**
 * DOC: pcie_replay_count
 *
 * The amdgpu driver provides a sysfs API for reporting the total number
 * of PCIe replays (NAKs)
 * The file pcie_replay_count is used for this and returns the total
 * number of replays as a sum of the NAKs generated and NAKs received
 */

static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	struct drm_device *ddev = dev_get_drvdata(dev);
	struct amdgpu_device *adev = ddev->dev_private;
	uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);

	return snprintf(buf, PAGE_SIZE, "%llu\n", cnt);
}

static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
		amdgpu_device_get_pcie_replay_count, NULL);

135 136
static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);

137 138 139 140 141 142 143 144
/**
 * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control
 *
 * @dev: drm_device pointer
 *
 * Returns true if the device is a dGPU with HG/PX power control,
 * otherwise return false.
 */
A
Alex Deucher 已提交
145 146 147 148
bool amdgpu_device_is_px(struct drm_device *dev)
{
	struct amdgpu_device *adev = dev->dev_private;

149
	if (adev->flags & AMD_IS_PX)
A
Alex Deucher 已提交
150 151 152 153 154 155 156
		return true;
	return false;
}

/*
 * MMIO register access helper functions.
 */
157 158 159 160 161 162 163 164 165
/**
 * amdgpu_mm_rreg - read a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 * @acc_flags: access flags which require special behavior
 *
 * Returns the 32 bit value from the offset specified.
 */
A
Alex Deucher 已提交
166
uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
M
Monk Liu 已提交
167
			uint32_t acc_flags)
A
Alex Deucher 已提交
168
{
169 170
	uint32_t ret;

171
	if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
172 173
		return amdgpu_virt_kiq_rreg(adev, reg);

M
Monk Liu 已提交
174
	if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
175
		ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
A
Alex Deucher 已提交
176 177 178 179 180 181 182 183
	else {
		unsigned long flags;

		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
		writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
		ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
	}
184 185
	trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
	return ret;
A
Alex Deucher 已提交
186 187
}

188 189 190 191 192 193
/*
 * MMIO register read with bytes helper functions
 * @offset:bytes offset from MMIO start
 *
*/

194 195 196 197 198 199 200 201
/**
 * amdgpu_mm_rreg8 - read a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @offset: byte aligned register offset
 *
 * Returns the 8 bit value from the offset specified.
 */
202 203 204 205 206 207 208 209 210 211 212 213
uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) {
	if (offset < adev->rmmio_size)
		return (readb(adev->rmmio + offset));
	BUG();
}

/*
 * MMIO register write with bytes helper functions
 * @offset:bytes offset from MMIO start
 * @value: the value want to be written to the register
 *
*/
214 215 216 217 218 219 220 221 222
/**
 * amdgpu_mm_wreg8 - read a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @offset: byte aligned register offset
 * @value: 8 bit value to write
 *
 * Writes the value specified to the offset specified.
 */
223 224 225 226 227 228 229
void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) {
	if (offset < adev->rmmio_size)
		writeb(value, adev->rmmio + offset);
	else
		BUG();
}

230 231 232 233 234 235 236 237 238 239
/**
 * amdgpu_mm_wreg - write to a memory mapped IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 * @v: 32 bit value to write to the register
 * @acc_flags: access flags which require special behavior
 *
 * Writes the value specified to the offset specified.
 */
A
Alex Deucher 已提交
240
void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
M
Monk Liu 已提交
241
		    uint32_t acc_flags)
A
Alex Deucher 已提交
242
{
243
	trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
244

245 246 247 248
	if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
		adev->last_mm_index = v;
	}

249
	if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
250 251
		return amdgpu_virt_kiq_wreg(adev, reg, v);

M
Monk Liu 已提交
252
	if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
A
Alex Deucher 已提交
253 254 255 256 257 258 259 260 261
		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
	else {
		unsigned long flags;

		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
		writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
		writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
	}
262 263 264 265

	if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
		udelay(500);
	}
A
Alex Deucher 已提交
266 267
}

268 269 270 271 272 273 274 275
/**
 * amdgpu_io_rreg - read an IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 *
 * Returns the 32 bit value from the offset specified.
 */
A
Alex Deucher 已提交
276 277 278 279 280 281 282 283 284 285
u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
{
	if ((reg * 4) < adev->rio_mem_size)
		return ioread32(adev->rio_mem + (reg * 4));
	else {
		iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
		return ioread32(adev->rio_mem + (mmMM_DATA * 4));
	}
}

286 287 288 289 290 291 292 293 294
/**
 * amdgpu_io_wreg - write to an IO register
 *
 * @adev: amdgpu_device pointer
 * @reg: dword aligned register offset
 * @v: 32 bit value to write to the register
 *
 * Writes the value specified to the offset specified.
 */
A
Alex Deucher 已提交
295 296
void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
{
297 298 299
	if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
		adev->last_mm_index = v;
	}
A
Alex Deucher 已提交
300 301 302 303 304 305 306

	if ((reg * 4) < adev->rio_mem_size)
		iowrite32(v, adev->rio_mem + (reg * 4));
	else {
		iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
		iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
	}
307 308 309 310

	if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
		udelay(500);
	}
A
Alex Deucher 已提交
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
}

/**
 * amdgpu_mm_rdoorbell - read a doorbell dword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 *
 * Returns the value in the doorbell aperture at the
 * requested doorbell index (CIK).
 */
u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
{
	if (index < adev->doorbell.num_doorbells) {
		return readl(adev->doorbell.ptr + index);
	} else {
		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
		return 0;
	}
}

/**
 * amdgpu_mm_wdoorbell - write a doorbell dword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 * @v: value to write
 *
 * Writes @v to the doorbell aperture at the
 * requested doorbell index (CIK).
 */
void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
{
	if (index < adev->doorbell.num_doorbells) {
		writel(v, adev->doorbell.ptr + index);
	} else {
		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
	}
}

351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
/**
 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 *
 * Returns the value in the doorbell aperture at the
 * requested doorbell index (VEGA10+).
 */
u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
{
	if (index < adev->doorbell.num_doorbells) {
		return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
	} else {
		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
		return 0;
	}
}

/**
 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 *
 * @adev: amdgpu_device pointer
 * @index: doorbell index
 * @v: value to write
 *
 * Writes @v to the doorbell aperture at the
 * requested doorbell index (VEGA10+).
 */
void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
{
	if (index < adev->doorbell.num_doorbells) {
		atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
	} else {
		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
	}
}

A
Alex Deucher 已提交
389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
/**
 * amdgpu_invalid_rreg - dummy reg read function
 *
 * @adev: amdgpu device pointer
 * @reg: offset of register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 * Returns the value in the register.
 */
static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
{
	DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
	BUG();
	return 0;
}

/**
 * amdgpu_invalid_wreg - dummy reg write function
 *
 * @adev: amdgpu device pointer
 * @reg: offset of register
 * @v: value to write to the register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 */
static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
{
	DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
		  reg, v);
	BUG();
}

423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
/**
 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
 *
 * @adev: amdgpu device pointer
 * @reg: offset of register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 * Returns the value in the register.
 */
static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
{
	DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
	BUG();
	return 0;
}

/**
 * amdgpu_invalid_wreg64 - dummy reg write function
 *
 * @adev: amdgpu device pointer
 * @reg: offset of register
 * @v: value to write to the register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 */
static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
{
	DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
		  reg, v);
	BUG();
}

A
Alex Deucher 已提交
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
/**
 * amdgpu_block_invalid_rreg - dummy reg read function
 *
 * @adev: amdgpu device pointer
 * @block: offset of instance
 * @reg: offset of register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 * Returns the value in the register.
 */
static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
					  uint32_t block, uint32_t reg)
{
	DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
		  reg, block);
	BUG();
	return 0;
}

/**
 * amdgpu_block_invalid_wreg - dummy reg write function
 *
 * @adev: amdgpu device pointer
 * @block: offset of instance
 * @reg: offset of register
 * @v: value to write to the register
 *
 * Dummy register read function.  Used for register blocks
 * that certain asics don't have (all asics).
 */
static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
				      uint32_t block,
				      uint32_t reg, uint32_t v)
{
	DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
		  reg, block, v);
	BUG();
}

497 498 499 500 501 502 503 504
/**
 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
 *
 * @adev: amdgpu device pointer
 *
 * Allocates a scratch page of VRAM for use by various things in the
 * driver.
 */
505
static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
506
{
507 508 509 510 511
	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
				       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
				       &adev->vram_scratch.robj,
				       &adev->vram_scratch.gpu_addr,
				       (void **)&adev->vram_scratch.ptr);
A
Alex Deucher 已提交
512 513
}

514 515 516 517 518 519 520
/**
 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
 *
 * @adev: amdgpu device pointer
 *
 * Frees the VRAM scratch page.
 */
521
static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
A
Alex Deucher 已提交
522
{
523
	amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
A
Alex Deucher 已提交
524 525 526
}

/**
527
 * amdgpu_device_program_register_sequence - program an array of registers.
A
Alex Deucher 已提交
528 529 530 531 532 533 534 535
 *
 * @adev: amdgpu_device pointer
 * @registers: pointer to the register array
 * @array_size: size of the register array
 *
 * Programs an array or registers with and and or masks.
 * This is a helper for setting golden registers.
 */
536 537 538
void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
					     const u32 *registers,
					     const u32 array_size)
A
Alex Deucher 已提交
539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
{
	u32 tmp, reg, and_mask, or_mask;
	int i;

	if (array_size % 3)
		return;

	for (i = 0; i < array_size; i +=3) {
		reg = registers[i + 0];
		and_mask = registers[i + 1];
		or_mask = registers[i + 2];

		if (and_mask == 0xffffffff) {
			tmp = or_mask;
		} else {
			tmp = RREG32(reg);
			tmp &= ~and_mask;
556 557 558 559
			if (adev->family >= AMDGPU_FAMILY_AI)
				tmp |= (or_mask & and_mask);
			else
				tmp |= or_mask;
A
Alex Deucher 已提交
560 561 562 563 564
		}
		WREG32(reg, tmp);
	}
}

565 566 567 568 569 570 571 572
/**
 * amdgpu_device_pci_config_reset - reset the GPU
 *
 * @adev: amdgpu_device pointer
 *
 * Resets the GPU using the pci config reset sequence.
 * Only applicable to asics prior to vega10.
 */
573
void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
A
Alex Deucher 已提交
574 575 576 577 578 579 580 581
{
	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
}

/*
 * GPU doorbell aperture helpers function.
 */
/**
582
 * amdgpu_device_doorbell_init - Init doorbell driver information.
A
Alex Deucher 已提交
583 584 585 586 587 588
 *
 * @adev: amdgpu_device pointer
 *
 * Init doorbell driver information (CIK)
 * Returns 0 on success, error on failure.
 */
589
static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
590
{
591

592 593 594 595 596 597 598 599 600
	/* No doorbell on SI hardware generation */
	if (adev->asic_type < CHIP_BONAIRE) {
		adev->doorbell.base = 0;
		adev->doorbell.size = 0;
		adev->doorbell.num_doorbells = 0;
		adev->doorbell.ptr = NULL;
		return 0;
	}

601 602 603
	if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
		return -EINVAL;

604 605
	amdgpu_asic_init_doorbell_index(adev);

A
Alex Deucher 已提交
606 607 608 609
	/* doorbell bar mapping */
	adev->doorbell.base = pci_resource_start(adev->pdev, 2);
	adev->doorbell.size = pci_resource_len(adev->pdev, 2);

610
	adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
611
					     adev->doorbell_index.max_assignment+1);
A
Alex Deucher 已提交
612 613 614
	if (adev->doorbell.num_doorbells == 0)
		return -EINVAL;

615
	/* For Vega, reserve and map two pages on doorbell BAR since SDMA
616 617 618 619
	 * paging queue doorbell use the second page. The
	 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
	 * doorbells are in the first page. So with paging queue enabled,
	 * the max num_doorbells should + 1 page (0x400 in dword)
620 621
	 */
	if (adev->asic_type >= CHIP_VEGA10)
622
		adev->doorbell.num_doorbells += 0x400;
623

624 625 626 627
	adev->doorbell.ptr = ioremap(adev->doorbell.base,
				     adev->doorbell.num_doorbells *
				     sizeof(u32));
	if (adev->doorbell.ptr == NULL)
A
Alex Deucher 已提交
628 629 630 631 632 633
		return -ENOMEM;

	return 0;
}

/**
634
 * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
A
Alex Deucher 已提交
635 636 637 638 639
 *
 * @adev: amdgpu_device pointer
 *
 * Tear down doorbell driver information (CIK)
 */
640
static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
A
Alex Deucher 已提交
641 642 643 644 645
{
	iounmap(adev->doorbell.ptr);
	adev->doorbell.ptr = NULL;
}

646

A
Alex Deucher 已提交
647 648

/*
649
 * amdgpu_device_wb_*()
650
 * Writeback is the method by which the GPU updates special pages in memory
A
Alex Xie 已提交
651
 * with the status of certain GPU events (fences, ring pointers,etc.).
A
Alex Deucher 已提交
652 653 654
 */

/**
655
 * amdgpu_device_wb_fini - Disable Writeback and free memory
A
Alex Deucher 已提交
656 657 658 659 660 661
 *
 * @adev: amdgpu_device pointer
 *
 * Disables Writeback and frees the Writeback memory (all asics).
 * Used at driver shutdown.
 */
662
static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
A
Alex Deucher 已提交
663 664
{
	if (adev->wb.wb_obj) {
665 666 667
		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
				      &adev->wb.gpu_addr,
				      (void **)&adev->wb.wb);
A
Alex Deucher 已提交
668 669 670 671 672
		adev->wb.wb_obj = NULL;
	}
}

/**
673
 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
A
Alex Deucher 已提交
674 675 676
 *
 * @adev: amdgpu_device pointer
 *
677
 * Initializes writeback and allocates writeback memory (all asics).
A
Alex Deucher 已提交
678 679 680
 * Used at driver startup.
 * Returns 0 on success or an -error on failure.
 */
681
static int amdgpu_device_wb_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
682 683 684 685
{
	int r;

	if (adev->wb.wb_obj == NULL) {
686 687
		/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
688 689 690
					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
					    (void **)&adev->wb.wb);
A
Alex Deucher 已提交
691 692 693 694 695 696 697 698 699
		if (r) {
			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
			return r;
		}

		adev->wb.num_wb = AMDGPU_MAX_WB;
		memset(&adev->wb.used, 0, sizeof(adev->wb.used));

		/* clear wb memory */
M
Monk Liu 已提交
700
		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
A
Alex Deucher 已提交
701 702 703 704 705 706
	}

	return 0;
}

/**
707
 * amdgpu_device_wb_get - Allocate a wb entry
A
Alex Deucher 已提交
708 709 710 711 712 713 714
 *
 * @adev: amdgpu_device pointer
 * @wb: wb index
 *
 * Allocate a wb slot for use by the driver (all asics).
 * Returns 0 on success or -EINVAL on failure.
 */
715
int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
A
Alex Deucher 已提交
716 717 718
{
	unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);

719
	if (offset < adev->wb.num_wb) {
K
Ken Wang 已提交
720
		__set_bit(offset, adev->wb.used);
M
Monk Liu 已提交
721
		*wb = offset << 3; /* convert to dw offset */
722 723 724 725 726 727
		return 0;
	} else {
		return -EINVAL;
	}
}

A
Alex Deucher 已提交
728
/**
729
 * amdgpu_device_wb_free - Free a wb entry
A
Alex Deucher 已提交
730 731 732 733 734 735
 *
 * @adev: amdgpu_device pointer
 * @wb: wb index
 *
 * Free a wb slot allocated for use by the driver (all asics)
 */
736
void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
A
Alex Deucher 已提交
737
{
M
Monk Liu 已提交
738
	wb >>= 3;
A
Alex Deucher 已提交
739
	if (wb < adev->wb.num_wb)
M
Monk Liu 已提交
740
		__clear_bit(wb, adev->wb.used);
A
Alex Deucher 已提交
741 742
}

743 744 745 746 747 748 749 750 751 752 753
/**
 * amdgpu_device_resize_fb_bar - try to resize FB BAR
 *
 * @adev: amdgpu_device pointer
 *
 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
 * to fail, but if any of the BARs is not accessible after the size we abort
 * driver loading by returning -ENODEV.
 */
int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
{
754
	u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
755
	u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
756 757 758
	struct pci_bus *root;
	struct resource *res;
	unsigned i;
759 760 761
	u16 cmd;
	int r;

762 763 764 765
	/* Bypass for VF */
	if (amdgpu_sriov_vf(adev))
		return 0;

766 767 768 769 770 771
	/* Check if the root BUS has 64bit memory resources */
	root = adev->pdev->bus;
	while (root->parent)
		root = root->parent;

	pci_bus_for_each_resource(root, res, i) {
772
		if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
773 774 775 776 777 778 779 780
		    res->start > 0x100000000ull)
			break;
	}

	/* Trying to resize is pointless without a root hub window above 4GB */
	if (!res)
		return 0;

781 782 783 784 785 786
	/* Disable memory decoding while we change the BAR addresses and size */
	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
	pci_write_config_word(adev->pdev, PCI_COMMAND,
			      cmd & ~PCI_COMMAND_MEMORY);

	/* Free the VRAM and doorbell BAR, we most likely need to move both. */
787
	amdgpu_device_doorbell_fini(adev);
788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803
	if (adev->asic_type >= CHIP_BONAIRE)
		pci_release_resource(adev->pdev, 2);

	pci_release_resource(adev->pdev, 0);

	r = pci_resize_resource(adev->pdev, 0, rbar_size);
	if (r == -ENOSPC)
		DRM_INFO("Not enough PCI address space for a large BAR.");
	else if (r && r != -ENOTSUPP)
		DRM_ERROR("Problem resizing BAR0 (%d).", r);

	pci_assign_unassigned_bus_resources(adev->pdev->bus);

	/* When the doorbell or fb BAR isn't available we have no chance of
	 * using the device.
	 */
804
	r = amdgpu_device_doorbell_init(adev);
805 806 807 808 809 810 811
	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
		return -ENODEV;

	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);

	return 0;
}
812

A
Alex Deucher 已提交
813 814 815 816
/*
 * GPU helpers function.
 */
/**
A
Alex Deucher 已提交
817
 * amdgpu_device_need_post - check if the hw need post or not
A
Alex Deucher 已提交
818 819 820
 *
 * @adev: amdgpu_device pointer
 *
821 822 823
 * Check if the asic has been initialized (all asics) at driver startup
 * or post is needed if  hw reset is performed.
 * Returns true if need or false if not.
A
Alex Deucher 已提交
824
 */
A
Alex Deucher 已提交
825
bool amdgpu_device_need_post(struct amdgpu_device *adev)
A
Alex Deucher 已提交
826 827 828
{
	uint32_t reg;

829 830 831 832
	if (amdgpu_sriov_vf(adev))
		return false;

	if (amdgpu_passthrough(adev)) {
M
Monk Liu 已提交
833 834 835 836
		/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
		 * some old smc fw still need driver do vPost otherwise gpu hang, while
		 * those smc fw version above 22.15 doesn't have this flaw, so we force
		 * vpost executed for smc version below 22.15
837 838 839 840 841 842 843 844 845 846
		 */
		if (adev->asic_type == CHIP_FIJI) {
			int err;
			uint32_t fw_ver;
			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
			/* force vPost if error occured */
			if (err)
				return true;

			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
M
Monk Liu 已提交
847 848
			if (fw_ver < 0x00160e00)
				return true;
849 850
		}
	}
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867

	if (adev->has_hw_reset) {
		adev->has_hw_reset = false;
		return true;
	}

	/* bios scratch used on CIK+ */
	if (adev->asic_type >= CHIP_BONAIRE)
		return amdgpu_atombios_scratch_need_asic_init(adev);

	/* check MEM_SIZE for older asics */
	reg = amdgpu_asic_get_config_memsize(adev);

	if ((reg != 0) && (reg != 0xffffffff))
		return false;

	return true;
868 869
}

A
Alex Deucher 已提交
870 871
/* if we get transitioned to only one device, take VGA back */
/**
872
 * amdgpu_device_vga_set_decode - enable/disable vga decode
A
Alex Deucher 已提交
873 874 875 876 877 878 879
 *
 * @cookie: amdgpu_device pointer
 * @state: enable/disable vga decode
 *
 * Enable/disable vga decode (all asics).
 * Returns VGA resource flags.
 */
880
static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
A
Alex Deucher 已提交
881 882 883 884 885 886 887 888 889 890
{
	struct amdgpu_device *adev = cookie;
	amdgpu_asic_set_vga_state(adev, state);
	if (state)
		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
	else
		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
}

891 892 893 894 895 896 897 898 899 900
/**
 * amdgpu_device_check_block_size - validate the vm block size
 *
 * @adev: amdgpu_device pointer
 *
 * Validates the vm block size specified via module parameter.
 * The vm block size defines number of bits in page table versus page directory,
 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
 * page table and the remaining bits are in the page directory.
 */
901
static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
902 903 904 905
{
	/* defines number of bits in page table versus page directory,
	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
	 * page table and the remaining bits are in the page directory */
906 907
	if (amdgpu_vm_block_size == -1)
		return;
908

909
	if (amdgpu_vm_block_size < 9) {
910 911
		dev_warn(adev->dev, "VM page table size (%d) too small\n",
			 amdgpu_vm_block_size);
912
		amdgpu_vm_block_size = -1;
913 914 915
	}
}

916 917 918 919 920 921 922 923
/**
 * amdgpu_device_check_vm_size - validate the vm size
 *
 * @adev: amdgpu_device pointer
 *
 * Validates the vm size in GB specified via module parameter.
 * The VM size is the size of the GPU virtual memory space in GB.
 */
924
static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
925
{
926 927 928 929
	/* no need to check the default value */
	if (amdgpu_vm_size == -1)
		return;

930 931 932
	if (amdgpu_vm_size < 1) {
		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
			 amdgpu_vm_size);
933
		amdgpu_vm_size = -1;
934 935 936
	}
}

937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
{
	struct sysinfo si;
	bool is_os_64 = (sizeof(void *) == 8) ? true : false;
	uint64_t total_memory;
	uint64_t dram_size_seven_GB = 0x1B8000000;
	uint64_t dram_size_three_GB = 0xB8000000;

	if (amdgpu_smu_memory_pool_size == 0)
		return;

	if (!is_os_64) {
		DRM_WARN("Not 64-bit OS, feature not supported\n");
		goto def_value;
	}
	si_meminfo(&si);
	total_memory = (uint64_t)si.totalram * si.mem_unit;

	if ((amdgpu_smu_memory_pool_size == 1) ||
		(amdgpu_smu_memory_pool_size == 2)) {
		if (total_memory < dram_size_three_GB)
			goto def_value1;
	} else if ((amdgpu_smu_memory_pool_size == 4) ||
		(amdgpu_smu_memory_pool_size == 8)) {
		if (total_memory < dram_size_seven_GB)
			goto def_value1;
	} else {
		DRM_WARN("Smu memory pool size not supported\n");
		goto def_value;
	}
	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;

	return;

def_value1:
	DRM_WARN("No enough system memory\n");
def_value:
	adev->pm.smu_prv_buffer_size = 0;
}

A
Alex Deucher 已提交
977
/**
978
 * amdgpu_device_check_arguments - validate module params
A
Alex Deucher 已提交
979 980 981 982 983 984
 *
 * @adev: amdgpu_device pointer
 *
 * Validates certain module parameters and updates
 * the associated values used by the driver (all asics).
 */
985
static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
A
Alex Deucher 已提交
986
{
987 988
	int ret = 0;

989 990 991 992
	if (amdgpu_sched_jobs < 4) {
		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
			 amdgpu_sched_jobs);
		amdgpu_sched_jobs = 4;
993
	} else if (!is_power_of_2(amdgpu_sched_jobs)){
994 995 996 997
		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
			 amdgpu_sched_jobs);
		amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
	}
A
Alex Deucher 已提交
998

999
	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1000 1001 1002
		/* gart size must be greater or equal to 32M */
		dev_warn(adev->dev, "gart size (%d) too small\n",
			 amdgpu_gart_size);
1003
		amdgpu_gart_size = -1;
A
Alex Deucher 已提交
1004 1005
	}

1006
	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1007
		/* gtt size must be greater or equal to 32M */
1008 1009 1010
		dev_warn(adev->dev, "gtt size (%d) too small\n",
				 amdgpu_gtt_size);
		amdgpu_gtt_size = -1;
A
Alex Deucher 已提交
1011 1012
	}

1013 1014 1015 1016 1017 1018 1019
	/* valid range is between 4 and 9 inclusive */
	if (amdgpu_vm_fragment_size != -1 &&
	    (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
		dev_warn(adev->dev, "valid range is between 4 and 9\n");
		amdgpu_vm_fragment_size = -1;
	}

1020 1021
	amdgpu_device_check_smu_prv_buffer_size(adev);

1022
	amdgpu_device_check_vm_size(adev);
A
Alex Deucher 已提交
1023

1024
	amdgpu_device_check_block_size(adev);
C
Christian König 已提交
1025

1026 1027 1028 1029
	ret = amdgpu_device_get_job_timeout_settings(adev);
	if (ret) {
		dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
		return ret;
1030
	}
1031 1032

	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1033 1034

	return ret;
A
Alex Deucher 已提交
1035 1036 1037 1038 1039 1040
}

/**
 * amdgpu_switcheroo_set_state - set switcheroo state
 *
 * @pdev: pci dev pointer
1041
 * @state: vga_switcheroo state
A
Alex Deucher 已提交
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
 *
 * Callback for the switcheroo driver.  Suspends or resumes the
 * the asics before or after it is powered up using ACPI methods.
 */
static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
{
	struct drm_device *dev = pci_get_drvdata(pdev);

	if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
		return;

	if (state == VGA_SWITCHEROO_ON) {
1054
		pr_info("amdgpu: switched on\n");
A
Alex Deucher 已提交
1055 1056 1057
		/* don't suspend or resume card normally */
		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;

1058
		amdgpu_device_resume(dev, true, true);
A
Alex Deucher 已提交
1059 1060 1061 1062

		dev->switch_power_state = DRM_SWITCH_POWER_ON;
		drm_kms_helper_poll_enable(dev);
	} else {
1063
		pr_info("amdgpu: switched off\n");
A
Alex Deucher 已提交
1064 1065
		drm_kms_helper_poll_disable(dev);
		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1066
		amdgpu_device_suspend(dev, true, true);
A
Alex Deucher 已提交
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
	}
}

/**
 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
 *
 * @pdev: pci dev pointer
 *
 * Callback for the switcheroo driver.  Check of the switcheroo
 * state can be changed.
 * Returns true if the state can be changed, false if not.
 */
static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
{
	struct drm_device *dev = pci_get_drvdata(pdev);

	/*
	* FIXME: open_count is protected by drm_global_mutex but that would lead to
	* locking inversion with the driver load path. And the access here is
	* completely racy anyway. So don't bother with locking for now.
	*/
	return dev->open_count == 0;
}

static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
	.set_gpu_state = amdgpu_switcheroo_set_state,
	.reprobe = NULL,
	.can_switch = amdgpu_switcheroo_can_switch,
};

1098 1099 1100
/**
 * amdgpu_device_ip_set_clockgating_state - set the CG state
 *
1101
 * @dev: amdgpu_device pointer
1102 1103 1104 1105 1106 1107 1108
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 * @state: clockgating state (gate or ungate)
 *
 * Sets the requested clockgating state for all instances of
 * the hardware IP specified.
 * Returns the error code from the last instance.
 */
1109
int amdgpu_device_ip_set_clockgating_state(void *dev,
1110 1111
					   enum amd_ip_block_type block_type,
					   enum amd_clockgating_state state)
A
Alex Deucher 已提交
1112
{
1113
	struct amdgpu_device *adev = dev;
A
Alex Deucher 已提交
1114 1115 1116
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1117
		if (!adev->ip_blocks[i].status.valid)
1118
			continue;
1119 1120 1121 1122 1123 1124 1125 1126 1127
		if (adev->ip_blocks[i].version->type != block_type)
			continue;
		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
			continue;
		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
			(void *)adev, state);
		if (r)
			DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
A
Alex Deucher 已提交
1128 1129 1130 1131
	}
	return r;
}

1132 1133 1134
/**
 * amdgpu_device_ip_set_powergating_state - set the PG state
 *
1135
 * @dev: amdgpu_device pointer
1136 1137 1138 1139 1140 1141 1142
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 * @state: powergating state (gate or ungate)
 *
 * Sets the requested powergating state for all instances of
 * the hardware IP specified.
 * Returns the error code from the last instance.
 */
1143
int amdgpu_device_ip_set_powergating_state(void *dev,
1144 1145
					   enum amd_ip_block_type block_type,
					   enum amd_powergating_state state)
A
Alex Deucher 已提交
1146
{
1147
	struct amdgpu_device *adev = dev;
A
Alex Deucher 已提交
1148 1149 1150
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1151
		if (!adev->ip_blocks[i].status.valid)
1152
			continue;
1153 1154 1155 1156 1157 1158 1159 1160 1161
		if (adev->ip_blocks[i].version->type != block_type)
			continue;
		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
			continue;
		r = adev->ip_blocks[i].version->funcs->set_powergating_state(
			(void *)adev, state);
		if (r)
			DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
A
Alex Deucher 已提交
1162 1163 1164 1165
	}
	return r;
}

1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
/**
 * amdgpu_device_ip_get_clockgating_state - get the CG state
 *
 * @adev: amdgpu_device pointer
 * @flags: clockgating feature flags
 *
 * Walks the list of IPs on the device and updates the clockgating
 * flags for each IP.
 * Updates @flags with the feature flags for each hardware IP where
 * clockgating is enabled.
 */
1177 1178
void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
					    u32 *flags)
1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189
{
	int i;

	for (i = 0; i < adev->num_ip_blocks; i++) {
		if (!adev->ip_blocks[i].status.valid)
			continue;
		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
			adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
	}
}

1190 1191 1192 1193 1194 1195 1196 1197 1198
/**
 * amdgpu_device_ip_wait_for_idle - wait for idle
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 *
 * Waits for the request hardware IP to be idle.
 * Returns 0 for success or a negative error code on failure.
 */
1199 1200
int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
				   enum amd_ip_block_type block_type)
1201 1202 1203 1204
{
	int i, r;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1205
		if (!adev->ip_blocks[i].status.valid)
1206
			continue;
1207 1208
		if (adev->ip_blocks[i].version->type == block_type) {
			r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1209 1210 1211 1212 1213 1214 1215 1216 1217
			if (r)
				return r;
			break;
		}
	}
	return 0;

}

1218 1219 1220 1221 1222 1223 1224 1225 1226
/**
 * amdgpu_device_ip_is_idle - is the hardware IP idle
 *
 * @adev: amdgpu_device pointer
 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
 *
 * Check if the hardware IP is idle or not.
 * Returns true if it the IP is idle, false if not.
 */
1227 1228
bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
			      enum amd_ip_block_type block_type)
1229 1230 1231 1232
{
	int i;

	for (i = 0; i < adev->num_ip_blocks; i++) {
1233
		if (!adev->ip_blocks[i].status.valid)
1234
			continue;
1235 1236
		if (adev->ip_blocks[i].version->type == block_type)
			return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1237 1238 1239 1240 1241
	}
	return true;

}

1242 1243 1244 1245
/**
 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
 *
 * @adev: amdgpu_device pointer
1246
 * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1247 1248 1249 1250
 *
 * Returns a pointer to the hardware IP block structure
 * if it exists for the asic, otherwise NULL.
 */
1251 1252 1253
struct amdgpu_ip_block *
amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
			      enum amd_ip_block_type type)
A
Alex Deucher 已提交
1254 1255 1256 1257
{
	int i;

	for (i = 0; i < adev->num_ip_blocks; i++)
1258
		if (adev->ip_blocks[i].version->type == type)
A
Alex Deucher 已提交
1259 1260 1261 1262 1263 1264
			return &adev->ip_blocks[i];

	return NULL;
}

/**
1265
 * amdgpu_device_ip_block_version_cmp
A
Alex Deucher 已提交
1266 1267
 *
 * @adev: amdgpu_device pointer
1268
 * @type: enum amd_ip_block_type
A
Alex Deucher 已提交
1269 1270 1271 1272 1273 1274
 * @major: major version
 * @minor: minor version
 *
 * return 0 if equal or greater
 * return 1 if smaller or the ip_block doesn't exist
 */
1275 1276 1277
int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
				       enum amd_ip_block_type type,
				       u32 major, u32 minor)
A
Alex Deucher 已提交
1278
{
1279
	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
A
Alex Deucher 已提交
1280

1281 1282 1283
	if (ip_block && ((ip_block->version->major > major) ||
			((ip_block->version->major == major) &&
			(ip_block->version->minor >= minor))))
A
Alex Deucher 已提交
1284 1285 1286 1287 1288
		return 0;

	return 1;
}

1289
/**
1290
 * amdgpu_device_ip_block_add
1291 1292 1293 1294 1295 1296 1297
 *
 * @adev: amdgpu_device pointer
 * @ip_block_version: pointer to the IP to add
 *
 * Adds the IP block driver information to the collection of IPs
 * on the asic.
 */
1298 1299
int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
			       const struct amdgpu_ip_block_version *ip_block_version)
1300 1301 1302 1303
{
	if (!ip_block_version)
		return -EINVAL;

1304
	DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1305 1306
		  ip_block_version->funcs->name);

1307 1308 1309 1310 1311
	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;

	return 0;
}

1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
/**
 * amdgpu_device_enable_virtual_display - enable virtual display feature
 *
 * @adev: amdgpu_device pointer
 *
 * Enabled the virtual display feature if the user has enabled it via
 * the module parameter virtual_display.  This feature provides a virtual
 * display hardware on headless boards or in virtualized environments.
 * This function parses and validates the configuration string specified by
 * the user and configues the virtual display configuration (number of
 * virtual connectors, crtcs, etc.) specified.
 */
1324
static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1325 1326 1327 1328 1329 1330
{
	adev->enable_virtual_display = false;

	if (amdgpu_virtual_display) {
		struct drm_device *ddev = adev->ddev;
		const char *pci_address_name = pci_name(ddev->pdev);
1331
		char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1332 1333 1334

		pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
		pciaddstr_tmp = pciaddstr;
1335 1336
		while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
			pciaddname = strsep(&pciaddname_tmp, ",");
1337 1338
			if (!strcmp("all", pciaddname)
			    || !strcmp(pci_address_name, pciaddname)) {
1339 1340 1341
				long num_crtc;
				int res = -1;

1342
				adev->enable_virtual_display = true;
1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356

				if (pciaddname_tmp)
					res = kstrtol(pciaddname_tmp, 10,
						      &num_crtc);

				if (!res) {
					if (num_crtc < 1)
						num_crtc = 1;
					if (num_crtc > 6)
						num_crtc = 6;
					adev->mode_info.num_crtc = num_crtc;
				} else {
					adev->mode_info.num_crtc = 1;
				}
1357 1358 1359 1360
				break;
			}
		}

1361 1362 1363
		DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
			 amdgpu_virtual_display, pci_address_name,
			 adev->enable_virtual_display, adev->mode_info.num_crtc);
1364 1365 1366 1367 1368

		kfree(pciaddstr);
	}
}

1369 1370 1371 1372 1373 1374 1375 1376 1377 1378
/**
 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
 *
 * @adev: amdgpu_device pointer
 *
 * Parses the asic configuration parameters specified in the gpu info
 * firmware and makes them availale to the driver for use in configuring
 * the asic.
 * Returns 0 on success, -EINVAL on failure.
 */
1379 1380 1381 1382 1383 1384 1385
static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
{
	const char *chip_name;
	char fw_name[30];
	int err;
	const struct gpu_info_firmware_header_v1_0 *hdr;

1386 1387
	adev->firmware.gpu_info_fw = NULL;

1388 1389 1390 1391 1392
	switch (adev->asic_type) {
	case CHIP_TOPAZ:
	case CHIP_TONGA:
	case CHIP_FIJI:
	case CHIP_POLARIS10:
1393
	case CHIP_POLARIS11:
1394
	case CHIP_POLARIS12:
1395
	case CHIP_VEGAM:
1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
	case CHIP_CARRIZO:
	case CHIP_STONEY:
#ifdef CONFIG_DRM_AMDGPU_SI
	case CHIP_VERDE:
	case CHIP_TAHITI:
	case CHIP_PITCAIRN:
	case CHIP_OLAND:
	case CHIP_HAINAN:
#endif
#ifdef CONFIG_DRM_AMDGPU_CIK
	case CHIP_BONAIRE:
	case CHIP_HAWAII:
	case CHIP_KAVERI:
	case CHIP_KABINI:
	case CHIP_MULLINS:
#endif
1412
	case CHIP_VEGA20:
1413 1414 1415 1416 1417
	default:
		return 0;
	case CHIP_VEGA10:
		chip_name = "vega10";
		break;
1418 1419 1420
	case CHIP_VEGA12:
		chip_name = "vega12";
		break;
1421
	case CHIP_RAVEN:
1422 1423
		if (adev->rev_id >= 8)
			chip_name = "raven2";
1424 1425
		else if (adev->pdev->device == 0x15d8)
			chip_name = "picasso";
1426 1427
		else
			chip_name = "raven";
1428
		break;
1429 1430 1431
	case CHIP_ARCTURUS:
		chip_name = "arcturus";
		break;
1432 1433 1434
	case CHIP_RENOIR:
		chip_name = "renoir";
		break;
1435 1436 1437
	case CHIP_NAVI10:
		chip_name = "navi10";
		break;
1438 1439 1440
	case CHIP_NAVI14:
		chip_name = "navi14";
		break;
1441 1442 1443
	case CHIP_NAVI12:
		chip_name = "navi12";
		break;
1444 1445 1446
	}

	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1447
	err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1448 1449 1450 1451 1452 1453
	if (err) {
		dev_err(adev->dev,
			"Failed to load gpu_info firmware \"%s\"\n",
			fw_name);
		goto out;
	}
1454
	err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1455 1456 1457 1458 1459 1460 1461
	if (err) {
		dev_err(adev->dev,
			"Failed to validate gpu_info firmware \"%s\"\n",
			fw_name);
		goto out;
	}

1462
	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1463 1464 1465 1466 1467 1468
	amdgpu_ucode_print_gpu_info_hdr(&hdr->header);

	switch (hdr->version_major) {
	case 1:
	{
		const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1469
			(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1470 1471
								le32_to_cpu(hdr->header.ucode_array_offset_bytes));

1472 1473 1474 1475
		adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
		adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
		adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
		adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1476
		adev->gfx.config.max_texture_channel_caches =
1477 1478 1479 1480 1481
			le32_to_cpu(gpu_info_fw->gc_num_tccs);
		adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
		adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
		adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
		adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1482
		adev->gfx.config.double_offchip_lds_buf =
1483 1484
			le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
		adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1485 1486 1487 1488 1489
		adev->gfx.cu_info.max_waves_per_simd =
			le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
		adev->gfx.cu_info.max_scratch_slots_per_cu =
			le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
		adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1490
		if (hdr->version_minor >= 1) {
1491 1492 1493 1494 1495 1496 1497 1498
			const struct gpu_info_firmware_v1_1 *gpu_info_fw =
				(const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
			adev->gfx.config.num_sc_per_sh =
				le32_to_cpu(gpu_info_fw->num_sc_per_sh);
			adev->gfx.config.num_packer_per_sc =
				le32_to_cpu(gpu_info_fw->num_packer_per_sc);
		}
1499 1500 1501 1502 1503 1504 1505 1506
#ifdef CONFIG_DRM_AMD_DC_DCN2_0
		if (hdr->version_minor == 2) {
			const struct gpu_info_firmware_v1_2 *gpu_info_fw =
				(const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
			adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
		}
#endif
1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518
		break;
	}
	default:
		dev_err(adev->dev,
			"Unsupported gpu_info table %d\n", hdr->header.ucode_version);
		err = -EINVAL;
		goto out;
	}
out:
	return err;
}

1519 1520 1521 1522 1523 1524 1525 1526 1527 1528
/**
 * amdgpu_device_ip_early_init - run early init for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Early initialization pass for hardware IPs.  The hardware IPs that make
 * up each asic are discovered each IP's early_init callback is run.  This
 * is the first stage in initializing the asic.
 * Returns 0 on success, negative error code on failure.
 */
1529
static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
1530
{
1531
	int i, r;
A
Alex Deucher 已提交
1532

1533
	amdgpu_device_enable_virtual_display(adev);
1534

A
Alex Deucher 已提交
1535
	switch (adev->asic_type) {
1536 1537
	case CHIP_TOPAZ:
	case CHIP_TONGA:
1538
	case CHIP_FIJI:
1539
	case CHIP_POLARIS10:
1540
	case CHIP_POLARIS11:
1541
	case CHIP_POLARIS12:
1542
	case CHIP_VEGAM:
1543
	case CHIP_CARRIZO:
1544 1545
	case CHIP_STONEY:
		if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1546 1547 1548 1549 1550 1551 1552 1553
			adev->family = AMDGPU_FAMILY_CZ;
		else
			adev->family = AMDGPU_FAMILY_VI;

		r = vi_set_ip_blocks(adev);
		if (r)
			return r;
		break;
K
Ken Wang 已提交
1554 1555 1556 1557 1558 1559
#ifdef CONFIG_DRM_AMDGPU_SI
	case CHIP_VERDE:
	case CHIP_TAHITI:
	case CHIP_PITCAIRN:
	case CHIP_OLAND:
	case CHIP_HAINAN:
K
Ken Wang 已提交
1560
		adev->family = AMDGPU_FAMILY_SI;
K
Ken Wang 已提交
1561 1562 1563 1564 1565
		r = si_set_ip_blocks(adev);
		if (r)
			return r;
		break;
#endif
1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581
#ifdef CONFIG_DRM_AMDGPU_CIK
	case CHIP_BONAIRE:
	case CHIP_HAWAII:
	case CHIP_KAVERI:
	case CHIP_KABINI:
	case CHIP_MULLINS:
		if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
			adev->family = AMDGPU_FAMILY_CI;
		else
			adev->family = AMDGPU_FAMILY_KV;

		r = cik_set_ip_blocks(adev);
		if (r)
			return r;
		break;
#endif
1582 1583
	case CHIP_VEGA10:
	case CHIP_VEGA12:
1584
	case CHIP_VEGA20:
1585
	case CHIP_RAVEN:
1586
	case CHIP_ARCTURUS:
1587 1588 1589
	case CHIP_RENOIR:
		if (adev->asic_type == CHIP_RAVEN ||
		    adev->asic_type == CHIP_RENOIR)
1590 1591 1592
			adev->family = AMDGPU_FAMILY_RV;
		else
			adev->family = AMDGPU_FAMILY_AI;
1593 1594 1595 1596 1597

		r = soc15_set_ip_blocks(adev);
		if (r)
			return r;
		break;
1598
	case  CHIP_NAVI10:
1599
	case  CHIP_NAVI14:
1600
	case  CHIP_NAVI12:
1601 1602 1603 1604 1605 1606
		adev->family = AMDGPU_FAMILY_NV;

		r = nv_set_ip_blocks(adev);
		if (r)
			return r;
		break;
A
Alex Deucher 已提交
1607 1608 1609 1610 1611
	default:
		/* FIXME: not supported yet */
		return -EINVAL;
	}

1612 1613 1614 1615
	r = amdgpu_device_parse_gpu_info_fw(adev);
	if (r)
		return r;

1616 1617
	amdgpu_amdkfd_device_probe(adev);

1618 1619 1620
	if (amdgpu_sriov_vf(adev)) {
		r = amdgpu_virt_request_full_gpu(adev, true);
		if (r)
1621
			return -EAGAIN;
1622 1623
	}

1624
	adev->pm.pp_feature = amdgpu_pp_feature_mask;
1625 1626
	if (amdgpu_sriov_vf(adev))
		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1627

A
Alex Deucher 已提交
1628 1629
	for (i = 0; i < adev->num_ip_blocks; i++) {
		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1630 1631
			DRM_ERROR("disabled ip block: %d <%s>\n",
				  i, adev->ip_blocks[i].version->funcs->name);
1632
			adev->ip_blocks[i].status.valid = false;
A
Alex Deucher 已提交
1633
		} else {
1634 1635
			if (adev->ip_blocks[i].version->funcs->early_init) {
				r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1636
				if (r == -ENOENT) {
1637
					adev->ip_blocks[i].status.valid = false;
1638
				} else if (r) {
1639 1640
					DRM_ERROR("early_init of IP block <%s> failed %d\n",
						  adev->ip_blocks[i].version->funcs->name, r);
A
Alex Deucher 已提交
1641
					return r;
1642
				} else {
1643
					adev->ip_blocks[i].status.valid = true;
1644
				}
1645
			} else {
1646
				adev->ip_blocks[i].status.valid = true;
A
Alex Deucher 已提交
1647 1648
			}
		}
1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
		/* get the vbios after the asic_funcs are set up */
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
			/* Read BIOS */
			if (!amdgpu_get_bios(adev))
				return -EINVAL;

			r = amdgpu_atombios_init(adev);
			if (r) {
				dev_err(adev->dev, "amdgpu_atombios_init failed\n");
				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
				return r;
			}
		}
A
Alex Deucher 已提交
1662 1663
	}

1664 1665 1666
	adev->cg_flags &= amdgpu_cg_mask;
	adev->pg_flags &= amdgpu_pg_mask;

A
Alex Deucher 已提交
1667 1668 1669
	return 0;
}

1670 1671 1672 1673 1674 1675 1676 1677 1678 1679
static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
{
	int i, r;

	for (i = 0; i < adev->num_ip_blocks; i++) {
		if (!adev->ip_blocks[i].status.sw)
			continue;
		if (adev->ip_blocks[i].status.hw)
			continue;
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1680
		    (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715
		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
			r = adev->ip_blocks[i].version->funcs->hw_init(adev);
			if (r) {
				DRM_ERROR("hw_init of IP block <%s> failed %d\n",
					  adev->ip_blocks[i].version->funcs->name, r);
				return r;
			}
			adev->ip_blocks[i].status.hw = true;
		}
	}

	return 0;
}

static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
{
	int i, r;

	for (i = 0; i < adev->num_ip_blocks; i++) {
		if (!adev->ip_blocks[i].status.sw)
			continue;
		if (adev->ip_blocks[i].status.hw)
			continue;
		r = adev->ip_blocks[i].version->funcs->hw_init(adev);
		if (r) {
			DRM_ERROR("hw_init of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
			return r;
		}
		adev->ip_blocks[i].status.hw = true;
	}

	return 0;
}

1716 1717 1718 1719
static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
{
	int r = 0;
	int i;
1720
	uint32_t smu_version;
1721 1722 1723

	if (adev->asic_type >= CHIP_VEGA10) {
		for (i = 0; i < adev->num_ip_blocks; i++) {
1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
			if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
				continue;

			/* no need to do the fw loading again if already done*/
			if (adev->ip_blocks[i].status.hw == true)
				break;

			if (adev->in_gpu_reset || adev->in_suspend) {
				r = adev->ip_blocks[i].version->funcs->resume(adev);
				if (r) {
					DRM_ERROR("resume of IP block <%s> failed %d\n",
1735
							  adev->ip_blocks[i].version->funcs->name, r);
1736 1737 1738 1739 1740 1741 1742 1743
					return r;
				}
			} else {
				r = adev->ip_blocks[i].version->funcs->hw_init(adev);
				if (r) {
					DRM_ERROR("hw_init of IP block <%s> failed %d\n",
							  adev->ip_blocks[i].version->funcs->name, r);
					return r;
1744 1745
				}
			}
1746 1747 1748

			adev->ip_blocks[i].status.hw = true;
			break;
1749 1750
		}
	}
1751

1752
	r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
1753

1754
	return r;
1755 1756
}

1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767
/**
 * amdgpu_device_ip_init - run init for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Main initialization pass for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
 * are run.  sw_init initializes the software state associated with each IP
 * and hw_init initializes the hardware associated with each IP.
 * Returns 0 on success, negative error code on failure.
 */
1768
static int amdgpu_device_ip_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
1769 1770 1771
{
	int i, r;

1772 1773 1774 1775
	r = amdgpu_ras_init(adev);
	if (r)
		return r;

A
Alex Deucher 已提交
1776
	for (i = 0; i < adev->num_ip_blocks; i++) {
1777
		if (!adev->ip_blocks[i].status.valid)
A
Alex Deucher 已提交
1778
			continue;
1779
		r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1780
		if (r) {
1781 1782
			DRM_ERROR("sw_init of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
1783
			goto init_failed;
1784
		}
1785
		adev->ip_blocks[i].status.sw = true;
1786

A
Alex Deucher 已提交
1787
		/* need to do gmc hw init early so we can allocate gpu mem */
1788
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1789
			r = amdgpu_device_vram_scratch_init(adev);
1790 1791
			if (r) {
				DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1792
				goto init_failed;
1793
			}
1794
			r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1795 1796
			if (r) {
				DRM_ERROR("hw_init %d failed %d\n", i, r);
1797
				goto init_failed;
1798
			}
1799
			r = amdgpu_device_wb_init(adev);
1800
			if (r) {
1801
				DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
1802
				goto init_failed;
1803
			}
1804
			adev->ip_blocks[i].status.hw = true;
M
Monk Liu 已提交
1805 1806

			/* right after GMC hw init, we create CSA */
1807
			if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
R
Rex Zhu 已提交
1808 1809 1810
				r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
								AMDGPU_GEM_DOMAIN_VRAM,
								AMDGPU_CSA_SIZE);
M
Monk Liu 已提交
1811 1812
				if (r) {
					DRM_ERROR("allocate CSA failed %d\n", r);
1813
					goto init_failed;
M
Monk Liu 已提交
1814 1815
				}
			}
A
Alex Deucher 已提交
1816 1817 1818
		}
	}

1819 1820 1821 1822 1823 1824 1825
	r = amdgpu_ib_pool_init(adev);
	if (r) {
		dev_err(adev->dev, "IB initialization failed (%d).\n", r);
		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
		goto init_failed;
	}

1826 1827
	r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
	if (r)
1828
		goto init_failed;
1829 1830 1831

	r = amdgpu_device_ip_hw_init_phase1(adev);
	if (r)
1832
		goto init_failed;
1833

1834 1835
	r = amdgpu_device_fw_loading(adev);
	if (r)
1836
		goto init_failed;
1837

1838 1839
	r = amdgpu_device_ip_hw_init_phase2(adev);
	if (r)
1840
		goto init_failed;
A
Alex Deucher 已提交
1841

1842 1843
	if (adev->gmc.xgmi.num_physical_nodes > 1)
		amdgpu_xgmi_add_device(adev);
1844
	amdgpu_amdkfd_device_init(adev);
1845

1846
init_failed:
1847
	if (amdgpu_sriov_vf(adev)) {
1848 1849
		if (!r)
			amdgpu_virt_init_data_exchange(adev);
1850
		amdgpu_virt_release_full_gpu(adev, true);
1851
	}
1852

1853
	return r;
A
Alex Deucher 已提交
1854 1855
}

1856 1857 1858 1859 1860 1861 1862 1863 1864
/**
 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
 *
 * @adev: amdgpu_device pointer
 *
 * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
 * this function before a GPU reset.  If the value is retained after a
 * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
 */
1865
static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
1866 1867 1868 1869
{
	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
}

1870 1871 1872 1873 1874 1875 1876 1877 1878 1879
/**
 * amdgpu_device_check_vram_lost - check if vram is valid
 *
 * @adev: amdgpu_device pointer
 *
 * Checks the reset magic value written to the gart pointer in VRAM.
 * The driver calls this after a GPU reset to see if the contents of
 * VRAM is lost or now.
 * returns true if vram is lost, false if not.
 */
1880
static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
1881 1882 1883 1884 1885
{
	return !!memcmp(adev->gart.ptr, adev->reset_magic,
			AMDGPU_RESET_MAGIC_NUM);
}

1886
/**
1887
 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
1888 1889 1890 1891
 *
 * @adev: amdgpu_device pointer
 *
 * The list of all the hardware IPs that make up the asic is walked and the
1892 1893 1894
 * set_clockgating_state callbacks are run.
 * Late initialization pass enabling clockgating for hardware IPs.
 * Fini or suspend, pass disabling clockgating for hardware IPs.
1895 1896
 * Returns 0 on success, negative error code on failure.
 */
1897

1898 1899
static int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
						enum amd_clockgating_state state)
A
Alex Deucher 已提交
1900
{
1901
	int i, j, r;
A
Alex Deucher 已提交
1902

1903 1904 1905
	if (amdgpu_emu_mode == 1)
		return 0;

1906 1907
	for (j = 0; j < adev->num_ip_blocks; j++) {
		i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
1908
		if (!adev->ip_blocks[i].status.late_initialized)
A
Alex Deucher 已提交
1909
			continue;
1910
		/* skip CG for VCE/UVD, it's handled specially */
1911
		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1912
		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
1913
		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
1914
		    adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1915
			/* enable clockgating to save power */
1916
			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1917
										     state);
1918 1919
			if (r) {
				DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1920
					  adev->ip_blocks[i].version->funcs->name, r);
1921 1922
				return r;
			}
1923
		}
A
Alex Deucher 已提交
1924
	}
1925

1926 1927 1928
	return 0;
}

1929
static int amdgpu_device_set_pg_state(struct amdgpu_device *adev, enum amd_powergating_state state)
1930
{
1931
	int i, j, r;
1932

1933 1934 1935
	if (amdgpu_emu_mode == 1)
		return 0;

1936 1937
	for (j = 0; j < adev->num_ip_blocks; j++) {
		i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
1938
		if (!adev->ip_blocks[i].status.late_initialized)
1939 1940 1941 1942 1943 1944 1945 1946
			continue;
		/* skip CG for VCE/UVD, it's handled specially */
		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
		    adev->ip_blocks[i].version->funcs->set_powergating_state) {
			/* enable powergating to save power */
			r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
1947
											state);
1948 1949 1950 1951 1952 1953 1954
			if (r) {
				DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
					  adev->ip_blocks[i].version->funcs->name, r);
				return r;
			}
		}
	}
1955 1956 1957
	return 0;
}

1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994
static int amdgpu_device_enable_mgpu_fan_boost(void)
{
	struct amdgpu_gpu_instance *gpu_ins;
	struct amdgpu_device *adev;
	int i, ret = 0;

	mutex_lock(&mgpu_info.mutex);

	/*
	 * MGPU fan boost feature should be enabled
	 * only when there are two or more dGPUs in
	 * the system
	 */
	if (mgpu_info.num_dgpu < 2)
		goto out;

	for (i = 0; i < mgpu_info.num_dgpu; i++) {
		gpu_ins = &(mgpu_info.gpu_ins[i]);
		adev = gpu_ins->adev;
		if (!(adev->flags & AMD_IS_APU) &&
		    !gpu_ins->mgpu_fan_enabled &&
		    adev->powerplay.pp_funcs &&
		    adev->powerplay.pp_funcs->enable_mgpu_fan_boost) {
			ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
			if (ret)
				break;

			gpu_ins->mgpu_fan_enabled = 1;
		}
	}

out:
	mutex_unlock(&mgpu_info.mutex);

	return ret;
}

1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006
/**
 * amdgpu_device_ip_late_init - run late init for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Late initialization pass for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked and the late_init callbacks are run.
 * late_init covers any special initialization that an IP requires
 * after all of the have been initialized or something that needs to happen
 * late in the init process.
 * Returns 0 on success, negative error code on failure.
 */
2007
static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2008 2009 2010 2011
{
	int i = 0, r;

	for (i = 0; i < adev->num_ip_blocks; i++) {
2012
		if (!adev->ip_blocks[i].status.hw)
2013 2014 2015 2016 2017 2018 2019 2020 2021
			continue;
		if (adev->ip_blocks[i].version->funcs->late_init) {
			r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
			if (r) {
				DRM_ERROR("late_init of IP block <%s> failed %d\n",
					  adev->ip_blocks[i].version->funcs->name, r);
				return r;
			}
		}
2022
		adev->ip_blocks[i].status.late_initialized = true;
2023 2024
	}

2025 2026
	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2027

2028
	amdgpu_device_fill_reset_magic(adev);
A
Alex Deucher 已提交
2029

2030 2031 2032 2033 2034 2035 2036
	r = amdgpu_device_enable_mgpu_fan_boost();
	if (r)
		DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);

	/* set to low pstate by default */
	amdgpu_xgmi_set_pstate(adev, 0);

A
Alex Deucher 已提交
2037 2038 2039
	return 0;
}

2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
/**
 * amdgpu_device_ip_fini - run fini for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Main teardown pass for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
 * are run.  hw_fini tears down the hardware associated with each IP
 * and sw_fini tears down any software state associated with each IP.
 * Returns 0 on success, negative error code on failure.
 */
2051
static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
A
Alex Deucher 已提交
2052 2053 2054
{
	int i, r;

2055 2056
	amdgpu_ras_pre_fini(adev);

2057 2058 2059
	if (adev->gmc.xgmi.num_physical_nodes > 1)
		amdgpu_xgmi_remove_device(adev);

2060
	amdgpu_amdkfd_device_fini(adev);
2061 2062

	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2063 2064
	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);

2065 2066
	/* need to disable SMC first */
	for (i = 0; i < adev->num_ip_blocks; i++) {
2067
		if (!adev->ip_blocks[i].status.hw)
2068
			continue;
2069
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2070
			r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2071 2072 2073
			/* XXX handle errors */
			if (r) {
				DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2074
					  adev->ip_blocks[i].version->funcs->name, r);
2075
			}
2076
			adev->ip_blocks[i].status.hw = false;
2077 2078 2079 2080
			break;
		}
	}

A
Alex Deucher 已提交
2081
	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2082
		if (!adev->ip_blocks[i].status.hw)
A
Alex Deucher 已提交
2083
			continue;
2084

2085
		r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
A
Alex Deucher 已提交
2086
		/* XXX handle errors */
2087
		if (r) {
2088 2089
			DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
2090
		}
2091

2092
		adev->ip_blocks[i].status.hw = false;
A
Alex Deucher 已提交
2093 2094
	}

2095

A
Alex Deucher 已提交
2096
	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2097
		if (!adev->ip_blocks[i].status.sw)
A
Alex Deucher 已提交
2098
			continue;
2099 2100

		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2101
			amdgpu_ucode_free_bo(adev);
R
Rex Zhu 已提交
2102
			amdgpu_free_static_csa(&adev->virt.csa_obj);
2103 2104
			amdgpu_device_wb_fini(adev);
			amdgpu_device_vram_scratch_fini(adev);
2105
			amdgpu_ib_pool_fini(adev);
2106 2107
		}

2108
		r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
A
Alex Deucher 已提交
2109
		/* XXX handle errors */
2110
		if (r) {
2111 2112
			DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
2113
		}
2114 2115
		adev->ip_blocks[i].status.sw = false;
		adev->ip_blocks[i].status.valid = false;
A
Alex Deucher 已提交
2116 2117
	}

M
Monk Liu 已提交
2118
	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2119
		if (!adev->ip_blocks[i].status.late_initialized)
2120
			continue;
2121 2122 2123
		if (adev->ip_blocks[i].version->funcs->late_fini)
			adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
		adev->ip_blocks[i].status.late_initialized = false;
M
Monk Liu 已提交
2124 2125
	}

2126 2127
	amdgpu_ras_fini(adev);

2128
	if (amdgpu_sriov_vf(adev))
2129 2130
		if (amdgpu_virt_release_full_gpu(adev, false))
			DRM_ERROR("failed to release exclusive mode on fini\n");
M
Monk Liu 已提交
2131

A
Alex Deucher 已提交
2132 2133 2134
	return 0;
}

2135
/**
2136
 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2137
 *
2138
 * @work: work_struct.
2139
 */
2140
static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2141 2142
{
	struct amdgpu_device *adev =
2143
		container_of(work, struct amdgpu_device, delayed_init_work.work);
2144 2145 2146 2147 2148
	int r;

	r = amdgpu_ib_ring_tests(adev);
	if (r)
		DRM_ERROR("ib ring test failed (%d).\n", r);
2149 2150
}

2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163
static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
{
	struct amdgpu_device *adev =
		container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);

	mutex_lock(&adev->gfx.gfx_off_mutex);
	if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) {
		if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
			adev->gfx.gfx_off_state = true;
	}
	mutex_unlock(&adev->gfx.gfx_off_mutex);
}

2164
/**
2165
 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2166 2167 2168 2169 2170 2171 2172 2173 2174
 *
 * @adev: amdgpu_device pointer
 *
 * Main suspend function for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked, clockgating is disabled and the
 * suspend callbacks are run.  suspend puts the hardware and software state
 * in each IP into a state suitable for suspend.
 * Returns 0 on success, negative error code on failure.
 */
2175 2176 2177 2178
static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
{
	int i, r;

2179
	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2180
	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2181

2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192
	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
		if (!adev->ip_blocks[i].status.valid)
			continue;
		/* displays are handled separately */
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {
			/* XXX handle errors */
			r = adev->ip_blocks[i].version->funcs->suspend(adev);
			/* XXX handle errors */
			if (r) {
				DRM_ERROR("suspend of IP block <%s> failed %d\n",
					  adev->ip_blocks[i].version->funcs->name, r);
2193
				return r;
2194
			}
2195
			adev->ip_blocks[i].status.hw = false;
2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213
		}
	}

	return 0;
}

/**
 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
 *
 * @adev: amdgpu_device pointer
 *
 * Main suspend function for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked, clockgating is disabled and the
 * suspend callbacks are run.  suspend puts the hardware and software state
 * in each IP into a state suitable for suspend.
 * Returns 0 on success, negative error code on failure.
 */
static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
A
Alex Deucher 已提交
2214 2215 2216 2217
{
	int i, r;

	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2218
		if (!adev->ip_blocks[i].status.valid)
A
Alex Deucher 已提交
2219
			continue;
2220 2221 2222
		/* displays are handled in phase1 */
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
			continue;
A
Alex Deucher 已提交
2223
		/* XXX handle errors */
2224
		r = adev->ip_blocks[i].version->funcs->suspend(adev);
A
Alex Deucher 已提交
2225
		/* XXX handle errors */
2226
		if (r) {
2227 2228
			DRM_ERROR("suspend of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
2229
		}
2230
		adev->ip_blocks[i].status.hw = false;
2231 2232 2233 2234 2235
		/* handle putting the SMC in the appropriate state */
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
			if (is_support_sw_smu(adev)) {
				/* todo */
			} else if (adev->powerplay.pp_funcs &&
2236
					   adev->powerplay.pp_funcs->set_mp1_state) {
2237 2238 2239 2240 2241 2242
				r = adev->powerplay.pp_funcs->set_mp1_state(
					adev->powerplay.pp_handle,
					adev->mp1_state);
				if (r) {
					DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
						  adev->mp1_state, r);
2243
					return r;
2244 2245 2246
				}
			}
		}
2247 2248

		adev->ip_blocks[i].status.hw = false;
A
Alex Deucher 已提交
2249 2250 2251 2252 2253
	}

	return 0;
}

2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268
/**
 * amdgpu_device_ip_suspend - run suspend for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Main suspend function for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked, clockgating is disabled and the
 * suspend callbacks are run.  suspend puts the hardware and software state
 * in each IP into a state suitable for suspend.
 * Returns 0 on success, negative error code on failure.
 */
int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
{
	int r;

2269 2270 2271
	if (amdgpu_sriov_vf(adev))
		amdgpu_virt_request_full_gpu(adev, false);

2272 2273 2274 2275 2276
	r = amdgpu_device_ip_suspend_phase1(adev);
	if (r)
		return r;
	r = amdgpu_device_ip_suspend_phase2(adev);

2277 2278 2279
	if (amdgpu_sriov_vf(adev))
		amdgpu_virt_release_full_gpu(adev, false);

2280 2281 2282
	return r;
}

2283
static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2284 2285 2286
{
	int i, r;

2287 2288 2289
	static enum amd_ip_block_type ip_order[] = {
		AMD_IP_BLOCK_TYPE_GMC,
		AMD_IP_BLOCK_TYPE_COMMON,
2290
		AMD_IP_BLOCK_TYPE_PSP,
2291 2292
		AMD_IP_BLOCK_TYPE_IH,
	};
2293

2294 2295 2296
	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
		int j;
		struct amdgpu_ip_block *block;
2297

2298 2299 2300
		for (j = 0; j < adev->num_ip_blocks; j++) {
			block = &adev->ip_blocks[j];

2301
			block->status.hw = false;
2302 2303 2304 2305 2306
			if (block->version->type != ip_order[i] ||
				!block->status.valid)
				continue;

			r = block->version->funcs->hw_init(adev);
2307
			DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2308 2309
			if (r)
				return r;
2310
			block->status.hw = true;
2311 2312 2313 2314 2315 2316
		}
	}

	return 0;
}

2317
static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
2318 2319 2320
{
	int i, r;

2321 2322 2323 2324 2325
	static enum amd_ip_block_type ip_order[] = {
		AMD_IP_BLOCK_TYPE_SMC,
		AMD_IP_BLOCK_TYPE_DCE,
		AMD_IP_BLOCK_TYPE_GFX,
		AMD_IP_BLOCK_TYPE_SDMA,
2326 2327
		AMD_IP_BLOCK_TYPE_UVD,
		AMD_IP_BLOCK_TYPE_VCE
2328
	};
2329

2330 2331 2332
	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
		int j;
		struct amdgpu_ip_block *block;
2333

2334 2335 2336 2337
		for (j = 0; j < adev->num_ip_blocks; j++) {
			block = &adev->ip_blocks[j];

			if (block->version->type != ip_order[i] ||
2338 2339
				!block->status.valid ||
				block->status.hw)
2340 2341 2342
				continue;

			r = block->version->funcs->hw_init(adev);
2343
			DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2344 2345
			if (r)
				return r;
2346
			block->status.hw = true;
2347 2348 2349 2350 2351 2352
		}
	}

	return 0;
}

2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364
/**
 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * First resume function for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked and the resume callbacks are run for
 * COMMON, GMC, and IH.  resume puts the hardware into a functional state
 * after a suspend and updates the software state as necessary.  This
 * function is also used for restoring the GPU after a GPU reset.
 * Returns 0 on success, negative error code on failure.
 */
2365
static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
A
Alex Deucher 已提交
2366 2367 2368
{
	int i, r;

2369
	for (i = 0; i < adev->num_ip_blocks; i++) {
2370
		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2371 2372
			continue;
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2373 2374
		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2375

2376 2377 2378 2379 2380 2381
			r = adev->ip_blocks[i].version->funcs->resume(adev);
			if (r) {
				DRM_ERROR("resume of IP block <%s> failed %d\n",
					  adev->ip_blocks[i].version->funcs->name, r);
				return r;
			}
2382
			adev->ip_blocks[i].status.hw = true;
2383 2384 2385 2386 2387 2388
		}
	}

	return 0;
}

2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401
/**
 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * First resume function for hardware IPs.  The list of all the hardware
 * IPs that make up the asic is walked and the resume callbacks are run for
 * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
 * functional state after a suspend and updates the software state as
 * necessary.  This function is also used for restoring the GPU after a GPU
 * reset.
 * Returns 0 on success, negative error code on failure.
 */
2402
static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
A
Alex Deucher 已提交
2403 2404 2405 2406
{
	int i, r;

	for (i = 0; i < adev->num_ip_blocks; i++) {
2407
		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
A
Alex Deucher 已提交
2408
			continue;
2409
		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2410
		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2411 2412
		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
2413
			continue;
2414
		r = adev->ip_blocks[i].version->funcs->resume(adev);
2415
		if (r) {
2416 2417
			DRM_ERROR("resume of IP block <%s> failed %d\n",
				  adev->ip_blocks[i].version->funcs->name, r);
A
Alex Deucher 已提交
2418
			return r;
2419
		}
2420
		adev->ip_blocks[i].status.hw = true;
A
Alex Deucher 已提交
2421 2422 2423 2424 2425
	}

	return 0;
}

2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437
/**
 * amdgpu_device_ip_resume - run resume for hardware IPs
 *
 * @adev: amdgpu_device pointer
 *
 * Main resume function for hardware IPs.  The hardware IPs
 * are split into two resume functions because they are
 * are also used in in recovering from a GPU reset and some additional
 * steps need to be take between them.  In this case (S3/S4) they are
 * run sequentially.
 * Returns 0 on success, negative error code on failure.
 */
2438
static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
2439 2440 2441
{
	int r;

2442
	r = amdgpu_device_ip_resume_phase1(adev);
2443 2444
	if (r)
		return r;
2445 2446 2447 2448 2449

	r = amdgpu_device_fw_loading(adev);
	if (r)
		return r;

2450
	r = amdgpu_device_ip_resume_phase2(adev);
2451 2452 2453 2454

	return r;
}

2455 2456 2457 2458 2459 2460 2461
/**
 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
 *
 * @adev: amdgpu_device pointer
 *
 * Query the VBIOS data tables to determine if the board supports SR-IOV.
 */
2462
static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2463
{
M
Monk Liu 已提交
2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474
	if (amdgpu_sriov_vf(adev)) {
		if (adev->is_atom_fw) {
			if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
		} else {
			if (amdgpu_atombios_has_gpu_virtualization_table(adev))
				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
		}

		if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2475
	}
2476 2477
}

2478 2479 2480 2481 2482 2483 2484 2485
/**
 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
 *
 * @asic_type: AMD asic type
 *
 * Check if there is DC (new modesetting infrastructre) support for an asic.
 * returns true if DC has support, false if not.
 */
2486 2487 2488 2489 2490
bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
{
	switch (asic_type) {
#if defined(CONFIG_DRM_AMD_DC)
	case CHIP_BONAIRE:
2491
	case CHIP_KAVERI:
2492 2493
	case CHIP_KABINI:
	case CHIP_MULLINS:
2494 2495 2496 2497 2498 2499 2500 2501 2502
		/*
		 * We have systems in the wild with these ASICs that require
		 * LVDS and VGA support which is not supported with DC.
		 *
		 * Fallback to the non-DC driver here by default so as not to
		 * cause regressions.
		 */
		return amdgpu_dc > 0;
	case CHIP_HAWAII:
2503 2504 2505
	case CHIP_CARRIZO:
	case CHIP_STONEY:
	case CHIP_POLARIS10:
L
Leo Liu 已提交
2506
	case CHIP_POLARIS11:
2507
	case CHIP_POLARIS12:
L
Leo Liu 已提交
2508
	case CHIP_VEGAM:
2509 2510
	case CHIP_TONGA:
	case CHIP_FIJI:
2511
	case CHIP_VEGA10:
2512
	case CHIP_VEGA12:
2513
	case CHIP_VEGA20:
2514
#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
2515
	case CHIP_RAVEN:
2516 2517 2518
#endif
#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
	case CHIP_NAVI10:
2519
	case CHIP_NAVI14:
L
Leo Li 已提交
2520
	case CHIP_NAVI12:
R
Roman Li 已提交
2521 2522 2523
#endif
#if defined(CONFIG_DRM_AMD_DC_DCN2_1)
	case CHIP_RENOIR:
2524
#endif
2525
		return amdgpu_dc != 0;
2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540
#endif
	default:
		return false;
	}
}

/**
 * amdgpu_device_has_dc_support - check if dc is supported
 *
 * @adev: amdgpu_device_pointer
 *
 * Returns true for supported, false for not supported
 */
bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
{
X
Xiangliang Yu 已提交
2541 2542 2543
	if (amdgpu_sriov_vf(adev))
		return false;

2544 2545 2546
	return amdgpu_device_asic_has_dc_support(adev->asic_type);
}

2547 2548 2549 2550 2551 2552 2553 2554

static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
{
	struct amdgpu_device *adev =
		container_of(__work, struct amdgpu_device, xgmi_reset_work);

	adev->asic_reset_res =  amdgpu_asic_reset(adev);
	if (adev->asic_reset_res)
E
Evan Quan 已提交
2555
		DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
2556 2557 2558 2559
			 adev->asic_reset_res, adev->ddev->unique);
}


A
Alex Deucher 已提交
2560 2561 2562 2563
/**
 * amdgpu_device_init - initialize the driver
 *
 * @adev: amdgpu_device pointer
2564
 * @ddev: drm dev pointer
A
Alex Deucher 已提交
2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578
 * @pdev: pci dev pointer
 * @flags: driver flags
 *
 * Initializes the driver info and hw (all asics).
 * Returns 0 for success or an error on failure.
 * Called at driver startup.
 */
int amdgpu_device_init(struct amdgpu_device *adev,
		       struct drm_device *ddev,
		       struct pci_dev *pdev,
		       uint32_t flags)
{
	int r, i;
	bool runtime = false;
2579
	u32 max_MBps;
A
Alex Deucher 已提交
2580 2581 2582 2583 2584 2585

	adev->shutdown = false;
	adev->dev = &pdev->dev;
	adev->ddev = ddev;
	adev->pdev = pdev;
	adev->flags = flags;
2586
	adev->asic_type = flags & AMD_ASIC_MASK;
A
Alex Deucher 已提交
2587
	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2588 2589
	if (amdgpu_emu_mode == 1)
		adev->usec_timeout *= 2;
2590
	adev->gmc.gart_size = 512 * 1024 * 1024;
A
Alex Deucher 已提交
2591 2592 2593 2594 2595
	adev->accel_working = false;
	adev->num_rings = 0;
	adev->mman.buffer_funcs = NULL;
	adev->mman.buffer_funcs_ring = NULL;
	adev->vm_manager.vm_pte_funcs = NULL;
2596
	adev->vm_manager.vm_pte_num_rqs = 0;
2597
	adev->gmc.gmc_funcs = NULL;
2598
	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2599
	bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
A
Alex Deucher 已提交
2600 2601 2602 2603 2604

	adev->smc_rreg = &amdgpu_invalid_rreg;
	adev->smc_wreg = &amdgpu_invalid_wreg;
	adev->pcie_rreg = &amdgpu_invalid_rreg;
	adev->pcie_wreg = &amdgpu_invalid_wreg;
2605 2606
	adev->pciep_rreg = &amdgpu_invalid_rreg;
	adev->pciep_wreg = &amdgpu_invalid_wreg;
2607 2608
	adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
	adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
A
Alex Deucher 已提交
2609 2610 2611 2612
	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
	adev->didt_rreg = &amdgpu_invalid_rreg;
	adev->didt_wreg = &amdgpu_invalid_wreg;
2613 2614
	adev->gc_cac_rreg = &amdgpu_invalid_rreg;
	adev->gc_cac_wreg = &amdgpu_invalid_wreg;
A
Alex Deucher 已提交
2615 2616 2617
	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;

2618 2619 2620
	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
		 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
		 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
A
Alex Deucher 已提交
2621 2622 2623 2624

	/* mutex initialization are all done here so we
	 * can recall function without having locking issues */
	atomic_set(&adev->irq.ih.lock, 0);
2625
	mutex_init(&adev->firmware.mutex);
A
Alex Deucher 已提交
2626 2627 2628
	mutex_init(&adev->pm.mutex);
	mutex_init(&adev->gfx.gpu_clock_mutex);
	mutex_init(&adev->srbm_mutex);
2629
	mutex_init(&adev->gfx.pipe_reserve_mutex);
2630
	mutex_init(&adev->gfx.gfx_off_mutex);
A
Alex Deucher 已提交
2631 2632
	mutex_init(&adev->grbm_idx_mutex);
	mutex_init(&adev->mn_lock);
A
Alex Deucher 已提交
2633
	mutex_init(&adev->virt.vf_errors.lock);
A
Alex Deucher 已提交
2634
	hash_init(adev->mn_hash);
2635
	mutex_init(&adev->lock_reset);
2636
	mutex_init(&adev->virt.dpm_mutex);
2637
	mutex_init(&adev->psp.mutex);
A
Alex Deucher 已提交
2638

2639 2640 2641
	r = amdgpu_device_check_arguments(adev);
	if (r)
		return r;
A
Alex Deucher 已提交
2642 2643 2644 2645 2646 2647

	spin_lock_init(&adev->mmio_idx_lock);
	spin_lock_init(&adev->smc_idx_lock);
	spin_lock_init(&adev->pcie_idx_lock);
	spin_lock_init(&adev->uvd_ctx_idx_lock);
	spin_lock_init(&adev->didt_idx_lock);
2648
	spin_lock_init(&adev->gc_cac_idx_lock);
2649
	spin_lock_init(&adev->se_cac_idx_lock);
A
Alex Deucher 已提交
2650
	spin_lock_init(&adev->audio_endpt_idx_lock);
2651
	spin_lock_init(&adev->mm_stats.lock);
A
Alex Deucher 已提交
2652

2653 2654 2655
	INIT_LIST_HEAD(&adev->shadow_list);
	mutex_init(&adev->shadow_list_lock);

2656 2657 2658
	INIT_LIST_HEAD(&adev->ring_lru_list);
	spin_lock_init(&adev->ring_lru_list_lock);

2659 2660
	INIT_DELAYED_WORK(&adev->delayed_init_work,
			  amdgpu_device_delayed_init_work_handler);
2661 2662
	INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
			  amdgpu_device_delay_enable_gfx_off);
2663

2664 2665
	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);

2666
	adev->gfx.gfx_off_req_count = 1;
2667 2668
	adev->pm.ac_power = power_supply_is_system_supplied() > 0 ? true : false;

2669 2670
	/* Registers mapping */
	/* TODO: block userspace mapping of io register */
2671 2672 2673 2674 2675 2676 2677
	if (adev->asic_type >= CHIP_BONAIRE) {
		adev->rmmio_base = pci_resource_start(adev->pdev, 5);
		adev->rmmio_size = pci_resource_len(adev->pdev, 5);
	} else {
		adev->rmmio_base = pci_resource_start(adev->pdev, 2);
		adev->rmmio_size = pci_resource_len(adev->pdev, 2);
	}
A
Alex Deucher 已提交
2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694

	adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
	if (adev->rmmio == NULL) {
		return -ENOMEM;
	}
	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
	DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);

	/* io port mapping */
	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
		if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
			adev->rio_mem_size = pci_resource_len(adev->pdev, i);
			adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
			break;
		}
	}
	if (adev->rio_mem == NULL)
2695
		DRM_INFO("PCI I/O BAR is not found.\n");
A
Alex Deucher 已提交
2696

2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707
	/* enable PCIE atomic ops */
	r = pci_enable_atomic_ops_to_root(adev->pdev,
					  PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
					  PCI_EXP_DEVCAP2_ATOMIC_COMP64);
	if (r) {
		adev->have_atomics_support = false;
		DRM_INFO("PCIE atomic ops is not supported\n");
	} else {
		adev->have_atomics_support = true;
	}

2708 2709
	amdgpu_device_get_pcie_info(adev);

2710 2711 2712
	if (amdgpu_mcbp)
		DRM_INFO("MCBP is enabled\n");

2713 2714 2715
	if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
		adev->enable_mes = true;

2716
	if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10) {
2717 2718 2719 2720 2721 2722 2723
		r = amdgpu_discovery_init(adev);
		if (r) {
			dev_err(adev->dev, "amdgpu_discovery_init failed\n");
			return r;
		}
	}

A
Alex Deucher 已提交
2724
	/* early init functions */
2725
	r = amdgpu_device_ip_early_init(adev);
A
Alex Deucher 已提交
2726 2727 2728
	if (r)
		return r;

2729 2730 2731
	/* doorbell bar mapping and doorbell index init*/
	amdgpu_device_doorbell_init(adev);

A
Alex Deucher 已提交
2732 2733 2734
	/* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
	/* this will fail for cards that aren't VGA class devices, just
	 * ignore it */
2735
	vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
A
Alex Deucher 已提交
2736

2737
	if (amdgpu_device_is_px(ddev))
A
Alex Deucher 已提交
2738
		runtime = true;
2739 2740 2741
	if (!pci_is_thunderbolt_attached(adev->pdev))
		vga_switcheroo_register_client(adev->pdev,
					       &amdgpu_switcheroo_ops, runtime);
A
Alex Deucher 已提交
2742 2743 2744
	if (runtime)
		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);

2745 2746 2747
	if (amdgpu_emu_mode == 1) {
		/* post the asic on emulation mode */
		emu_soc_asic_init(adev);
2748
		goto fence_driver_init;
2749
	}
2750

2751 2752
	/* detect if we are with an SRIOV vbios */
	amdgpu_device_detect_sriov_bios(adev);
2753

2754 2755 2756
	/* check if we need to reset the asic
	 *  E.g., driver was not cleanly unloaded previously, etc.
	 */
2757
	if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
2758 2759 2760 2761 2762 2763 2764
		r = amdgpu_asic_reset(adev);
		if (r) {
			dev_err(adev->dev, "asic reset on init failed\n");
			goto failed;
		}
	}

A
Alex Deucher 已提交
2765
	/* Post card if necessary */
A
Alex Deucher 已提交
2766
	if (amdgpu_device_need_post(adev)) {
A
Alex Deucher 已提交
2767
		if (!adev->bios) {
2768
			dev_err(adev->dev, "no vBIOS found\n");
2769 2770
			r = -EINVAL;
			goto failed;
A
Alex Deucher 已提交
2771
		}
2772
		DRM_INFO("GPU posting now...\n");
2773 2774 2775 2776 2777
		r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
		if (r) {
			dev_err(adev->dev, "gpu post error!\n");
			goto failed;
		}
A
Alex Deucher 已提交
2778 2779
	}

2780 2781 2782 2783 2784
	if (adev->is_atom_fw) {
		/* Initialize clocks */
		r = amdgpu_atomfirmware_get_clock_info(adev);
		if (r) {
			dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
A
Alex Deucher 已提交
2785
			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2786 2787 2788
			goto failed;
		}
	} else {
2789 2790 2791 2792
		/* Initialize clocks */
		r = amdgpu_atombios_get_clock_info(adev);
		if (r) {
			dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
A
Alex Deucher 已提交
2793
			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2794
			goto failed;
2795 2796
		}
		/* init i2c buses */
2797 2798
		if (!amdgpu_device_has_dc_support(adev))
			amdgpu_atombios_i2c_init(adev);
2799
	}
A
Alex Deucher 已提交
2800

2801
fence_driver_init:
A
Alex Deucher 已提交
2802 2803
	/* Fence driver */
	r = amdgpu_fence_driver_init(adev);
2804 2805
	if (r) {
		dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
A
Alex Deucher 已提交
2806
		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2807
		goto failed;
2808
	}
A
Alex Deucher 已提交
2809 2810 2811 2812

	/* init the mode config */
	drm_mode_config_init(adev->ddev);

2813
	r = amdgpu_device_ip_init(adev);
A
Alex Deucher 已提交
2814
	if (r) {
2815 2816 2817 2818 2819 2820
		/* failed in exclusive mode due to timeout */
		if (amdgpu_sriov_vf(adev) &&
		    !amdgpu_sriov_runtime(adev) &&
		    amdgpu_virt_mmio_blocked(adev) &&
		    !amdgpu_virt_wait_reset(adev)) {
			dev_err(adev->dev, "VF exclusive mode timeout\n");
2821 2822 2823
			/* Don't send request since VF is inactive. */
			adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
			adev->virt.ops = NULL;
2824 2825 2826
			r = -EAGAIN;
			goto failed;
		}
2827
		dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
A
Alex Deucher 已提交
2828
		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2829 2830
		if (amdgpu_virt_request_full_gpu(adev, false))
			amdgpu_virt_release_full_gpu(adev, false);
2831
		goto failed;
A
Alex Deucher 已提交
2832 2833 2834 2835
	}

	adev->accel_working = true;

2836 2837
	amdgpu_vm_check_compute_bug(adev);

2838 2839 2840 2841 2842 2843 2844 2845
	/* Initialize the buffer migration limit. */
	if (amdgpu_moverate >= 0)
		max_MBps = amdgpu_moverate;
	else
		max_MBps = 8; /* Allow 8 MB/s. */
	/* Get a log2 for easy divisions. */
	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));

2846 2847
	amdgpu_fbdev_init(adev);

2848 2849 2850
	if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))
		amdgpu_pm_virt_sysfs_init(adev);

2851 2852 2853 2854
	r = amdgpu_pm_sysfs_init(adev);
	if (r)
		DRM_ERROR("registering pm debugfs failed (%d).\n", r);

2855 2856 2857 2858
	r = amdgpu_ucode_sysfs_init(adev);
	if (r)
		DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);

2859
	r = amdgpu_debugfs_gem_init(adev);
M
Monk Liu 已提交
2860
	if (r)
A
Alex Deucher 已提交
2861 2862 2863
		DRM_ERROR("registering gem debugfs failed (%d).\n", r);

	r = amdgpu_debugfs_regs_init(adev);
M
Monk Liu 已提交
2864
	if (r)
A
Alex Deucher 已提交
2865 2866
		DRM_ERROR("registering register debugfs failed (%d).\n", r);

2867
	r = amdgpu_debugfs_firmware_init(adev);
M
Monk Liu 已提交
2868
	if (r)
2869 2870
		DRM_ERROR("registering firmware debugfs failed (%d).\n", r);

2871
	r = amdgpu_debugfs_init(adev);
2872
	if (r)
2873
		DRM_ERROR("Creating debugfs files failed (%d).\n", r);
2874

A
Alex Deucher 已提交
2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887
	if ((amdgpu_testing & 1)) {
		if (adev->accel_working)
			amdgpu_test_moves(adev);
		else
			DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
	}
	if (amdgpu_benchmarking) {
		if (adev->accel_working)
			amdgpu_benchmark(adev, amdgpu_benchmarking);
		else
			DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
	}

2888 2889 2890 2891 2892 2893 2894
	/*
	 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
	 * Otherwise the mgpu fan boost feature will be skipped due to the
	 * gpu instance is counted less.
	 */
	amdgpu_register_gpu_instance(adev);

A
Alex Deucher 已提交
2895 2896 2897
	/* enable clockgating, etc. after ib tests, etc. since some blocks require
	 * explicit gating rather than handling it automatically.
	 */
2898
	r = amdgpu_device_ip_late_init(adev);
2899
	if (r) {
2900
		dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
A
Alex Deucher 已提交
2901
		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2902
		goto failed;
2903
	}
A
Alex Deucher 已提交
2904

2905
	/* must succeed. */
2906
	amdgpu_ras_resume(adev);
2907

2908 2909 2910
	queue_delayed_work(system_wq, &adev->delayed_init_work,
			   msecs_to_jiffies(AMDGPU_RESUME_MS));

2911 2912 2913 2914 2915
	r = device_create_file(adev->dev, &dev_attr_pcie_replay_count);
	if (r) {
		dev_err(adev->dev, "Could not create pcie_replay_count");
		return r;
	}
2916

2917 2918
	if (IS_ENABLED(CONFIG_PERF_EVENTS))
		r = amdgpu_pmu_init(adev);
J
Jonathan Kim 已提交
2919 2920 2921
	if (r)
		dev_err(adev->dev, "amdgpu_pmu_init failed\n");

A
Alex Deucher 已提交
2922
	return 0;
2923 2924

failed:
2925
	amdgpu_vf_error_trans_all(adev);
2926 2927
	if (runtime)
		vga_switcheroo_fini_domain_pm_ops(adev->dev);
2928

2929
	return r;
A
Alex Deucher 已提交
2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945
}

/**
 * amdgpu_device_fini - tear down the driver
 *
 * @adev: amdgpu_device pointer
 *
 * Tear down the driver info (all asics).
 * Called at driver shutdown.
 */
void amdgpu_device_fini(struct amdgpu_device *adev)
{
	int r;

	DRM_INFO("amdgpu: finishing device.\n");
	adev->shutdown = true;
2946 2947
	/* disable all interrupts */
	amdgpu_irq_disable_all(adev);
2948 2949
	if (adev->mode_info.mode_config_initialized){
		if (!amdgpu_device_has_dc_support(adev))
2950
			drm_helper_force_disable_all(adev->ddev);
2951 2952 2953
		else
			drm_atomic_helper_shutdown(adev->ddev);
	}
A
Alex Deucher 已提交
2954
	amdgpu_fence_driver_fini(adev);
2955
	amdgpu_pm_sysfs_fini(adev);
A
Alex Deucher 已提交
2956
	amdgpu_fbdev_fini(adev);
2957
	r = amdgpu_device_ip_fini(adev);
2958 2959 2960 2961
	if (adev->firmware.gpu_info_fw) {
		release_firmware(adev->firmware.gpu_info_fw);
		adev->firmware.gpu_info_fw = NULL;
	}
A
Alex Deucher 已提交
2962
	adev->accel_working = false;
2963
	cancel_delayed_work_sync(&adev->delayed_init_work);
A
Alex Deucher 已提交
2964
	/* free i2c buses */
2965 2966
	if (!amdgpu_device_has_dc_support(adev))
		amdgpu_i2c_fini(adev);
2967 2968 2969 2970

	if (amdgpu_emu_mode != 1)
		amdgpu_atombios_fini(adev);

A
Alex Deucher 已提交
2971 2972
	kfree(adev->bios);
	adev->bios = NULL;
2973 2974
	if (!pci_is_thunderbolt_attached(adev->pdev))
		vga_switcheroo_unregister_client(adev->pdev);
2975 2976
	if (adev->flags & AMD_IS_PX)
		vga_switcheroo_fini_domain_pm_ops(adev->dev);
A
Alex Deucher 已提交
2977 2978 2979 2980 2981 2982
	vga_client_register(adev->pdev, NULL, NULL, NULL);
	if (adev->rio_mem)
		pci_iounmap(adev->pdev, adev->rio_mem);
	adev->rio_mem = NULL;
	iounmap(adev->rmmio);
	adev->rmmio = NULL;
2983
	amdgpu_device_doorbell_fini(adev);
2984 2985 2986
	if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))
		amdgpu_pm_virt_sysfs_fini(adev);

A
Alex Deucher 已提交
2987
	amdgpu_debugfs_regs_cleanup(adev);
2988
	device_remove_file(adev->dev, &dev_attr_pcie_replay_count);
2989
	amdgpu_ucode_sysfs_fini(adev);
2990 2991
	if (IS_ENABLED(CONFIG_PERF_EVENTS))
		amdgpu_pmu_fini(adev);
2992
	amdgpu_debugfs_preempt_cleanup(adev);
2993
	if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10)
2994
		amdgpu_discovery_fini(adev);
A
Alex Deucher 已提交
2995 2996 2997 2998 2999 3000 3001
}


/*
 * Suspend & resume.
 */
/**
3002
 * amdgpu_device_suspend - initiate device suspend
A
Alex Deucher 已提交
3003
 *
3004 3005 3006
 * @dev: drm dev pointer
 * @suspend: suspend state
 * @fbcon : notify the fbdev of suspend
A
Alex Deucher 已提交
3007 3008 3009 3010 3011
 *
 * Puts the hw in the suspend state (all asics).
 * Returns 0 for success or an error on failure.
 * Called at driver suspend.
 */
3012
int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
A
Alex Deucher 已提交
3013 3014 3015 3016
{
	struct amdgpu_device *adev;
	struct drm_crtc *crtc;
	struct drm_connector *connector;
3017
	int r;
A
Alex Deucher 已提交
3018 3019 3020 3021 3022 3023 3024 3025 3026 3027

	if (dev == NULL || dev->dev_private == NULL) {
		return -ENODEV;
	}

	adev = dev->dev_private;

	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
		return 0;

3028
	adev->in_suspend = true;
A
Alex Deucher 已提交
3029 3030
	drm_kms_helper_poll_disable(dev);

3031 3032 3033
	if (fbcon)
		amdgpu_fbdev_set_suspend(adev, 1);

3034
	cancel_delayed_work_sync(&adev->delayed_init_work);
3035

3036 3037 3038 3039 3040 3041 3042
	if (!amdgpu_device_has_dc_support(adev)) {
		/* turn off display hw */
		drm_modeset_lock_all(dev);
		list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
			drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
		}
		drm_modeset_unlock_all(dev);
3043 3044 3045 3046 3047 3048
			/* unpin the front buffers and cursors */
		list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
			struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
			struct drm_framebuffer *fb = crtc->primary->fb;
			struct amdgpu_bo *robj;

3049
			if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3050 3051 3052 3053 3054 3055
				struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
				r = amdgpu_bo_reserve(aobj, true);
				if (r == 0) {
					amdgpu_bo_unpin(aobj);
					amdgpu_bo_unreserve(aobj);
				}
3056 3057
			}

3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068
			if (fb == NULL || fb->obj[0] == NULL) {
				continue;
			}
			robj = gem_to_amdgpu_bo(fb->obj[0]);
			/* don't unpin kernel fb objects */
			if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
				r = amdgpu_bo_reserve(robj, true);
				if (r == 0) {
					amdgpu_bo_unpin(robj);
					amdgpu_bo_unreserve(robj);
				}
A
Alex Deucher 已提交
3069 3070 3071
			}
		}
	}
3072 3073 3074

	amdgpu_amdkfd_suspend(adev);

3075 3076
	amdgpu_ras_suspend(adev);

3077 3078
	r = amdgpu_device_ip_suspend_phase1(adev);

A
Alex Deucher 已提交
3079 3080 3081
	/* evict vram memory */
	amdgpu_bo_evict_vram(adev);

3082
	amdgpu_fence_driver_suspend(adev);
A
Alex Deucher 已提交
3083

3084
	r = amdgpu_device_ip_suspend_phase2(adev);
A
Alex Deucher 已提交
3085

3086 3087 3088 3089
	/* evict remaining vram memory
	 * This second call to evict vram is to evict the gart page table
	 * using the CPU.
	 */
A
Alex Deucher 已提交
3090 3091 3092 3093 3094 3095 3096
	amdgpu_bo_evict_vram(adev);

	pci_save_state(dev->pdev);
	if (suspend) {
		/* Shut down the device */
		pci_disable_device(dev->pdev);
		pci_set_power_state(dev->pdev, PCI_D3hot);
J
jimqu 已提交
3097 3098 3099 3100
	} else {
		r = amdgpu_asic_reset(adev);
		if (r)
			DRM_ERROR("amdgpu asic reset failed\n");
A
Alex Deucher 已提交
3101 3102 3103 3104 3105 3106
	}

	return 0;
}

/**
3107
 * amdgpu_device_resume - initiate device resume
A
Alex Deucher 已提交
3108
 *
3109 3110 3111
 * @dev: drm dev pointer
 * @resume: resume state
 * @fbcon : notify the fbdev of resume
A
Alex Deucher 已提交
3112 3113 3114 3115 3116
 *
 * Bring the hw back to operating state (all asics).
 * Returns 0 for success or an error on failure.
 * Called at driver resume.
 */
3117
int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
A
Alex Deucher 已提交
3118 3119 3120
{
	struct drm_connector *connector;
	struct amdgpu_device *adev = dev->dev_private;
3121
	struct drm_crtc *crtc;
3122
	int r = 0;
A
Alex Deucher 已提交
3123 3124 3125 3126 3127 3128 3129

	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
		return 0;

	if (resume) {
		pci_set_power_state(dev->pdev, PCI_D0);
		pci_restore_state(dev->pdev);
J
jimqu 已提交
3130
		r = pci_enable_device(dev->pdev);
3131
		if (r)
3132
			return r;
A
Alex Deucher 已提交
3133 3134 3135
	}

	/* post card */
A
Alex Deucher 已提交
3136
	if (amdgpu_device_need_post(adev)) {
J
jimqu 已提交
3137 3138 3139 3140
		r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
		if (r)
			DRM_ERROR("amdgpu asic init failed\n");
	}
A
Alex Deucher 已提交
3141

3142
	r = amdgpu_device_ip_resume(adev);
3143
	if (r) {
3144
		DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
3145
		return r;
3146
	}
3147 3148
	amdgpu_fence_driver_resume(adev);

A
Alex Deucher 已提交
3149

3150
	r = amdgpu_device_ip_late_init(adev);
3151
	if (r)
3152
		return r;
A
Alex Deucher 已提交
3153

3154 3155 3156
	queue_delayed_work(system_wq, &adev->delayed_init_work,
			   msecs_to_jiffies(AMDGPU_RESUME_MS));

3157 3158 3159 3160 3161
	if (!amdgpu_device_has_dc_support(adev)) {
		/* pin cursors */
		list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
			struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);

3162
			if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3163 3164 3165 3166 3167 3168 3169 3170 3171
				struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
				r = amdgpu_bo_reserve(aobj, true);
				if (r == 0) {
					r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
					if (r != 0)
						DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
					amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
					amdgpu_bo_unreserve(aobj);
				}
3172 3173 3174
			}
		}
	}
3175 3176 3177
	r = amdgpu_amdkfd_resume(adev);
	if (r)
		return r;
3178

3179
	/* Make sure IB tests flushed */
3180
	flush_delayed_work(&adev->delayed_init_work);
3181

A
Alex Deucher 已提交
3182 3183
	/* blat the mode back in */
	if (fbcon) {
3184 3185 3186 3187 3188 3189 3190 3191 3192 3193
		if (!amdgpu_device_has_dc_support(adev)) {
			/* pre DCE11 */
			drm_helper_resume_force_mode(dev);

			/* turn on display hw */
			drm_modeset_lock_all(dev);
			list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
				drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
			}
			drm_modeset_unlock_all(dev);
A
Alex Deucher 已提交
3194
		}
3195
		amdgpu_fbdev_set_suspend(adev, 0);
A
Alex Deucher 已提交
3196 3197 3198
	}

	drm_kms_helper_poll_enable(dev);
3199

3200 3201
	amdgpu_ras_resume(adev);

3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213
	/*
	 * Most of the connector probing functions try to acquire runtime pm
	 * refs to ensure that the GPU is powered on when connector polling is
	 * performed. Since we're calling this from a runtime PM callback,
	 * trying to acquire rpm refs will cause us to deadlock.
	 *
	 * Since we're guaranteed to be holding the rpm lock, it's safe to
	 * temporarily disable the rpm helpers so this doesn't deadlock us.
	 */
#ifdef CONFIG_PM
	dev->dev->power.disable_depth++;
#endif
3214 3215 3216 3217
	if (!amdgpu_device_has_dc_support(adev))
		drm_helper_hpd_irq_event(dev);
	else
		drm_kms_helper_hotplug_event(dev);
3218 3219 3220
#ifdef CONFIG_PM
	dev->dev->power.disable_depth--;
#endif
3221 3222
	adev->in_suspend = false;

3223
	return 0;
A
Alex Deucher 已提交
3224 3225
}

3226 3227 3228 3229 3230 3231 3232 3233 3234 3235
/**
 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
 *
 * @adev: amdgpu_device pointer
 *
 * The list of all the hardware IPs that make up the asic is walked and
 * the check_soft_reset callbacks are run.  check_soft_reset determines
 * if the asic is still hung or not.
 * Returns true if any of the IPs are still in a hung state, false if not.
 */
3236
static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
3237 3238 3239 3240
{
	int i;
	bool asic_hang = false;

3241 3242 3243
	if (amdgpu_sriov_vf(adev))
		return true;

3244 3245 3246
	if (amdgpu_asic_need_full_reset(adev))
		return true;

3247
	for (i = 0; i < adev->num_ip_blocks; i++) {
3248
		if (!adev->ip_blocks[i].status.valid)
3249
			continue;
3250 3251 3252 3253 3254
		if (adev->ip_blocks[i].version->funcs->check_soft_reset)
			adev->ip_blocks[i].status.hang =
				adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
		if (adev->ip_blocks[i].status.hang) {
			DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
3255 3256 3257 3258 3259 3260
			asic_hang = true;
		}
	}
	return asic_hang;
}

3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271
/**
 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
 *
 * @adev: amdgpu_device pointer
 *
 * The list of all the hardware IPs that make up the asic is walked and the
 * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
 * handles any IP specific hardware or software state changes that are
 * necessary for a soft reset to succeed.
 * Returns 0 on success, negative error code on failure.
 */
3272
static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
3273 3274 3275 3276
{
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
3277
		if (!adev->ip_blocks[i].status.valid)
3278
			continue;
3279 3280 3281
		if (adev->ip_blocks[i].status.hang &&
		    adev->ip_blocks[i].version->funcs->pre_soft_reset) {
			r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
3282 3283 3284 3285 3286 3287 3288 3289
			if (r)
				return r;
		}
	}

	return 0;
}

3290 3291 3292 3293 3294 3295 3296 3297 3298
/**
 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
 *
 * @adev: amdgpu_device pointer
 *
 * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
 * reset is necessary to recover.
 * Returns true if a full asic reset is required, false if not.
 */
3299
static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
3300
{
3301 3302
	int i;

3303 3304 3305
	if (amdgpu_asic_need_full_reset(adev))
		return true;

3306
	for (i = 0; i < adev->num_ip_blocks; i++) {
3307
		if (!adev->ip_blocks[i].status.valid)
3308
			continue;
3309 3310 3311
		if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
3312 3313
		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3314
			if (adev->ip_blocks[i].status.hang) {
3315 3316 3317 3318
				DRM_INFO("Some block need full reset!\n");
				return true;
			}
		}
3319 3320 3321 3322
	}
	return false;
}

3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333
/**
 * amdgpu_device_ip_soft_reset - do a soft reset
 *
 * @adev: amdgpu_device pointer
 *
 * The list of all the hardware IPs that make up the asic is walked and the
 * soft_reset callbacks are run if the block is hung.  soft_reset handles any
 * IP specific hardware or software state changes that are necessary to soft
 * reset the IP.
 * Returns 0 on success, negative error code on failure.
 */
3334
static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
3335 3336 3337 3338
{
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
3339
		if (!adev->ip_blocks[i].status.valid)
3340
			continue;
3341 3342 3343
		if (adev->ip_blocks[i].status.hang &&
		    adev->ip_blocks[i].version->funcs->soft_reset) {
			r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
3344 3345 3346 3347 3348 3349 3350 3351
			if (r)
				return r;
		}
	}

	return 0;
}

3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362
/**
 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
 *
 * @adev: amdgpu_device pointer
 *
 * The list of all the hardware IPs that make up the asic is walked and the
 * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
 * handles any IP specific hardware or software state changes that are
 * necessary after the IP has been soft reset.
 * Returns 0 on success, negative error code on failure.
 */
3363
static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
3364 3365 3366 3367
{
	int i, r = 0;

	for (i = 0; i < adev->num_ip_blocks; i++) {
3368
		if (!adev->ip_blocks[i].status.valid)
3369
			continue;
3370 3371 3372
		if (adev->ip_blocks[i].status.hang &&
		    adev->ip_blocks[i].version->funcs->post_soft_reset)
			r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
3373 3374 3375 3376 3377 3378 3379
		if (r)
			return r;
	}

	return 0;
}

3380
/**
3381
 * amdgpu_device_recover_vram - Recover some VRAM contents
3382 3383 3384 3385 3386 3387
 *
 * @adev: amdgpu_device pointer
 *
 * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
 * restore things like GPUVM page tables after a GPU reset where
 * the contents of VRAM might be lost.
3388 3389 3390
 *
 * Returns:
 * 0 on success, negative error code on failure.
3391
 */
3392
static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
3393 3394
{
	struct dma_fence *fence = NULL, *next = NULL;
3395 3396
	struct amdgpu_bo *shadow;
	long r = 1, tmo;
3397 3398

	if (amdgpu_sriov_runtime(adev))
3399
		tmo = msecs_to_jiffies(8000);
3400 3401 3402 3403 3404
	else
		tmo = msecs_to_jiffies(100);

	DRM_INFO("recover vram bo from shadow start\n");
	mutex_lock(&adev->shadow_list_lock);
3405 3406 3407 3408
	list_for_each_entry(shadow, &adev->shadow_list, shadow_list) {

		/* No need to recover an evicted BO */
		if (shadow->tbo.mem.mem_type != TTM_PL_TT ||
3409
		    shadow->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET ||
3410 3411 3412 3413 3414 3415 3416
		    shadow->parent->tbo.mem.mem_type != TTM_PL_VRAM)
			continue;

		r = amdgpu_bo_restore_shadow(shadow, &next);
		if (r)
			break;

3417
		if (fence) {
3418
			tmo = dma_fence_wait_timeout(fence, false, tmo);
3419 3420
			dma_fence_put(fence);
			fence = next;
3421 3422
			if (tmo == 0) {
				r = -ETIMEDOUT;
3423
				break;
3424 3425 3426 3427
			} else if (tmo < 0) {
				r = tmo;
				break;
			}
3428 3429
		} else {
			fence = next;
3430 3431 3432 3433
		}
	}
	mutex_unlock(&adev->shadow_list_lock);

3434 3435
	if (fence)
		tmo = dma_fence_wait_timeout(fence, false, tmo);
3436 3437
	dma_fence_put(fence);

3438 3439
	if (r < 0 || tmo <= 0) {
		DRM_ERROR("recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
3440 3441
		return -EIO;
	}
3442

3443 3444
	DRM_INFO("recover vram bo from shadow done\n");
	return 0;
3445 3446
}

3447

3448
/**
3449
 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
3450 3451
 *
 * @adev: amdgpu device pointer
3452
 * @from_hypervisor: request from hypervisor
3453 3454
 *
 * do VF FLR and reinitialize Asic
3455
 * return 0 means succeeded otherwise failed
3456 3457 3458
 */
static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
				     bool from_hypervisor)
3459 3460 3461 3462 3463 3464 3465 3466 3467
{
	int r;

	if (from_hypervisor)
		r = amdgpu_virt_request_full_gpu(adev, true);
	else
		r = amdgpu_virt_reset_gpu(adev);
	if (r)
		return r;
3468

3469 3470
	amdgpu_amdkfd_pre_reset(adev);

3471
	/* Resume IP prior to SMC */
3472
	r = amdgpu_device_ip_reinit_early_sriov(adev);
3473 3474
	if (r)
		goto error;
3475 3476

	/* we need recover gart prior to run SMC/CP/SDMA resume */
3477
	amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
3478

3479 3480 3481 3482
	r = amdgpu_device_fw_loading(adev);
	if (r)
		return r;

3483
	/* now we are okay to resume SMC/CP/SDMA */
3484
	r = amdgpu_device_ip_reinit_late_sriov(adev);
3485 3486
	if (r)
		goto error;
3487 3488

	amdgpu_irq_gpu_reset_resume_helper(adev);
3489
	r = amdgpu_ib_ring_tests(adev);
3490
	amdgpu_amdkfd_post_reset(adev);
3491

3492
error:
3493
	amdgpu_virt_init_data_exchange(adev);
3494
	amdgpu_virt_release_full_gpu(adev, true);
3495
	if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
3496
		amdgpu_inc_vram_lost(adev);
3497
		r = amdgpu_device_recover_vram(adev);
3498 3499 3500 3501 3502
	}

	return r;
}

3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517
/**
 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
 *
 * @adev: amdgpu device pointer
 *
 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
 * a hung GPU.
 */
bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
{
	if (!amdgpu_device_ip_check_soft_reset(adev)) {
		DRM_INFO("Timeout, but no hardware hang detected.\n");
		return false;
	}

3518 3519 3520 3521 3522 3523 3524 3525
	if (amdgpu_gpu_recovery == 0)
		goto disabled;

	if (amdgpu_sriov_vf(adev))
		return true;

	if (amdgpu_gpu_recovery == -1) {
		switch (adev->asic_type) {
3526 3527
		case CHIP_BONAIRE:
		case CHIP_HAWAII:
3528 3529 3530 3531 3532 3533 3534 3535 3536 3537
		case CHIP_TOPAZ:
		case CHIP_TONGA:
		case CHIP_FIJI:
		case CHIP_POLARIS10:
		case CHIP_POLARIS11:
		case CHIP_POLARIS12:
		case CHIP_VEGAM:
		case CHIP_VEGA20:
		case CHIP_VEGA10:
		case CHIP_VEGA12:
3538
		case CHIP_RAVEN:
3539 3540 3541 3542
			break;
		default:
			goto disabled;
		}
3543 3544 3545
	}

	return true;
3546 3547 3548 3549

disabled:
		DRM_INFO("GPU recovery disabled.\n");
		return false;
3550 3551
}

3552

3553 3554 3555 3556 3557 3558
static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
					struct amdgpu_job *job,
					bool *need_full_reset_arg)
{
	int i, r = 0;
	bool need_full_reset  = *need_full_reset_arg;
3559 3560

	/* block all schedulers and reset given job's ring */
3561 3562 3563
	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
		struct amdgpu_ring *ring = adev->rings[i];

C
Chunming Zhou 已提交
3564
		if (!ring || !ring->sched.thread)
3565
			continue;
3566

M
Monk Liu 已提交
3567 3568
		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
		amdgpu_fence_driver_force_completion(ring);
3569
	}
A
Alex Deucher 已提交
3570

3571 3572 3573
	if(job)
		drm_sched_increase_karma(&job->base);

3574
	/* Don't suspend on bare metal if we are not going to HW reset the ASIC */
3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612
	if (!amdgpu_sriov_vf(adev)) {

		if (!need_full_reset)
			need_full_reset = amdgpu_device_ip_need_full_reset(adev);

		if (!need_full_reset) {
			amdgpu_device_ip_pre_soft_reset(adev);
			r = amdgpu_device_ip_soft_reset(adev);
			amdgpu_device_ip_post_soft_reset(adev);
			if (r || amdgpu_device_ip_check_soft_reset(adev)) {
				DRM_INFO("soft reset failed, will fallback to full reset!\n");
				need_full_reset = true;
			}
		}

		if (need_full_reset)
			r = amdgpu_device_ip_suspend(adev);

		*need_full_reset_arg = need_full_reset;
	}

	return r;
}

static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
			       struct list_head *device_list_handle,
			       bool *need_full_reset_arg)
{
	struct amdgpu_device *tmp_adev = NULL;
	bool need_full_reset = *need_full_reset_arg, vram_lost = false;
	int r = 0;

	/*
	 * ASIC reset has to be done on all HGMI hive nodes ASAP
	 * to allow proper links negotiation in FW (within 1 sec)
	 */
	if (need_full_reset) {
		list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3613 3614 3615 3616 3617 3618 3619 3620
			/* For XGMI run all resets in parallel to speed up the process */
			if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
				if (!queue_work(system_highpri_wq, &tmp_adev->xgmi_reset_work))
					r = -EALREADY;
			} else
				r = amdgpu_asic_reset(tmp_adev);

			if (r) {
E
Evan Quan 已提交
3621
				DRM_ERROR("ASIC reset failed with error, %d for drm dev, %s",
3622
					 r, tmp_adev->ddev->unique);
3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637
				break;
			}
		}

		/* For XGMI wait for all PSP resets to complete before proceed */
		if (!r) {
			list_for_each_entry(tmp_adev, device_list_handle,
					    gmc.xgmi.head) {
				if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
					flush_work(&tmp_adev->xgmi_reset_work);
					r = tmp_adev->asic_reset_res;
					if (r)
						break;
				}
			}
3638 3639 3640 3641 3642

			list_for_each_entry(tmp_adev, device_list_handle,
					gmc.xgmi.head) {
				amdgpu_ras_reserve_bad_pages(tmp_adev);
			}
3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660
		}
	}


	list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
		if (need_full_reset) {
			/* post card */
			if (amdgpu_atom_asic_init(tmp_adev->mode_info.atom_context))
				DRM_WARN("asic atom init failed!");

			if (!r) {
				dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
				r = amdgpu_device_ip_resume_phase1(tmp_adev);
				if (r)
					goto out;

				vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
				if (vram_lost) {
3661
					DRM_INFO("VRAM is lost due to GPU reset!\n");
3662
					amdgpu_inc_vram_lost(tmp_adev);
3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680
				}

				r = amdgpu_gtt_mgr_recover(
					&tmp_adev->mman.bdev.man[TTM_PL_TT]);
				if (r)
					goto out;

				r = amdgpu_device_fw_loading(tmp_adev);
				if (r)
					return r;

				r = amdgpu_device_ip_resume_phase2(tmp_adev);
				if (r)
					goto out;

				if (vram_lost)
					amdgpu_device_fill_reset_magic(tmp_adev);

3681 3682 3683 3684 3685 3686
				/*
				 * Add this ASIC as tracked as reset was already
				 * complete successfully.
				 */
				amdgpu_register_gpu_instance(tmp_adev);

3687 3688 3689 3690
				r = amdgpu_device_ip_late_init(tmp_adev);
				if (r)
					goto out;

3691
				/* must succeed. */
3692
				amdgpu_ras_resume(tmp_adev);
3693

3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724
				/* Update PSP FW topology after reset */
				if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1)
					r = amdgpu_xgmi_update_topology(hive, tmp_adev);
			}
		}


out:
		if (!r) {
			amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
			r = amdgpu_ib_ring_tests(tmp_adev);
			if (r) {
				dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
				r = amdgpu_device_ip_suspend(tmp_adev);
				need_full_reset = true;
				r = -EAGAIN;
				goto end;
			}
		}

		if (!r)
			r = amdgpu_device_recover_vram(tmp_adev);
		else
			tmp_adev->asic_reset_res = r;
	}

end:
	*need_full_reset_arg = need_full_reset;
	return r;
}

3725
static bool amdgpu_device_lock_adev(struct amdgpu_device *adev, bool trylock)
3726
{
3727 3728 3729 3730 3731
	if (trylock) {
		if (!mutex_trylock(&adev->lock_reset))
			return false;
	} else
		mutex_lock(&adev->lock_reset);
3732

3733 3734
	atomic_inc(&adev->gpu_reset_counter);
	adev->in_gpu_reset = 1;
3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745
	switch (amdgpu_asic_reset_method(adev)) {
	case AMD_RESET_METHOD_MODE1:
		adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
		break;
	case AMD_RESET_METHOD_MODE2:
		adev->mp1_state = PP_MP1_STATE_RESET;
		break;
	default:
		adev->mp1_state = PP_MP1_STATE_NONE;
		break;
	}
3746 3747 3748
	/* Block kfd: SRIOV would do it separately */
	if (!amdgpu_sriov_vf(adev))
                amdgpu_amdkfd_pre_reset(adev);
3749 3750

	return true;
3751
}
A
Alex Deucher 已提交
3752

3753 3754
static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
{
3755 3756 3757
	/*unlock kfd: SRIOV would do it separately */
	if (!amdgpu_sriov_vf(adev))
                amdgpu_amdkfd_post_reset(adev);
3758
	amdgpu_vf_error_trans_all(adev);
3759
	adev->mp1_state = PP_MP1_STATE_NONE;
3760 3761
	adev->in_gpu_reset = 0;
	mutex_unlock(&adev->lock_reset);
3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778
}


/**
 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
 *
 * @adev: amdgpu device pointer
 * @job: which job trigger hang
 *
 * Attempt to reset the GPU if it has hung (all asics).
 * Attempt to do soft-reset or full-reset and reinitialize Asic
 * Returns 0 for success or an error on failure.
 */

int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
			      struct amdgpu_job *job)
{
3779 3780
	struct list_head device_list, *device_list_handle =  NULL;
	bool need_full_reset, job_signaled;
3781 3782
	struct amdgpu_hive_info *hive = NULL;
	struct amdgpu_device *tmp_adev = NULL;
3783
	int i, r = 0;
3784

3785
	need_full_reset = job_signaled = false;
3786 3787 3788 3789
	INIT_LIST_HEAD(&device_list);

	dev_info(adev->dev, "GPU reset begin!\n");

3790
	cancel_delayed_work_sync(&adev->delayed_init_work);
3791

3792 3793
	hive = amdgpu_get_xgmi_hive(adev, false);

3794
	/*
3795 3796 3797 3798 3799
	 * Here we trylock to avoid chain of resets executing from
	 * either trigger by jobs on different adevs in XGMI hive or jobs on
	 * different schedulers for same device while this TO handler is running.
	 * We always reset all schedulers for device and all devices for XGMI
	 * hive so that should take care of them too.
3800
	 */
3801 3802 3803

	if (hive && !mutex_trylock(&hive->reset_lock)) {
		DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
3804
			  job ? job->base.id : -1, hive->hive_id);
3805
		return 0;
3806
	}
3807 3808

	/* Start with adev pre asic reset first for soft reset check.*/
3809 3810
	if (!amdgpu_device_lock_adev(adev, !hive)) {
		DRM_INFO("Bailing on TDR for s_job:%llx, as another already in progress",
3811
			  job ? job->base.id : -1);
3812
		return 0;
3813 3814 3815
	}

	/* Build list of devices to reset */
3816
	if  (adev->gmc.xgmi.num_physical_nodes > 1) {
3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832
		if (!hive) {
			amdgpu_device_unlock_adev(adev);
			return -ENODEV;
		}

		/*
		 * In case we are in XGMI hive mode device reset is done for all the
		 * nodes in the hive to retrain all XGMI links and hence the reset
		 * sequence is executed in loop on all nodes.
		 */
		device_list_handle = &hive->device_list;
	} else {
		list_add_tail(&adev->gmc.xgmi.head, &device_list);
		device_list_handle = &device_list;
	}

3833 3834 3835 3836 3837 3838 3839
	/*
	 * Mark these ASICs to be reseted as untracked first
	 * And add them back after reset completed
	 */
	list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head)
		amdgpu_unregister_gpu_instance(tmp_adev);

3840 3841
	/* block all schedulers and reset given job's ring */
	list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3842 3843 3844 3845
		/* disable ras on ALL IPs */
		if (amdgpu_device_ip_need_full_reset(tmp_adev))
			amdgpu_ras_suspend(tmp_adev);

3846 3847 3848 3849 3850 3851
		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
			struct amdgpu_ring *ring = tmp_adev->rings[i];

			if (!ring || !ring->sched.thread)
				continue;

3852
			drm_sched_stop(&ring->sched, job ? &job->base : NULL);
3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876
		}
	}


	/*
	 * Must check guilty signal here since after this point all old
	 * HW fences are force signaled.
	 *
	 * job->base holds a reference to parent fence
	 */
	if (job && job->base.s_fence->parent &&
	    dma_fence_is_signaled(job->base.s_fence->parent))
		job_signaled = true;

	if (!amdgpu_device_ip_need_full_reset(adev))
		device_list_handle = &device_list;

	if (job_signaled) {
		dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
		goto skip_hw_reset;
	}


	/* Guilty job will be freed after this*/
3877
	r = amdgpu_device_pre_asic_reset(adev, job, &need_full_reset);
3878 3879 3880 3881 3882 3883 3884
	if (r) {
		/*TODO Should we stop ?*/
		DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
			  r, adev->ddev->unique);
		adev->asic_reset_res = r;
	}

3885 3886 3887 3888 3889 3890
retry:	/* Rest of adevs pre asic reset from XGMI hive. */
	list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {

		if (tmp_adev == adev)
			continue;

3891
		amdgpu_device_lock_adev(tmp_adev, false);
3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914
		r = amdgpu_device_pre_asic_reset(tmp_adev,
						 NULL,
						 &need_full_reset);
		/*TODO Should we stop ?*/
		if (r) {
			DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
				  r, tmp_adev->ddev->unique);
			tmp_adev->asic_reset_res = r;
		}
	}

	/* Actual ASIC resets if needed.*/
	/* TODO Implement XGMI hive reset logic for SRIOV */
	if (amdgpu_sriov_vf(adev)) {
		r = amdgpu_device_reset_sriov(adev, job ? false : true);
		if (r)
			adev->asic_reset_res = r;
	} else {
		r  = amdgpu_do_asic_reset(hive, device_list_handle, &need_full_reset);
		if (r && r == -EAGAIN)
			goto retry;
	}

3915 3916
skip_hw_reset:

3917 3918
	/* Post ASIC reset for all devs .*/
	list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936
		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
			struct amdgpu_ring *ring = tmp_adev->rings[i];

			if (!ring || !ring->sched.thread)
				continue;

			/* No point to resubmit jobs if we didn't HW reset*/
			if (!tmp_adev->asic_reset_res && !job_signaled)
				drm_sched_resubmit_jobs(&ring->sched);

			drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
		}

		if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
			drm_helper_resume_force_mode(tmp_adev->ddev);
		}

		tmp_adev->asic_reset_res = 0;
3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948

		if (r) {
			/* bad news, how to tell it to userspace ? */
			dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
			amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
		} else {
			dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&adev->gpu_reset_counter));
		}

		amdgpu_device_unlock_adev(tmp_adev);
	}

3949
	if (hive)
3950
		mutex_unlock(&hive->reset_lock);
3951 3952 3953

	if (r)
		dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
A
Alex Deucher 已提交
3954 3955 3956
	return r;
}

3957 3958 3959 3960 3961 3962 3963 3964 3965
/**
 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
 *
 * @adev: amdgpu_device pointer
 *
 * Fetchs and stores in the driver the PCIE capabilities (gen speed
 * and lanes) of the slot the device is in. Handles APUs and
 * virtualized environments where PCIE config space may not be available.
 */
3966
static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
3967
{
3968
	struct pci_dev *pdev;
3969 3970
	enum pci_bus_speed speed_cap, platform_speed_cap;
	enum pcie_link_width platform_link_width;
3971

3972 3973
	if (amdgpu_pcie_gen_cap)
		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
3974

3975 3976
	if (amdgpu_pcie_lane_cap)
		adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
3977

3978 3979 3980 3981 3982 3983
	/* covers APUs as well */
	if (pci_is_root_bus(adev->pdev->bus)) {
		if (adev->pm.pcie_gen_mask == 0)
			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
		if (adev->pm.pcie_mlw_mask == 0)
			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3984
		return;
3985
	}
3986

3987 3988 3989
	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
		return;

3990 3991
	pcie_bandwidth_available(adev->pdev, NULL,
				 &platform_speed_cap, &platform_link_width);
3992

3993
	if (adev->pm.pcie_gen_mask == 0) {
3994 3995 3996 3997 3998
		/* asic caps */
		pdev = adev->pdev;
		speed_cap = pcie_get_speed_cap(pdev);
		if (speed_cap == PCI_SPEED_UNKNOWN) {
			adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3999 4000 4001
						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
		} else {
4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017
			if (speed_cap == PCIE_SPEED_16_0GT)
				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
			else if (speed_cap == PCIE_SPEED_8_0GT)
				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
			else if (speed_cap == PCIE_SPEED_5_0GT)
				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
			else
				adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
		}
		/* platform caps */
4018
		if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
4019 4020 4021
			adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
						   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
		} else {
4022
			if (platform_speed_cap == PCIE_SPEED_16_0GT)
4023 4024 4025 4026
				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
4027
			else if (platform_speed_cap == PCIE_SPEED_8_0GT)
4028 4029 4030
				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
4031
			else if (platform_speed_cap == PCIE_SPEED_5_0GT)
4032 4033 4034 4035 4036
				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
			else
				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;

4037 4038 4039
		}
	}
	if (adev->pm.pcie_mlw_mask == 0) {
4040
		if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
4041 4042
			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
		} else {
4043
			switch (platform_link_width) {
4044
			case PCIE_LNK_X32:
4045 4046 4047 4048 4049 4050 4051 4052
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
4053
			case PCIE_LNK_X16:
4054 4055 4056 4057 4058 4059 4060
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
4061
			case PCIE_LNK_X12:
4062 4063 4064 4065 4066 4067
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
4068
			case PCIE_LNK_X8:
4069 4070 4071 4072 4073
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
4074
			case PCIE_LNK_X4:
4075 4076 4077 4078
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
4079
			case PCIE_LNK_X2:
4080 4081 4082
				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
				break;
4083
			case PCIE_LNK_X1:
4084 4085 4086 4087 4088
				adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
				break;
			default:
				break;
			}
4089 4090 4091
		}
	}
}
A
Alex Deucher 已提交
4092