amdgpu_object.c 25.0 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
/*
 * Copyright 2009 Jerome Glisse.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 */
/*
 * Authors:
 *    Jerome Glisse <glisse@freedesktop.org>
 *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
 *    Dave Airlie
 */
#include <linux/list.h>
#include <linux/slab.h>
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
36
#include <drm/drm_cache.h>
A
Alex Deucher 已提交
37 38 39
#include "amdgpu.h"
#include "amdgpu_trace.h"

40 41 42 43 44 45 46 47
static bool amdgpu_need_backup(struct amdgpu_device *adev)
{
	if (adev->flags & AMD_IS_APU)
		return false;

	return amdgpu_gpu_recovery;
}

A
Alex Deucher 已提交
48 49
static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
{
50
	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
51
	struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo);
A
Alex Deucher 已提交
52

53
	amdgpu_bo_kunmap(bo);
A
Alex Deucher 已提交
54 55

	drm_gem_object_release(&bo->gem_base);
56
	amdgpu_bo_unref(&bo->parent);
57
	if (!list_empty(&bo->shadow_list)) {
58
		mutex_lock(&adev->shadow_list_lock);
59
		list_del_init(&bo->shadow_list);
60
		mutex_unlock(&adev->shadow_list_lock);
61
	}
A
Alex Deucher 已提交
62 63 64 65 66 67 68 69 70 71 72
	kfree(bo->metadata);
	kfree(bo);
}

bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
{
	if (bo->destroy == &amdgpu_ttm_bo_destroy)
		return true;
	return false;
}

73
void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
A
Alex Deucher 已提交
74
{
75 76 77 78
	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
	struct ttm_placement *placement = &abo->placement;
	struct ttm_place *places = abo->placements;
	u64 flags = abo->flags;
79
	u32 c = 0;
80

A
Alex Deucher 已提交
81
	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
82 83 84
		unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT;

		places[c].fpfn = 0;
85
		places[c].lpfn = 0;
86
		places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
87
			TTM_PL_FLAG_VRAM;
88

89 90 91 92
		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
			places[c].lpfn = visible_pfn;
		else
			places[c].flags |= TTM_PL_FLAG_TOPDOWN;
93 94 95

		if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
			places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
96
		c++;
A
Alex Deucher 已提交
97 98 99
	}

	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
100
		places[c].fpfn = 0;
101 102 103 104
		if (flags & AMDGPU_GEM_CREATE_SHADOW)
			places[c].lpfn = adev->mc.gart_size >> PAGE_SHIFT;
		else
			places[c].lpfn = 0;
105 106 107 108 109 110 111
		places[c].flags = TTM_PL_FLAG_TT;
		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
			places[c].flags |= TTM_PL_FLAG_WC |
				TTM_PL_FLAG_UNCACHED;
		else
			places[c].flags |= TTM_PL_FLAG_CACHED;
		c++;
A
Alex Deucher 已提交
112 113 114
	}

	if (domain & AMDGPU_GEM_DOMAIN_CPU) {
115 116 117 118 119 120 121 122 123
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_SYSTEM;
		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
			places[c].flags |= TTM_PL_FLAG_WC |
				TTM_PL_FLAG_UNCACHED;
		else
			places[c].flags |= TTM_PL_FLAG_CACHED;
		c++;
A
Alex Deucher 已提交
124 125 126
	}

	if (domain & AMDGPU_GEM_DOMAIN_GDS) {
127 128 129 130
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS;
		c++;
A
Alex Deucher 已提交
131
	}
132

A
Alex Deucher 已提交
133
	if (domain & AMDGPU_GEM_DOMAIN_GWS) {
134 135 136 137
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS;
		c++;
A
Alex Deucher 已提交
138
	}
139

A
Alex Deucher 已提交
140
	if (domain & AMDGPU_GEM_DOMAIN_OA) {
141 142 143 144
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA;
		c++;
A
Alex Deucher 已提交
145 146 147
	}

	if (!c) {
148 149 150 151
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
		c++;
A
Alex Deucher 已提交
152
	}
153

154
	placement->num_placement = c;
155
	placement->placement = places;
A
Alex Deucher 已提交
156

157 158
	placement->num_busy_placement = c;
	placement->busy_placement = places;
A
Alex Deucher 已提交
159 160
}

161
/**
162
 * amdgpu_bo_create_reserved - create reserved BO for kernel use
163 164 165 166 167 168 169 170 171
 *
 * @adev: amdgpu device object
 * @size: size for the new BO
 * @align: alignment for the new BO
 * @domain: where to place it
 * @bo_ptr: resulting BO
 * @gpu_addr: GPU addr of the pinned BO
 * @cpu_addr: optional CPU address mapping
 *
172 173
 * Allocates and pins a BO for kernel internal use, and returns it still
 * reserved.
174 175 176
 *
 * Returns 0 on success, negative error code otherwise.
 */
177 178 179 180
int amdgpu_bo_create_reserved(struct amdgpu_device *adev,
			      unsigned long size, int align,
			      u32 domain, struct amdgpu_bo **bo_ptr,
			      u64 *gpu_addr, void **cpu_addr)
181
{
182
	bool free = false;
183 184
	int r;

185 186 187 188
	if (!*bo_ptr) {
		r = amdgpu_bo_create(adev, size, align, true, domain,
				     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
189
				     NULL, NULL, 0, bo_ptr);
190 191 192 193 194 195
		if (r) {
			dev_err(adev->dev, "(%d) failed to allocate kernel bo\n",
				r);
			return r;
		}
		free = true;
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
	}

	r = amdgpu_bo_reserve(*bo_ptr, false);
	if (r) {
		dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
		goto error_free;
	}

	r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr);
	if (r) {
		dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
		goto error_unreserve;
	}

	if (cpu_addr) {
		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
		if (r) {
			dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
			goto error_unreserve;
		}
	}

	return 0;

error_unreserve:
	amdgpu_bo_unreserve(*bo_ptr);

error_free:
224 225
	if (free)
		amdgpu_bo_unref(bo_ptr);
226 227 228 229

	return r;
}

230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
/**
 * amdgpu_bo_create_kernel - create BO for kernel use
 *
 * @adev: amdgpu device object
 * @size: size for the new BO
 * @align: alignment for the new BO
 * @domain: where to place it
 * @bo_ptr: resulting BO
 * @gpu_addr: GPU addr of the pinned BO
 * @cpu_addr: optional CPU address mapping
 *
 * Allocates and pins a BO for kernel internal use.
 *
 * Returns 0 on success, negative error code otherwise.
 */
int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
			    unsigned long size, int align,
			    u32 domain, struct amdgpu_bo **bo_ptr,
			    u64 *gpu_addr, void **cpu_addr)
{
	int r;

	r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr,
				      gpu_addr, cpu_addr);

	if (r)
		return r;

	amdgpu_bo_unreserve(*bo_ptr);

	return 0;
}

263 264 265 266 267 268 269 270 271 272 273 274 275
/**
 * amdgpu_bo_free_kernel - free BO for kernel use
 *
 * @bo: amdgpu BO to free
 *
 * unmaps and unpin a BO for kernel internal use.
 */
void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
			   void **cpu_addr)
{
	if (*bo == NULL)
		return;

276
	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
		if (cpu_addr)
			amdgpu_bo_kunmap(*bo);

		amdgpu_bo_unpin(*bo);
		amdgpu_bo_unreserve(*bo);
	}
	amdgpu_bo_unref(bo);

	if (gpu_addr)
		*gpu_addr = 0;

	if (cpu_addr)
		*cpu_addr = NULL;
}

292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
/* Validate bo size is bit bigger then the request domain */
static bool amdgpu_bo_validate_size(struct amdgpu_device *adev,
					  unsigned long size, u32 domain)
{
	struct ttm_mem_type_manager *man = NULL;

	/*
	 * If GTT is part of requested domains the check must succeed to
	 * allow fall back to GTT
	 */
	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
		man = &adev->mman.bdev.man[TTM_PL_TT];

		if (size < (man->size << PAGE_SHIFT))
			return true;
		else
			goto fail;
	}

	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
		man = &adev->mman.bdev.man[TTM_PL_VRAM];

		if (size < (man->size << PAGE_SHIFT))
			return true;
		else
			goto fail;
	}


	/* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */
	return true;

fail:
325 326
	DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size,
		  man->size << PAGE_SHIFT);
327 328 329
	return false;
}

330 331 332 333 334 335 336
static int amdgpu_bo_do_create(struct amdgpu_device *adev,
			       unsigned long size, int byte_align,
			       bool kernel, u32 domain, u64 flags,
			       struct sg_table *sg,
			       struct reservation_object *resv,
			       uint64_t init_value,
			       struct amdgpu_bo **bo_ptr)
A
Alex Deucher 已提交
337
{
338 339 340 341 342 343
	struct ttm_operation_ctx ctx = {
		.interruptible = !kernel,
		.no_wait_gpu = false,
		.allow_reserved_eviction = true,
		.resv = resv
	};
A
Alex Deucher 已提交
344 345 346 347 348 349 350 351 352
	struct amdgpu_bo *bo;
	enum ttm_bo_type type;
	unsigned long page_align;
	size_t acc_size;
	int r;

	page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
	size = ALIGN(size, PAGE_SIZE);

353 354 355
	if (!amdgpu_bo_validate_size(adev, size, domain))
		return -ENOMEM;

A
Alex Deucher 已提交
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
	if (kernel) {
		type = ttm_bo_type_kernel;
	} else if (sg) {
		type = ttm_bo_type_sg;
	} else {
		type = ttm_bo_type_device;
	}
	*bo_ptr = NULL;

	acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
				       sizeof(struct amdgpu_bo));

	bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);
	if (bo == NULL)
		return -ENOMEM;
	r = drm_gem_object_init(adev->ddev, &bo->gem_base, size);
	if (unlikely(r)) {
		kfree(bo);
		return r;
	}
376
	INIT_LIST_HEAD(&bo->shadow_list);
A
Alex Deucher 已提交
377
	INIT_LIST_HEAD(&bo->va);
K
Kent Russell 已提交
378
	bo->preferred_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
379 380 381 382 383
					 AMDGPU_GEM_DOMAIN_GTT |
					 AMDGPU_GEM_DOMAIN_CPU |
					 AMDGPU_GEM_DOMAIN_GDS |
					 AMDGPU_GEM_DOMAIN_GWS |
					 AMDGPU_GEM_DOMAIN_OA);
K
Kent Russell 已提交
384
	bo->allowed_domains = bo->preferred_domains;
385 386
	if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
A
Alex Deucher 已提交
387 388

	bo->flags = flags;
389

390 391 392 393 394 395 396 397 398 399 400
#ifdef CONFIG_X86_32
	/* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
	 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
	 */
	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
	/* Don't try to enable write-combining when it can't work, or things
	 * may be slow
	 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
	 */

401
#ifndef CONFIG_COMPILE_TEST
402 403
#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
	 thanks to write-combining
404
#endif
405 406 407 408 409 410

	if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
		DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
			      "better performance thanks to write-combining\n");
	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
#else
411 412 413 414 415
	/* For architectures that don't support WC memory,
	 * mask out the WC flag from the BO
	 */
	if (!drm_arch_can_wc_memory())
		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
416
#endif
417

418 419
	bo->tbo.bdev = &adev->mman.bdev;
	amdgpu_ttm_placement_from_domain(bo, domain);
420

421
	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
422
				 &bo->placement, page_align, &ctx, NULL,
423
				 acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
424 425 426
	if (unlikely(r != 0))
		return r;

427 428 429
	if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
	    bo->tbo.mem.mem_type == TTM_PL_VRAM &&
	    bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)
430 431
		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved,
					     ctx.bytes_moved);
432
	else
433
		amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0);
434

435
	if (kernel)
436
		bo->tbo.priority = 1;
437

438 439
	if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
	    bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
440
		struct dma_fence *fence;
441

442
		r = amdgpu_fill_buffer(bo, init_value, bo->tbo.resv, &fence);
443 444 445
		if (unlikely(r))
			goto fail_unreserve;

446
		amdgpu_bo_fence(bo, fence, false);
447 448 449
		dma_fence_put(bo->tbo.moving);
		bo->tbo.moving = dma_fence_get(fence);
		dma_fence_put(fence);
450
	}
451
	if (!resv)
452
		amdgpu_bo_unreserve(bo);
A
Alex Deucher 已提交
453 454 455 456
	*bo_ptr = bo;

	trace_amdgpu_bo_create(bo);

457 458 459 460
	/* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
	if (type == ttm_bo_type_device)
		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;

A
Alex Deucher 已提交
461
	return 0;
462 463

fail_unreserve:
464 465
	if (!resv)
		ww_mutex_unlock(&bo->tbo.resv->lock);
466 467
	amdgpu_bo_unref(&bo);
	return r;
A
Alex Deucher 已提交
468 469
}

470 471 472 473 474 475 476 477 478
static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
				   unsigned long size, int byte_align,
				   struct amdgpu_bo *bo)
{
	int r;

	if (bo->shadow)
		return 0;

479 480 481 482 483 484
	r = amdgpu_bo_do_create(adev, size, byte_align, true,
				AMDGPU_GEM_DOMAIN_GTT,
				AMDGPU_GEM_CREATE_CPU_GTT_USWC |
				AMDGPU_GEM_CREATE_SHADOW,
				NULL, bo->tbo.resv, 0,
				&bo->shadow);
485
	if (!r) {
486
		bo->shadow->parent = amdgpu_bo_ref(bo);
487 488 489 490
		mutex_lock(&adev->shadow_list_lock);
		list_add_tail(&bo->shadow_list, &adev->shadow_list);
		mutex_unlock(&adev->shadow_list_lock);
	}
491 492 493 494

	return r;
}

495 496 497
/* init_value will only take effect when flags contains
 * AMDGPU_GEM_CREATE_VRAM_CLEARED.
 */
498 499 500
int amdgpu_bo_create(struct amdgpu_device *adev,
		     unsigned long size, int byte_align,
		     bool kernel, u32 domain, u64 flags,
501 502
		     struct sg_table *sg,
		     struct reservation_object *resv,
503
		     uint64_t init_value,
504
		     struct amdgpu_bo **bo_ptr)
505
{
506
	uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW;
507
	int r;
508

509 510
	r = amdgpu_bo_do_create(adev, size, byte_align, kernel, domain,
				parent_flags, sg, resv, init_value, bo_ptr);
511 512 513
	if (r)
		return r;

514 515 516 517
	if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_need_backup(adev)) {
		if (!resv)
			WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv,
							NULL));
518

519
		r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
520 521

		if (!resv)
522
			reservation_object_unlock((*bo_ptr)->tbo.resv);
523

524 525 526 527 528
		if (r)
			amdgpu_bo_unref(bo_ptr);
	}

	return r;
529 530
}

531 532 533 534
int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
			       struct amdgpu_ring *ring,
			       struct amdgpu_bo *bo,
			       struct reservation_object *resv,
535
			       struct dma_fence **fence,
536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
			       bool direct)

{
	struct amdgpu_bo *shadow = bo->shadow;
	uint64_t bo_addr, shadow_addr;
	int r;

	if (!shadow)
		return -EINVAL;

	bo_addr = amdgpu_bo_gpu_offset(bo);
	shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);

	r = reservation_object_reserve_shared(bo->tbo.resv);
	if (r)
		goto err;

	r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr,
			       amdgpu_bo_size(bo), resv, fence,
555
			       direct, false);
556 557 558 559 560 561 562
	if (!r)
		amdgpu_bo_fence(bo, *fence, true);

err:
	return r;
}

563 564
int amdgpu_bo_validate(struct amdgpu_bo *bo)
{
565
	struct ttm_operation_ctx ctx = { false, false };
566 567 568 569 570 571
	uint32_t domain;
	int r;

	if (bo->pin_count)
		return 0;

K
Kent Russell 已提交
572
	domain = bo->preferred_domains;
573 574 575

retry:
	amdgpu_ttm_placement_from_domain(bo, domain);
576
	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
577 578 579 580 581 582 583 584
	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
		domain = bo->allowed_domains;
		goto retry;
	}

	return r;
}

585 586 587 588
int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
				  struct amdgpu_ring *ring,
				  struct amdgpu_bo *bo,
				  struct reservation_object *resv,
589
				  struct dma_fence **fence,
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608
				  bool direct)

{
	struct amdgpu_bo *shadow = bo->shadow;
	uint64_t bo_addr, shadow_addr;
	int r;

	if (!shadow)
		return -EINVAL;

	bo_addr = amdgpu_bo_gpu_offset(bo);
	shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);

	r = reservation_object_reserve_shared(bo->tbo.resv);
	if (r)
		goto err;

	r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr,
			       amdgpu_bo_size(bo), resv, fence,
609
			       direct, false);
610 611 612 613 614 615 616
	if (!r)
		amdgpu_bo_fence(bo, *fence, true);

err:
	return r;
}

A
Alex Deucher 已提交
617 618
int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
{
619
	void *kptr;
620
	long r;
A
Alex Deucher 已提交
621

622 623 624
	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
		return -EPERM;

625 626 627 628
	kptr = amdgpu_bo_kptr(bo);
	if (kptr) {
		if (ptr)
			*ptr = kptr;
A
Alex Deucher 已提交
629 630
		return 0;
	}
631 632 633 634 635 636

	r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false,
						MAX_SCHEDULE_TIMEOUT);
	if (r < 0)
		return r;

A
Alex Deucher 已提交
637
	r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
638
	if (r)
A
Alex Deucher 已提交
639
		return r;
640 641

	if (ptr)
642
		*ptr = amdgpu_bo_kptr(bo);
643

A
Alex Deucher 已提交
644 645 646
	return 0;
}

647 648 649 650 651 652 653
void *amdgpu_bo_kptr(struct amdgpu_bo *bo)
{
	bool is_iomem;

	return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
}

A
Alex Deucher 已提交
654 655
void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
{
656 657
	if (bo->kmap.bo)
		ttm_bo_kunmap(&bo->kmap);
A
Alex Deucher 已提交
658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681
}

struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
{
	if (bo == NULL)
		return NULL;

	ttm_bo_reference(&bo->tbo);
	return bo;
}

void amdgpu_bo_unref(struct amdgpu_bo **bo)
{
	struct ttm_buffer_object *tbo;

	if ((*bo) == NULL)
		return;

	tbo = &((*bo)->tbo);
	ttm_bo_unref(&tbo);
	if (tbo == NULL)
		*bo = NULL;
}

682 683
int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
			     u64 min_offset, u64 max_offset,
A
Alex Deucher 已提交
684 685
			     u64 *gpu_addr)
{
686
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
687
	struct ttm_operation_ctx ctx = { false, false };
A
Alex Deucher 已提交
688 689
	int r, i;

690
	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
A
Alex Deucher 已提交
691 692
		return -EPERM;

693 694 695
	if (WARN_ON_ONCE(min_offset > max_offset))
		return -EINVAL;

696 697 698 699
	/* A shared bo cannot be migrated to VRAM */
	if (bo->prime_shared_count && (domain == AMDGPU_GEM_DOMAIN_VRAM))
		return -EINVAL;

A
Alex Deucher 已提交
700
	if (bo->pin_count) {
701 702
		uint32_t mem_type = bo->tbo.mem.mem_type;

703
		if (!(domain & amdgpu_mem_type_to_domain(mem_type)))
704 705
			return -EINVAL;

A
Alex Deucher 已提交
706 707 708 709 710
		bo->pin_count++;
		if (gpu_addr)
			*gpu_addr = amdgpu_bo_gpu_offset(bo);

		if (max_offset != 0) {
711
			u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset;
A
Alex Deucher 已提交
712 713 714 715 716 717
			WARN_ON_ONCE(max_offset <
				     (amdgpu_bo_gpu_offset(bo) - domain_start));
		}

		return 0;
	}
718 719

	bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
720 721 722
	/* force to pin into visible video ram */
	if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS))
		bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
A
Alex Deucher 已提交
723 724
	amdgpu_ttm_placement_from_domain(bo, domain);
	for (i = 0; i < bo->placement.num_placement; i++) {
725 726 727 728 729
		unsigned fpfn, lpfn;

		fpfn = min_offset >> PAGE_SHIFT;
		lpfn = max_offset >> PAGE_SHIFT;

730 731
		if (fpfn > bo->placements[i].fpfn)
			bo->placements[i].fpfn = fpfn;
732 733
		if (!bo->placements[i].lpfn ||
		    (lpfn && lpfn < bo->placements[i].lpfn))
734
			bo->placements[i].lpfn = lpfn;
A
Alex Deucher 已提交
735 736 737
		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
	}

738
	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
739
	if (unlikely(r)) {
740
		dev_err(adev->dev, "%p pin failed\n", bo);
741 742 743
		goto error;
	}

744
	r = amdgpu_ttm_alloc_gart(&bo->tbo);
745 746 747 748 749
	if (unlikely(r)) {
		dev_err(adev->dev, "%p bind failed\n", bo);
		goto error;
	}

750
	bo->pin_count = 1;
751
	if (gpu_addr != NULL)
752
		*gpu_addr = amdgpu_bo_gpu_offset(bo);
753 754

	domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
755
	if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
756
		adev->vram_pin_size += amdgpu_bo_size(bo);
757
		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
758
			adev->invisible_pin_size += amdgpu_bo_size(bo);
759
	} else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
760
		adev->gart_pin_size += amdgpu_bo_size(bo);
A
Alex Deucher 已提交
761
	}
762 763

error:
A
Alex Deucher 已提交
764 765 766 767 768
	return r;
}

int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
{
769
	return amdgpu_bo_pin_restricted(bo, domain, 0, 0, gpu_addr);
A
Alex Deucher 已提交
770 771 772 773
}

int amdgpu_bo_unpin(struct amdgpu_bo *bo)
{
774
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
775
	struct ttm_operation_ctx ctx = { false, false };
A
Alex Deucher 已提交
776 777 778
	int r, i;

	if (!bo->pin_count) {
779
		dev_warn(adev->dev, "%p unpin not necessary\n", bo);
A
Alex Deucher 已提交
780 781 782 783 784 785 786 787 788
		return 0;
	}
	bo->pin_count--;
	if (bo->pin_count)
		return 0;
	for (i = 0; i < bo->placement.num_placement; i++) {
		bo->placements[i].lpfn = 0;
		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
	}
789
	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
790
	if (unlikely(r)) {
791
		dev_err(adev->dev, "%p validate failed for unpin\n", bo);
792
		goto error;
A
Alex Deucher 已提交
793
	}
794 795

	if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
796
		adev->vram_pin_size -= amdgpu_bo_size(bo);
797
		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
798
			adev->invisible_pin_size -= amdgpu_bo_size(bo);
799
	} else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
800
		adev->gart_pin_size -= amdgpu_bo_size(bo);
801 802 803
	}

error:
A
Alex Deucher 已提交
804 805 806 807 808 809
	return r;
}

int amdgpu_bo_evict_vram(struct amdgpu_device *adev)
{
	/* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */
810
	if (0 && (adev->flags & AMD_IS_APU)) {
A
Alex Deucher 已提交
811 812 813 814 815 816
		/* Useless to evict on IGP chips */
		return 0;
	}
	return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM);
}

817 818 819 820 821 822 823 824 825 826 827
static const char *amdgpu_vram_names[] = {
	"UNKNOWN",
	"GDDR1",
	"DDR2",
	"GDDR3",
	"GDDR4",
	"GDDR5",
	"HBM",
	"DDR3"
};

A
Alex Deucher 已提交
828 829
int amdgpu_bo_init(struct amdgpu_device *adev)
{
830 831 832 833
	/* reserve PAT memory space to WC for VRAM */
	arch_io_reserve_memtype_wc(adev->mc.aper_base,
				   adev->mc.aper_size);

A
Alex Deucher 已提交
834 835 836 837
	/* Add an MTRR for the VRAM */
	adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base,
					      adev->mc.aper_size);
	DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
838 839
		 adev->mc.mc_vram_size >> 20,
		 (unsigned long long)adev->mc.aper_size >> 20);
840 841
	DRM_INFO("RAM width %dbits %s\n",
		 adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]);
A
Alex Deucher 已提交
842 843 844 845 846 847 848
	return amdgpu_ttm_init(adev);
}

void amdgpu_bo_fini(struct amdgpu_device *adev)
{
	amdgpu_ttm_fini(adev);
	arch_phys_wc_del(adev->mc.vram_mtrr);
849
	arch_io_free_memtype_wc(adev->mc.aper_base, adev->mc.aper_size);
A
Alex Deucher 已提交
850 851 852 853 854 855 856 857 858 859
}

int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo,
			     struct vm_area_struct *vma)
{
	return ttm_fbdev_mmap(vma, &bo->tbo);
}

int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
{
860 861 862 863
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);

	if (adev->family <= AMDGPU_FAMILY_CZ &&
	    AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
A
Alex Deucher 已提交
864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885
		return -EINVAL;

	bo->tiling_flags = tiling_flags;
	return 0;
}

void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
{
	lockdep_assert_held(&bo->tbo.resv->lock.base);

	if (tiling_flags)
		*tiling_flags = bo->tiling_flags;
}

int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
			    uint32_t metadata_size, uint64_t flags)
{
	void *buffer;

	if (!metadata_size) {
		if (bo->metadata_size) {
			kfree(bo->metadata);
886
			bo->metadata = NULL;
A
Alex Deucher 已提交
887 888 889 890 891 892 893 894
			bo->metadata_size = 0;
		}
		return 0;
	}

	if (metadata == NULL)
		return -EINVAL;

895
	buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
A
Alex Deucher 已提交
896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930
	if (buffer == NULL)
		return -ENOMEM;

	kfree(bo->metadata);
	bo->metadata_flags = flags;
	bo->metadata = buffer;
	bo->metadata_size = metadata_size;

	return 0;
}

int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
			   size_t buffer_size, uint32_t *metadata_size,
			   uint64_t *flags)
{
	if (!buffer && !metadata_size)
		return -EINVAL;

	if (buffer) {
		if (buffer_size < bo->metadata_size)
			return -EINVAL;

		if (bo->metadata_size)
			memcpy(buffer, bo->metadata, bo->metadata_size);
	}

	if (metadata_size)
		*metadata_size = bo->metadata_size;
	if (flags)
		*flags = bo->metadata_flags;

	return 0;
}

void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
931
			   bool evict,
A
Alex Deucher 已提交
932 933
			   struct ttm_mem_reg *new_mem)
{
934
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
935
	struct amdgpu_bo *abo;
936
	struct ttm_mem_reg *old_mem = &bo->mem;
A
Alex Deucher 已提交
937 938 939 940

	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
		return;

941
	abo = ttm_to_amdgpu_bo(bo);
942
	amdgpu_vm_bo_invalidate(adev, abo, evict);
A
Alex Deucher 已提交
943

944 945
	amdgpu_bo_kunmap(abo);

946 947 948 949
	/* remember the eviction */
	if (evict)
		atomic64_inc(&adev->num_evictions);

A
Alex Deucher 已提交
950 951 952 953 954
	/* update statistics */
	if (!new_mem)
		return;

	/* move_notify is called before move happens */
955
	trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
A
Alex Deucher 已提交
956 957 958 959
}

int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
{
960
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
961
	struct ttm_operation_ctx ctx = { false, false };
962
	struct amdgpu_bo *abo;
963 964
	unsigned long offset, size;
	int r;
A
Alex Deucher 已提交
965 966 967

	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
		return 0;
968

969
	abo = ttm_to_amdgpu_bo(bo);
970 971 972 973

	/* Remember that this BO was accessed by the CPU */
	abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;

974 975 976 977 978
	if (bo->mem.mem_type != TTM_PL_VRAM)
		return 0;

	size = bo->mem.num_pages << PAGE_SHIFT;
	offset = bo->mem.start << PAGE_SHIFT;
979
	if ((offset + size) <= adev->mc.visible_vram_size)
980 981
		return 0;

982 983 984 985
	/* Can't move a pinned BO to visible VRAM */
	if (abo->pin_count > 0)
		return -EINVAL;

986
	/* hurrah the memory is not visible ! */
987
	atomic64_inc(&adev->num_vram_cpu_page_faults);
988 989 990 991 992 993 994
	amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
					 AMDGPU_GEM_DOMAIN_GTT);

	/* Avoid costly evictions; only set GTT as a busy placement */
	abo->placement.num_busy_placement = 1;
	abo->placement.busy_placement = &abo->placements[1];

995
	r = ttm_bo_validate(bo, &abo->placement, &ctx);
996
	if (unlikely(r != 0))
997 998 999 1000
		return r;

	offset = bo->mem.start << PAGE_SHIFT;
	/* this should never happen */
1001 1002
	if (bo->mem.mem_type == TTM_PL_VRAM &&
	    (offset + size) > adev->mc.visible_vram_size)
1003 1004
		return -EINVAL;

A
Alex Deucher 已提交
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
	return 0;
}

/**
 * amdgpu_bo_fence - add fence to buffer object
 *
 * @bo: buffer object in question
 * @fence: fence to add
 * @shared: true if fence should be added shared
 *
 */
1016
void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
A
Alex Deucher 已提交
1017 1018 1019 1020 1021
		     bool shared)
{
	struct reservation_object *resv = bo->tbo.resv;

	if (shared)
1022
		reservation_object_add_shared_fence(resv, fence);
A
Alex Deucher 已提交
1023
	else
1024
		reservation_object_add_excl_fence(resv, fence);
A
Alex Deucher 已提交
1025
}
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038

/**
 * amdgpu_bo_gpu_offset - return GPU offset of bo
 * @bo:	amdgpu object for which we query the offset
 *
 * Returns current GPU offset of the object.
 *
 * Note: object should either be pinned or reserved when calling this
 * function, it might be useful to add check for this for debugging.
 */
u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
{
	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
1039
	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT &&
1040
		     !amdgpu_gtt_mgr_has_gart_addr(&bo->tbo.mem));
1041 1042
	WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
		     !bo->pin_count);
1043
	WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
1044 1045
	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM &&
		     !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1046 1047 1048

	return bo->tbo.offset;
}