amdgpu_object.c 25.3 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
/*
 * Copyright 2009 Jerome Glisse.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 */
/*
 * Authors:
 *    Jerome Glisse <glisse@freedesktop.org>
 *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
 *    Dave Airlie
 */
#include <linux/list.h>
#include <linux/slab.h>
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
36
#include <drm/drm_cache.h>
A
Alex Deucher 已提交
37 38 39 40 41 42
#include "amdgpu.h"
#include "amdgpu_trace.h"



static u64 amdgpu_get_vis_part_size(struct amdgpu_device *adev,
43
						struct ttm_mem_reg *mem)
A
Alex Deucher 已提交
44
{
45 46 47 48 49 50 51
	if (mem->start << PAGE_SHIFT >= adev->mc.visible_vram_size)
		return 0;

	return ((mem->start << PAGE_SHIFT) + mem->size) >
		adev->mc.visible_vram_size ?
		adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT) :
		mem->size;
A
Alex Deucher 已提交
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
}

static void amdgpu_update_memory_usage(struct amdgpu_device *adev,
		       struct ttm_mem_reg *old_mem,
		       struct ttm_mem_reg *new_mem)
{
	u64 vis_size;
	if (!adev)
		return;

	if (new_mem) {
		switch (new_mem->mem_type) {
		case TTM_PL_TT:
			atomic64_add(new_mem->size, &adev->gtt_usage);
			break;
		case TTM_PL_VRAM:
			atomic64_add(new_mem->size, &adev->vram_usage);
			vis_size = amdgpu_get_vis_part_size(adev, new_mem);
			atomic64_add(vis_size, &adev->vram_vis_usage);
			break;
		}
	}

	if (old_mem) {
		switch (old_mem->mem_type) {
		case TTM_PL_TT:
			atomic64_sub(old_mem->size, &adev->gtt_usage);
			break;
		case TTM_PL_VRAM:
			atomic64_sub(old_mem->size, &adev->vram_usage);
			vis_size = amdgpu_get_vis_part_size(adev, old_mem);
			atomic64_sub(vis_size, &adev->vram_vis_usage);
			break;
		}
	}
}

static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
{
91
	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
A
Alex Deucher 已提交
92 93 94 95
	struct amdgpu_bo *bo;

	bo = container_of(tbo, struct amdgpu_bo, tbo);

96
	amdgpu_update_memory_usage(adev, &bo->tbo.mem, NULL);
A
Alex Deucher 已提交
97 98

	drm_gem_object_release(&bo->gem_base);
99
	amdgpu_bo_unref(&bo->parent);
100
	if (!list_empty(&bo->shadow_list)) {
101
		mutex_lock(&adev->shadow_list_lock);
102
		list_del_init(&bo->shadow_list);
103
		mutex_unlock(&adev->shadow_list_lock);
104
	}
A
Alex Deucher 已提交
105 106 107 108 109 110 111 112 113 114 115
	kfree(bo->metadata);
	kfree(bo);
}

bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
{
	if (bo->destroy == &amdgpu_ttm_bo_destroy)
		return true;
	return false;
}

116 117
static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
				      struct ttm_placement *placement,
118
				      struct ttm_place *places,
119
				      u32 domain, u64 flags)
A
Alex Deucher 已提交
120
{
121
	u32 c = 0;
122

A
Alex Deucher 已提交
123
	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
124 125 126
		unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT;

		places[c].fpfn = 0;
127
		places[c].lpfn = 0;
128
		places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
129
			TTM_PL_FLAG_VRAM;
130

131 132 133 134
		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
			places[c].lpfn = visible_pfn;
		else
			places[c].flags |= TTM_PL_FLAG_TOPDOWN;
135 136 137

		if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
			places[c].flags |= TTM_PL_FLAG_CONTIGUOUS;
138
		c++;
A
Alex Deucher 已提交
139 140 141
	}

	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
142 143 144 145 146 147 148 149 150
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_TT;
		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
			places[c].flags |= TTM_PL_FLAG_WC |
				TTM_PL_FLAG_UNCACHED;
		else
			places[c].flags |= TTM_PL_FLAG_CACHED;
		c++;
A
Alex Deucher 已提交
151 152 153
	}

	if (domain & AMDGPU_GEM_DOMAIN_CPU) {
154 155 156 157 158 159 160 161 162
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_SYSTEM;
		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
			places[c].flags |= TTM_PL_FLAG_WC |
				TTM_PL_FLAG_UNCACHED;
		else
			places[c].flags |= TTM_PL_FLAG_CACHED;
		c++;
A
Alex Deucher 已提交
163 164 165
	}

	if (domain & AMDGPU_GEM_DOMAIN_GDS) {
166 167 168 169
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS;
		c++;
A
Alex Deucher 已提交
170
	}
171

A
Alex Deucher 已提交
172
	if (domain & AMDGPU_GEM_DOMAIN_GWS) {
173 174 175 176
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS;
		c++;
A
Alex Deucher 已提交
177
	}
178

A
Alex Deucher 已提交
179
	if (domain & AMDGPU_GEM_DOMAIN_OA) {
180 181 182 183
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA;
		c++;
A
Alex Deucher 已提交
184 185 186
	}

	if (!c) {
187 188 189 190
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
		c++;
A
Alex Deucher 已提交
191
	}
192

193
	placement->num_placement = c;
194
	placement->placement = places;
A
Alex Deucher 已提交
195

196 197
	placement->num_busy_placement = c;
	placement->busy_placement = places;
A
Alex Deucher 已提交
198 199
}

200
void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
201
{
202 203 204 205
	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);

	amdgpu_ttm_placement_init(adev, &abo->placement, abo->placements,
				  domain, abo->flags);
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
}

static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
					struct ttm_placement *placement)
{
	BUG_ON(placement->num_placement > (AMDGPU_GEM_DOMAIN_MAX + 1));

	memcpy(bo->placements, placement->placement,
	       placement->num_placement * sizeof(struct ttm_place));
	bo->placement.num_placement = placement->num_placement;
	bo->placement.num_busy_placement = placement->num_busy_placement;
	bo->placement.placement = bo->placements;
	bo->placement.busy_placement = bo->placements;
}

221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243
/**
 * amdgpu_bo_create_kernel - create BO for kernel use
 *
 * @adev: amdgpu device object
 * @size: size for the new BO
 * @align: alignment for the new BO
 * @domain: where to place it
 * @bo_ptr: resulting BO
 * @gpu_addr: GPU addr of the pinned BO
 * @cpu_addr: optional CPU address mapping
 *
 * Allocates and pins a BO for kernel internal use.
 *
 * Returns 0 on success, negative error code otherwise.
 */
int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
			    unsigned long size, int align,
			    u32 domain, struct amdgpu_bo **bo_ptr,
			    u64 *gpu_addr, void **cpu_addr)
{
	int r;

	r = amdgpu_bo_create(adev, size, align, true, domain,
244 245
			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
			     NULL, NULL, bo_ptr);
	if (r) {
		dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", r);
		return r;
	}

	r = amdgpu_bo_reserve(*bo_ptr, false);
	if (r) {
		dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
		goto error_free;
	}

	r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr);
	if (r) {
		dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
		goto error_unreserve;
	}

	if (cpu_addr) {
		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
		if (r) {
			dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
			goto error_unreserve;
		}
	}

	amdgpu_bo_unreserve(*bo_ptr);

	return 0;

error_unreserve:
	amdgpu_bo_unreserve(*bo_ptr);

error_free:
	amdgpu_bo_unref(bo_ptr);

	return r;
}

285 286 287 288 289 290 291 292 293 294 295 296 297
/**
 * amdgpu_bo_free_kernel - free BO for kernel use
 *
 * @bo: amdgpu BO to free
 *
 * unmaps and unpin a BO for kernel internal use.
 */
void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
			   void **cpu_addr)
{
	if (*bo == NULL)
		return;

298
	if (likely(amdgpu_bo_reserve(*bo, true) == 0)) {
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
		if (cpu_addr)
			amdgpu_bo_kunmap(*bo);

		amdgpu_bo_unpin(*bo);
		amdgpu_bo_unreserve(*bo);
	}
	amdgpu_bo_unref(bo);

	if (gpu_addr)
		*gpu_addr = 0;

	if (cpu_addr)
		*cpu_addr = NULL;
}

314 315 316 317 318
int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
				unsigned long size, int byte_align,
				bool kernel, u32 domain, u64 flags,
				struct sg_table *sg,
				struct ttm_placement *placement,
319
				struct reservation_object *resv,
320
				struct amdgpu_bo **bo_ptr)
A
Alex Deucher 已提交
321 322 323 324
{
	struct amdgpu_bo *bo;
	enum ttm_bo_type type;
	unsigned long page_align;
325
	u64 initial_bytes_moved, bytes_moved;
A
Alex Deucher 已提交
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
	size_t acc_size;
	int r;

	page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
	size = ALIGN(size, PAGE_SIZE);

	if (kernel) {
		type = ttm_bo_type_kernel;
	} else if (sg) {
		type = ttm_bo_type_sg;
	} else {
		type = ttm_bo_type_device;
	}
	*bo_ptr = NULL;

	acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
				       sizeof(struct amdgpu_bo));

	bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);
	if (bo == NULL)
		return -ENOMEM;
	r = drm_gem_object_init(adev->ddev, &bo->gem_base, size);
	if (unlikely(r)) {
		kfree(bo);
		return r;
	}
352
	INIT_LIST_HEAD(&bo->shadow_list);
A
Alex Deucher 已提交
353
	INIT_LIST_HEAD(&bo->va);
354 355 356 357 358 359 360 361 362
	bo->prefered_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
					 AMDGPU_GEM_DOMAIN_GTT |
					 AMDGPU_GEM_DOMAIN_CPU |
					 AMDGPU_GEM_DOMAIN_GDS |
					 AMDGPU_GEM_DOMAIN_GWS |
					 AMDGPU_GEM_DOMAIN_OA);
	bo->allowed_domains = bo->prefered_domains;
	if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
A
Alex Deucher 已提交
363 364

	bo->flags = flags;
365

366 367 368 369 370 371 372 373 374 375 376
#ifdef CONFIG_X86_32
	/* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit
	 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627
	 */
	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
#elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT)
	/* Don't try to enable write-combining when it can't work, or things
	 * may be slow
	 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758
	 */

377
#ifndef CONFIG_COMPILE_TEST
378 379
#warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \
	 thanks to write-combining
380
#endif
381 382 383 384 385 386

	if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
		DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for "
			      "better performance thanks to write-combining\n");
	bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
#else
387 388 389 390 391
	/* For architectures that don't support WC memory,
	 * mask out the WC flag from the BO
	 */
	if (!drm_arch_can_wc_memory())
		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;
392
#endif
393

394
	amdgpu_fill_placement_to_bo(bo, placement);
A
Alex Deucher 已提交
395
	/* Kernel allocation are uninterruptible */
396

397
	initial_bytes_moved = atomic64_read(&adev->num_bytes_moved);
398 399 400
	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type,
				 &bo->placement, page_align, !kernel, NULL,
				 acc_size, sg, resv, &amdgpu_ttm_bo_destroy);
401 402 403 404 405 406 407 408
	bytes_moved = atomic64_read(&adev->num_bytes_moved) -
		      initial_bytes_moved;
	if (adev->mc.visible_vram_size < adev->mc.real_vram_size &&
	    bo->tbo.mem.mem_type == TTM_PL_VRAM &&
	    bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT)
		amdgpu_cs_report_moved_bytes(adev, bytes_moved, bytes_moved);
	else
		amdgpu_cs_report_moved_bytes(adev, bytes_moved, 0);
409

410
	if (unlikely(r != 0))
A
Alex Deucher 已提交
411
		return r;
412

413
	if (kernel)
414
		bo->tbo.priority = 1;
415

416 417
	if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
	    bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
418
		struct dma_fence *fence;
419

420 421 422 423
		r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence);
		if (unlikely(r))
			goto fail_unreserve;

424
		amdgpu_bo_fence(bo, fence, false);
425 426 427
		dma_fence_put(bo->tbo.moving);
		bo->tbo.moving = dma_fence_get(fence);
		dma_fence_put(fence);
428
	}
429
	if (!resv)
430
		amdgpu_bo_unreserve(bo);
A
Alex Deucher 已提交
431 432 433 434
	*bo_ptr = bo;

	trace_amdgpu_bo_create(bo);

435 436 437 438
	/* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */
	if (type == ttm_bo_type_device)
		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;

A
Alex Deucher 已提交
439
	return 0;
440 441

fail_unreserve:
442 443
	if (!resv)
		ww_mutex_unlock(&bo->tbo.resv->lock);
444 445
	amdgpu_bo_unref(&bo);
	return r;
A
Alex Deucher 已提交
446 447
}

448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
				   unsigned long size, int byte_align,
				   struct amdgpu_bo *bo)
{
	struct ttm_placement placement = {0};
	struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
	int r;

	if (bo->shadow)
		return 0;

	bo->flags |= AMDGPU_GEM_CREATE_SHADOW;
	memset(&placements, 0,
	       (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));

	amdgpu_ttm_placement_init(adev, &placement,
				  placements, AMDGPU_GEM_DOMAIN_GTT,
				  AMDGPU_GEM_CREATE_CPU_GTT_USWC);

	r = amdgpu_bo_create_restricted(adev, size, byte_align, true,
					AMDGPU_GEM_DOMAIN_GTT,
					AMDGPU_GEM_CREATE_CPU_GTT_USWC,
					NULL, &placement,
					bo->tbo.resv,
					&bo->shadow);
473
	if (!r) {
474
		bo->shadow->parent = amdgpu_bo_ref(bo);
475 476 477 478
		mutex_lock(&adev->shadow_list_lock);
		list_add_tail(&bo->shadow_list, &adev->shadow_list);
		mutex_unlock(&adev->shadow_list_lock);
	}
479 480 481 482

	return r;
}

483 484 485
int amdgpu_bo_create(struct amdgpu_device *adev,
		     unsigned long size, int byte_align,
		     bool kernel, u32 domain, u64 flags,
486 487 488
		     struct sg_table *sg,
		     struct reservation_object *resv,
		     struct amdgpu_bo **bo_ptr)
489 490 491
{
	struct ttm_placement placement = {0};
	struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
492
	int r;
493 494 495 496 497 498 499

	memset(&placements, 0,
	       (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));

	amdgpu_ttm_placement_init(adev, &placement,
				  placements, domain, flags);

500 501 502 503 504 505
	r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel,
					domain, flags, sg, &placement,
					resv, bo_ptr);
	if (r)
		return r;

506
	if (amdgpu_need_backup(adev) && (flags & AMDGPU_GEM_CREATE_SHADOW)) {
507 508 509 510 511
		if (!resv) {
			r = ww_mutex_lock(&(*bo_ptr)->tbo.resv->lock, NULL);
			WARN_ON(r != 0);
		}

512
		r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
513 514 515 516

		if (!resv)
			ww_mutex_unlock(&(*bo_ptr)->tbo.resv->lock);

517 518 519 520 521
		if (r)
			amdgpu_bo_unref(bo_ptr);
	}

	return r;
522 523
}

524 525 526 527
int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
			       struct amdgpu_ring *ring,
			       struct amdgpu_bo *bo,
			       struct reservation_object *resv,
528
			       struct dma_fence **fence,
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547
			       bool direct)

{
	struct amdgpu_bo *shadow = bo->shadow;
	uint64_t bo_addr, shadow_addr;
	int r;

	if (!shadow)
		return -EINVAL;

	bo_addr = amdgpu_bo_gpu_offset(bo);
	shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);

	r = reservation_object_reserve_shared(bo->tbo.resv);
	if (r)
		goto err;

	r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr,
			       amdgpu_bo_size(bo), resv, fence,
548
			       direct, false);
549 550 551 552 553 554 555
	if (!r)
		amdgpu_bo_fence(bo, *fence, true);

err:
	return r;
}

556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
int amdgpu_bo_validate(struct amdgpu_bo *bo)
{
	uint32_t domain;
	int r;

	if (bo->pin_count)
		return 0;

	domain = bo->prefered_domains;

retry:
	amdgpu_ttm_placement_from_domain(bo, domain);
	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
		domain = bo->allowed_domains;
		goto retry;
	}

	return r;
}

577 578 579 580
int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
				  struct amdgpu_ring *ring,
				  struct amdgpu_bo *bo,
				  struct reservation_object *resv,
581
				  struct dma_fence **fence,
582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
				  bool direct)

{
	struct amdgpu_bo *shadow = bo->shadow;
	uint64_t bo_addr, shadow_addr;
	int r;

	if (!shadow)
		return -EINVAL;

	bo_addr = amdgpu_bo_gpu_offset(bo);
	shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);

	r = reservation_object_reserve_shared(bo->tbo.resv);
	if (r)
		goto err;

	r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr,
			       amdgpu_bo_size(bo), resv, fence,
601
			       direct, false);
602 603 604 605 606 607 608
	if (!r)
		amdgpu_bo_fence(bo, *fence, true);

err:
	return r;
}

A
Alex Deucher 已提交
609 610 611
int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
{
	bool is_iomem;
612
	long r;
A
Alex Deucher 已提交
613

614 615 616
	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
		return -EPERM;

A
Alex Deucher 已提交
617 618 619 620 621 622
	if (bo->kptr) {
		if (ptr) {
			*ptr = bo->kptr;
		}
		return 0;
	}
623 624 625 626 627 628

	r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false,
						MAX_SCHEDULE_TIMEOUT);
	if (r < 0)
		return r;

A
Alex Deucher 已提交
629
	r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
630
	if (r)
A
Alex Deucher 已提交
631
		return r;
632

A
Alex Deucher 已提交
633
	bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
634
	if (ptr)
A
Alex Deucher 已提交
635
		*ptr = bo->kptr;
636

A
Alex Deucher 已提交
637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669
	return 0;
}

void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
{
	if (bo->kptr == NULL)
		return;
	bo->kptr = NULL;
	ttm_bo_kunmap(&bo->kmap);
}

struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
{
	if (bo == NULL)
		return NULL;

	ttm_bo_reference(&bo->tbo);
	return bo;
}

void amdgpu_bo_unref(struct amdgpu_bo **bo)
{
	struct ttm_buffer_object *tbo;

	if ((*bo) == NULL)
		return;

	tbo = &((*bo)->tbo);
	ttm_bo_unref(&tbo);
	if (tbo == NULL)
		*bo = NULL;
}

670 671
int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
			     u64 min_offset, u64 max_offset,
A
Alex Deucher 已提交
672 673
			     u64 *gpu_addr)
{
674
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
A
Alex Deucher 已提交
675
	int r, i;
676
	unsigned fpfn, lpfn;
A
Alex Deucher 已提交
677

678
	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
A
Alex Deucher 已提交
679 680
		return -EPERM;

681 682 683
	if (WARN_ON_ONCE(min_offset > max_offset))
		return -EINVAL;

684 685 686 687
	/* A shared bo cannot be migrated to VRAM */
	if (bo->prime_shared_count && (domain == AMDGPU_GEM_DOMAIN_VRAM))
		return -EINVAL;

A
Alex Deucher 已提交
688
	if (bo->pin_count) {
689 690 691 692 693
		uint32_t mem_type = bo->tbo.mem.mem_type;

		if (domain != amdgpu_mem_type_to_domain(mem_type))
			return -EINVAL;

A
Alex Deucher 已提交
694 695 696 697 698
		bo->pin_count++;
		if (gpu_addr)
			*gpu_addr = amdgpu_bo_gpu_offset(bo);

		if (max_offset != 0) {
699
			u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset;
A
Alex Deucher 已提交
700 701 702 703 704 705
			WARN_ON_ONCE(max_offset <
				     (amdgpu_bo_gpu_offset(bo) - domain_start));
		}

		return 0;
	}
706 707

	bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
A
Alex Deucher 已提交
708 709 710 711
	amdgpu_ttm_placement_from_domain(bo, domain);
	for (i = 0; i < bo->placement.num_placement; i++) {
		/* force to pin into visible video ram */
		if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
712
		    !(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) &&
713
		    (!max_offset || max_offset >
714
		     adev->mc.visible_vram_size)) {
715
			if (WARN_ON_ONCE(min_offset >
716
					 adev->mc.visible_vram_size))
717 718
				return -EINVAL;
			fpfn = min_offset >> PAGE_SHIFT;
719
			lpfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
720 721 722 723 724 725
		} else {
			fpfn = min_offset >> PAGE_SHIFT;
			lpfn = max_offset >> PAGE_SHIFT;
		}
		if (fpfn > bo->placements[i].fpfn)
			bo->placements[i].fpfn = fpfn;
726 727
		if (!bo->placements[i].lpfn ||
		    (lpfn && lpfn < bo->placements[i].lpfn))
728
			bo->placements[i].lpfn = lpfn;
A
Alex Deucher 已提交
729 730 731 732
		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
	}

	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
733
	if (unlikely(r)) {
734
		dev_err(adev->dev, "%p pin failed\n", bo);
735 736
		goto error;
	}
737
	r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
738
	if (unlikely(r)) {
739
		dev_err(adev->dev, "%p bind failed\n", bo);
740 741
		goto error;
	}
742 743 744 745 746

	bo->pin_count = 1;
	if (gpu_addr != NULL)
		*gpu_addr = amdgpu_bo_gpu_offset(bo);
	if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
747
		adev->vram_pin_size += amdgpu_bo_size(bo);
748
		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
749
			adev->invisible_pin_size += amdgpu_bo_size(bo);
750
	} else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
751
		adev->gart_pin_size += amdgpu_bo_size(bo);
A
Alex Deucher 已提交
752
	}
753 754

error:
A
Alex Deucher 已提交
755 756 757 758 759
	return r;
}

int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
{
760
	return amdgpu_bo_pin_restricted(bo, domain, 0, 0, gpu_addr);
A
Alex Deucher 已提交
761 762 763 764
}

int amdgpu_bo_unpin(struct amdgpu_bo *bo)
{
765
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
A
Alex Deucher 已提交
766 767 768
	int r, i;

	if (!bo->pin_count) {
769
		dev_warn(adev->dev, "%p unpin not necessary\n", bo);
A
Alex Deucher 已提交
770 771 772 773 774 775 776 777 778 779
		return 0;
	}
	bo->pin_count--;
	if (bo->pin_count)
		return 0;
	for (i = 0; i < bo->placement.num_placement; i++) {
		bo->placements[i].lpfn = 0;
		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
	}
	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
780
	if (unlikely(r)) {
781
		dev_err(adev->dev, "%p validate failed for unpin\n", bo);
782
		goto error;
A
Alex Deucher 已提交
783
	}
784 785

	if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
786
		adev->vram_pin_size -= amdgpu_bo_size(bo);
787
		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
788
			adev->invisible_pin_size -= amdgpu_bo_size(bo);
789
	} else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
790
		adev->gart_pin_size -= amdgpu_bo_size(bo);
791 792 793
	}

error:
A
Alex Deucher 已提交
794 795 796 797 798 799
	return r;
}

int amdgpu_bo_evict_vram(struct amdgpu_device *adev)
{
	/* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */
800
	if (0 && (adev->flags & AMD_IS_APU)) {
A
Alex Deucher 已提交
801 802 803 804 805 806
		/* Useless to evict on IGP chips */
		return 0;
	}
	return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM);
}

807 808 809 810 811 812 813 814 815 816 817
static const char *amdgpu_vram_names[] = {
	"UNKNOWN",
	"GDDR1",
	"DDR2",
	"GDDR3",
	"GDDR4",
	"GDDR5",
	"HBM",
	"DDR3"
};

A
Alex Deucher 已提交
818 819
int amdgpu_bo_init(struct amdgpu_device *adev)
{
820 821 822 823
	/* reserve PAT memory space to WC for VRAM */
	arch_io_reserve_memtype_wc(adev->mc.aper_base,
				   adev->mc.aper_size);

A
Alex Deucher 已提交
824 825 826 827 828 829
	/* Add an MTRR for the VRAM */
	adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base,
					      adev->mc.aper_size);
	DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
		adev->mc.mc_vram_size >> 20,
		(unsigned long long)adev->mc.aper_size >> 20);
830 831
	DRM_INFO("RAM width %dbits %s\n",
		 adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]);
A
Alex Deucher 已提交
832 833 834 835 836 837 838
	return amdgpu_ttm_init(adev);
}

void amdgpu_bo_fini(struct amdgpu_device *adev)
{
	amdgpu_ttm_fini(adev);
	arch_phys_wc_del(adev->mc.vram_mtrr);
839
	arch_io_free_memtype_wc(adev->mc.aper_base, adev->mc.aper_size);
A
Alex Deucher 已提交
840 841 842 843 844 845 846 847 848 849
}

int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo,
			     struct vm_area_struct *vma)
{
	return ttm_fbdev_mmap(vma, &bo->tbo);
}

int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
{
850 851 852 853
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);

	if (adev->family <= AMDGPU_FAMILY_CZ &&
	    AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
A
Alex Deucher 已提交
854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875
		return -EINVAL;

	bo->tiling_flags = tiling_flags;
	return 0;
}

void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
{
	lockdep_assert_held(&bo->tbo.resv->lock.base);

	if (tiling_flags)
		*tiling_flags = bo->tiling_flags;
}

int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
			    uint32_t metadata_size, uint64_t flags)
{
	void *buffer;

	if (!metadata_size) {
		if (bo->metadata_size) {
			kfree(bo->metadata);
876
			bo->metadata = NULL;
A
Alex Deucher 已提交
877 878 879 880 881 882 883 884
			bo->metadata_size = 0;
		}
		return 0;
	}

	if (metadata == NULL)
		return -EINVAL;

885
	buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
A
Alex Deucher 已提交
886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
	if (buffer == NULL)
		return -ENOMEM;

	kfree(bo->metadata);
	bo->metadata_flags = flags;
	bo->metadata = buffer;
	bo->metadata_size = metadata_size;

	return 0;
}

int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
			   size_t buffer_size, uint32_t *metadata_size,
			   uint64_t *flags)
{
	if (!buffer && !metadata_size)
		return -EINVAL;

	if (buffer) {
		if (buffer_size < bo->metadata_size)
			return -EINVAL;

		if (bo->metadata_size)
			memcpy(buffer, bo->metadata, bo->metadata_size);
	}

	if (metadata_size)
		*metadata_size = bo->metadata_size;
	if (flags)
		*flags = bo->metadata_flags;

	return 0;
}

void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
921
			   bool evict,
A
Alex Deucher 已提交
922 923
			   struct ttm_mem_reg *new_mem)
{
924
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
925
	struct amdgpu_bo *abo;
926
	struct ttm_mem_reg *old_mem = &bo->mem;
A
Alex Deucher 已提交
927 928 929 930

	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
		return;

931
	abo = container_of(bo, struct amdgpu_bo, tbo);
932
	amdgpu_vm_bo_invalidate(adev, abo);
A
Alex Deucher 已提交
933

934 935 936 937
	/* remember the eviction */
	if (evict)
		atomic64_inc(&adev->num_evictions);

A
Alex Deucher 已提交
938 939 940 941 942
	/* update statistics */
	if (!new_mem)
		return;

	/* move_notify is called before move happens */
943
	amdgpu_update_memory_usage(adev, &bo->mem, new_mem);
944

945
	trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
A
Alex Deucher 已提交
946 947 948 949
}

int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
{
950
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
951
	struct amdgpu_bo *abo;
952 953
	unsigned long offset, size;
	int r;
A
Alex Deucher 已提交
954 955 956

	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
		return 0;
957 958

	abo = container_of(bo, struct amdgpu_bo, tbo);
959 960 961 962

	/* Remember that this BO was accessed by the CPU */
	abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;

963 964 965 966 967
	if (bo->mem.mem_type != TTM_PL_VRAM)
		return 0;

	size = bo->mem.num_pages << PAGE_SHIFT;
	offset = bo->mem.start << PAGE_SHIFT;
968
	if ((offset + size) <= adev->mc.visible_vram_size)
969 970
		return 0;

971 972 973 974
	/* Can't move a pinned BO to visible VRAM */
	if (abo->pin_count > 0)
		return -EINVAL;

975
	/* hurrah the memory is not visible ! */
976
	atomic64_inc(&adev->num_vram_cpu_page_faults);
977 978 979 980 981 982 983
	amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
					 AMDGPU_GEM_DOMAIN_GTT);

	/* Avoid costly evictions; only set GTT as a busy placement */
	abo->placement.num_busy_placement = 1;
	abo->placement.busy_placement = &abo->placements[1];

984
	r = ttm_bo_validate(bo, &abo->placement, false, false);
985
	if (unlikely(r != 0))
986 987 988 989
		return r;

	offset = bo->mem.start << PAGE_SHIFT;
	/* this should never happen */
990 991
	if (bo->mem.mem_type == TTM_PL_VRAM &&
	    (offset + size) > adev->mc.visible_vram_size)
992 993
		return -EINVAL;

A
Alex Deucher 已提交
994 995 996 997 998 999 1000 1001 1002 1003 1004
	return 0;
}

/**
 * amdgpu_bo_fence - add fence to buffer object
 *
 * @bo: buffer object in question
 * @fence: fence to add
 * @shared: true if fence should be added shared
 *
 */
1005
void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
A
Alex Deucher 已提交
1006 1007 1008 1009 1010
		     bool shared)
{
	struct reservation_object *resv = bo->tbo.resv;

	if (shared)
1011
		reservation_object_add_shared_fence(resv, fence);
A
Alex Deucher 已提交
1012
	else
1013
		reservation_object_add_excl_fence(resv, fence);
A
Alex Deucher 已提交
1014
}
1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027

/**
 * amdgpu_bo_gpu_offset - return GPU offset of bo
 * @bo:	amdgpu object for which we query the offset
 *
 * Returns current GPU offset of the object.
 *
 * Note: object should either be pinned or reserved when calling this
 * function, it might be useful to add check for this for debugging.
 */
u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
{
	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
1028 1029
	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT &&
		     !amdgpu_ttm_is_bound(bo->tbo.ttm));
1030 1031
	WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
		     !bo->pin_count);
1032
	WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
1033 1034
	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM &&
		     !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
1035 1036 1037

	return bo->tbo.offset;
}