amdgpu_object.c 23.5 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
/*
 * Copyright 2009 Jerome Glisse.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 */
/*
 * Authors:
 *    Jerome Glisse <glisse@freedesktop.org>
 *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
 *    Dave Airlie
 */
#include <linux/list.h>
#include <linux/slab.h>
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
36
#include <drm/drm_cache.h>
A
Alex Deucher 已提交
37 38 39 40 41 42
#include "amdgpu.h"
#include "amdgpu_trace.h"



static u64 amdgpu_get_vis_part_size(struct amdgpu_device *adev,
43
						struct ttm_mem_reg *mem)
A
Alex Deucher 已提交
44
{
45 46 47 48 49 50 51
	if (mem->start << PAGE_SHIFT >= adev->mc.visible_vram_size)
		return 0;

	return ((mem->start << PAGE_SHIFT) + mem->size) >
		adev->mc.visible_vram_size ?
		adev->mc.visible_vram_size - (mem->start << PAGE_SHIFT) :
		mem->size;
A
Alex Deucher 已提交
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
}

static void amdgpu_update_memory_usage(struct amdgpu_device *adev,
		       struct ttm_mem_reg *old_mem,
		       struct ttm_mem_reg *new_mem)
{
	u64 vis_size;
	if (!adev)
		return;

	if (new_mem) {
		switch (new_mem->mem_type) {
		case TTM_PL_TT:
			atomic64_add(new_mem->size, &adev->gtt_usage);
			break;
		case TTM_PL_VRAM:
			atomic64_add(new_mem->size, &adev->vram_usage);
			vis_size = amdgpu_get_vis_part_size(adev, new_mem);
			atomic64_add(vis_size, &adev->vram_vis_usage);
			break;
		}
	}

	if (old_mem) {
		switch (old_mem->mem_type) {
		case TTM_PL_TT:
			atomic64_sub(old_mem->size, &adev->gtt_usage);
			break;
		case TTM_PL_VRAM:
			atomic64_sub(old_mem->size, &adev->vram_usage);
			vis_size = amdgpu_get_vis_part_size(adev, old_mem);
			atomic64_sub(vis_size, &adev->vram_vis_usage);
			break;
		}
	}
}

static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo)
{
91
	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
A
Alex Deucher 已提交
92 93 94 95
	struct amdgpu_bo *bo;

	bo = container_of(tbo, struct amdgpu_bo, tbo);

96
	amdgpu_update_memory_usage(adev, &bo->tbo.mem, NULL);
A
Alex Deucher 已提交
97 98

	drm_gem_object_release(&bo->gem_base);
99
	amdgpu_bo_unref(&bo->parent);
100
	if (!list_empty(&bo->shadow_list)) {
101
		mutex_lock(&adev->shadow_list_lock);
102
		list_del_init(&bo->shadow_list);
103
		mutex_unlock(&adev->shadow_list_lock);
104
	}
A
Alex Deucher 已提交
105 106 107 108 109 110 111 112 113 114 115
	kfree(bo->metadata);
	kfree(bo);
}

bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo)
{
	if (bo->destroy == &amdgpu_ttm_bo_destroy)
		return true;
	return false;
}

116 117
static void amdgpu_ttm_placement_init(struct amdgpu_device *adev,
				      struct ttm_placement *placement,
118
				      struct ttm_place *places,
119
				      u32 domain, u64 flags)
A
Alex Deucher 已提交
120
{
121
	u32 c = 0;
122

A
Alex Deucher 已提交
123
	if (domain & AMDGPU_GEM_DOMAIN_VRAM) {
124
		unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
125 126 127 128 129
		unsigned lpfn = 0;

		/* This forces a reallocation if the flag wasn't set before */
		if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)
			lpfn = adev->mc.real_vram_size >> PAGE_SHIFT;
130 131

		places[c].fpfn = 0;
132
		places[c].lpfn = lpfn;
133
		places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED |
134
			TTM_PL_FLAG_VRAM;
135 136 137 138 139
		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)
			places[c].lpfn = visible_pfn;
		else
			places[c].flags |= TTM_PL_FLAG_TOPDOWN;
		c++;
A
Alex Deucher 已提交
140 141 142
	}

	if (domain & AMDGPU_GEM_DOMAIN_GTT) {
143 144 145 146 147 148 149 150 151
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_TT;
		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
			places[c].flags |= TTM_PL_FLAG_WC |
				TTM_PL_FLAG_UNCACHED;
		else
			places[c].flags |= TTM_PL_FLAG_CACHED;
		c++;
A
Alex Deucher 已提交
152 153 154
	}

	if (domain & AMDGPU_GEM_DOMAIN_CPU) {
155 156 157 158 159 160 161 162 163
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_SYSTEM;
		if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC)
			places[c].flags |= TTM_PL_FLAG_WC |
				TTM_PL_FLAG_UNCACHED;
		else
			places[c].flags |= TTM_PL_FLAG_CACHED;
		c++;
A
Alex Deucher 已提交
164 165 166
	}

	if (domain & AMDGPU_GEM_DOMAIN_GDS) {
167 168 169 170
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS;
		c++;
A
Alex Deucher 已提交
171
	}
172

A
Alex Deucher 已提交
173
	if (domain & AMDGPU_GEM_DOMAIN_GWS) {
174 175 176 177
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS;
		c++;
A
Alex Deucher 已提交
178
	}
179

A
Alex Deucher 已提交
180
	if (domain & AMDGPU_GEM_DOMAIN_OA) {
181 182 183 184
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA;
		c++;
A
Alex Deucher 已提交
185 186 187
	}

	if (!c) {
188 189 190 191
		places[c].fpfn = 0;
		places[c].lpfn = 0;
		places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
		c++;
A
Alex Deucher 已提交
192
	}
193

194
	placement->num_placement = c;
195
	placement->placement = places;
A
Alex Deucher 已提交
196

197 198
	placement->num_busy_placement = c;
	placement->busy_placement = places;
A
Alex Deucher 已提交
199 200
}

201
void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain)
202
{
203 204 205 206
	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);

	amdgpu_ttm_placement_init(adev, &abo->placement, abo->placements,
				  domain, abo->flags);
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
}

static void amdgpu_fill_placement_to_bo(struct amdgpu_bo *bo,
					struct ttm_placement *placement)
{
	BUG_ON(placement->num_placement > (AMDGPU_GEM_DOMAIN_MAX + 1));

	memcpy(bo->placements, placement->placement,
	       placement->num_placement * sizeof(struct ttm_place));
	bo->placement.num_placement = placement->num_placement;
	bo->placement.num_busy_placement = placement->num_busy_placement;
	bo->placement.placement = bo->placements;
	bo->placement.busy_placement = bo->placements;
}

222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
/**
 * amdgpu_bo_create_kernel - create BO for kernel use
 *
 * @adev: amdgpu device object
 * @size: size for the new BO
 * @align: alignment for the new BO
 * @domain: where to place it
 * @bo_ptr: resulting BO
 * @gpu_addr: GPU addr of the pinned BO
 * @cpu_addr: optional CPU address mapping
 *
 * Allocates and pins a BO for kernel internal use.
 *
 * Returns 0 on success, negative error code otherwise.
 */
int amdgpu_bo_create_kernel(struct amdgpu_device *adev,
			    unsigned long size, int align,
			    u32 domain, struct amdgpu_bo **bo_ptr,
			    u64 *gpu_addr, void **cpu_addr)
{
	int r;

	r = amdgpu_bo_create(adev, size, align, true, domain,
245 246
			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
			     NULL, NULL, bo_ptr);
	if (r) {
		dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", r);
		return r;
	}

	r = amdgpu_bo_reserve(*bo_ptr, false);
	if (r) {
		dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r);
		goto error_free;
	}

	r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr);
	if (r) {
		dev_err(adev->dev, "(%d) kernel bo pin failed\n", r);
		goto error_unreserve;
	}

	if (cpu_addr) {
		r = amdgpu_bo_kmap(*bo_ptr, cpu_addr);
		if (r) {
			dev_err(adev->dev, "(%d) kernel bo map failed\n", r);
			goto error_unreserve;
		}
	}

	amdgpu_bo_unreserve(*bo_ptr);

	return 0;

error_unreserve:
	amdgpu_bo_unreserve(*bo_ptr);

error_free:
	amdgpu_bo_unref(bo_ptr);

	return r;
}

286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
/**
 * amdgpu_bo_free_kernel - free BO for kernel use
 *
 * @bo: amdgpu BO to free
 *
 * unmaps and unpin a BO for kernel internal use.
 */
void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
			   void **cpu_addr)
{
	if (*bo == NULL)
		return;

	if (likely(amdgpu_bo_reserve(*bo, false) == 0)) {
		if (cpu_addr)
			amdgpu_bo_kunmap(*bo);

		amdgpu_bo_unpin(*bo);
		amdgpu_bo_unreserve(*bo);
	}
	amdgpu_bo_unref(bo);

	if (gpu_addr)
		*gpu_addr = 0;

	if (cpu_addr)
		*cpu_addr = NULL;
}

315 316 317 318 319
int amdgpu_bo_create_restricted(struct amdgpu_device *adev,
				unsigned long size, int byte_align,
				bool kernel, u32 domain, u64 flags,
				struct sg_table *sg,
				struct ttm_placement *placement,
320
				struct reservation_object *resv,
321
				struct amdgpu_bo **bo_ptr)
A
Alex Deucher 已提交
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
{
	struct amdgpu_bo *bo;
	enum ttm_bo_type type;
	unsigned long page_align;
	size_t acc_size;
	int r;

	page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT;
	size = ALIGN(size, PAGE_SIZE);

	if (kernel) {
		type = ttm_bo_type_kernel;
	} else if (sg) {
		type = ttm_bo_type_sg;
	} else {
		type = ttm_bo_type_device;
	}
	*bo_ptr = NULL;

	acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
				       sizeof(struct amdgpu_bo));

	bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL);
	if (bo == NULL)
		return -ENOMEM;
	r = drm_gem_object_init(adev->ddev, &bo->gem_base, size);
	if (unlikely(r)) {
		kfree(bo);
		return r;
	}
352
	INIT_LIST_HEAD(&bo->shadow_list);
A
Alex Deucher 已提交
353
	INIT_LIST_HEAD(&bo->va);
354 355 356 357 358 359 360 361 362
	bo->prefered_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM |
					 AMDGPU_GEM_DOMAIN_GTT |
					 AMDGPU_GEM_DOMAIN_CPU |
					 AMDGPU_GEM_DOMAIN_GDS |
					 AMDGPU_GEM_DOMAIN_GWS |
					 AMDGPU_GEM_DOMAIN_OA);
	bo->allowed_domains = bo->prefered_domains;
	if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
		bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
A
Alex Deucher 已提交
363 364

	bo->flags = flags;
365 366 367 368 369 370 371

	/* For architectures that don't support WC memory,
	 * mask out the WC flag from the BO
	 */
	if (!drm_arch_can_wc_memory())
		bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC;

372
	amdgpu_fill_placement_to_bo(bo, placement);
A
Alex Deucher 已提交
373
	/* Kernel allocation are uninterruptible */
374 375 376 377 378 379 380 381

	if (!resv) {
		bool locked;

		reservation_object_init(&bo->tbo.ttm_resv);
		locked = ww_mutex_trylock(&bo->tbo.ttm_resv.lock);
		WARN_ON(!locked);
	}
A
Alex Deucher 已提交
382 383
	r = ttm_bo_init(&adev->mman.bdev, &bo->tbo, size, type,
			&bo->placement, page_align, !kernel, NULL,
384 385 386
			acc_size, sg, resv ? resv : &bo->tbo.ttm_resv,
			&amdgpu_ttm_bo_destroy);
	if (unlikely(r != 0))
A
Alex Deucher 已提交
387
		return r;
388 389 390

	if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED &&
	    bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) {
391
		struct dma_fence *fence;
392

393 394 395 396
		r = amdgpu_fill_buffer(bo, 0, bo->tbo.resv, &fence);
		if (unlikely(r))
			goto fail_unreserve;

397
		amdgpu_bo_fence(bo, fence, false);
398 399 400
		dma_fence_put(bo->tbo.moving);
		bo->tbo.moving = dma_fence_get(fence);
		dma_fence_put(fence);
401
	}
402 403
	if (!resv)
		ww_mutex_unlock(&bo->tbo.resv->lock);
A
Alex Deucher 已提交
404 405 406 407 408
	*bo_ptr = bo;

	trace_amdgpu_bo_create(bo);

	return 0;
409 410

fail_unreserve:
411
	ww_mutex_unlock(&bo->tbo.resv->lock);
412 413
	amdgpu_bo_unref(&bo);
	return r;
A
Alex Deucher 已提交
414 415
}

416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
				   unsigned long size, int byte_align,
				   struct amdgpu_bo *bo)
{
	struct ttm_placement placement = {0};
	struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
	int r;

	if (bo->shadow)
		return 0;

	bo->flags |= AMDGPU_GEM_CREATE_SHADOW;
	memset(&placements, 0,
	       (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));

	amdgpu_ttm_placement_init(adev, &placement,
				  placements, AMDGPU_GEM_DOMAIN_GTT,
				  AMDGPU_GEM_CREATE_CPU_GTT_USWC);

	r = amdgpu_bo_create_restricted(adev, size, byte_align, true,
					AMDGPU_GEM_DOMAIN_GTT,
					AMDGPU_GEM_CREATE_CPU_GTT_USWC,
					NULL, &placement,
					bo->tbo.resv,
					&bo->shadow);
441
	if (!r) {
442
		bo->shadow->parent = amdgpu_bo_ref(bo);
443 444 445 446
		mutex_lock(&adev->shadow_list_lock);
		list_add_tail(&bo->shadow_list, &adev->shadow_list);
		mutex_unlock(&adev->shadow_list_lock);
	}
447 448 449 450

	return r;
}

451 452 453
int amdgpu_bo_create(struct amdgpu_device *adev,
		     unsigned long size, int byte_align,
		     bool kernel, u32 domain, u64 flags,
454 455 456
		     struct sg_table *sg,
		     struct reservation_object *resv,
		     struct amdgpu_bo **bo_ptr)
457 458 459
{
	struct ttm_placement placement = {0};
	struct ttm_place placements[AMDGPU_GEM_DOMAIN_MAX + 1];
460
	int r;
461 462 463 464 465 466 467

	memset(&placements, 0,
	       (AMDGPU_GEM_DOMAIN_MAX + 1) * sizeof(struct ttm_place));

	amdgpu_ttm_placement_init(adev, &placement,
				  placements, domain, flags);

468 469 470 471 472 473
	r = amdgpu_bo_create_restricted(adev, size, byte_align, kernel,
					domain, flags, sg, &placement,
					resv, bo_ptr);
	if (r)
		return r;

474
	if (amdgpu_need_backup(adev) && (flags & AMDGPU_GEM_CREATE_SHADOW)) {
475 476 477 478 479 480
		r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr));
		if (r)
			amdgpu_bo_unref(bo_ptr);
	}

	return r;
481 482
}

483 484 485 486
int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev,
			       struct amdgpu_ring *ring,
			       struct amdgpu_bo *bo,
			       struct reservation_object *resv,
487
			       struct dma_fence **fence,
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518
			       bool direct)

{
	struct amdgpu_bo *shadow = bo->shadow;
	uint64_t bo_addr, shadow_addr;
	int r;

	if (!shadow)
		return -EINVAL;

	bo_addr = amdgpu_bo_gpu_offset(bo);
	shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);

	r = reservation_object_reserve_shared(bo->tbo.resv);
	if (r)
		goto err;

	r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr,
			       amdgpu_bo_size(bo), resv, fence,
			       direct);
	if (!r)
		amdgpu_bo_fence(bo, *fence, true);

err:
	return r;
}

int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev,
				  struct amdgpu_ring *ring,
				  struct amdgpu_bo *bo,
				  struct reservation_object *resv,
519
				  struct dma_fence **fence,
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546
				  bool direct)

{
	struct amdgpu_bo *shadow = bo->shadow;
	uint64_t bo_addr, shadow_addr;
	int r;

	if (!shadow)
		return -EINVAL;

	bo_addr = amdgpu_bo_gpu_offset(bo);
	shadow_addr = amdgpu_bo_gpu_offset(bo->shadow);

	r = reservation_object_reserve_shared(bo->tbo.resv);
	if (r)
		goto err;

	r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr,
			       amdgpu_bo_size(bo), resv, fence,
			       direct);
	if (!r)
		amdgpu_bo_fence(bo, *fence, true);

err:
	return r;
}

A
Alex Deucher 已提交
547 548 549
int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr)
{
	bool is_iomem;
550
	long r;
A
Alex Deucher 已提交
551

552 553 554
	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
		return -EPERM;

A
Alex Deucher 已提交
555 556 557 558 559 560
	if (bo->kptr) {
		if (ptr) {
			*ptr = bo->kptr;
		}
		return 0;
	}
561 562 563 564 565 566

	r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false,
						MAX_SCHEDULE_TIMEOUT);
	if (r < 0)
		return r;

A
Alex Deucher 已提交
567
	r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap);
568
	if (r)
A
Alex Deucher 已提交
569
		return r;
570

A
Alex Deucher 已提交
571
	bo->kptr = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem);
572
	if (ptr)
A
Alex Deucher 已提交
573
		*ptr = bo->kptr;
574

A
Alex Deucher 已提交
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607
	return 0;
}

void amdgpu_bo_kunmap(struct amdgpu_bo *bo)
{
	if (bo->kptr == NULL)
		return;
	bo->kptr = NULL;
	ttm_bo_kunmap(&bo->kmap);
}

struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo)
{
	if (bo == NULL)
		return NULL;

	ttm_bo_reference(&bo->tbo);
	return bo;
}

void amdgpu_bo_unref(struct amdgpu_bo **bo)
{
	struct ttm_buffer_object *tbo;

	if ((*bo) == NULL)
		return;

	tbo = &((*bo)->tbo);
	ttm_bo_unref(&tbo);
	if (tbo == NULL)
		*bo = NULL;
}

608 609
int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain,
			     u64 min_offset, u64 max_offset,
A
Alex Deucher 已提交
610 611
			     u64 *gpu_addr)
{
612
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
A
Alex Deucher 已提交
613
	int r, i;
614
	unsigned fpfn, lpfn;
A
Alex Deucher 已提交
615

616
	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
A
Alex Deucher 已提交
617 618
		return -EPERM;

619 620 621
	if (WARN_ON_ONCE(min_offset > max_offset))
		return -EINVAL;

A
Alex Deucher 已提交
622
	if (bo->pin_count) {
623 624 625 626 627
		uint32_t mem_type = bo->tbo.mem.mem_type;

		if (domain != amdgpu_mem_type_to_domain(mem_type))
			return -EINVAL;

A
Alex Deucher 已提交
628 629 630 631 632
		bo->pin_count++;
		if (gpu_addr)
			*gpu_addr = amdgpu_bo_gpu_offset(bo);

		if (max_offset != 0) {
633
			u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset;
A
Alex Deucher 已提交
634 635 636 637 638 639
			WARN_ON_ONCE(max_offset <
				     (amdgpu_bo_gpu_offset(bo) - domain_start));
		}

		return 0;
	}
640 641

	bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
A
Alex Deucher 已提交
642 643 644 645
	amdgpu_ttm_placement_from_domain(bo, domain);
	for (i = 0; i < bo->placement.num_placement; i++) {
		/* force to pin into visible video ram */
		if ((bo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
646
		    !(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) &&
647
		    (!max_offset || max_offset >
648
		     adev->mc.visible_vram_size)) {
649
			if (WARN_ON_ONCE(min_offset >
650
					 adev->mc.visible_vram_size))
651 652
				return -EINVAL;
			fpfn = min_offset >> PAGE_SHIFT;
653
			lpfn = adev->mc.visible_vram_size >> PAGE_SHIFT;
654 655 656 657 658 659
		} else {
			fpfn = min_offset >> PAGE_SHIFT;
			lpfn = max_offset >> PAGE_SHIFT;
		}
		if (fpfn > bo->placements[i].fpfn)
			bo->placements[i].fpfn = fpfn;
660 661
		if (!bo->placements[i].lpfn ||
		    (lpfn && lpfn < bo->placements[i].lpfn))
662
			bo->placements[i].lpfn = lpfn;
A
Alex Deucher 已提交
663 664 665 666
		bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT;
	}

	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
667
	if (unlikely(r)) {
668
		dev_err(adev->dev, "%p pin failed\n", bo);
669 670
		goto error;
	}
671
	r = amdgpu_ttm_bind(&bo->tbo, &bo->tbo.mem);
672
	if (unlikely(r)) {
673
		dev_err(adev->dev, "%p bind failed\n", bo);
674 675
		goto error;
	}
676 677 678 679 680

	bo->pin_count = 1;
	if (gpu_addr != NULL)
		*gpu_addr = amdgpu_bo_gpu_offset(bo);
	if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
681
		adev->vram_pin_size += amdgpu_bo_size(bo);
682
		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
683
			adev->invisible_pin_size += amdgpu_bo_size(bo);
684
	} else if (domain == AMDGPU_GEM_DOMAIN_GTT) {
685
		adev->gart_pin_size += amdgpu_bo_size(bo);
A
Alex Deucher 已提交
686
	}
687 688

error:
A
Alex Deucher 已提交
689 690 691 692 693
	return r;
}

int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr)
{
694
	return amdgpu_bo_pin_restricted(bo, domain, 0, 0, gpu_addr);
A
Alex Deucher 已提交
695 696 697 698
}

int amdgpu_bo_unpin(struct amdgpu_bo *bo)
{
699
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
A
Alex Deucher 已提交
700 701 702
	int r, i;

	if (!bo->pin_count) {
703
		dev_warn(adev->dev, "%p unpin not necessary\n", bo);
A
Alex Deucher 已提交
704 705 706 707 708 709 710 711 712 713
		return 0;
	}
	bo->pin_count--;
	if (bo->pin_count)
		return 0;
	for (i = 0; i < bo->placement.num_placement; i++) {
		bo->placements[i].lpfn = 0;
		bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT;
	}
	r = ttm_bo_validate(&bo->tbo, &bo->placement, false, false);
714
	if (unlikely(r)) {
715
		dev_err(adev->dev, "%p validate failed for unpin\n", bo);
716
		goto error;
A
Alex Deucher 已提交
717
	}
718 719

	if (bo->tbo.mem.mem_type == TTM_PL_VRAM) {
720
		adev->vram_pin_size -= amdgpu_bo_size(bo);
721
		if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
722
			adev->invisible_pin_size -= amdgpu_bo_size(bo);
723
	} else if (bo->tbo.mem.mem_type == TTM_PL_TT) {
724
		adev->gart_pin_size -= amdgpu_bo_size(bo);
725 726 727
	}

error:
A
Alex Deucher 已提交
728 729 730 731 732 733
	return r;
}

int amdgpu_bo_evict_vram(struct amdgpu_device *adev)
{
	/* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */
734
	if (0 && (adev->flags & AMD_IS_APU)) {
A
Alex Deucher 已提交
735 736 737 738 739 740
		/* Useless to evict on IGP chips */
		return 0;
	}
	return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM);
}

741 742 743 744 745 746 747 748 749 750 751
static const char *amdgpu_vram_names[] = {
	"UNKNOWN",
	"GDDR1",
	"DDR2",
	"GDDR3",
	"GDDR4",
	"GDDR5",
	"HBM",
	"DDR3"
};

A
Alex Deucher 已提交
752 753
int amdgpu_bo_init(struct amdgpu_device *adev)
{
754 755 756 757
	/* reserve PAT memory space to WC for VRAM */
	arch_io_reserve_memtype_wc(adev->mc.aper_base,
				   adev->mc.aper_size);

A
Alex Deucher 已提交
758 759 760 761 762 763
	/* Add an MTRR for the VRAM */
	adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base,
					      adev->mc.aper_size);
	DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n",
		adev->mc.mc_vram_size >> 20,
		(unsigned long long)adev->mc.aper_size >> 20);
764 765
	DRM_INFO("RAM width %dbits %s\n",
		 adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]);
A
Alex Deucher 已提交
766 767 768 769 770 771 772
	return amdgpu_ttm_init(adev);
}

void amdgpu_bo_fini(struct amdgpu_device *adev)
{
	amdgpu_ttm_fini(adev);
	arch_phys_wc_del(adev->mc.vram_mtrr);
773
	arch_io_free_memtype_wc(adev->mc.aper_base, adev->mc.aper_size);
A
Alex Deucher 已提交
774 775 776 777 778 779 780 781 782 783
}

int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo,
			     struct vm_area_struct *vma)
{
	return ttm_fbdev_mmap(vma, &bo->tbo);
}

int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags)
{
M
Marek Olšák 已提交
784
	if (AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6)
A
Alex Deucher 已提交
785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806
		return -EINVAL;

	bo->tiling_flags = tiling_flags;
	return 0;
}

void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags)
{
	lockdep_assert_held(&bo->tbo.resv->lock.base);

	if (tiling_flags)
		*tiling_flags = bo->tiling_flags;
}

int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata,
			    uint32_t metadata_size, uint64_t flags)
{
	void *buffer;

	if (!metadata_size) {
		if (bo->metadata_size) {
			kfree(bo->metadata);
807
			bo->metadata = NULL;
A
Alex Deucher 已提交
808 809 810 811 812 813 814 815
			bo->metadata_size = 0;
		}
		return 0;
	}

	if (metadata == NULL)
		return -EINVAL;

816
	buffer = kmemdup(metadata, metadata_size, GFP_KERNEL);
A
Alex Deucher 已提交
817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853
	if (buffer == NULL)
		return -ENOMEM;

	kfree(bo->metadata);
	bo->metadata_flags = flags;
	bo->metadata = buffer;
	bo->metadata_size = metadata_size;

	return 0;
}

int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer,
			   size_t buffer_size, uint32_t *metadata_size,
			   uint64_t *flags)
{
	if (!buffer && !metadata_size)
		return -EINVAL;

	if (buffer) {
		if (buffer_size < bo->metadata_size)
			return -EINVAL;

		if (bo->metadata_size)
			memcpy(buffer, bo->metadata, bo->metadata_size);
	}

	if (metadata_size)
		*metadata_size = bo->metadata_size;
	if (flags)
		*flags = bo->metadata_flags;

	return 0;
}

void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
			   struct ttm_mem_reg *new_mem)
{
854
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
855
	struct amdgpu_bo *abo;
856
	struct ttm_mem_reg *old_mem = &bo->mem;
A
Alex Deucher 已提交
857 858 859 860

	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
		return;

861
	abo = container_of(bo, struct amdgpu_bo, tbo);
862
	amdgpu_vm_bo_invalidate(adev, abo);
A
Alex Deucher 已提交
863 864 865 866 867 868

	/* update statistics */
	if (!new_mem)
		return;

	/* move_notify is called before move happens */
869
	amdgpu_update_memory_usage(adev, &bo->mem, new_mem);
870

871
	trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type);
A
Alex Deucher 已提交
872 873 874 875
}

int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
{
876
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
877 878 879
	struct amdgpu_bo *abo;
	unsigned long offset, size, lpfn;
	int i, r;
A
Alex Deucher 已提交
880 881 882

	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
		return 0;
883 884 885 886 887 888 889

	abo = container_of(bo, struct amdgpu_bo, tbo);
	if (bo->mem.mem_type != TTM_PL_VRAM)
		return 0;

	size = bo->mem.num_pages << PAGE_SHIFT;
	offset = bo->mem.start << PAGE_SHIFT;
890 891 892
	/* TODO: figure out how to map scattered VRAM to the CPU */
	if ((offset + size) <= adev->mc.visible_vram_size &&
	    (abo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS))
893 894
		return 0;

895 896 897 898
	/* Can't move a pinned BO to visible VRAM */
	if (abo->pin_count > 0)
		return -EINVAL;

899
	/* hurrah the memory is not visible ! */
900
	abo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
901 902 903 904 905
	amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM);
	lpfn =	adev->mc.visible_vram_size >> PAGE_SHIFT;
	for (i = 0; i < abo->placement.num_placement; i++) {
		/* Force into visible VRAM */
		if ((abo->placements[i].flags & TTM_PL_FLAG_VRAM) &&
906 907
		    (!abo->placements[i].lpfn ||
		     abo->placements[i].lpfn > lpfn))
908 909 910 911 912 913 914 915
			abo->placements[i].lpfn = lpfn;
	}
	r = ttm_bo_validate(bo, &abo->placement, false, false);
	if (unlikely(r == -ENOMEM)) {
		amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
		return ttm_bo_validate(bo, &abo->placement, false, false);
	} else if (unlikely(r != 0)) {
		return r;
A
Alex Deucher 已提交
916
	}
917 918 919 920 921 922

	offset = bo->mem.start << PAGE_SHIFT;
	/* this should never happen */
	if ((offset + size) > adev->mc.visible_vram_size)
		return -EINVAL;

A
Alex Deucher 已提交
923 924 925 926 927 928 929 930 931 932 933
	return 0;
}

/**
 * amdgpu_bo_fence - add fence to buffer object
 *
 * @bo: buffer object in question
 * @fence: fence to add
 * @shared: true if fence should be added shared
 *
 */
934
void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence,
A
Alex Deucher 已提交
935 936 937 938 939
		     bool shared)
{
	struct reservation_object *resv = bo->tbo.resv;

	if (shared)
940
		reservation_object_add_shared_fence(resv, fence);
A
Alex Deucher 已提交
941
	else
942
		reservation_object_add_excl_fence(resv, fence);
A
Alex Deucher 已提交
943
}
944 945 946 947 948 949 950 951 952 953 954 955 956

/**
 * amdgpu_bo_gpu_offset - return GPU offset of bo
 * @bo:	amdgpu object for which we query the offset
 *
 * Returns current GPU offset of the object.
 *
 * Note: object should either be pinned or reserved when calling this
 * function, it might be useful to add check for this for debugging.
 */
u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
{
	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM);
957 958
	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT &&
		     !amdgpu_ttm_is_bound(bo->tbo.ttm));
959 960
	WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) &&
		     !bo->pin_count);
961
	WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET);
962 963
	WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM &&
		     !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS));
964 965 966

	return bo->tbo.offset;
}