amdgpu_ttm.c 64.8 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
 * Copyright 2009 Jerome Glisse.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 */
/*
 * Authors:
 *    Jerome Glisse <glisse@freedesktop.org>
 *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
 *    Dave Airlie
 */
32 33 34 35 36
#include <drm/ttm/ttm_bo_api.h>
#include <drm/ttm/ttm_bo_driver.h>
#include <drm/ttm/ttm_placement.h>
#include <drm/ttm/ttm_module.h>
#include <drm/ttm/ttm_page_alloc.h>
A
Alex Deucher 已提交
37 38 39 40 41 42 43 44
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/swiotlb.h>
#include <linux/swap.h>
#include <linux/pagemap.h>
#include <linux/debugfs.h>
45
#include <linux/iommu.h>
A
Alex Deucher 已提交
46
#include "amdgpu.h"
47
#include "amdgpu_object.h"
48
#include "amdgpu_trace.h"
49
#include "amdgpu_amdkfd.h"
A
Alex Deucher 已提交
50 51 52 53
#include "bif/bif_4_1_d.h"

#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)

54 55 56 57 58 59
static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
			     struct ttm_mem_reg *mem, unsigned num_pages,
			     uint64_t offset, unsigned window,
			     struct amdgpu_ring *ring,
			     uint64_t *addr);

A
Alex Deucher 已提交
60 61 62 63 64 65
static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);

/*
 * Global memory.
 */
66 67 68 69 70 71 72 73 74 75

/**
 * amdgpu_ttm_mem_global_init - Initialize and acquire reference to
 * memory object
 *
 * @ref: Object for initialization.
 *
 * This is called by drm_global_item_ref() when an object is being
 * initialized.
 */
A
Alex Deucher 已提交
76 77 78 79 80
static int amdgpu_ttm_mem_global_init(struct drm_global_reference *ref)
{
	return ttm_mem_global_init(ref->object);
}

81 82 83 84 85 86 87 88
/**
 * amdgpu_ttm_mem_global_release - Drop reference to a memory object
 *
 * @ref: Object being removed
 *
 * This is called by drm_global_item_unref() when an object is being
 * released.
 */
A
Alex Deucher 已提交
89 90 91 92 93
static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref)
{
	ttm_mem_global_release(ref->object);
}

94 95 96 97 98 99 100 101 102 103
/**
 * amdgpu_ttm_global_init - Initialize global TTM memory reference
 * 							structures.
 *
 * @adev:  	AMDGPU device for which the global structures need to be
 *			registered.
 *
 * This is called as part of the AMDGPU ttm init from amdgpu_ttm_init()
 * during bring up.
 */
104
static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
105 106 107 108
{
	struct drm_global_reference *global_ref;
	int r;

109
	/* ensure reference is false in case init fails */
A
Alex Deucher 已提交
110
	adev->mman.mem_global_referenced = false;
111

A
Alex Deucher 已提交
112 113 114 115 116 117
	global_ref = &adev->mman.mem_global_ref;
	global_ref->global_type = DRM_GLOBAL_TTM_MEM;
	global_ref->size = sizeof(struct ttm_mem_global);
	global_ref->init = &amdgpu_ttm_mem_global_init;
	global_ref->release = &amdgpu_ttm_mem_global_release;
	r = drm_global_item_ref(global_ref);
118
	if (r) {
A
Alex Deucher 已提交
119 120
		DRM_ERROR("Failed setting up TTM memory accounting "
			  "subsystem.\n");
121
		goto error_mem;
A
Alex Deucher 已提交
122 123 124 125 126 127 128 129 130 131
	}

	adev->mman.bo_global_ref.mem_glob =
		adev->mman.mem_global_ref.object;
	global_ref = &adev->mman.bo_global_ref.ref;
	global_ref->global_type = DRM_GLOBAL_TTM_BO;
	global_ref->size = sizeof(struct ttm_bo_global);
	global_ref->init = &ttm_bo_global_init;
	global_ref->release = &ttm_bo_global_release;
	r = drm_global_item_ref(global_ref);
132
	if (r) {
A
Alex Deucher 已提交
133
		DRM_ERROR("Failed setting up TTM BO subsystem.\n");
134
		goto error_bo;
A
Alex Deucher 已提交
135 136
	}

137 138
	mutex_init(&adev->mman.gtt_window_lock);

A
Alex Deucher 已提交
139
	adev->mman.mem_global_referenced = true;
140

A
Alex Deucher 已提交
141
	return 0;
142 143 144 145 146

error_bo:
	drm_global_item_unref(&adev->mman.mem_global_ref);
error_mem:
	return r;
A
Alex Deucher 已提交
147 148 149 150 151
}

static void amdgpu_ttm_global_fini(struct amdgpu_device *adev)
{
	if (adev->mman.mem_global_referenced) {
152
		mutex_destroy(&adev->mman.gtt_window_lock);
A
Alex Deucher 已提交
153 154 155 156 157 158 159 160 161 162 163
		drm_global_item_unref(&adev->mman.bo_global_ref.ref);
		drm_global_item_unref(&adev->mman.mem_global_ref);
		adev->mman.mem_global_referenced = false;
	}
}

static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
{
	return 0;
}

164 165 166 167 168 169 170 171 172 173 174 175
/**
 * amdgpu_init_mem_type - 	Initialize a memory manager for a specific
 * 							type of memory request.
 *
 * @bdev:	The TTM BO device object (contains a reference to
 * 			amdgpu_device)
 * @type:	The type of memory requested
 * @man:
 *
 * This is called by ttm_bo_init_mm() when a buffer object is being
 * initialized.
 */
A
Alex Deucher 已提交
176 177 178 179 180
static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
				struct ttm_mem_type_manager *man)
{
	struct amdgpu_device *adev;

181
	adev = amdgpu_ttm_adev(bdev);
A
Alex Deucher 已提交
182 183 184 185 186 187 188 189 190

	switch (type) {
	case TTM_PL_SYSTEM:
		/* System memory */
		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
		man->available_caching = TTM_PL_MASK_CACHING;
		man->default_caching = TTM_PL_FLAG_CACHED;
		break;
	case TTM_PL_TT:
191
		/* GTT memory  */
192
		man->func = &amdgpu_gtt_mgr_func;
193
		man->gpu_offset = adev->gmc.gart_start;
A
Alex Deucher 已提交
194 195 196 197 198 199
		man->available_caching = TTM_PL_MASK_CACHING;
		man->default_caching = TTM_PL_FLAG_CACHED;
		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
		break;
	case TTM_PL_VRAM:
		/* "On-card" video ram */
C
Christian König 已提交
200
		man->func = &amdgpu_vram_mgr_func;
201
		man->gpu_offset = adev->gmc.vram_start;
A
Alex Deucher 已提交
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
		man->flags = TTM_MEMTYPE_FLAG_FIXED |
			     TTM_MEMTYPE_FLAG_MAPPABLE;
		man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
		man->default_caching = TTM_PL_FLAG_WC;
		break;
	case AMDGPU_PL_GDS:
	case AMDGPU_PL_GWS:
	case AMDGPU_PL_OA:
		/* On-chip GDS memory*/
		man->func = &ttm_bo_manager_func;
		man->gpu_offset = 0;
		man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;
		man->available_caching = TTM_PL_FLAG_UNCACHED;
		man->default_caching = TTM_PL_FLAG_UNCACHED;
		break;
	default:
		DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
		return -EINVAL;
	}
	return 0;
}

224 225 226 227 228 229 230 231
/**
 * amdgpu_evict_flags - Compute placement flags
 *
 * @bo: The buffer object to evict
 * @placement: Possible destination(s) for evicted BO
 *
 * Fill in placement data when ttm_bo_evict() is called
 */
A
Alex Deucher 已提交
232 233 234
static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
				struct ttm_placement *placement)
{
235
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
236
	struct amdgpu_bo *abo;
237
	static const struct ttm_place placements = {
A
Alex Deucher 已提交
238 239 240 241 242
		.fpfn = 0,
		.lpfn = 0,
		.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
	};

243
	/* Don't handle scatter gather BOs */
244 245 246 247 248 249
	if (bo->type == ttm_bo_type_sg) {
		placement->num_placement = 0;
		placement->num_busy_placement = 0;
		return;
	}

250
	/* Object isn't an AMDGPU object so ignore */
A
Alex Deucher 已提交
251 252 253 254 255 256 257
	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) {
		placement->placement = &placements;
		placement->busy_placement = &placements;
		placement->num_placement = 1;
		placement->num_busy_placement = 1;
		return;
	}
258

259
	abo = ttm_to_amdgpu_bo(bo);
A
Alex Deucher 已提交
260 261
	switch (bo->mem.mem_type) {
	case TTM_PL_VRAM:
262
		if (!adev->mman.buffer_funcs_enabled) {
263
			/* Move to system memory */
264
			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
265
		} else if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
266 267
			   !(abo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) &&
			   amdgpu_bo_in_cpu_visible_vram(abo)) {
268 269 270 271 272 273 274 275

			/* Try evicting to the CPU inaccessible part of VRAM
			 * first, but only set GTT as busy placement, so this
			 * BO will be evicted to GTT rather than causing other
			 * BOs to be evicted from VRAM
			 */
			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM |
							 AMDGPU_GEM_DOMAIN_GTT);
276
			abo->placements[0].fpfn = adev->gmc.visible_vram_size >> PAGE_SHIFT;
277 278 279
			abo->placements[0].lpfn = 0;
			abo->placement.busy_placement = &abo->placements[1];
			abo->placement.num_busy_placement = 1;
280
		} else {
281
			/* Move to GTT memory */
282
			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
283
		}
A
Alex Deucher 已提交
284 285 286
		break;
	case TTM_PL_TT:
	default:
287
		amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
A
Alex Deucher 已提交
288
	}
289
	*placement = abo->placement;
A
Alex Deucher 已提交
290 291
}

292 293 294 295 296 297 298 299 300
/**
 * amdgpu_verify_access - Verify access for a mmap call
 *
 * @bo:		The buffer object to map
 * @filp:	The file pointer from the process performing the mmap
 *
 * This is called by ttm_bo_mmap() to verify whether a process
 * has the right to mmap a BO to their process space.
 */
A
Alex Deucher 已提交
301 302
static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
{
303
	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
A
Alex Deucher 已提交
304

305 306 307 308 309 310 311
	/*
	 * Don't verify access for KFD BOs. They don't have a GEM
	 * object associated with them.
	 */
	if (abo->kfd_bo)
		return 0;

312 313
	if (amdgpu_ttm_tt_get_usermm(bo->ttm))
		return -EPERM;
314
	return drm_vma_node_verify_access(&abo->gem_base.vma_node,
D
David Herrmann 已提交
315
					  filp->private_data);
A
Alex Deucher 已提交
316 317
}

318 319 320 321 322 323 324 325 326
/**
 * amdgpu_move_null - Register memory for a buffer object
 *
 * @bo:			The bo to assign the memory to
 * @new_mem:	The memory to be assigned.
 *
 * Assign the memory from new_mem to the memory of the buffer object
 * bo.
 */
A
Alex Deucher 已提交
327 328 329 330 331 332 333 334 335 336
static void amdgpu_move_null(struct ttm_buffer_object *bo,
			     struct ttm_mem_reg *new_mem)
{
	struct ttm_mem_reg *old_mem = &bo->mem;

	BUG_ON(old_mem->mm_node != NULL);
	*old_mem = *new_mem;
	new_mem->mm_node = NULL;
}

337 338 339 340
/**
 * amdgpu_mm_node_addr -	Compute the GPU relative offset of a GTT
 * 							buffer.
 */
341 342 343
static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
				    struct drm_mm_node *mm_node,
				    struct ttm_mem_reg *mem)
A
Alex Deucher 已提交
344
{
345
	uint64_t addr = 0;
346

347
	if (mem->mem_type != TTM_PL_TT || amdgpu_gtt_mgr_has_gart_addr(mem)) {
348 349 350
		addr = mm_node->start << PAGE_SHIFT;
		addr += bo->bdev->man[mem->mem_type].gpu_offset;
	}
351
	return addr;
352 353
}

354
/**
355 356 357 358
 * amdgpu_find_mm_node -	Helper function finds the drm_mm_node
 *  						corresponding to @offset. It also modifies
 * 							the offset to be within the drm_mm_node
 * 							returned
359 360 361
 */
static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
					       unsigned long *offset)
362
{
363
	struct drm_mm_node *mm_node = mem->mm_node;
364

365 366 367 368 369 370
	while (*offset >= (mm_node->size << PAGE_SHIFT)) {
		*offset -= (mm_node->size << PAGE_SHIFT);
		++mm_node;
	}
	return mm_node;
}
371

372 373
/**
 * amdgpu_copy_ttm_mem_to_mem - Helper function for copy
374 375 376 377 378 379 380 381 382 383 384 385 386
 *
 * The function copies @size bytes from {src->mem + src->offset} to
 * {dst->mem + dst->offset}. src->bo and dst->bo could be same BO for a
 * move and different for a BO to BO copy.
 *
 * @f: Returns the last fence if multiple jobs are submitted.
 */
int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
			       struct amdgpu_copy_mem *src,
			       struct amdgpu_copy_mem *dst,
			       uint64_t size,
			       struct reservation_object *resv,
			       struct dma_fence **f)
387 388
{
	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
389 390 391
	struct drm_mm_node *src_mm, *dst_mm;
	uint64_t src_node_start, dst_node_start, src_node_size,
		 dst_node_size, src_page_offset, dst_page_offset;
392
	struct dma_fence *fence = NULL;
393 394 395
	int r = 0;
	const uint64_t GTT_MAX_BYTES = (AMDGPU_GTT_MAX_TRANSFER_SIZE *
					AMDGPU_GPU_PAGE_SIZE);
396

397
	if (!adev->mman.buffer_funcs_enabled) {
A
Alex Deucher 已提交
398 399 400 401
		DRM_ERROR("Trying to move memory with ring turned off.\n");
		return -EINVAL;
	}

402
	src_mm = amdgpu_find_mm_node(src->mem, &src->offset);
403 404 405 406
	src_node_start = amdgpu_mm_node_addr(src->bo, src_mm, src->mem) +
					     src->offset;
	src_node_size = (src_mm->size << PAGE_SHIFT) - src->offset;
	src_page_offset = src_node_start & (PAGE_SIZE - 1);
407

408
	dst_mm = amdgpu_find_mm_node(dst->mem, &dst->offset);
409 410 411 412
	dst_node_start = amdgpu_mm_node_addr(dst->bo, dst_mm, dst->mem) +
					     dst->offset;
	dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst->offset;
	dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
413

414
	mutex_lock(&adev->mman.gtt_window_lock);
415 416 417 418

	while (size) {
		unsigned long cur_size;
		uint64_t from = src_node_start, to = dst_node_start;
419
		struct dma_fence *next;
420

421 422 423 424 425 426 427 428 429 430 431 432 433
		/* Copy size cannot exceed GTT_MAX_BYTES. So if src or dst
		 * begins at an offset, then adjust the size accordingly
		 */
		cur_size = min3(min(src_node_size, dst_node_size), size,
				GTT_MAX_BYTES);
		if (cur_size + src_page_offset > GTT_MAX_BYTES ||
		    cur_size + dst_page_offset > GTT_MAX_BYTES)
			cur_size -= max(src_page_offset, dst_page_offset);

		/* Map only what needs to be accessed. Map src to window 0 and
		 * dst to window 1
		 */
		if (src->mem->mem_type == TTM_PL_TT &&
434
		    !amdgpu_gtt_mgr_has_gart_addr(src->mem)) {
435 436 437 438
			r = amdgpu_map_buffer(src->bo, src->mem,
					PFN_UP(cur_size + src_page_offset),
					src_node_start, 0, ring,
					&from);
439 440
			if (r)
				goto error;
441 442 443 444
			/* Adjust the offset because amdgpu_map_buffer returns
			 * start of mapped page
			 */
			from += src_page_offset;
445 446
		}

447
		if (dst->mem->mem_type == TTM_PL_TT &&
448
		    !amdgpu_gtt_mgr_has_gart_addr(dst->mem)) {
449 450 451 452
			r = amdgpu_map_buffer(dst->bo, dst->mem,
					PFN_UP(cur_size + dst_page_offset),
					dst_node_start, 1, ring,
					&to);
453 454
			if (r)
				goto error;
455
			to += dst_page_offset;
456 457
		}

458 459
		r = amdgpu_copy_buffer(ring, from, to, cur_size,
				       resv, &next, false, true);
460 461 462
		if (r)
			goto error;

463
		dma_fence_put(fence);
464 465
		fence = next;

466 467
		size -= cur_size;
		if (!size)
468 469
			break;

470 471 472 473 474
		src_node_size -= cur_size;
		if (!src_node_size) {
			src_node_start = amdgpu_mm_node_addr(src->bo, ++src_mm,
							     src->mem);
			src_node_size = (src_mm->size << PAGE_SHIFT);
475
		} else {
476 477
			src_node_start += cur_size;
			src_page_offset = src_node_start & (PAGE_SIZE - 1);
478
		}
479 480 481 482 483
		dst_node_size -= cur_size;
		if (!dst_node_size) {
			dst_node_start = amdgpu_mm_node_addr(dst->bo, ++dst_mm,
							     dst->mem);
			dst_node_size = (dst_mm->size << PAGE_SHIFT);
484
		} else {
485 486
			dst_node_start += cur_size;
			dst_page_offset = dst_node_start & (PAGE_SIZE - 1);
487 488
		}
	}
489
error:
490
	mutex_unlock(&adev->mman.gtt_window_lock);
491 492 493 494 495 496
	if (f)
		*f = dma_fence_get(fence);
	dma_fence_put(fence);
	return r;
}

497 498 499 500 501 502
/**
 * amdgpu_move_blit - Copy an entire buffer to another buffer
 *
 * This is a helper called by amdgpu_bo_move() and
 * amdgpu_move_vram_ram() to help move buffers to and from VRAM.
 */
503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
static int amdgpu_move_blit(struct ttm_buffer_object *bo,
			    bool evict, bool no_wait_gpu,
			    struct ttm_mem_reg *new_mem,
			    struct ttm_mem_reg *old_mem)
{
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
	struct amdgpu_copy_mem src, dst;
	struct dma_fence *fence = NULL;
	int r;

	src.bo = bo;
	dst.bo = bo;
	src.mem = old_mem;
	dst.mem = new_mem;
	src.offset = 0;
	dst.offset = 0;

	r = amdgpu_ttm_copy_mem_to_mem(adev, &src, &dst,
				       new_mem->num_pages << PAGE_SHIFT,
				       bo->resv, &fence);
	if (r)
		goto error;
525 526

	r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
527
	dma_fence_put(fence);
A
Alex Deucher 已提交
528
	return r;
529 530 531

error:
	if (fence)
532 533
		dma_fence_wait(fence, false);
	dma_fence_put(fence);
534
	return r;
A
Alex Deucher 已提交
535 536
}

537 538 539 540 541
/**
 * amdgpu_move_vram_ram - Copy VRAM buffer to RAM buffer
 *
 * Called by amdgpu_bo_move().
 */
542 543
static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
				struct ttm_operation_ctx *ctx,
A
Alex Deucher 已提交
544 545 546 547 548 549 550 551 552
				struct ttm_mem_reg *new_mem)
{
	struct amdgpu_device *adev;
	struct ttm_mem_reg *old_mem = &bo->mem;
	struct ttm_mem_reg tmp_mem;
	struct ttm_place placements;
	struct ttm_placement placement;
	int r;

553
	adev = amdgpu_ttm_adev(bo->bdev);
554 555

	/* create space/pages for new_mem in GTT space */
A
Alex Deucher 已提交
556 557 558 559 560 561 562
	tmp_mem = *new_mem;
	tmp_mem.mm_node = NULL;
	placement.num_placement = 1;
	placement.placement = &placements;
	placement.num_busy_placement = 1;
	placement.busy_placement = &placements;
	placements.fpfn = 0;
563
	placements.lpfn = 0;
A
Alex Deucher 已提交
564
	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
565
	r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
A
Alex Deucher 已提交
566 567 568 569
	if (unlikely(r)) {
		return r;
	}

570
	/* set caching flags */
A
Alex Deucher 已提交
571 572 573 574 575
	r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
	if (unlikely(r)) {
		goto out_cleanup;
	}

576
	/* Bind the memory to the GTT space */
577
	r = ttm_tt_bind(bo->ttm, &tmp_mem, ctx);
A
Alex Deucher 已提交
578 579 580
	if (unlikely(r)) {
		goto out_cleanup;
	}
581 582

	/* blit VRAM to GTT */
583
	r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, &tmp_mem, old_mem);
A
Alex Deucher 已提交
584 585 586
	if (unlikely(r)) {
		goto out_cleanup;
	}
587 588

	/* move BO (in tmp_mem) to new_mem */
589
	r = ttm_bo_move_ttm(bo, ctx, new_mem);
A
Alex Deucher 已提交
590 591 592 593 594
out_cleanup:
	ttm_bo_mem_put(bo, &tmp_mem);
	return r;
}

595 596 597 598 599
/**
 * amdgpu_move_ram_vram - Copy buffer from RAM to VRAM
 *
 * Called by amdgpu_bo_move().
 */
600 601
static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
				struct ttm_operation_ctx *ctx,
A
Alex Deucher 已提交
602 603 604 605 606 607 608 609 610
				struct ttm_mem_reg *new_mem)
{
	struct amdgpu_device *adev;
	struct ttm_mem_reg *old_mem = &bo->mem;
	struct ttm_mem_reg tmp_mem;
	struct ttm_placement placement;
	struct ttm_place placements;
	int r;

611
	adev = amdgpu_ttm_adev(bo->bdev);
612 613

	/* make space in GTT for old_mem buffer */
A
Alex Deucher 已提交
614 615 616 617 618 619 620
	tmp_mem = *new_mem;
	tmp_mem.mm_node = NULL;
	placement.num_placement = 1;
	placement.placement = &placements;
	placement.num_busy_placement = 1;
	placement.busy_placement = &placements;
	placements.fpfn = 0;
621
	placements.lpfn = 0;
A
Alex Deucher 已提交
622
	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
623
	r = ttm_bo_mem_space(bo, &placement, &tmp_mem, ctx);
A
Alex Deucher 已提交
624 625 626
	if (unlikely(r)) {
		return r;
	}
627 628

	/* move/bind old memory to GTT space */
629
	r = ttm_bo_move_ttm(bo, ctx, &tmp_mem);
A
Alex Deucher 已提交
630 631 632
	if (unlikely(r)) {
		goto out_cleanup;
	}
633 634

	/* copy to VRAM */
635
	r = amdgpu_move_blit(bo, true, ctx->no_wait_gpu, new_mem, old_mem);
A
Alex Deucher 已提交
636 637 638 639 640 641 642 643
	if (unlikely(r)) {
		goto out_cleanup;
	}
out_cleanup:
	ttm_bo_mem_put(bo, &tmp_mem);
	return r;
}

644 645 646 647 648
/**
 * amdgpu_bo_move - Move a buffer object to a new memory location
 *
 * Called by ttm_bo_handle_move_mem()
 */
649 650 651
static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
			  struct ttm_operation_ctx *ctx,
			  struct ttm_mem_reg *new_mem)
A
Alex Deucher 已提交
652 653
{
	struct amdgpu_device *adev;
654
	struct amdgpu_bo *abo;
A
Alex Deucher 已提交
655 656 657
	struct ttm_mem_reg *old_mem = &bo->mem;
	int r;

658
	/* Can't move a pinned BO */
659
	abo = ttm_to_amdgpu_bo(bo);
660 661 662
	if (WARN_ON_ONCE(abo->pin_count > 0))
		return -EINVAL;

663
	adev = amdgpu_ttm_adev(bo->bdev);
664

A
Alex Deucher 已提交
665 666 667 668 669 670 671 672 673 674 675 676
	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
		amdgpu_move_null(bo, new_mem);
		return 0;
	}
	if ((old_mem->mem_type == TTM_PL_TT &&
	     new_mem->mem_type == TTM_PL_SYSTEM) ||
	    (old_mem->mem_type == TTM_PL_SYSTEM &&
	     new_mem->mem_type == TTM_PL_TT)) {
		/* bind is enough */
		amdgpu_move_null(bo, new_mem);
		return 0;
	}
677 678

	if (!adev->mman.buffer_funcs_enabled)
A
Alex Deucher 已提交
679 680 681 682
		goto memcpy;

	if (old_mem->mem_type == TTM_PL_VRAM &&
	    new_mem->mem_type == TTM_PL_SYSTEM) {
683
		r = amdgpu_move_vram_ram(bo, evict, ctx, new_mem);
A
Alex Deucher 已提交
684 685
	} else if (old_mem->mem_type == TTM_PL_SYSTEM &&
		   new_mem->mem_type == TTM_PL_VRAM) {
686
		r = amdgpu_move_ram_vram(bo, evict, ctx, new_mem);
A
Alex Deucher 已提交
687
	} else {
688 689
		r = amdgpu_move_blit(bo, evict, ctx->no_wait_gpu,
				     new_mem, old_mem);
A
Alex Deucher 已提交
690 691 692 693
	}

	if (r) {
memcpy:
694
		r = ttm_bo_move_memcpy(bo, ctx, new_mem);
A
Alex Deucher 已提交
695 696 697 698 699
		if (r) {
			return r;
		}
	}

700 701 702 703 704 705 706 707 708
	if (bo->type == ttm_bo_type_device &&
	    new_mem->mem_type == TTM_PL_VRAM &&
	    old_mem->mem_type != TTM_PL_VRAM) {
		/* amdgpu_bo_fault_reserve_notify will re-set this if the CPU
		 * accesses the BO after it's moved.
		 */
		abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
	}

A
Alex Deucher 已提交
709 710 711 712 713
	/* update statistics */
	atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);
	return 0;
}

714 715 716 717 718
/**
 * amdgpu_ttm_io_mem_reserve - Reserve a block of memory during a fault
 *
 * Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
 */
A
Alex Deucher 已提交
719 720 721
static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
722
	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
723
	struct drm_mm_node *mm_node = mem->mm_node;
A
Alex Deucher 已提交
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740

	mem->bus.addr = NULL;
	mem->bus.offset = 0;
	mem->bus.size = mem->num_pages << PAGE_SHIFT;
	mem->bus.base = 0;
	mem->bus.is_iomem = false;
	if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
		return -EINVAL;
	switch (mem->mem_type) {
	case TTM_PL_SYSTEM:
		/* system memory */
		return 0;
	case TTM_PL_TT:
		break;
	case TTM_PL_VRAM:
		mem->bus.offset = mem->start << PAGE_SHIFT;
		/* check if it's visible */
741
		if ((mem->bus.offset + mem->bus.size) > adev->gmc.visible_vram_size)
A
Alex Deucher 已提交
742
			return -EINVAL;
743 744 745 746 747 748 749 750 751
		/* Only physically contiguous buffers apply. In a contiguous
		 * buffer, size of the first mm_node would match the number of
		 * pages in ttm_mem_reg.
		 */
		if (adev->mman.aper_base_kaddr &&
		    (mm_node->size == mem->num_pages))
			mem->bus.addr = (u8 *)adev->mman.aper_base_kaddr +
					mem->bus.offset;

752
		mem->bus.base = adev->gmc.aper_base;
A
Alex Deucher 已提交
753 754 755 756 757 758 759 760 761 762 763 764
		mem->bus.is_iomem = true;
		break;
	default:
		return -EINVAL;
	}
	return 0;
}

static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
}

765 766 767
static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
					   unsigned long page_offset)
{
768 769
	struct drm_mm_node *mm;
	unsigned long offset = (page_offset << PAGE_SHIFT);
770

771 772 773
	mm = amdgpu_find_mm_node(&bo->mem, &offset);
	return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start +
		(offset >> PAGE_SHIFT);
774 775
}

A
Alex Deucher 已提交
776 777 778
/*
 * TTM backend functions.
 */
779 780 781 782 783
struct amdgpu_ttm_gup_task_list {
	struct list_head	list;
	struct task_struct	*task;
};

A
Alex Deucher 已提交
784
struct amdgpu_ttm_tt {
785 786 787
	struct ttm_dma_tt	ttm;
	u64			offset;
	uint64_t		userptr;
788
	struct task_struct	*usertask;
789 790 791
	uint32_t		userflags;
	spinlock_t              guptasklock;
	struct list_head        guptasks;
792
	atomic_t		mmu_invalidations;
793
	uint32_t		last_set_pages;
A
Alex Deucher 已提交
794 795
};

796 797 798 799 800 801 802 803
/**
 * amdgpu_ttm_tt_get_user_pages - 	Pin pages of memory pointed to
 * 									by a USERPTR pointer to memory
 *
 * Called by amdgpu_gem_userptr_ioctl() and amdgpu_cs_parser_bos().
 * This provides a wrapper around the get_user_pages() call to provide
 * device accessible pages that back user memory.
 */
804
int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
A
Alex Deucher 已提交
805 806
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
807
	struct mm_struct *mm = gtt->usertask->mm;
808
	unsigned int flags = 0;
809 810
	unsigned pinned = 0;
	int r;
A
Alex Deucher 已提交
811

812 813 814
	if (!mm) /* Happens during process shutdown */
		return -ESRCH;

815 816 817
	if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
		flags |= FOLL_WRITE;

818
	down_read(&mm->mmap_sem);
819

A
Alex Deucher 已提交
820
	if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
821
		/* check that we only use anonymous memory
A
Alex Deucher 已提交
822 823 824 825
		   to prevent problems with writeback */
		unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
		struct vm_area_struct *vma;

826
		vma = find_vma(mm, gtt->userptr);
827
		if (!vma || vma->vm_file || vma->vm_end < end) {
828
			up_read(&mm->mmap_sem);
A
Alex Deucher 已提交
829
			return -EPERM;
830
		}
A
Alex Deucher 已提交
831 832
	}

833
	/* loop enough times using contiguous pages of memory */
A
Alex Deucher 已提交
834 835 836
	do {
		unsigned num_pages = ttm->num_pages - pinned;
		uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
837
		struct page **p = pages + pinned;
838 839 840 841 842 843
		struct amdgpu_ttm_gup_task_list guptask;

		guptask.task = current;
		spin_lock(&gtt->guptasklock);
		list_add(&guptask.list, &gtt->guptasks);
		spin_unlock(&gtt->guptasklock);
A
Alex Deucher 已提交
844

845 846 847 848 849 850
		if (mm == current->mm)
			r = get_user_pages(userptr, num_pages, flags, p, NULL);
		else
			r = get_user_pages_remote(gtt->usertask,
					mm, userptr, num_pages,
					flags, p, NULL, NULL);
851 852 853 854

		spin_lock(&gtt->guptasklock);
		list_del(&guptask.list);
		spin_unlock(&gtt->guptasklock);
A
Alex Deucher 已提交
855 856 857 858 859 860 861 862

		if (r < 0)
			goto release_pages;

		pinned += r;

	} while (pinned < ttm->num_pages);

863
	up_read(&mm->mmap_sem);
864 865 866
	return 0;

release_pages:
867
	release_pages(pages, pinned);
868
	up_read(&mm->mmap_sem);
869 870 871
	return r;
}

872 873 874 875 876 877 878 879
/**
 * amdgpu_ttm_tt_set_user_pages - 	Copy pages in, putting old pages
 * 									as necessary.
 *
 * Called by amdgpu_cs_list_validate().  This creates the page list
 * that backs user memory and will ultimately be mapped into the device
 * address space.
 */
880
void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
881 882 883 884
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
	unsigned i;

885
	gtt->last_set_pages = atomic_read(&gtt->mmu_invalidations);
886 887 888 889 890
	for (i = 0; i < ttm->num_pages; ++i) {
		if (ttm->pages[i])
			put_page(ttm->pages[i]);

		ttm->pages[i] = pages ? pages[i] : NULL;
891 892 893
	}
}

894 895 896 897 898
/**
 * amdgpu_ttm_tt_mark_user_page - Mark pages as dirty
 *
 * Called while unpinning userptr pages
 */
899
void amdgpu_ttm_tt_mark_user_pages(struct ttm_tt *ttm)
900 901 902 903
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
	unsigned i;

904 905 906 907 908 909 910 911 912 913
	for (i = 0; i < ttm->num_pages; ++i) {
		struct page *page = ttm->pages[i];

		if (!page)
			continue;

		if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
			set_page_dirty(page);

		mark_page_accessed(page);
914 915 916
	}
}

917 918 919 920 921 922
/**
 * amdgpu_ttm_tt_pin_userptr - 	prepare the sg table with the
 * 								user pages
 *
 * Called by amdgpu_ttm_backend_bind()
 **/
923 924
static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
{
925
	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
926 927 928 929 930 931 932 933
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
	unsigned nents;
	int r;

	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
	enum dma_data_direction direction = write ?
		DMA_BIDIRECTIONAL : DMA_TO_DEVICE;

934
	/* Allocate an SG array and squash pages into it */
A
Alex Deucher 已提交
935 936 937 938 939 940
	r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
				      ttm->num_pages << PAGE_SHIFT,
				      GFP_KERNEL);
	if (r)
		goto release_sg;

941
	/* Map SG to device */
A
Alex Deucher 已提交
942 943 944 945 946
	r = -ENOMEM;
	nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
	if (nents != ttm->sg->nents)
		goto release_sg;

947
	/* convert SG to linear array of pages and dma addresses */
A
Alex Deucher 已提交
948 949 950 951 952 953 954 955 956 957
	drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
					 gtt->ttm.dma_address, ttm->num_pages);

	return 0;

release_sg:
	kfree(ttm->sg);
	return r;
}

958 959 960
/**
 * amdgpu_ttm_tt_unpin_userptr - Unpin and unmap userptr pages
 */
A
Alex Deucher 已提交
961 962
static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
{
963
	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
A
Alex Deucher 已提交
964 965 966 967 968 969 970 971 972 973
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
	enum dma_data_direction direction = write ?
		DMA_BIDIRECTIONAL : DMA_TO_DEVICE;

	/* double check that we don't free the table twice */
	if (!ttm->sg->sgl)
		return;

974
	/* unmap the pages mapped to the device */
A
Alex Deucher 已提交
975 976
	dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);

977
	/* mark the pages as dirty */
978
	amdgpu_ttm_tt_mark_user_pages(ttm);
979

A
Alex Deucher 已提交
980 981 982
	sg_free_table(ttm->sg);
}

983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
int amdgpu_ttm_gart_bind(struct amdgpu_device *adev,
				struct ttm_buffer_object *tbo,
				uint64_t flags)
{
	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(tbo);
	struct ttm_tt *ttm = tbo->ttm;
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
	int r;

	if (abo->flags & AMDGPU_GEM_CREATE_MQD_GFX9) {
		uint64_t page_idx = 1;

		r = amdgpu_gart_bind(adev, gtt->offset, page_idx,
				ttm->pages, gtt->ttm.dma_address, flags);
		if (r)
			goto gart_bind_fail;

		/* Patch mtype of the second part BO */
		flags &=  ~AMDGPU_PTE_MTYPE_MASK;
		flags |= AMDGPU_PTE_MTYPE(AMDGPU_MTYPE_NC);

		r = amdgpu_gart_bind(adev,
				gtt->offset + (page_idx << PAGE_SHIFT),
				ttm->num_pages - page_idx,
				&ttm->pages[page_idx],
				&(gtt->ttm.dma_address[page_idx]), flags);
	} else {
		r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
				     ttm->pages, gtt->ttm.dma_address, flags);
	}

gart_bind_fail:
	if (r)
		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
			  ttm->num_pages, gtt->offset);

	return r;
}

1022 1023 1024 1025 1026 1027
/**
 * amdgpu_ttm_backend_bind - Bind GTT memory
 *
 * Called by ttm_tt_bind() on behalf of ttm_bo_handle_move_mem().
 * This handles binding GTT memory to the device address space.
 */
A
Alex Deucher 已提交
1028 1029 1030
static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
				   struct ttm_mem_reg *bo_mem)
{
C
Christian König 已提交
1031
	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
A
Alex Deucher 已提交
1032
	struct amdgpu_ttm_tt *gtt = (void*)ttm;
1033
	uint64_t flags;
1034
	int r = 0;
A
Alex Deucher 已提交
1035

1036 1037 1038 1039 1040 1041 1042
	if (gtt->userptr) {
		r = amdgpu_ttm_tt_pin_userptr(ttm);
		if (r) {
			DRM_ERROR("failed to pin userptr\n");
			return r;
		}
	}
A
Alex Deucher 已提交
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
	if (!ttm->num_pages) {
		WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
		     ttm->num_pages, bo_mem, ttm);
	}

	if (bo_mem->mem_type == AMDGPU_PL_GDS ||
	    bo_mem->mem_type == AMDGPU_PL_GWS ||
	    bo_mem->mem_type == AMDGPU_PL_OA)
		return -EINVAL;

1053 1054
	if (!amdgpu_gtt_mgr_has_gart_addr(bo_mem)) {
		gtt->offset = AMDGPU_BO_INVALID_OFFSET;
1055
		return 0;
1056
	}
1057

1058
	/* compute PTE flags relevant to this BO memory */
C
Christian König 已提交
1059
	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, bo_mem);
1060 1061

	/* bind pages into GART page tables */
1062
	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
C
Christian König 已提交
1063
	r = amdgpu_gart_bind(adev, gtt->offset, ttm->num_pages,
1064 1065
		ttm->pages, gtt->ttm.dma_address, flags);

1066
	if (r)
1067 1068
		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
			  ttm->num_pages, gtt->offset);
1069
	return r;
1070 1071
}

1072 1073 1074
/**
 * amdgpu_ttm_alloc_gart - Allocate GART memory for buffer object
 */
1075
int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
1076
{
1077
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
1078
	struct ttm_operation_ctx ctx = { false, false };
1079
	struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;
1080 1081 1082
	struct ttm_mem_reg tmp;
	struct ttm_placement placement;
	struct ttm_place placements;
1083
	uint64_t flags;
1084 1085
	int r;

1086 1087
	if (bo->mem.mem_type != TTM_PL_TT ||
	    amdgpu_gtt_mgr_has_gart_addr(&bo->mem))
1088 1089
		return 0;

1090
	/* allocate GTT space */
1091 1092 1093 1094 1095 1096 1097
	tmp = bo->mem;
	tmp.mm_node = NULL;
	placement.num_placement = 1;
	placement.placement = &placements;
	placement.num_busy_placement = 1;
	placement.busy_placement = &placements;
	placements.fpfn = 0;
1098
	placements.lpfn = adev->gmc.gart_size >> PAGE_SHIFT;
1099 1100
	placements.flags = (bo->mem.placement & ~TTM_PL_MASK_MEM) |
		TTM_PL_FLAG_TT;
1101

1102
	r = ttm_bo_mem_space(bo, &placement, &tmp, &ctx);
1103
	if (unlikely(r))
1104 1105
		return r;

1106
	/* compute PTE flags for this buffer object */
1107
	flags = amdgpu_ttm_tt_pte_flags(adev, bo->ttm, &tmp);
1108 1109

	/* Bind pages */
1110
	gtt->offset = (u64)tmp.start << PAGE_SHIFT;
1111
	r = amdgpu_ttm_gart_bind(adev, bo, flags);
1112
	if (unlikely(r)) {
1113
		ttm_bo_mem_put(bo, &tmp);
1114 1115
		return r;
	}
1116

1117 1118 1119 1120 1121 1122
	ttm_bo_mem_put(bo, &bo->mem);
	bo->mem = tmp;
	bo->offset = (bo->mem.start << PAGE_SHIFT) +
		bo->bdev->man[bo->mem.mem_type].gpu_offset;

	return 0;
A
Alex Deucher 已提交
1123 1124
}

1125 1126 1127 1128 1129 1130
/**
 * amdgpu_ttm_recover_gart - Rebind GTT pages
 *
 * Called by amdgpu_gtt_mgr_recover() from amdgpu_device_reset() to
 * rebind GTT pages during a GPU reset.
 */
1131
int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
1132
{
1133
	struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev);
1134
	uint64_t flags;
1135 1136
	int r;

1137
	if (!tbo->ttm)
1138 1139
		return 0;

1140 1141 1142
	flags = amdgpu_ttm_tt_pte_flags(adev, tbo->ttm, &tbo->mem);
	r = amdgpu_ttm_gart_bind(adev, tbo, flags);

1143
	return r;
1144 1145
}

1146 1147 1148 1149 1150 1151
/**
 * amdgpu_ttm_backend_unbind - Unbind GTT mapped pages
 *
 * Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
 * ttm_tt_destroy().
 */
A
Alex Deucher 已提交
1152 1153
static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
{
C
Christian König 已提交
1154
	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
A
Alex Deucher 已提交
1155
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1156
	int r;
A
Alex Deucher 已提交
1157

1158
	/* if the pages have userptr pinning then clear that first */
1159 1160 1161
	if (gtt->userptr)
		amdgpu_ttm_tt_unpin_userptr(ttm);

1162
	if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
1163 1164
		return 0;

A
Alex Deucher 已提交
1165
	/* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
C
Christian König 已提交
1166
	r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);
1167
	if (r)
1168 1169 1170
		DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n",
			  gtt->ttm.ttm.num_pages, gtt->offset);
	return r;
A
Alex Deucher 已提交
1171 1172 1173 1174 1175 1176
}

static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

1177 1178 1179
	if (gtt->usertask)
		put_task_struct(gtt->usertask);

A
Alex Deucher 已提交
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189
	ttm_dma_tt_fini(&gtt->ttm);
	kfree(gtt);
}

static struct ttm_backend_func amdgpu_backend_func = {
	.bind = &amdgpu_ttm_backend_bind,
	.unbind = &amdgpu_ttm_backend_unbind,
	.destroy = &amdgpu_ttm_backend_destroy,
};

1190 1191 1192 1193 1194 1195 1196
/**
 * amdgpu_ttm_tt_create - Create a ttm_tt object for a given BO
 *
 * @bo: The buffer object to create a GTT ttm_tt object around
 *
 * Called by ttm_tt_create().
 */
1197 1198
static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_buffer_object *bo,
					   uint32_t page_flags)
A
Alex Deucher 已提交
1199 1200 1201 1202
{
	struct amdgpu_device *adev;
	struct amdgpu_ttm_tt *gtt;

1203
	adev = amdgpu_ttm_adev(bo->bdev);
A
Alex Deucher 已提交
1204 1205 1206 1207 1208 1209

	gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
	if (gtt == NULL) {
		return NULL;
	}
	gtt->ttm.ttm.func = &amdgpu_backend_func;
1210 1211

	/* allocate space for the uninitialized page entries */
1212
	if (ttm_sg_tt_init(&gtt->ttm, bo, page_flags)) {
A
Alex Deucher 已提交
1213 1214 1215 1216 1217 1218
		kfree(gtt);
		return NULL;
	}
	return &gtt->ttm.ttm;
}

1219 1220 1221 1222 1223 1224
/**
 * amdgpu_ttm_tt_populate - Map GTT pages visible to the device
 *
 * Map the pages of a ttm_tt object to an address space visible
 * to the underlying device.
 */
1225 1226
static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm,
			struct ttm_operation_ctx *ctx)
A
Alex Deucher 已提交
1227
{
1228
	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
A
Alex Deucher 已提交
1229 1230 1231
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);

1232
	/* user pages are bound by amdgpu_ttm_tt_pin_userptr() */
A
Alex Deucher 已提交
1233
	if (gtt && gtt->userptr) {
1234
		ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
A
Alex Deucher 已提交
1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
		if (!ttm->sg)
			return -ENOMEM;

		ttm->page_flags |= TTM_PAGE_FLAG_SG;
		ttm->state = tt_unbound;
		return 0;
	}

	if (slave && ttm->sg) {
		drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
1245 1246
						 gtt->ttm.dma_address,
						 ttm->num_pages);
A
Alex Deucher 已提交
1247
		ttm->state = tt_unbound;
1248
		return 0;
A
Alex Deucher 已提交
1249 1250 1251
	}

#ifdef CONFIG_SWIOTLB
1252
	if (adev->need_swiotlb && swiotlb_nr_tbl()) {
1253
		return ttm_dma_populate(&gtt->ttm, adev->dev, ctx);
A
Alex Deucher 已提交
1254 1255 1256
	}
#endif

1257 1258
	/* fall back to generic helper to populate the page array
	 * and map them to the device */
1259
	return ttm_populate_and_map_pages(adev->dev, &gtt->ttm, ctx);
A
Alex Deucher 已提交
1260 1261
}

1262 1263 1264 1265 1266 1267
/**
 * amdgpu_ttm_tt_unpopulate - unmap GTT pages and unpopulate page arrays
 *
 * Unmaps pages of a ttm_tt object from the device address space and
 * unpopulates the page array backing it.
 */
A
Alex Deucher 已提交
1268 1269 1270 1271 1272 1273 1274
static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
{
	struct amdgpu_device *adev;
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);

	if (gtt && gtt->userptr) {
1275
		amdgpu_ttm_tt_set_user_pages(ttm, NULL);
A
Alex Deucher 已提交
1276 1277 1278 1279 1280 1281 1282 1283
		kfree(ttm->sg);
		ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
		return;
	}

	if (slave)
		return;

1284
	adev = amdgpu_ttm_adev(ttm->bdev);
A
Alex Deucher 已提交
1285 1286

#ifdef CONFIG_SWIOTLB
1287
	if (adev->need_swiotlb && swiotlb_nr_tbl()) {
A
Alex Deucher 已提交
1288 1289 1290 1291 1292
		ttm_dma_unpopulate(&gtt->ttm, adev->dev);
		return;
	}
#endif

1293
	/* fall back to generic helper to unmap and unpopulate array */
1294
	ttm_unmap_and_unpopulate_pages(adev->dev, &gtt->ttm);
A
Alex Deucher 已提交
1295 1296
}

1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
/**
 * amdgpu_ttm_tt_set_userptr -	Initialize userptr GTT ttm_tt
 * 								for the current task
 *
 * @ttm: The ttm_tt object to bind this userptr object to
 * @addr:  The address in the current tasks VM space to use
 * @flags: Requirements of userptr object.
 *
 * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages
 * to current task
 */
A
Alex Deucher 已提交
1308 1309 1310 1311 1312 1313 1314 1315 1316 1317
int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
			      uint32_t flags)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

	if (gtt == NULL)
		return -EINVAL;

	gtt->userptr = addr;
	gtt->userflags = flags;
1318 1319 1320 1321 1322 1323

	if (gtt->usertask)
		put_task_struct(gtt->usertask);
	gtt->usertask = current->group_leader;
	get_task_struct(gtt->usertask);

1324 1325
	spin_lock_init(&gtt->guptasklock);
	INIT_LIST_HEAD(&gtt->guptasks);
1326
	atomic_set(&gtt->mmu_invalidations, 0);
1327
	gtt->last_set_pages = 0;
1328

A
Alex Deucher 已提交
1329 1330 1331
	return 0;
}

1332 1333 1334
/**
 * amdgpu_ttm_tt_get_usermm - Return memory manager for ttm_tt object
 */
1335
struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
A
Alex Deucher 已提交
1336 1337 1338 1339
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

	if (gtt == NULL)
1340
		return NULL;
A
Alex Deucher 已提交
1341

1342 1343 1344 1345
	if (gtt->usertask == NULL)
		return NULL;

	return gtt->usertask->mm;
A
Alex Deucher 已提交
1346 1347
}

1348 1349 1350 1351 1352 1353
/**
 * amdgpu_ttm_tt_affect_userptr -	Determine if a ttm_tt object lays
 * 									inside an address range for the
 * 									current task.
 *
 */
1354 1355 1356 1357
bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
				  unsigned long end)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
1358
	struct amdgpu_ttm_gup_task_list *entry;
1359 1360
	unsigned long size;

1361
	if (gtt == NULL || !gtt->userptr)
1362 1363
		return false;

1364 1365 1366
	/* Return false if no part of the ttm_tt object lies within
	 * the range
	 */
1367 1368 1369 1370
	size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;
	if (gtt->userptr > end || gtt->userptr + size <= start)
		return false;

1371 1372 1373
	/* Search the lists of tasks that hold this mapping and see
	 * if current is one of them.  If it is return false.
	 */
1374 1375 1376 1377 1378 1379 1380 1381 1382
	spin_lock(&gtt->guptasklock);
	list_for_each_entry(entry, &gtt->guptasks, list) {
		if (entry->task == current) {
			spin_unlock(&gtt->guptasklock);
			return false;
		}
	}
	spin_unlock(&gtt->guptasklock);

1383 1384
	atomic_inc(&gtt->mmu_invalidations);

1385 1386 1387
	return true;
}

1388 1389 1390 1391
/**
 * amdgpu_ttm_tt_userptr_invalidated -	Has the ttm_tt object been
 * 										invalidated?
 */
1392 1393 1394 1395 1396 1397 1398 1399 1400 1401
bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
				       int *last_invalidated)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
	int prev_invalidated = *last_invalidated;

	*last_invalidated = atomic_read(&gtt->mmu_invalidations);
	return prev_invalidated != *last_invalidated;
}

1402 1403 1404 1405 1406 1407
/**
 * amdgpu_ttm_tt_userptr_needs_pages -	Have the pages backing this
 * 										ttm_tt object been invalidated
 * 										since the last time they've
 * 										been set?
 */
1408 1409 1410 1411 1412 1413 1414 1415 1416 1417
bool amdgpu_ttm_tt_userptr_needs_pages(struct ttm_tt *ttm)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

	if (gtt == NULL || !gtt->userptr)
		return false;

	return atomic_read(&gtt->mmu_invalidations) != gtt->last_set_pages;
}

1418 1419 1420
/**
 * amdgpu_ttm_tt_is_readonly - Is the ttm_tt object read only?
 */
A
Alex Deucher 已提交
1421 1422 1423 1424 1425 1426 1427 1428 1429 1430
bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

	if (gtt == NULL)
		return false;

	return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
}

1431 1432 1433 1434 1435 1436
/**
 * amdgpu_ttm_tt_pte_flags - Compute PTE flags for ttm_tt object
 *
 * @ttm: The ttm_tt object to compute the flags for
 * @mem: The memory registry backing this ttm_tt object
 */
1437
uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
A
Alex Deucher 已提交
1438 1439
				 struct ttm_mem_reg *mem)
{
1440
	uint64_t flags = 0;
A
Alex Deucher 已提交
1441 1442 1443 1444

	if (mem && mem->mem_type != TTM_PL_SYSTEM)
		flags |= AMDGPU_PTE_VALID;

1445
	if (mem && mem->mem_type == TTM_PL_TT) {
A
Alex Deucher 已提交
1446 1447
		flags |= AMDGPU_PTE_SYSTEM;

1448 1449 1450
		if (ttm->caching_state == tt_cached)
			flags |= AMDGPU_PTE_SNOOPED;
	}
A
Alex Deucher 已提交
1451

1452
	flags |= adev->gart.gart_pte_flags;
A
Alex Deucher 已提交
1453 1454 1455 1456 1457 1458 1459 1460
	flags |= AMDGPU_PTE_READABLE;

	if (!amdgpu_ttm_tt_is_readonly(ttm))
		flags |= AMDGPU_PTE_WRITEABLE;

	return flags;
}

1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
/**
 * amdgpu_ttm_bo_eviction_valuable -	Check to see if we can evict
 * 										a buffer object.
 *
 * Return true if eviction is sensible.  Called by
 * ttm_mem_evict_first() on behalf of ttm_bo_mem_force_space()
 * which tries to evict buffer objects until it can find space
 * for a new object and by ttm_bo_force_list_clean() which is
 * used to clean out a memory space.
 */
1471 1472 1473
static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
					    const struct ttm_place *place)
{
1474 1475
	unsigned long num_pages = bo->mem.num_pages;
	struct drm_mm_node *node = bo->mem.mm_node;
1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492
	struct reservation_object_list *flist;
	struct dma_fence *f;
	int i;

	/* If bo is a KFD BO, check if the bo belongs to the current process.
	 * If true, then return false as any KFD process needs all its BOs to
	 * be resident to run successfully
	 */
	flist = reservation_object_get_list(bo->resv);
	if (flist) {
		for (i = 0; i < flist->shared_count; ++i) {
			f = rcu_dereference_protected(flist->shared[i],
				reservation_object_held(bo->resv));
			if (amdkfd_fence_check_mm(f, current->mm))
				return false;
		}
	}
1493

1494 1495 1496
	switch (bo->mem.mem_type) {
	case TTM_PL_TT:
		return true;
1497

1498
	case TTM_PL_VRAM:
1499 1500 1501 1502 1503 1504 1505 1506 1507
		/* Check each drm MM node individually */
		while (num_pages) {
			if (place->fpfn < (node->start + node->size) &&
			    !(place->lpfn && place->lpfn <= node->start))
				return true;

			num_pages -= node->size;
			++node;
		}
1508
		return false;
1509

1510 1511
	default:
		break;
1512 1513 1514 1515 1516
	}

	return ttm_bo_eviction_valuable(bo, place);
}

1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529
/**
 * amdgpu_ttm_access_memory -	Read or Write memory that backs a
 * 								buffer object.
 *
 * @bo:  The buffer object to read/write
 * @offset:  Offset into buffer object
 * @buf:  Secondary buffer to write/read from
 * @len: Length in bytes of access
 * @write:  true if writing
 *
 * This is used to access VRAM that backs a buffer object via MMIO
 * access for debugging purposes.
 */
1530 1531 1532 1533
static int amdgpu_ttm_access_memory(struct ttm_buffer_object *bo,
				    unsigned long offset,
				    void *buf, int len, int write)
{
1534
	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
1535
	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
1536
	struct drm_mm_node *nodes;
1537 1538 1539 1540 1541 1542 1543 1544
	uint32_t value = 0;
	int ret = 0;
	uint64_t pos;
	unsigned long flags;

	if (bo->mem.mem_type != TTM_PL_VRAM)
		return -EIO;

1545
	nodes = amdgpu_find_mm_node(&abo->tbo.mem, &offset);
1546 1547
	pos = (nodes->start << PAGE_SHIFT) + offset;

1548
	while (len && pos < adev->gmc.mc_vram_size) {
1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559
		uint64_t aligned_pos = pos & ~(uint64_t)3;
		uint32_t bytes = 4 - (pos & 3);
		uint32_t shift = (pos & 3) * 8;
		uint32_t mask = 0xffffffff << shift;

		if (len < bytes) {
			mask &= 0xffffffff >> (bytes - len) * 8;
			bytes = len;
		}

		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
1560 1561
		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)aligned_pos) | 0x80000000);
		WREG32_NO_KIQ(mmMM_INDEX_HI, aligned_pos >> 31);
1562
		if (!write || mask != 0xffffffff)
1563
			value = RREG32_NO_KIQ(mmMM_DATA);
1564 1565 1566
		if (write) {
			value &= ~mask;
			value |= (*(uint32_t *)buf << shift) & mask;
1567
			WREG32_NO_KIQ(mmMM_DATA, value);
1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587
		}
		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
		if (!write) {
			value = (value & mask) >> shift;
			memcpy(buf, &value, bytes);
		}

		ret += bytes;
		buf = (uint8_t *)buf + bytes;
		pos += bytes;
		len -= bytes;
		if (pos >= (nodes->start + nodes->size) << PAGE_SHIFT) {
			++nodes;
			pos = (nodes->start << PAGE_SHIFT);
		}
	}

	return ret;
}

A
Alex Deucher 已提交
1588 1589 1590 1591 1592 1593
static struct ttm_bo_driver amdgpu_bo_driver = {
	.ttm_tt_create = &amdgpu_ttm_tt_create,
	.ttm_tt_populate = &amdgpu_ttm_tt_populate,
	.ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
	.invalidate_caches = &amdgpu_invalidate_caches,
	.init_mem_type = &amdgpu_init_mem_type,
1594
	.eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
A
Alex Deucher 已提交
1595 1596 1597 1598 1599 1600 1601
	.evict_flags = &amdgpu_evict_flags,
	.move = &amdgpu_bo_move,
	.verify_access = &amdgpu_verify_access,
	.move_notify = &amdgpu_bo_move_notify,
	.fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
	.io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
	.io_mem_free = &amdgpu_ttm_io_mem_free,
1602
	.io_mem_pfn = amdgpu_ttm_io_mem_pfn,
1603
	.access_memory = &amdgpu_ttm_access_memory
A
Alex Deucher 已提交
1604 1605
};

1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631
/*
 * Firmware Reservation functions
 */
/**
 * amdgpu_ttm_fw_reserve_vram_fini - free fw reserved vram
 *
 * @adev: amdgpu_device pointer
 *
 * free fw reserved vram if it has been reserved.
 */
static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev)
{
	amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
		NULL, &adev->fw_vram_usage.va);
}

/**
 * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw
 *
 * @adev: amdgpu_device pointer
 *
 * create bo vram reservation from fw.
 */
static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev)
{
	struct ttm_operation_ctx ctx = { false, false };
1632
	struct amdgpu_bo_param bp;
1633 1634
	int r = 0;
	int i;
1635
	u64 vram_size = adev->gmc.visible_vram_size;
1636 1637 1638 1639
	u64 offset = adev->fw_vram_usage.start_offset;
	u64 size = adev->fw_vram_usage.size;
	struct amdgpu_bo *bo;

1640 1641 1642 1643 1644 1645 1646 1647
	memset(&bp, 0, sizeof(bp));
	bp.size = adev->fw_vram_usage.size;
	bp.byte_align = PAGE_SIZE;
	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
	bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
		AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
	bp.type = ttm_bo_type_kernel;
	bp.resv = NULL;
1648 1649 1650 1651 1652 1653
	adev->fw_vram_usage.va = NULL;
	adev->fw_vram_usage.reserved_bo = NULL;

	if (adev->fw_vram_usage.size > 0 &&
		adev->fw_vram_usage.size <= vram_size) {

1654
		r = amdgpu_bo_create(adev, &bp,
1655
				     &adev->fw_vram_usage.reserved_bo);
1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682
		if (r)
			goto error_create;

		r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
		if (r)
			goto error_reserve;

		/* remove the original mem node and create a new one at the
		 * request position
		 */
		bo = adev->fw_vram_usage.reserved_bo;
		offset = ALIGN(offset, PAGE_SIZE);
		for (i = 0; i < bo->placement.num_placement; ++i) {
			bo->placements[i].fpfn = offset >> PAGE_SHIFT;
			bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
		}

		ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem);
		r = ttm_bo_mem_space(&bo->tbo, &bo->placement,
				     &bo->tbo.mem, &ctx);
		if (r)
			goto error_pin;

		r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
			AMDGPU_GEM_DOMAIN_VRAM,
			adev->fw_vram_usage.start_offset,
			(adev->fw_vram_usage.start_offset +
1683
			adev->fw_vram_usage.size));
1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705
		if (r)
			goto error_pin;
		r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
			&adev->fw_vram_usage.va);
		if (r)
			goto error_kmap;

		amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
	}
	return r;

error_kmap:
	amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
error_pin:
	amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
error_reserve:
	amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
error_create:
	adev->fw_vram_usage.va = NULL;
	adev->fw_vram_usage.reserved_bo = NULL;
	return r;
}
1706 1707 1708 1709 1710 1711 1712 1713 1714
/**
 * amdgpu_ttm_init -	Init the memory management (ttm) as well as
 * 						various gtt/vram related fields.
 *
 * This initializes all of the memory space pools that the TTM layer
 * will need such as the GTT space (system memory mapped to the device),
 * VRAM (on-board memory), and on-chip memories (GDS, GWS, OA) which
 * can be mapped per VMID.
 */
A
Alex Deucher 已提交
1715 1716
int amdgpu_ttm_init(struct amdgpu_device *adev)
{
1717
	uint64_t gtt_size;
A
Alex Deucher 已提交
1718
	int r;
1719
	u64 vis_vram_limit;
A
Alex Deucher 已提交
1720

1721
	/* initialize global references for vram/gtt */
1722 1723 1724 1725
	r = amdgpu_ttm_global_init(adev);
	if (r) {
		return r;
	}
A
Alex Deucher 已提交
1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737
	/* No others user of address space so set it to 0 */
	r = ttm_bo_device_init(&adev->mman.bdev,
			       adev->mman.bo_global_ref.ref.object,
			       &amdgpu_bo_driver,
			       adev->ddev->anon_inode->i_mapping,
			       DRM_FILE_PAGE_OFFSET,
			       adev->need_dma32);
	if (r) {
		DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
		return r;
	}
	adev->mman.initialized = true;
1738 1739 1740 1741

	/* We opt to avoid OOM on system pages allocations */
	adev->mman.bdev.no_retry = true;

1742
	/* Initialize VRAM pool with all of VRAM divided into pages */
A
Alex Deucher 已提交
1743
	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
1744
				adev->gmc.real_vram_size >> PAGE_SHIFT);
A
Alex Deucher 已提交
1745 1746 1747 1748
	if (r) {
		DRM_ERROR("Failed initializing VRAM heap.\n");
		return r;
	}
1749 1750 1751 1752

	/* Reduce size of CPU-visible VRAM if requested */
	vis_vram_limit = (u64)amdgpu_vis_vram_limit * 1024 * 1024;
	if (amdgpu_vis_vram_limit > 0 &&
1753 1754
	    vis_vram_limit <= adev->gmc.visible_vram_size)
		adev->gmc.visible_vram_size = vis_vram_limit;
1755

A
Alex Deucher 已提交
1756
	/* Change the size here instead of the init above so only lpfn is affected */
1757
	amdgpu_ttm_set_buffer_funcs_status(adev, false);
1758 1759 1760 1761
#ifdef CONFIG_64BIT
	adev->mman.aper_base_kaddr = ioremap_wc(adev->gmc.aper_base,
						adev->gmc.visible_vram_size);
#endif
A
Alex Deucher 已提交
1762

1763 1764 1765 1766
	/*
	 *The reserved vram for firmware must be pinned to the specified
	 *place on the VRAM, so reserve it early.
	 */
1767
	r = amdgpu_ttm_fw_reserve_vram_init(adev);
1768 1769 1770 1771
	if (r) {
		return r;
	}

1772 1773 1774 1775
	/* allocate memory as required for VGA
	 * This is used for VGA emulation and pre-OS scanout buffers to
	 * avoid display artifacts while transitioning between pre-OS
	 * and driver.  */
1776 1777 1778 1779 1780 1781 1782 1783
	if (adev->gmc.stolen_size) {
		r = amdgpu_bo_create_kernel(adev, adev->gmc.stolen_size, PAGE_SIZE,
					    AMDGPU_GEM_DOMAIN_VRAM,
					    &adev->stolen_vga_memory,
					    NULL, NULL);
		if (r)
			return r;
	}
A
Alex Deucher 已提交
1784
	DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
1785
		 (unsigned) (adev->gmc.real_vram_size / (1024 * 1024)));
1786

1787 1788
	/* Compute GTT size, either bsaed on 3/4th the size of RAM size
	 * or whatever the user passed on module init */
1789 1790 1791 1792
	if (amdgpu_gtt_size == -1) {
		struct sysinfo si;

		si_meminfo(&si);
1793
		gtt_size = min(max((AMDGPU_DEFAULT_GTT_SIZE_MB << 20),
1794
			       adev->gmc.mc_vram_size),
1795 1796 1797
			       ((uint64_t)si.totalram * si.mem_unit * 3/4));
	}
	else
1798
		gtt_size = (uint64_t)amdgpu_gtt_size << 20;
1799 1800

	/* Initialize GTT memory pool */
1801
	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
A
Alex Deucher 已提交
1802 1803 1804 1805 1806
	if (r) {
		DRM_ERROR("Failed initializing GTT heap.\n");
		return r;
	}
	DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
1807
		 (unsigned)(gtt_size / (1024 * 1024)));
A
Alex Deucher 已提交
1808

1809
	/* Initialize various on-chip memory pools */
A
Alex Deucher 已提交
1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
	adev->gds.mem.total_size = adev->gds.mem.total_size << AMDGPU_GDS_SHIFT;
	adev->gds.mem.gfx_partition_size = adev->gds.mem.gfx_partition_size << AMDGPU_GDS_SHIFT;
	adev->gds.mem.cs_partition_size = adev->gds.mem.cs_partition_size << AMDGPU_GDS_SHIFT;
	adev->gds.gws.total_size = adev->gds.gws.total_size << AMDGPU_GWS_SHIFT;
	adev->gds.gws.gfx_partition_size = adev->gds.gws.gfx_partition_size << AMDGPU_GWS_SHIFT;
	adev->gds.gws.cs_partition_size = adev->gds.gws.cs_partition_size << AMDGPU_GWS_SHIFT;
	adev->gds.oa.total_size = adev->gds.oa.total_size << AMDGPU_OA_SHIFT;
	adev->gds.oa.gfx_partition_size = adev->gds.oa.gfx_partition_size << AMDGPU_OA_SHIFT;
	adev->gds.oa.cs_partition_size = adev->gds.oa.cs_partition_size << AMDGPU_OA_SHIFT;
	/* GDS Memory */
1820 1821 1822 1823 1824 1825 1826
	if (adev->gds.mem.total_size) {
		r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,
				   adev->gds.mem.total_size >> PAGE_SHIFT);
		if (r) {
			DRM_ERROR("Failed initializing GDS heap.\n");
			return r;
		}
A
Alex Deucher 已提交
1827 1828 1829
	}

	/* GWS */
1830 1831 1832 1833 1834 1835 1836
	if (adev->gds.gws.total_size) {
		r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,
				   adev->gds.gws.total_size >> PAGE_SHIFT);
		if (r) {
			DRM_ERROR("Failed initializing gws heap.\n");
			return r;
		}
A
Alex Deucher 已提交
1837 1838 1839
	}

	/* OA */
1840 1841 1842 1843 1844 1845 1846
	if (adev->gds.oa.total_size) {
		r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,
				   adev->gds.oa.total_size >> PAGE_SHIFT);
		if (r) {
			DRM_ERROR("Failed initializing oa heap.\n");
			return r;
		}
A
Alex Deucher 已提交
1847 1848
	}

1849
	/* Register debugfs entries for amdgpu_ttm */
A
Alex Deucher 已提交
1850 1851 1852 1853 1854 1855 1856 1857
	r = amdgpu_ttm_debugfs_init(adev);
	if (r) {
		DRM_ERROR("Failed to init debugfs\n");
		return r;
	}
	return 0;
}

1858 1859 1860 1861
/**
 * amdgpu_ttm_late_init -	Handle any late initialization for
 * 							amdgpu_ttm
 */
1862 1863
void amdgpu_ttm_late_init(struct amdgpu_device *adev)
{
1864
	/* return the VGA stolen memory (if any) back to VRAM */
1865 1866 1867
	amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
}

1868 1869 1870
/**
 * amdgpu_ttm_fini - De-initialize the TTM memory pools
 */
A
Alex Deucher 已提交
1871 1872 1873 1874
void amdgpu_ttm_fini(struct amdgpu_device *adev)
{
	if (!adev->mman.initialized)
		return;
1875

A
Alex Deucher 已提交
1876
	amdgpu_ttm_debugfs_fini(adev);
1877
	amdgpu_ttm_fw_reserve_vram_fini(adev);
1878 1879 1880
	if (adev->mman.aper_base_kaddr)
		iounmap(adev->mman.aper_base_kaddr);
	adev->mman.aper_base_kaddr = NULL;
1881

A
Alex Deucher 已提交
1882 1883
	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
1884 1885 1886 1887 1888 1889
	if (adev->gds.mem.total_size)
		ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS);
	if (adev->gds.gws.total_size)
		ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS);
	if (adev->gds.oa.total_size)
		ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
A
Alex Deucher 已提交
1890 1891 1892 1893 1894 1895
	ttm_bo_device_release(&adev->mman.bdev);
	amdgpu_ttm_global_fini(adev);
	adev->mman.initialized = false;
	DRM_INFO("amdgpu: ttm finalized\n");
}

1896 1897 1898 1899 1900 1901 1902 1903 1904 1905
/**
 * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
 *
 * @adev: amdgpu_device pointer
 * @enable: true when we can use buffer functions.
 *
 * Enable/disable use of buffer functions during suspend/resume. This should
 * only be called at bootup or when userspace isn't running.
 */
void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
A
Alex Deucher 已提交
1906
{
1907 1908
	struct ttm_mem_type_manager *man = &adev->mman.bdev.man[TTM_PL_VRAM];
	uint64_t size;
1909
	int r;
A
Alex Deucher 已提交
1910

1911 1912
	if (!adev->mman.initialized || adev->in_gpu_reset ||
	    adev->mman.buffer_funcs_enabled == enable)
A
Alex Deucher 已提交
1913 1914
		return;

1915 1916 1917 1918 1919 1920
	if (enable) {
		struct amdgpu_ring *ring;
		struct drm_sched_rq *rq;

		ring = adev->mman.buffer_funcs_ring;
		rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
1921
		r = drm_sched_entity_init(&adev->mman.entity, &rq, 1, NULL);
1922 1923 1924 1925 1926 1927 1928 1929 1930 1931
		if (r) {
			DRM_ERROR("Failed setting up TTM BO move entity (%d)\n",
				  r);
			return;
		}
	} else {
		drm_sched_entity_destroy(adev->mman.entity.sched,
					 &adev->mman.entity);
	}

A
Alex Deucher 已提交
1932
	/* this just adjusts TTM size idea, which sets lpfn to the correct value */
1933 1934 1935 1936
	if (enable)
		size = adev->gmc.real_vram_size;
	else
		size = adev->gmc.visible_vram_size;
A
Alex Deucher 已提交
1937
	man->size = size >> PAGE_SHIFT;
1938
	adev->mman.buffer_funcs_enabled = enable;
A
Alex Deucher 已提交
1939 1940 1941 1942 1943 1944 1945
}

int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct drm_file *file_priv;
	struct amdgpu_device *adev;

C
Christian König 已提交
1946
	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
A
Alex Deucher 已提交
1947 1948 1949 1950
		return -EINVAL;

	file_priv = filp->private_data;
	adev = file_priv->minor->dev->dev_private;
C
Christian König 已提交
1951
	if (adev == NULL)
A
Alex Deucher 已提交
1952
		return -EINVAL;
C
Christian König 已提交
1953 1954

	return ttm_bo_mmap(filp, vma, &adev->mman.bdev);
A
Alex Deucher 已提交
1955 1956
}

1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976
static int amdgpu_map_buffer(struct ttm_buffer_object *bo,
			     struct ttm_mem_reg *mem, unsigned num_pages,
			     uint64_t offset, unsigned window,
			     struct amdgpu_ring *ring,
			     uint64_t *addr)
{
	struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
	struct amdgpu_device *adev = ring->adev;
	struct ttm_tt *ttm = bo->ttm;
	struct amdgpu_job *job;
	unsigned num_dw, num_bytes;
	dma_addr_t *dma_address;
	struct dma_fence *fence;
	uint64_t src_addr, dst_addr;
	uint64_t flags;
	int r;

	BUG_ON(adev->mman.buffer_funcs->copy_max_bytes <
	       AMDGPU_GTT_MAX_TRANSFER_SIZE * 8);

1977
	*addr = adev->gmc.gart_start;
1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008
	*addr += (u64)window * AMDGPU_GTT_MAX_TRANSFER_SIZE *
		AMDGPU_GPU_PAGE_SIZE;

	num_dw = adev->mman.buffer_funcs->copy_num_dw;
	while (num_dw & 0x7)
		num_dw++;

	num_bytes = num_pages * 8;

	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes, &job);
	if (r)
		return r;

	src_addr = num_dw * 4;
	src_addr += job->ibs[0].gpu_addr;

	dst_addr = adev->gart.table_addr;
	dst_addr += window * AMDGPU_GTT_MAX_TRANSFER_SIZE * 8;
	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
				dst_addr, num_bytes);

	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
	WARN_ON(job->ibs[0].length_dw > num_dw);

	dma_address = &gtt->ttm.dma_address[offset >> PAGE_SHIFT];
	flags = amdgpu_ttm_tt_pte_flags(adev, ttm, mem);
	r = amdgpu_gart_map(adev, 0, num_pages, dma_address, flags,
			    &job->ibs[0].ptr[num_dw]);
	if (r)
		goto error_free;

2009
	r = amdgpu_job_submit(job, &adev->mman.entity,
2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022
			      AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
	if (r)
		goto error_free;

	dma_fence_put(fence);

	return r;

error_free:
	amdgpu_job_free(job);
	return r;
}

2023 2024
int amdgpu_copy_buffer(struct amdgpu_ring *ring, uint64_t src_offset,
		       uint64_t dst_offset, uint32_t byte_count,
A
Alex Deucher 已提交
2025
		       struct reservation_object *resv,
2026 2027
		       struct dma_fence **fence, bool direct_submit,
		       bool vm_needs_flush)
A
Alex Deucher 已提交
2028 2029
{
	struct amdgpu_device *adev = ring->adev;
2030 2031
	struct amdgpu_job *job;

A
Alex Deucher 已提交
2032 2033 2034 2035 2036
	uint32_t max_bytes;
	unsigned num_loops, num_dw;
	unsigned i;
	int r;

2037 2038 2039 2040 2041
	if (direct_submit && !ring->ready) {
		DRM_ERROR("Trying to move memory with ring turned off.\n");
		return -EINVAL;
	}

A
Alex Deucher 已提交
2042 2043 2044 2045
	max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
	num_loops = DIV_ROUND_UP(byte_count, max_bytes);
	num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw;

2046 2047 2048 2049
	/* for IB padding */
	while (num_dw & 0x7)
		num_dw++;

2050 2051
	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
	if (r)
2052
		return r;
2053

2054
	job->vm_needs_flush = vm_needs_flush;
2055
	if (resv) {
2056
		r = amdgpu_sync_resv(adev, &job->sync, resv,
2057 2058
				     AMDGPU_FENCE_OWNER_UNDEFINED,
				     false);
2059 2060 2061 2062
		if (r) {
			DRM_ERROR("sync failed (%d).\n", r);
			goto error_free;
		}
A
Alex Deucher 已提交
2063 2064 2065 2066 2067
	}

	for (i = 0; i < num_loops; i++) {
		uint32_t cur_size_in_bytes = min(byte_count, max_bytes);

2068 2069
		amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
					dst_offset, cur_size_in_bytes);
A
Alex Deucher 已提交
2070 2071 2072 2073 2074 2075

		src_offset += cur_size_in_bytes;
		dst_offset += cur_size_in_bytes;
		byte_count -= cur_size_in_bytes;
	}

2076 2077
	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
	WARN_ON(job->ibs[0].length_dw > num_dw);
2078 2079 2080
	if (direct_submit)
		r = amdgpu_job_submit_direct(job, ring, fence);
	else
2081
		r = amdgpu_job_submit(job, &adev->mman.entity,
2082
				      AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2083 2084
	if (r)
		goto error_free;
A
Alex Deucher 已提交
2085

2086
	return r;
2087

2088
error_free:
2089
	amdgpu_job_free(job);
2090
	DRM_ERROR("Error scheduling IBs (%d)\n", r);
2091
	return r;
A
Alex Deucher 已提交
2092 2093
}

2094
int amdgpu_fill_buffer(struct amdgpu_bo *bo,
2095
		       uint32_t src_data,
2096 2097
		       struct reservation_object *resv,
		       struct dma_fence **fence)
2098
{
2099
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
2100
	uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
2101 2102
	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;

2103 2104
	struct drm_mm_node *mm_node;
	unsigned long num_pages;
2105
	unsigned int num_loops, num_dw;
2106 2107

	struct amdgpu_job *job;
2108 2109
	int r;

2110
	if (!adev->mman.buffer_funcs_enabled) {
2111 2112 2113 2114
		DRM_ERROR("Trying to clear memory with ring turned off.\n");
		return -EINVAL;
	}

2115
	if (bo->tbo.mem.mem_type == TTM_PL_TT) {
2116
		r = amdgpu_ttm_alloc_gart(&bo->tbo);
2117 2118 2119 2120
		if (r)
			return r;
	}

2121 2122 2123 2124 2125 2126 2127 2128 2129 2130
	num_pages = bo->tbo.num_pages;
	mm_node = bo->tbo.mem.mm_node;
	num_loops = 0;
	while (num_pages) {
		uint32_t byte_count = mm_node->size << PAGE_SHIFT;

		num_loops += DIV_ROUND_UP(byte_count, max_bytes);
		num_pages -= mm_node->size;
		++mm_node;
	}
2131
	num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;
2132 2133

	/* for IB padding */
2134
	num_dw += 64;
2135 2136 2137 2138 2139 2140 2141

	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
	if (r)
		return r;

	if (resv) {
		r = amdgpu_sync_resv(adev, &job->sync, resv,
2142
				     AMDGPU_FENCE_OWNER_UNDEFINED, false);
2143 2144 2145 2146 2147 2148
		if (r) {
			DRM_ERROR("sync failed (%d).\n", r);
			goto error_free;
		}
	}

2149 2150
	num_pages = bo->tbo.num_pages;
	mm_node = bo->tbo.mem.mm_node;
2151

2152 2153 2154
	while (num_pages) {
		uint32_t byte_count = mm_node->size << PAGE_SHIFT;
		uint64_t dst_addr;
2155

2156
		dst_addr = amdgpu_mm_node_addr(&bo->tbo, mm_node, &bo->tbo.mem);
2157 2158 2159
		while (byte_count) {
			uint32_t cur_size_in_bytes = min(byte_count, max_bytes);

2160 2161
			amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
						dst_addr, cur_size_in_bytes);
2162 2163 2164 2165 2166 2167 2168

			dst_addr += cur_size_in_bytes;
			byte_count -= cur_size_in_bytes;
		}

		num_pages -= mm_node->size;
		++mm_node;
2169 2170 2171 2172
	}

	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
	WARN_ON(job->ibs[0].length_dw > num_dw);
2173
	r = amdgpu_job_submit(job, &adev->mman.entity,
2174
			      AMDGPU_FENCE_OWNER_UNDEFINED, fence);
2175 2176 2177 2178 2179 2180 2181 2182 2183 2184
	if (r)
		goto error_free;

	return 0;

error_free:
	amdgpu_job_free(job);
	return r;
}

A
Alex Deucher 已提交
2185 2186 2187 2188 2189 2190 2191 2192
#if defined(CONFIG_DEBUG_FS)

static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
{
	struct drm_info_node *node = (struct drm_info_node *)m->private;
	unsigned ttm_pl = *(int *)node->info_ent->data;
	struct drm_device *dev = node->minor->dev;
	struct amdgpu_device *adev = dev->dev_private;
2193
	struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl];
D
Daniel Vetter 已提交
2194
	struct drm_printer p = drm_seq_file_printer(m);
A
Alex Deucher 已提交
2195

2196
	man->func->debug(man, &p);
D
Daniel Vetter 已提交
2197
	return 0;
A
Alex Deucher 已提交
2198 2199 2200 2201 2202
}

static int ttm_pl_vram = TTM_PL_VRAM;
static int ttm_pl_tt = TTM_PL_TT;

2203
static const struct drm_info_list amdgpu_ttm_debugfs_list[] = {
A
Alex Deucher 已提交
2204 2205 2206 2207 2208 2209 2210 2211
	{"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, &ttm_pl_vram},
	{"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, &ttm_pl_tt},
	{"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL},
#ifdef CONFIG_SWIOTLB
	{"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL}
#endif
};

2212 2213 2214 2215 2216
/**
 * amdgpu_ttm_vram_read - Linear read access to VRAM
 *
 * Accesses VRAM via MMIO for debugging purposes.
 */
A
Alex Deucher 已提交
2217 2218 2219
static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
				    size_t size, loff_t *pos)
{
A
Al Viro 已提交
2220
	struct amdgpu_device *adev = file_inode(f)->i_private;
A
Alex Deucher 已提交
2221 2222 2223 2224 2225 2226
	ssize_t result = 0;
	int r;

	if (size & 0x3 || *pos & 0x3)
		return -EINVAL;

2227
	if (*pos >= adev->gmc.mc_vram_size)
2228 2229
		return -ENXIO;

A
Alex Deucher 已提交
2230 2231 2232 2233
	while (size) {
		unsigned long flags;
		uint32_t value;

2234
		if (*pos >= adev->gmc.mc_vram_size)
A
Alex Deucher 已提交
2235 2236 2237
			return result;

		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2238 2239 2240
		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
		WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
		value = RREG32_NO_KIQ(mmMM_DATA);
A
Alex Deucher 已提交
2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255
		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);

		r = put_user(value, (uint32_t *)buf);
		if (r)
			return r;

		result += 4;
		buf += 4;
		*pos += 4;
		size -= 4;
	}

	return result;
}

2256 2257 2258 2259 2260
/**
 * amdgpu_ttm_vram_write - Linear write access to VRAM
 *
 * Accesses VRAM via MMIO for debugging purposes.
 */
2261 2262 2263 2264 2265 2266 2267 2268 2269 2270
static ssize_t amdgpu_ttm_vram_write(struct file *f, const char __user *buf,
				    size_t size, loff_t *pos)
{
	struct amdgpu_device *adev = file_inode(f)->i_private;
	ssize_t result = 0;
	int r;

	if (size & 0x3 || *pos & 0x3)
		return -EINVAL;

2271
	if (*pos >= adev->gmc.mc_vram_size)
2272 2273 2274 2275 2276 2277
		return -ENXIO;

	while (size) {
		unsigned long flags;
		uint32_t value;

2278
		if (*pos >= adev->gmc.mc_vram_size)
2279 2280 2281 2282 2283 2284 2285
			return result;

		r = get_user(value, (uint32_t *)buf);
		if (r)
			return r;

		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
2286 2287 2288
		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
		WREG32_NO_KIQ(mmMM_INDEX_HI, *pos >> 31);
		WREG32_NO_KIQ(mmMM_DATA, value);
2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299
		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);

		result += 4;
		buf += 4;
		*pos += 4;
		size -= 4;
	}

	return result;
}

A
Alex Deucher 已提交
2300 2301 2302
static const struct file_operations amdgpu_ttm_vram_fops = {
	.owner = THIS_MODULE,
	.read = amdgpu_ttm_vram_read,
2303 2304
	.write = amdgpu_ttm_vram_write,
	.llseek = default_llseek,
A
Alex Deucher 已提交
2305 2306
};

2307 2308
#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS

2309 2310 2311
/**
 * amdgpu_ttm_gtt_read - Linear read access to GTT memory
 */
A
Alex Deucher 已提交
2312 2313 2314
static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf,
				   size_t size, loff_t *pos)
{
A
Al Viro 已提交
2315
	struct amdgpu_device *adev = file_inode(f)->i_private;
A
Alex Deucher 已提交
2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358
	ssize_t result = 0;
	int r;

	while (size) {
		loff_t p = *pos / PAGE_SIZE;
		unsigned off = *pos & ~PAGE_MASK;
		size_t cur_size = min_t(size_t, size, PAGE_SIZE - off);
		struct page *page;
		void *ptr;

		if (p >= adev->gart.num_cpu_pages)
			return result;

		page = adev->gart.pages[p];
		if (page) {
			ptr = kmap(page);
			ptr += off;

			r = copy_to_user(buf, ptr, cur_size);
			kunmap(adev->gart.pages[p]);
		} else
			r = clear_user(buf, cur_size);

		if (r)
			return -EFAULT;

		result += cur_size;
		buf += cur_size;
		*pos += cur_size;
		size -= cur_size;
	}

	return result;
}

static const struct file_operations amdgpu_ttm_gtt_fops = {
	.owner = THIS_MODULE,
	.read = amdgpu_ttm_gtt_read,
	.llseek = default_llseek
};

#endif

2359 2360 2361 2362 2363 2364 2365
/**
 * amdgpu_iomem_read - Virtual read access to GPU mapped memory
 *
 * This function is used to read memory that has been mapped to the
 * GPU and the known addresses are not physical addresses but instead
 * bus addresses (e.g., what you'd put in an IB or ring buffer).
 */
2366 2367
static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf,
				 size_t size, loff_t *pos)
2368 2369 2370
{
	struct amdgpu_device *adev = file_inode(f)->i_private;
	struct iommu_domain *dom;
2371 2372
	ssize_t result = 0;
	int r;
2373

2374
	/* retrieve the IOMMU domain if any for this device */
2375
	dom = iommu_get_domain_for_dev(adev->dev);
2376

2377 2378 2379 2380 2381 2382 2383 2384 2385 2386
	while (size) {
		phys_addr_t addr = *pos & PAGE_MASK;
		loff_t off = *pos & ~PAGE_MASK;
		size_t bytes = PAGE_SIZE - off;
		unsigned long pfn;
		struct page *p;
		void *ptr;

		bytes = bytes < size ? bytes : size;

2387 2388 2389 2390
		/* Translate the bus address to a physical address.  If
		 * the domain is NULL it means there is no IOMMU active
		 * and the address translation is the identity
		 */
2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401
		addr = dom ? iommu_iova_to_phys(dom, addr) : addr;

		pfn = addr >> PAGE_SHIFT;
		if (!pfn_valid(pfn))
			return -EPERM;

		p = pfn_to_page(pfn);
		if (p->mapping != adev->mman.bdev.dev_mapping)
			return -EPERM;

		ptr = kmap(p);
2402
		r = copy_to_user(buf, ptr + off, bytes);
2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414
		kunmap(p);
		if (r)
			return -EFAULT;

		size -= bytes;
		*pos += bytes;
		result += bytes;
	}

	return result;
}

2415 2416 2417 2418 2419 2420 2421
/**
 * amdgpu_iomem_write - Virtual write access to GPU mapped memory
 *
 * This function is used to write memory that has been mapped to the
 * GPU and the known addresses are not physical addresses but instead
 * bus addresses (e.g., what you'd put in an IB or ring buffer).
 */
2422 2423 2424 2425 2426 2427 2428
static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf,
				 size_t size, loff_t *pos)
{
	struct amdgpu_device *adev = file_inode(f)->i_private;
	struct iommu_domain *dom;
	ssize_t result = 0;
	int r;
2429 2430

	dom = iommu_get_domain_for_dev(adev->dev);
2431

2432 2433 2434 2435 2436 2437 2438 2439 2440
	while (size) {
		phys_addr_t addr = *pos & PAGE_MASK;
		loff_t off = *pos & ~PAGE_MASK;
		size_t bytes = PAGE_SIZE - off;
		unsigned long pfn;
		struct page *p;
		void *ptr;

		bytes = bytes < size ? bytes : size;
2441

2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452
		addr = dom ? iommu_iova_to_phys(dom, addr) : addr;

		pfn = addr >> PAGE_SHIFT;
		if (!pfn_valid(pfn))
			return -EPERM;

		p = pfn_to_page(pfn);
		if (p->mapping != adev->mman.bdev.dev_mapping)
			return -EPERM;

		ptr = kmap(p);
2453
		r = copy_from_user(ptr + off, buf, bytes);
2454 2455 2456 2457 2458 2459 2460 2461 2462 2463
		kunmap(p);
		if (r)
			return -EFAULT;

		size -= bytes;
		*pos += bytes;
		result += bytes;
	}

	return result;
2464 2465
}

2466
static const struct file_operations amdgpu_ttm_iomem_fops = {
2467
	.owner = THIS_MODULE,
2468 2469
	.read = amdgpu_iomem_read,
	.write = amdgpu_iomem_write,
2470 2471
	.llseek = default_llseek
};
2472 2473 2474 2475 2476 2477 2478 2479 2480 2481

static const struct {
	char *name;
	const struct file_operations *fops;
	int domain;
} ttm_debugfs_entries[] = {
	{ "amdgpu_vram", &amdgpu_ttm_vram_fops, TTM_PL_VRAM },
#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
	{ "amdgpu_gtt", &amdgpu_ttm_gtt_fops, TTM_PL_TT },
#endif
2482
	{ "amdgpu_iomem", &amdgpu_ttm_iomem_fops, TTM_PL_SYSTEM },
2483 2484
};

2485 2486
#endif

A
Alex Deucher 已提交
2487 2488 2489 2490 2491 2492 2493 2494
static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
{
#if defined(CONFIG_DEBUG_FS)
	unsigned count;

	struct drm_minor *minor = adev->ddev->primary;
	struct dentry *ent, *root = minor->debugfs_root;

2495 2496 2497 2498 2499 2500 2501 2502 2503
	for (count = 0; count < ARRAY_SIZE(ttm_debugfs_entries); count++) {
		ent = debugfs_create_file(
				ttm_debugfs_entries[count].name,
				S_IFREG | S_IRUGO, root,
				adev,
				ttm_debugfs_entries[count].fops);
		if (IS_ERR(ent))
			return PTR_ERR(ent);
		if (ttm_debugfs_entries[count].domain == TTM_PL_VRAM)
2504
			i_size_write(ent->d_inode, adev->gmc.mc_vram_size);
2505
		else if (ttm_debugfs_entries[count].domain == TTM_PL_TT)
2506
			i_size_write(ent->d_inode, adev->gmc.gart_size);
2507 2508
		adev->mman.debugfs_entries[count] = ent;
	}
A
Alex Deucher 已提交
2509 2510 2511 2512

	count = ARRAY_SIZE(amdgpu_ttm_debugfs_list);

#ifdef CONFIG_SWIOTLB
2513
	if (!(adev->need_swiotlb && swiotlb_nr_tbl()))
A
Alex Deucher 已提交
2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525
		--count;
#endif

	return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);
#else
	return 0;
#endif
}

static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
{
#if defined(CONFIG_DEBUG_FS)
2526
	unsigned i;
A
Alex Deucher 已提交
2527

2528 2529
	for (i = 0; i < ARRAY_SIZE(ttm_debugfs_entries); i++)
		debugfs_remove(adev->mman.debugfs_entries[i]);
2530
#endif
A
Alex Deucher 已提交
2531
}