amdgpu_ttm.c 39.3 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
/*
 * Copyright 2009 Jerome Glisse.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 */
/*
 * Authors:
 *    Jerome Glisse <glisse@freedesktop.org>
 *    Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
 *    Dave Airlie
 */
#include <ttm/ttm_bo_api.h>
#include <ttm/ttm_bo_driver.h>
#include <ttm/ttm_placement.h>
#include <ttm/ttm_module.h>
#include <ttm/ttm_page_alloc.h>
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/swiotlb.h>
#include <linux/swap.h>
#include <linux/pagemap.h>
#include <linux/debugfs.h>
#include "amdgpu.h"
#include "bif/bif_4_1_d.h"

#define DRM_FILE_PAGE_OFFSET (0x100000000ULL >> PAGE_SHIFT)

static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev);
static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev);


/*
 * Global memory.
 */
static int amdgpu_ttm_mem_global_init(struct drm_global_reference *ref)
{
	return ttm_mem_global_init(ref->object);
}

static void amdgpu_ttm_mem_global_release(struct drm_global_reference *ref)
{
	ttm_mem_global_release(ref->object);
}

67
static int amdgpu_ttm_global_init(struct amdgpu_device *adev)
A
Alex Deucher 已提交
68 69
{
	struct drm_global_reference *global_ref;
70 71
	struct amdgpu_ring *ring;
	struct amd_sched_rq *rq;
A
Alex Deucher 已提交
72 73 74 75 76 77 78 79 80
	int r;

	adev->mman.mem_global_referenced = false;
	global_ref = &adev->mman.mem_global_ref;
	global_ref->global_type = DRM_GLOBAL_TTM_MEM;
	global_ref->size = sizeof(struct ttm_mem_global);
	global_ref->init = &amdgpu_ttm_mem_global_init;
	global_ref->release = &amdgpu_ttm_mem_global_release;
	r = drm_global_item_ref(global_ref);
81
	if (r) {
A
Alex Deucher 已提交
82 83
		DRM_ERROR("Failed setting up TTM memory accounting "
			  "subsystem.\n");
84
		goto error_mem;
A
Alex Deucher 已提交
85 86 87 88 89 90 91 92 93 94
	}

	adev->mman.bo_global_ref.mem_glob =
		adev->mman.mem_global_ref.object;
	global_ref = &adev->mman.bo_global_ref.ref;
	global_ref->global_type = DRM_GLOBAL_TTM_BO;
	global_ref->size = sizeof(struct ttm_bo_global);
	global_ref->init = &ttm_bo_global_init;
	global_ref->release = &ttm_bo_global_release;
	r = drm_global_item_ref(global_ref);
95
	if (r) {
A
Alex Deucher 已提交
96
		DRM_ERROR("Failed setting up TTM BO subsystem.\n");
97
		goto error_bo;
A
Alex Deucher 已提交
98 99
	}

100 101 102 103
	ring = adev->mman.buffer_funcs_ring;
	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
	r = amd_sched_entity_init(&ring->sched, &adev->mman.entity,
				  rq, amdgpu_sched_jobs);
104
	if (r) {
105
		DRM_ERROR("Failed setting up TTM BO move run queue.\n");
106
		goto error_entity;
107 108
	}

A
Alex Deucher 已提交
109
	adev->mman.mem_global_referenced = true;
110

A
Alex Deucher 已提交
111
	return 0;
112 113 114 115 116 117 118

error_entity:
	drm_global_item_unref(&adev->mman.bo_global_ref.ref);
error_bo:
	drm_global_item_unref(&adev->mman.mem_global_ref);
error_mem:
	return r;
A
Alex Deucher 已提交
119 120 121 122 123
}

static void amdgpu_ttm_global_fini(struct amdgpu_device *adev)
{
	if (adev->mman.mem_global_referenced) {
124 125
		amd_sched_entity_fini(adev->mman.entity.sched,
				      &adev->mman.entity);
A
Alex Deucher 已提交
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
		drm_global_item_unref(&adev->mman.bo_global_ref.ref);
		drm_global_item_unref(&adev->mman.mem_global_ref);
		adev->mman.mem_global_referenced = false;
	}
}

static int amdgpu_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
{
	return 0;
}

static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
				struct ttm_mem_type_manager *man)
{
	struct amdgpu_device *adev;

142
	adev = amdgpu_ttm_adev(bdev);
A
Alex Deucher 已提交
143 144 145 146 147 148 149 150 151

	switch (type) {
	case TTM_PL_SYSTEM:
		/* System memory */
		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
		man->available_caching = TTM_PL_MASK_CACHING;
		man->default_caching = TTM_PL_FLAG_CACHED;
		break;
	case TTM_PL_TT:
152
		man->func = &amdgpu_gtt_mgr_func;
A
Alex Deucher 已提交
153 154 155 156 157 158 159
		man->gpu_offset = adev->mc.gtt_start;
		man->available_caching = TTM_PL_MASK_CACHING;
		man->default_caching = TTM_PL_FLAG_CACHED;
		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
		break;
	case TTM_PL_VRAM:
		/* "On-card" video ram */
C
Christian König 已提交
160
		man->func = &amdgpu_vram_mgr_func;
A
Alex Deucher 已提交
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
		man->gpu_offset = adev->mc.vram_start;
		man->flags = TTM_MEMTYPE_FLAG_FIXED |
			     TTM_MEMTYPE_FLAG_MAPPABLE;
		man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
		man->default_caching = TTM_PL_FLAG_WC;
		break;
	case AMDGPU_PL_GDS:
	case AMDGPU_PL_GWS:
	case AMDGPU_PL_OA:
		/* On-chip GDS memory*/
		man->func = &ttm_bo_manager_func;
		man->gpu_offset = 0;
		man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;
		man->available_caching = TTM_PL_FLAG_UNCACHED;
		man->default_caching = TTM_PL_FLAG_UNCACHED;
		break;
	default:
		DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
		return -EINVAL;
	}
	return 0;
}

static void amdgpu_evict_flags(struct ttm_buffer_object *bo,
				struct ttm_placement *placement)
{
187
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
188
	struct amdgpu_bo *abo;
A
Alex Deucher 已提交
189 190 191 192 193
	static struct ttm_place placements = {
		.fpfn = 0,
		.lpfn = 0,
		.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM
	};
194
	unsigned i;
A
Alex Deucher 已提交
195 196 197 198 199 200 201 202

	if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) {
		placement->placement = &placements;
		placement->busy_placement = &placements;
		placement->num_placement = 1;
		placement->num_busy_placement = 1;
		return;
	}
203
	abo = container_of(bo, struct amdgpu_bo, tbo);
A
Alex Deucher 已提交
204 205
	switch (bo->mem.mem_type) {
	case TTM_PL_VRAM:
206 207 208
		if (adev->mman.buffer_funcs &&
		    adev->mman.buffer_funcs_ring &&
		    adev->mman.buffer_funcs_ring->ready == false) {
209
			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
210
		} else {
211 212 213
			amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_GTT);
			for (i = 0; i < abo->placement.num_placement; ++i) {
				if (!(abo->placements[i].flags &
214 215 216
				      TTM_PL_FLAG_TT))
					continue;

217
				if (abo->placements[i].lpfn)
218 219 220 221 222
					continue;

				/* set an upper limit to force directly
				 * allocating address space for the BO.
				 */
223
				abo->placements[i].lpfn =
224
					adev->mc.gtt_size >> PAGE_SHIFT;
225 226
			}
		}
A
Alex Deucher 已提交
227 228 229
		break;
	case TTM_PL_TT:
	default:
230
		amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_CPU);
A
Alex Deucher 已提交
231
	}
232
	*placement = abo->placement;
A
Alex Deucher 已提交
233 234 235 236
}

static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
{
237
	struct amdgpu_bo *abo = container_of(bo, struct amdgpu_bo, tbo);
A
Alex Deucher 已提交
238

239 240
	if (amdgpu_ttm_tt_get_usermm(bo->ttm))
		return -EPERM;
241
	return drm_vma_node_verify_access(&abo->gem_base.vma_node,
D
David Herrmann 已提交
242
					  filp->private_data);
A
Alex Deucher 已提交
243 244 245 246 247 248 249 250 251 252 253 254
}

static void amdgpu_move_null(struct ttm_buffer_object *bo,
			     struct ttm_mem_reg *new_mem)
{
	struct ttm_mem_reg *old_mem = &bo->mem;

	BUG_ON(old_mem->mm_node != NULL);
	*old_mem = *new_mem;
	new_mem->mm_node = NULL;
}

255 256 257 258
static int amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
			       struct drm_mm_node *mm_node,
			       struct ttm_mem_reg *mem,
			       uint64_t *addr)
A
Alex Deucher 已提交
259 260 261
{
	int r;

262
	switch (mem->mem_type) {
A
Alex Deucher 已提交
263
	case TTM_PL_TT:
264
		r = amdgpu_ttm_bind(bo, mem);
265 266 267 268
		if (r)
			return r;

	case TTM_PL_VRAM:
269 270
		*addr = mm_node->start << PAGE_SHIFT;
		*addr += bo->bdev->man[mem->mem_type].gpu_offset;
A
Alex Deucher 已提交
271 272
		break;
	default:
273
		DRM_ERROR("Unknown placement %d\n", mem->mem_type);
A
Alex Deucher 已提交
274 275
		return -EINVAL;
	}
276

277 278 279 280 281 282 283 284
	return 0;
}

static int amdgpu_move_blit(struct ttm_buffer_object *bo,
			    bool evict, bool no_wait_gpu,
			    struct ttm_mem_reg *new_mem,
			    struct ttm_mem_reg *old_mem)
{
285
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
286 287 288 289 290
	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;

	struct drm_mm_node *old_mm, *new_mm;
	uint64_t old_start, old_size, new_start, new_size;
	unsigned long num_pages;
291
	struct dma_fence *fence = NULL;
292 293 294 295
	int r;

	BUILD_BUG_ON((PAGE_SIZE % AMDGPU_GPU_PAGE_SIZE) != 0);

A
Alex Deucher 已提交
296 297 298 299 300
	if (!ring->ready) {
		DRM_ERROR("Trying to move memory with ring turned off.\n");
		return -EINVAL;
	}

301 302 303 304 305 306
	old_mm = old_mem->mm_node;
	r = amdgpu_mm_node_addr(bo, old_mm, old_mem, &old_start);
	if (r)
		return r;
	old_size = old_mm->size;

A
Alex Deucher 已提交
307

308 309
	new_mm = new_mem->mm_node;
	r = amdgpu_mm_node_addr(bo, new_mm, new_mem, &new_start);
310 311
	if (r)
		return r;
312 313 314 315 316
	new_size = new_mm->size;

	num_pages = new_mem->num_pages;
	while (num_pages) {
		unsigned long cur_pages = min(old_size, new_size);
317
		struct dma_fence *next;
318 319 320 321 322 323 324

		r = amdgpu_copy_buffer(ring, old_start, new_start,
				       cur_pages * PAGE_SIZE,
				       bo->resv, &next, false);
		if (r)
			goto error;

325
		dma_fence_put(fence);
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
		fence = next;

		num_pages -= cur_pages;
		if (!num_pages)
			break;

		old_size -= cur_pages;
		if (!old_size) {
			r = amdgpu_mm_node_addr(bo, ++old_mm, old_mem,
						&old_start);
			if (r)
				goto error;
			old_size = old_mm->size;
		} else {
			old_start += cur_pages * PAGE_SIZE;
		}

		new_size -= cur_pages;
		if (!new_size) {
			r = amdgpu_mm_node_addr(bo, ++new_mm, new_mem,
						&new_start);
			if (r)
				goto error;

			new_size = new_mm->size;
		} else {
			new_start += cur_pages * PAGE_SIZE;
		}
	}
355 356

	r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
357
	dma_fence_put(fence);
A
Alex Deucher 已提交
358
	return r;
359 360 361

error:
	if (fence)
362 363
		dma_fence_wait(fence, false);
	dma_fence_put(fence);
364
	return r;
A
Alex Deucher 已提交
365 366 367 368 369 370 371 372 373 374 375 376 377 378
}

static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo,
				bool evict, bool interruptible,
				bool no_wait_gpu,
				struct ttm_mem_reg *new_mem)
{
	struct amdgpu_device *adev;
	struct ttm_mem_reg *old_mem = &bo->mem;
	struct ttm_mem_reg tmp_mem;
	struct ttm_place placements;
	struct ttm_placement placement;
	int r;

379
	adev = amdgpu_ttm_adev(bo->bdev);
A
Alex Deucher 已提交
380 381 382 383 384 385 386
	tmp_mem = *new_mem;
	tmp_mem.mm_node = NULL;
	placement.num_placement = 1;
	placement.placement = &placements;
	placement.num_busy_placement = 1;
	placement.busy_placement = &placements;
	placements.fpfn = 0;
387
	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
A
Alex Deucher 已提交
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407
	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
			     interruptible, no_wait_gpu);
	if (unlikely(r)) {
		return r;
	}

	r = ttm_tt_set_placement_caching(bo->ttm, tmp_mem.placement);
	if (unlikely(r)) {
		goto out_cleanup;
	}

	r = ttm_tt_bind(bo->ttm, &tmp_mem);
	if (unlikely(r)) {
		goto out_cleanup;
	}
	r = amdgpu_move_blit(bo, true, no_wait_gpu, &tmp_mem, old_mem);
	if (unlikely(r)) {
		goto out_cleanup;
	}
408
	r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, new_mem);
A
Alex Deucher 已提交
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425
out_cleanup:
	ttm_bo_mem_put(bo, &tmp_mem);
	return r;
}

static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo,
				bool evict, bool interruptible,
				bool no_wait_gpu,
				struct ttm_mem_reg *new_mem)
{
	struct amdgpu_device *adev;
	struct ttm_mem_reg *old_mem = &bo->mem;
	struct ttm_mem_reg tmp_mem;
	struct ttm_placement placement;
	struct ttm_place placements;
	int r;

426
	adev = amdgpu_ttm_adev(bo->bdev);
A
Alex Deucher 已提交
427 428 429 430 431 432 433
	tmp_mem = *new_mem;
	tmp_mem.mm_node = NULL;
	placement.num_placement = 1;
	placement.placement = &placements;
	placement.num_busy_placement = 1;
	placement.busy_placement = &placements;
	placements.fpfn = 0;
434
	placements.lpfn = adev->mc.gtt_size >> PAGE_SHIFT;
A
Alex Deucher 已提交
435 436 437 438 439 440
	placements.flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_TT;
	r = ttm_bo_mem_space(bo, &placement, &tmp_mem,
			     interruptible, no_wait_gpu);
	if (unlikely(r)) {
		return r;
	}
441
	r = ttm_bo_move_ttm(bo, interruptible, no_wait_gpu, &tmp_mem);
A
Alex Deucher 已提交
442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459
	if (unlikely(r)) {
		goto out_cleanup;
	}
	r = amdgpu_move_blit(bo, true, no_wait_gpu, new_mem, old_mem);
	if (unlikely(r)) {
		goto out_cleanup;
	}
out_cleanup:
	ttm_bo_mem_put(bo, &tmp_mem);
	return r;
}

static int amdgpu_bo_move(struct ttm_buffer_object *bo,
			bool evict, bool interruptible,
			bool no_wait_gpu,
			struct ttm_mem_reg *new_mem)
{
	struct amdgpu_device *adev;
460
	struct amdgpu_bo *abo;
A
Alex Deucher 已提交
461 462 463
	struct ttm_mem_reg *old_mem = &bo->mem;
	int r;

464 465 466 467 468
	/* Can't move a pinned BO */
	abo = container_of(bo, struct amdgpu_bo, tbo);
	if (WARN_ON_ONCE(abo->pin_count > 0))
		return -EINVAL;

469
	adev = amdgpu_ttm_adev(bo->bdev);
470

A
Alex Deucher 已提交
471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503
	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
		amdgpu_move_null(bo, new_mem);
		return 0;
	}
	if ((old_mem->mem_type == TTM_PL_TT &&
	     new_mem->mem_type == TTM_PL_SYSTEM) ||
	    (old_mem->mem_type == TTM_PL_SYSTEM &&
	     new_mem->mem_type == TTM_PL_TT)) {
		/* bind is enough */
		amdgpu_move_null(bo, new_mem);
		return 0;
	}
	if (adev->mman.buffer_funcs == NULL ||
	    adev->mman.buffer_funcs_ring == NULL ||
	    !adev->mman.buffer_funcs_ring->ready) {
		/* use memcpy */
		goto memcpy;
	}

	if (old_mem->mem_type == TTM_PL_VRAM &&
	    new_mem->mem_type == TTM_PL_SYSTEM) {
		r = amdgpu_move_vram_ram(bo, evict, interruptible,
					no_wait_gpu, new_mem);
	} else if (old_mem->mem_type == TTM_PL_SYSTEM &&
		   new_mem->mem_type == TTM_PL_VRAM) {
		r = amdgpu_move_ram_vram(bo, evict, interruptible,
					    no_wait_gpu, new_mem);
	} else {
		r = amdgpu_move_blit(bo, evict, no_wait_gpu, new_mem, old_mem);
	}

	if (r) {
memcpy:
504
		r = ttm_bo_move_memcpy(bo, interruptible, no_wait_gpu, new_mem);
A
Alex Deucher 已提交
505 506 507 508 509 510 511 512 513 514 515 516 517
		if (r) {
			return r;
		}
	}

	/* update statistics */
	atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved);
	return 0;
}

static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
518
	struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
A
Alex Deucher 已提交
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550

	mem->bus.addr = NULL;
	mem->bus.offset = 0;
	mem->bus.size = mem->num_pages << PAGE_SHIFT;
	mem->bus.base = 0;
	mem->bus.is_iomem = false;
	if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
		return -EINVAL;
	switch (mem->mem_type) {
	case TTM_PL_SYSTEM:
		/* system memory */
		return 0;
	case TTM_PL_TT:
		break;
	case TTM_PL_VRAM:
		mem->bus.offset = mem->start << PAGE_SHIFT;
		/* check if it's visible */
		if ((mem->bus.offset + mem->bus.size) > adev->mc.visible_vram_size)
			return -EINVAL;
		mem->bus.base = adev->mc.aper_base;
		mem->bus.is_iomem = true;
		break;
	default:
		return -EINVAL;
	}
	return 0;
}

static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
}

551 552 553 554 555
static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
					   unsigned long page_offset)
{
	struct drm_mm_node *mm = bo->mem.mm_node;
	uint64_t size = mm->size;
556
	uint64_t offset = page_offset;
557 558

	page_offset = do_div(offset, size);
559
	mm += offset;
560 561 562
	return (bo->mem.bus.base >> PAGE_SHIFT) + mm->start + page_offset;
}

A
Alex Deucher 已提交
563 564 565
/*
 * TTM backend functions.
 */
566 567 568 569 570
struct amdgpu_ttm_gup_task_list {
	struct list_head	list;
	struct task_struct	*task;
};

A
Alex Deucher 已提交
571
struct amdgpu_ttm_tt {
572 573 574 575 576 577 578 579
	struct ttm_dma_tt	ttm;
	struct amdgpu_device	*adev;
	u64			offset;
	uint64_t		userptr;
	struct mm_struct	*usermm;
	uint32_t		userflags;
	spinlock_t              guptasklock;
	struct list_head        guptasks;
580
	atomic_t		mmu_invalidations;
581
	struct list_head        list;
A
Alex Deucher 已提交
582 583
};

584
int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
A
Alex Deucher 已提交
585 586
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
587
	unsigned int flags = 0;
588 589
	unsigned pinned = 0;
	int r;
A
Alex Deucher 已提交
590

591 592 593
	if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
		flags |= FOLL_WRITE;

A
Alex Deucher 已提交
594
	if (gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) {
595
		/* check that we only use anonymous memory
A
Alex Deucher 已提交
596 597 598 599 600 601 602 603 604 605 606 607
		   to prevent problems with writeback */
		unsigned long end = gtt->userptr + ttm->num_pages * PAGE_SIZE;
		struct vm_area_struct *vma;

		vma = find_vma(gtt->usermm, gtt->userptr);
		if (!vma || vma->vm_file || vma->vm_end < end)
			return -EPERM;
	}

	do {
		unsigned num_pages = ttm->num_pages - pinned;
		uint64_t userptr = gtt->userptr + pinned * PAGE_SIZE;
608
		struct page **p = pages + pinned;
609 610 611 612 613 614
		struct amdgpu_ttm_gup_task_list guptask;

		guptask.task = current;
		spin_lock(&gtt->guptasklock);
		list_add(&guptask.list, &gtt->guptasks);
		spin_unlock(&gtt->guptasklock);
A
Alex Deucher 已提交
615

616
		r = get_user_pages(userptr, num_pages, flags, p, NULL);
617 618 619 620

		spin_lock(&gtt->guptasklock);
		list_del(&guptask.list);
		spin_unlock(&gtt->guptasklock);
A
Alex Deucher 已提交
621 622 623 624 625 626 627 628

		if (r < 0)
			goto release_pages;

		pinned += r;

	} while (pinned < ttm->num_pages);

629 630 631 632 633 634 635 636 637 638
	return 0;

release_pages:
	release_pages(pages, pinned, 0);
	return r;
}

/* prepare the sg table with the user pages */
static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
{
639
	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
640 641 642 643 644 645 646 647
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
	unsigned nents;
	int r;

	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
	enum dma_data_direction direction = write ?
		DMA_BIDIRECTIONAL : DMA_TO_DEVICE;

A
Alex Deucher 已提交
648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670
	r = sg_alloc_table_from_pages(ttm->sg, ttm->pages, ttm->num_pages, 0,
				      ttm->num_pages << PAGE_SHIFT,
				      GFP_KERNEL);
	if (r)
		goto release_sg;

	r = -ENOMEM;
	nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
	if (nents != ttm->sg->nents)
		goto release_sg;

	drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
					 gtt->ttm.dma_address, ttm->num_pages);

	return 0;

release_sg:
	kfree(ttm->sg);
	return r;
}

static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
{
671
	struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
A
Alex Deucher 已提交
672
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
673
	struct sg_page_iter sg_iter;
A
Alex Deucher 已提交
674 675 676 677 678 679 680 681 682 683 684 685

	int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
	enum dma_data_direction direction = write ?
		DMA_BIDIRECTIONAL : DMA_TO_DEVICE;

	/* double check that we don't free the table twice */
	if (!ttm->sg->sgl)
		return;

	/* free the sg table and pages again */
	dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);

686 687
	for_each_sg_page(ttm->sg->sgl, &sg_iter, ttm->sg->nents, 0) {
		struct page *page = sg_page_iter_page(&sg_iter);
A
Alex Deucher 已提交
688 689 690 691
		if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
			set_page_dirty(page);

		mark_page_accessed(page);
692
		put_page(page);
A
Alex Deucher 已提交
693 694 695 696 697 698 699 700 701 702 703
	}

	sg_free_table(ttm->sg);
}

static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
				   struct ttm_mem_reg *bo_mem)
{
	struct amdgpu_ttm_tt *gtt = (void*)ttm;
	int r;

704 705 706 707 708 709 710
	if (gtt->userptr) {
		r = amdgpu_ttm_tt_pin_userptr(ttm);
		if (r) {
			DRM_ERROR("failed to pin userptr\n");
			return r;
		}
	}
A
Alex Deucher 已提交
711 712 713 714 715 716 717 718 719 720
	if (!ttm->num_pages) {
		WARN(1, "nothing to bind %lu pages for mreg %p back %p!\n",
		     ttm->num_pages, bo_mem, ttm);
	}

	if (bo_mem->mem_type == AMDGPU_PL_GDS ||
	    bo_mem->mem_type == AMDGPU_PL_GWS ||
	    bo_mem->mem_type == AMDGPU_PL_OA)
		return -EINVAL;

721 722 723 724 725 726 727 728 729 730
	return 0;
}

bool amdgpu_ttm_is_bound(struct ttm_tt *ttm)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

	return gtt && !list_empty(&gtt->list);
}

731
int amdgpu_ttm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *bo_mem)
732
{
733 734
	struct ttm_tt *ttm = bo->ttm;
	struct amdgpu_ttm_tt *gtt = (void *)bo->ttm;
735
	uint64_t flags;
736 737 738 739 740
	int r;

	if (!ttm || amdgpu_ttm_is_bound(ttm))
		return 0;

741 742 743 744 745 746 747
	r = amdgpu_gtt_mgr_alloc(&bo->bdev->man[TTM_PL_TT], bo,
				 NULL, bo_mem);
	if (r) {
		DRM_ERROR("Failed to allocate GTT address space (%d)\n", r);
		return r;
	}

748
	flags = amdgpu_ttm_tt_pte_flags(gtt->adev, ttm, bo_mem);
749
	gtt->offset = (u64)bo_mem->start << PAGE_SHIFT;
A
Alex Deucher 已提交
750 751 752 753
	r = amdgpu_gart_bind(gtt->adev, gtt->offset, ttm->num_pages,
		ttm->pages, gtt->ttm.dma_address, flags);

	if (r) {
754 755
		DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
			  ttm->num_pages, gtt->offset);
A
Alex Deucher 已提交
756 757
		return r;
	}
758 759 760
	spin_lock(&gtt->adev->gtt_list_lock);
	list_add_tail(&gtt->list, &gtt->adev->gtt_list);
	spin_unlock(&gtt->adev->gtt_list_lock);
A
Alex Deucher 已提交
761 762 763
	return 0;
}

764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
int amdgpu_ttm_recover_gart(struct amdgpu_device *adev)
{
	struct amdgpu_ttm_tt *gtt, *tmp;
	struct ttm_mem_reg bo_mem;
	uint32_t flags;
	int r;

	bo_mem.mem_type = TTM_PL_TT;
	spin_lock(&adev->gtt_list_lock);
	list_for_each_entry_safe(gtt, tmp, &adev->gtt_list, list) {
		flags = amdgpu_ttm_tt_pte_flags(gtt->adev, &gtt->ttm.ttm, &bo_mem);
		r = amdgpu_gart_bind(adev, gtt->offset, gtt->ttm.ttm.num_pages,
				     gtt->ttm.ttm.pages, gtt->ttm.dma_address,
				     flags);
		if (r) {
			spin_unlock(&adev->gtt_list_lock);
780 781
			DRM_ERROR("failed to bind %lu pages at 0x%08llX\n",
				  gtt->ttm.ttm.num_pages, gtt->offset);
782 783 784 785 786 787 788
			return r;
		}
	}
	spin_unlock(&adev->gtt_list_lock);
	return 0;
}

A
Alex Deucher 已提交
789 790 791 792
static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

793 794 795
	if (gtt->userptr)
		amdgpu_ttm_tt_unpin_userptr(ttm);

796 797 798
	if (!amdgpu_ttm_is_bound(ttm))
		return 0;

A
Alex Deucher 已提交
799 800 801 802
	/* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
	if (gtt->adev->gart.ready)
		amdgpu_gart_unbind(gtt->adev, gtt->offset, ttm->num_pages);

803 804 805 806
	spin_lock(&gtt->adev->gtt_list_lock);
	list_del_init(&gtt->list);
	spin_unlock(&gtt->adev->gtt_list_lock);

A
Alex Deucher 已提交
807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
	return 0;
}

static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

	ttm_dma_tt_fini(&gtt->ttm);
	kfree(gtt);
}

static struct ttm_backend_func amdgpu_backend_func = {
	.bind = &amdgpu_ttm_backend_bind,
	.unbind = &amdgpu_ttm_backend_unbind,
	.destroy = &amdgpu_ttm_backend_destroy,
};

static struct ttm_tt *amdgpu_ttm_tt_create(struct ttm_bo_device *bdev,
				    unsigned long size, uint32_t page_flags,
				    struct page *dummy_read_page)
{
	struct amdgpu_device *adev;
	struct amdgpu_ttm_tt *gtt;

831
	adev = amdgpu_ttm_adev(bdev);
A
Alex Deucher 已提交
832 833 834 835 836 837 838 839 840 841 842

	gtt = kzalloc(sizeof(struct amdgpu_ttm_tt), GFP_KERNEL);
	if (gtt == NULL) {
		return NULL;
	}
	gtt->ttm.ttm.func = &amdgpu_backend_func;
	gtt->adev = adev;
	if (ttm_dma_tt_init(&gtt->ttm, bdev, size, page_flags, dummy_read_page)) {
		kfree(gtt);
		return NULL;
	}
843
	INIT_LIST_HEAD(&gtt->list);
A
Alex Deucher 已提交
844 845 846 847 848 849 850 851 852 853 854 855 856 857 858
	return &gtt->ttm.ttm;
}

static int amdgpu_ttm_tt_populate(struct ttm_tt *ttm)
{
	struct amdgpu_device *adev;
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
	unsigned i;
	int r;
	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);

	if (ttm->state != tt_unpopulated)
		return 0;

	if (gtt && gtt->userptr) {
859
		ttm->sg = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
A
Alex Deucher 已提交
860 861 862 863 864 865 866 867 868 869 870 871 872 873 874
		if (!ttm->sg)
			return -ENOMEM;

		ttm->page_flags |= TTM_PAGE_FLAG_SG;
		ttm->state = tt_unbound;
		return 0;
	}

	if (slave && ttm->sg) {
		drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
						 gtt->ttm.dma_address, ttm->num_pages);
		ttm->state = tt_unbound;
		return 0;
	}

875
	adev = amdgpu_ttm_adev(ttm->bdev);
A
Alex Deucher 已提交
876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892

#ifdef CONFIG_SWIOTLB
	if (swiotlb_nr_tbl()) {
		return ttm_dma_populate(&gtt->ttm, adev->dev);
	}
#endif

	r = ttm_pool_populate(ttm);
	if (r) {
		return r;
	}

	for (i = 0; i < ttm->num_pages; i++) {
		gtt->ttm.dma_address[i] = pci_map_page(adev->pdev, ttm->pages[i],
						       0, PAGE_SIZE,
						       PCI_DMA_BIDIRECTIONAL);
		if (pci_dma_mapping_error(adev->pdev, gtt->ttm.dma_address[i])) {
893
			while (i--) {
A
Alex Deucher 已提交
894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
				pci_unmap_page(adev->pdev, gtt->ttm.dma_address[i],
					       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
				gtt->ttm.dma_address[i] = 0;
			}
			ttm_pool_unpopulate(ttm);
			return -EFAULT;
		}
	}
	return 0;
}

static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
{
	struct amdgpu_device *adev;
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
	unsigned i;
	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);

	if (gtt && gtt->userptr) {
		kfree(ttm->sg);
		ttm->page_flags &= ~TTM_PAGE_FLAG_SG;
		return;
	}

	if (slave)
		return;

921
	adev = amdgpu_ttm_adev(ttm->bdev);
A
Alex Deucher 已提交
922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950

#ifdef CONFIG_SWIOTLB
	if (swiotlb_nr_tbl()) {
		ttm_dma_unpopulate(&gtt->ttm, adev->dev);
		return;
	}
#endif

	for (i = 0; i < ttm->num_pages; i++) {
		if (gtt->ttm.dma_address[i]) {
			pci_unmap_page(adev->pdev, gtt->ttm.dma_address[i],
				       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
		}
	}

	ttm_pool_unpopulate(ttm);
}

int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
			      uint32_t flags)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

	if (gtt == NULL)
		return -EINVAL;

	gtt->userptr = addr;
	gtt->usermm = current->mm;
	gtt->userflags = flags;
951 952
	spin_lock_init(&gtt->guptasklock);
	INIT_LIST_HEAD(&gtt->guptasks);
953
	atomic_set(&gtt->mmu_invalidations, 0);
954

A
Alex Deucher 已提交
955 956 957
	return 0;
}

958
struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm)
A
Alex Deucher 已提交
959 960 961 962
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

	if (gtt == NULL)
963
		return NULL;
A
Alex Deucher 已提交
964

965
	return gtt->usermm;
A
Alex Deucher 已提交
966 967
}

968 969 970 971
bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start,
				  unsigned long end)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
972
	struct amdgpu_ttm_gup_task_list *entry;
973 974
	unsigned long size;

975
	if (gtt == NULL || !gtt->userptr)
976 977 978 979 980 981
		return false;

	size = (unsigned long)gtt->ttm.ttm.num_pages * PAGE_SIZE;
	if (gtt->userptr > end || gtt->userptr + size <= start)
		return false;

982 983 984 985 986 987 988 989 990
	spin_lock(&gtt->guptasklock);
	list_for_each_entry(entry, &gtt->guptasks, list) {
		if (entry->task == current) {
			spin_unlock(&gtt->guptasklock);
			return false;
		}
	}
	spin_unlock(&gtt->guptasklock);

991 992
	atomic_inc(&gtt->mmu_invalidations);

993 994 995
	return true;
}

996 997 998 999 1000 1001 1002 1003 1004 1005
bool amdgpu_ttm_tt_userptr_invalidated(struct ttm_tt *ttm,
				       int *last_invalidated)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;
	int prev_invalidated = *last_invalidated;

	*last_invalidated = atomic_read(&gtt->mmu_invalidations);
	return prev_invalidated != *last_invalidated;
}

A
Alex Deucher 已提交
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
{
	struct amdgpu_ttm_tt *gtt = (void *)ttm;

	if (gtt == NULL)
		return false;

	return !!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
}

1016
uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
A
Alex Deucher 已提交
1017 1018
				 struct ttm_mem_reg *mem)
{
1019
	uint64_t flags = 0;
A
Alex Deucher 已提交
1020 1021 1022 1023

	if (mem && mem->mem_type != TTM_PL_SYSTEM)
		flags |= AMDGPU_PTE_VALID;

1024
	if (mem && mem->mem_type == TTM_PL_TT) {
A
Alex Deucher 已提交
1025 1026
		flags |= AMDGPU_PTE_SYSTEM;

1027 1028 1029
		if (ttm->caching_state == tt_cached)
			flags |= AMDGPU_PTE_SNOOPED;
	}
A
Alex Deucher 已提交
1030

1031
	flags |= adev->gart.gart_pte_flags;
A
Alex Deucher 已提交
1032 1033 1034 1035 1036 1037 1038 1039
	flags |= AMDGPU_PTE_READABLE;

	if (!amdgpu_ttm_tt_is_readonly(ttm))
		flags |= AMDGPU_PTE_WRITEABLE;

	return flags;
}

1040 1041 1042
static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
					    const struct ttm_place *place)
{
1043 1044
	unsigned long num_pages = bo->mem.num_pages;
	struct drm_mm_node *node = bo->mem.mm_node;
1045

1046 1047 1048 1049 1050 1051 1052 1053
	if (bo->mem.start != AMDGPU_BO_INVALID_OFFSET)
		return ttm_bo_eviction_valuable(bo, place);

	switch (bo->mem.mem_type) {
	case TTM_PL_TT:
		return true;

	case TTM_PL_VRAM:
1054 1055 1056 1057 1058 1059 1060 1061 1062
		/* Check each drm MM node individually */
		while (num_pages) {
			if (place->fpfn < (node->start + node->size) &&
			    !(place->lpfn && place->lpfn <= node->start))
				return true;

			num_pages -= node->size;
			++node;
		}
1063
		break;
1064

1065 1066
	default:
		break;
1067 1068 1069 1070 1071
	}

	return ttm_bo_eviction_valuable(bo, place);
}

A
Alex Deucher 已提交
1072 1073 1074 1075 1076 1077
static struct ttm_bo_driver amdgpu_bo_driver = {
	.ttm_tt_create = &amdgpu_ttm_tt_create,
	.ttm_tt_populate = &amdgpu_ttm_tt_populate,
	.ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
	.invalidate_caches = &amdgpu_invalidate_caches,
	.init_mem_type = &amdgpu_init_mem_type,
1078
	.eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
A
Alex Deucher 已提交
1079 1080 1081 1082 1083 1084 1085
	.evict_flags = &amdgpu_evict_flags,
	.move = &amdgpu_bo_move,
	.verify_access = &amdgpu_verify_access,
	.move_notify = &amdgpu_bo_move_notify,
	.fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
	.io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
	.io_mem_free = &amdgpu_ttm_io_mem_free,
1086
	.io_mem_pfn = amdgpu_ttm_io_mem_pfn,
A
Alex Deucher 已提交
1087 1088 1089 1090 1091 1092
};

int amdgpu_ttm_init(struct amdgpu_device *adev)
{
	int r;

1093 1094 1095 1096
	r = amdgpu_ttm_global_init(adev);
	if (r) {
		return r;
	}
A
Alex Deucher 已提交
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118
	/* No others user of address space so set it to 0 */
	r = ttm_bo_device_init(&adev->mman.bdev,
			       adev->mman.bo_global_ref.ref.object,
			       &amdgpu_bo_driver,
			       adev->ddev->anon_inode->i_mapping,
			       DRM_FILE_PAGE_OFFSET,
			       adev->need_dma32);
	if (r) {
		DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
		return r;
	}
	adev->mman.initialized = true;
	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
				adev->mc.real_vram_size >> PAGE_SHIFT);
	if (r) {
		DRM_ERROR("Failed initializing VRAM heap.\n");
		return r;
	}
	/* Change the size here instead of the init above so only lpfn is affected */
	amdgpu_ttm_set_active_vram_size(adev, adev->mc.visible_vram_size);

	r = amdgpu_bo_create(adev, 256 * 1024, PAGE_SIZE, true,
1119
			     AMDGPU_GEM_DOMAIN_VRAM,
1120 1121
			     AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
1122
			     NULL, NULL, &adev->stollen_vga_memory);
A
Alex Deucher 已提交
1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155
	if (r) {
		return r;
	}
	r = amdgpu_bo_reserve(adev->stollen_vga_memory, false);
	if (r)
		return r;
	r = amdgpu_bo_pin(adev->stollen_vga_memory, AMDGPU_GEM_DOMAIN_VRAM, NULL);
	amdgpu_bo_unreserve(adev->stollen_vga_memory);
	if (r) {
		amdgpu_bo_unref(&adev->stollen_vga_memory);
		return r;
	}
	DRM_INFO("amdgpu: %uM of VRAM memory ready\n",
		 (unsigned) (adev->mc.real_vram_size / (1024 * 1024)));
	r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT,
				adev->mc.gtt_size >> PAGE_SHIFT);
	if (r) {
		DRM_ERROR("Failed initializing GTT heap.\n");
		return r;
	}
	DRM_INFO("amdgpu: %uM of GTT memory ready.\n",
		 (unsigned)(adev->mc.gtt_size / (1024 * 1024)));

	adev->gds.mem.total_size = adev->gds.mem.total_size << AMDGPU_GDS_SHIFT;
	adev->gds.mem.gfx_partition_size = adev->gds.mem.gfx_partition_size << AMDGPU_GDS_SHIFT;
	adev->gds.mem.cs_partition_size = adev->gds.mem.cs_partition_size << AMDGPU_GDS_SHIFT;
	adev->gds.gws.total_size = adev->gds.gws.total_size << AMDGPU_GWS_SHIFT;
	adev->gds.gws.gfx_partition_size = adev->gds.gws.gfx_partition_size << AMDGPU_GWS_SHIFT;
	adev->gds.gws.cs_partition_size = adev->gds.gws.cs_partition_size << AMDGPU_GWS_SHIFT;
	adev->gds.oa.total_size = adev->gds.oa.total_size << AMDGPU_OA_SHIFT;
	adev->gds.oa.gfx_partition_size = adev->gds.oa.gfx_partition_size << AMDGPU_OA_SHIFT;
	adev->gds.oa.cs_partition_size = adev->gds.oa.cs_partition_size << AMDGPU_OA_SHIFT;
	/* GDS Memory */
1156 1157 1158 1159 1160 1161 1162
	if (adev->gds.mem.total_size) {
		r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,
				   adev->gds.mem.total_size >> PAGE_SHIFT);
		if (r) {
			DRM_ERROR("Failed initializing GDS heap.\n");
			return r;
		}
A
Alex Deucher 已提交
1163 1164 1165
	}

	/* GWS */
1166 1167 1168 1169 1170 1171 1172
	if (adev->gds.gws.total_size) {
		r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,
				   adev->gds.gws.total_size >> PAGE_SHIFT);
		if (r) {
			DRM_ERROR("Failed initializing gws heap.\n");
			return r;
		}
A
Alex Deucher 已提交
1173 1174 1175
	}

	/* OA */
1176 1177 1178 1179 1180 1181 1182
	if (adev->gds.oa.total_size) {
		r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,
				   adev->gds.oa.total_size >> PAGE_SHIFT);
		if (r) {
			DRM_ERROR("Failed initializing oa heap.\n");
			return r;
		}
A
Alex Deucher 已提交
1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209
	}

	r = amdgpu_ttm_debugfs_init(adev);
	if (r) {
		DRM_ERROR("Failed to init debugfs\n");
		return r;
	}
	return 0;
}

void amdgpu_ttm_fini(struct amdgpu_device *adev)
{
	int r;

	if (!adev->mman.initialized)
		return;
	amdgpu_ttm_debugfs_fini(adev);
	if (adev->stollen_vga_memory) {
		r = amdgpu_bo_reserve(adev->stollen_vga_memory, false);
		if (r == 0) {
			amdgpu_bo_unpin(adev->stollen_vga_memory);
			amdgpu_bo_unreserve(adev->stollen_vga_memory);
		}
		amdgpu_bo_unref(&adev->stollen_vga_memory);
	}
	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
	ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
1210 1211 1212 1213 1214 1215
	if (adev->gds.mem.total_size)
		ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS);
	if (adev->gds.gws.total_size)
		ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS);
	if (adev->gds.oa.total_size)
		ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
A
Alex Deucher 已提交
1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241
	ttm_bo_device_release(&adev->mman.bdev);
	amdgpu_gart_fini(adev);
	amdgpu_ttm_global_fini(adev);
	adev->mman.initialized = false;
	DRM_INFO("amdgpu: ttm finalized\n");
}

/* this should only be called at bootup or when userspace
 * isn't running */
void amdgpu_ttm_set_active_vram_size(struct amdgpu_device *adev, u64 size)
{
	struct ttm_mem_type_manager *man;

	if (!adev->mman.initialized)
		return;

	man = &adev->mman.bdev.man[TTM_PL_VRAM];
	/* this just adjusts TTM size idea, which sets lpfn to the correct value */
	man->size = size >> PAGE_SHIFT;
}

int amdgpu_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct drm_file *file_priv;
	struct amdgpu_device *adev;

C
Christian König 已提交
1242
	if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET))
A
Alex Deucher 已提交
1243 1244 1245 1246
		return -EINVAL;

	file_priv = filp->private_data;
	adev = file_priv->minor->dev->dev_private;
C
Christian König 已提交
1247
	if (adev == NULL)
A
Alex Deucher 已提交
1248
		return -EINVAL;
C
Christian König 已提交
1249 1250

	return ttm_bo_mmap(filp, vma, &adev->mman.bdev);
A
Alex Deucher 已提交
1251 1252 1253 1254 1255 1256 1257
}

int amdgpu_copy_buffer(struct amdgpu_ring *ring,
		       uint64_t src_offset,
		       uint64_t dst_offset,
		       uint32_t byte_count,
		       struct reservation_object *resv,
1258
		       struct dma_fence **fence, bool direct_submit)
A
Alex Deucher 已提交
1259 1260
{
	struct amdgpu_device *adev = ring->adev;
1261 1262
	struct amdgpu_job *job;

A
Alex Deucher 已提交
1263 1264 1265 1266 1267 1268 1269 1270 1271
	uint32_t max_bytes;
	unsigned num_loops, num_dw;
	unsigned i;
	int r;

	max_bytes = adev->mman.buffer_funcs->copy_max_bytes;
	num_loops = DIV_ROUND_UP(byte_count, max_bytes);
	num_dw = num_loops * adev->mman.buffer_funcs->copy_num_dw;

1272 1273 1274 1275
	/* for IB padding */
	while (num_dw & 0x7)
		num_dw++;

1276 1277
	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
	if (r)
1278
		return r;
1279 1280

	if (resv) {
1281
		r = amdgpu_sync_resv(adev, &job->sync, resv,
1282 1283 1284 1285 1286
				     AMDGPU_FENCE_OWNER_UNDEFINED);
		if (r) {
			DRM_ERROR("sync failed (%d).\n", r);
			goto error_free;
		}
A
Alex Deucher 已提交
1287 1288 1289 1290 1291
	}

	for (i = 0; i < num_loops; i++) {
		uint32_t cur_size_in_bytes = min(byte_count, max_bytes);

1292 1293
		amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_offset,
					dst_offset, cur_size_in_bytes);
A
Alex Deucher 已提交
1294 1295 1296 1297 1298 1299

		src_offset += cur_size_in_bytes;
		dst_offset += cur_size_in_bytes;
		byte_count -= cur_size_in_bytes;
	}

1300 1301
	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
	WARN_ON(job->ibs[0].length_dw > num_dw);
1302 1303
	if (direct_submit) {
		r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs,
1304
				       NULL, fence);
1305
		job->fence = dma_fence_get(*fence);
1306 1307 1308 1309 1310 1311 1312 1313 1314
		if (r)
			DRM_ERROR("Error scheduling IBs (%d)\n", r);
		amdgpu_job_free(job);
	} else {
		r = amdgpu_job_submit(job, ring, &adev->mman.entity,
				      AMDGPU_FENCE_OWNER_UNDEFINED, fence);
		if (r)
			goto error_free;
	}
A
Alex Deucher 已提交
1315

1316
	return r;
1317

1318
error_free:
1319
	amdgpu_job_free(job);
1320
	return r;
A
Alex Deucher 已提交
1321 1322
}

1323
int amdgpu_fill_buffer(struct amdgpu_bo *bo,
1324 1325 1326
		       uint32_t src_data,
		       struct reservation_object *resv,
		       struct dma_fence **fence)
1327
{
1328
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
1329
	uint32_t max_bytes = adev->mman.buffer_funcs->fill_max_bytes;
1330 1331
	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;

1332 1333
	struct drm_mm_node *mm_node;
	unsigned long num_pages;
1334
	unsigned int num_loops, num_dw;
1335 1336

	struct amdgpu_job *job;
1337 1338
	int r;

1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353
	if (!ring->ready) {
		DRM_ERROR("Trying to clear memory with ring turned off.\n");
		return -EINVAL;
	}

	num_pages = bo->tbo.num_pages;
	mm_node = bo->tbo.mem.mm_node;
	num_loops = 0;
	while (num_pages) {
		uint32_t byte_count = mm_node->size << PAGE_SHIFT;

		num_loops += DIV_ROUND_UP(byte_count, max_bytes);
		num_pages -= mm_node->size;
		++mm_node;
	}
1354 1355 1356
	num_dw = num_loops * adev->mman.buffer_funcs->fill_num_dw;

	/* for IB padding */
1357
	num_dw += 64;
1358 1359 1360 1361 1362 1363 1364

	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4, &job);
	if (r)
		return r;

	if (resv) {
		r = amdgpu_sync_resv(adev, &job->sync, resv,
1365
				     AMDGPU_FENCE_OWNER_UNDEFINED);
1366 1367 1368 1369 1370 1371
		if (r) {
			DRM_ERROR("sync failed (%d).\n", r);
			goto error_free;
		}
	}

1372 1373
	num_pages = bo->tbo.num_pages;
	mm_node = bo->tbo.mem.mm_node;
1374

1375 1376 1377
	while (num_pages) {
		uint32_t byte_count = mm_node->size << PAGE_SHIFT;
		uint64_t dst_addr;
1378

1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
		r = amdgpu_mm_node_addr(&bo->tbo, mm_node,
					&bo->tbo.mem, &dst_addr);
		if (r)
			return r;

		while (byte_count) {
			uint32_t cur_size_in_bytes = min(byte_count, max_bytes);

			amdgpu_emit_fill_buffer(adev, &job->ibs[0], src_data,
						dst_addr, cur_size_in_bytes);

			dst_addr += cur_size_in_bytes;
			byte_count -= cur_size_in_bytes;
		}

		num_pages -= mm_node->size;
		++mm_node;
1396 1397 1398 1399 1400
	}

	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
	WARN_ON(job->ibs[0].length_dw > num_dw);
	r = amdgpu_job_submit(job, ring, &adev->mman.entity,
1401
			      AMDGPU_FENCE_OWNER_UNDEFINED, fence);
1402 1403 1404 1405 1406 1407 1408 1409 1410 1411
	if (r)
		goto error_free;

	return 0;

error_free:
	amdgpu_job_free(job);
	return r;
}

A
Alex Deucher 已提交
1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
#if defined(CONFIG_DEBUG_FS)

static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
{
	struct drm_info_node *node = (struct drm_info_node *)m->private;
	unsigned ttm_pl = *(int *)node->info_ent->data;
	struct drm_device *dev = node->minor->dev;
	struct amdgpu_device *adev = dev->dev_private;
	struct drm_mm *mm = (struct drm_mm *)adev->mman.bdev.man[ttm_pl].priv;
	struct ttm_bo_global *glob = adev->mman.bdev.glob;
D
Daniel Vetter 已提交
1422
	struct drm_printer p = drm_seq_file_printer(m);
A
Alex Deucher 已提交
1423 1424

	spin_lock(&glob->lru_lock);
D
Daniel Vetter 已提交
1425
	drm_mm_print(mm, &p);
A
Alex Deucher 已提交
1426
	spin_unlock(&glob->lru_lock);
1427
	if (ttm_pl == TTM_PL_VRAM)
1428
		seq_printf(m, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n",
1429
			   adev->mman.bdev.man[ttm_pl].size,
1430 1431
			   (u64)atomic64_read(&adev->vram_usage) >> 20,
			   (u64)atomic64_read(&adev->vram_vis_usage) >> 20);
D
Daniel Vetter 已提交
1432
	return 0;
A
Alex Deucher 已提交
1433 1434 1435 1436 1437
}

static int ttm_pl_vram = TTM_PL_VRAM;
static int ttm_pl_tt = TTM_PL_TT;

1438
static const struct drm_info_list amdgpu_ttm_debugfs_list[] = {
A
Alex Deucher 已提交
1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449
	{"amdgpu_vram_mm", amdgpu_mm_dump_table, 0, &ttm_pl_vram},
	{"amdgpu_gtt_mm", amdgpu_mm_dump_table, 0, &ttm_pl_tt},
	{"ttm_page_pool", ttm_page_alloc_debugfs, 0, NULL},
#ifdef CONFIG_SWIOTLB
	{"ttm_dma_page_pool", ttm_dma_page_alloc_debugfs, 0, NULL}
#endif
};

static ssize_t amdgpu_ttm_vram_read(struct file *f, char __user *buf,
				    size_t size, loff_t *pos)
{
A
Al Viro 已提交
1450
	struct amdgpu_device *adev = file_inode(f)->i_private;
A
Alex Deucher 已提交
1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488
	ssize_t result = 0;
	int r;

	if (size & 0x3 || *pos & 0x3)
		return -EINVAL;

	while (size) {
		unsigned long flags;
		uint32_t value;

		if (*pos >= adev->mc.mc_vram_size)
			return result;

		spin_lock_irqsave(&adev->mmio_idx_lock, flags);
		WREG32(mmMM_INDEX, ((uint32_t)*pos) | 0x80000000);
		WREG32(mmMM_INDEX_HI, *pos >> 31);
		value = RREG32(mmMM_DATA);
		spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);

		r = put_user(value, (uint32_t *)buf);
		if (r)
			return r;

		result += 4;
		buf += 4;
		*pos += 4;
		size -= 4;
	}

	return result;
}

static const struct file_operations amdgpu_ttm_vram_fops = {
	.owner = THIS_MODULE,
	.read = amdgpu_ttm_vram_read,
	.llseek = default_llseek
};

1489 1490
#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS

A
Alex Deucher 已提交
1491 1492 1493
static ssize_t amdgpu_ttm_gtt_read(struct file *f, char __user *buf,
				   size_t size, loff_t *pos)
{
A
Al Viro 已提交
1494
	struct amdgpu_device *adev = file_inode(f)->i_private;
A
Alex Deucher 已提交
1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537
	ssize_t result = 0;
	int r;

	while (size) {
		loff_t p = *pos / PAGE_SIZE;
		unsigned off = *pos & ~PAGE_MASK;
		size_t cur_size = min_t(size_t, size, PAGE_SIZE - off);
		struct page *page;
		void *ptr;

		if (p >= adev->gart.num_cpu_pages)
			return result;

		page = adev->gart.pages[p];
		if (page) {
			ptr = kmap(page);
			ptr += off;

			r = copy_to_user(buf, ptr, cur_size);
			kunmap(adev->gart.pages[p]);
		} else
			r = clear_user(buf, cur_size);

		if (r)
			return -EFAULT;

		result += cur_size;
		buf += cur_size;
		*pos += cur_size;
		size -= cur_size;
	}

	return result;
}

static const struct file_operations amdgpu_ttm_gtt_fops = {
	.owner = THIS_MODULE,
	.read = amdgpu_ttm_gtt_read,
	.llseek = default_llseek
};

#endif

1538 1539
#endif

A
Alex Deucher 已提交
1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554
static int amdgpu_ttm_debugfs_init(struct amdgpu_device *adev)
{
#if defined(CONFIG_DEBUG_FS)
	unsigned count;

	struct drm_minor *minor = adev->ddev->primary;
	struct dentry *ent, *root = minor->debugfs_root;

	ent = debugfs_create_file("amdgpu_vram", S_IFREG | S_IRUGO, root,
				  adev, &amdgpu_ttm_vram_fops);
	if (IS_ERR(ent))
		return PTR_ERR(ent);
	i_size_write(ent->d_inode, adev->mc.mc_vram_size);
	adev->mman.vram = ent;

1555
#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
A
Alex Deucher 已提交
1556 1557 1558 1559 1560 1561 1562
	ent = debugfs_create_file("amdgpu_gtt", S_IFREG | S_IRUGO, root,
				  adev, &amdgpu_ttm_gtt_fops);
	if (IS_ERR(ent))
		return PTR_ERR(ent);
	i_size_write(ent->d_inode, adev->mc.gtt_size);
	adev->mman.gtt = ent;

1563
#endif
A
Alex Deucher 已提交
1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584
	count = ARRAY_SIZE(amdgpu_ttm_debugfs_list);

#ifdef CONFIG_SWIOTLB
	if (!swiotlb_nr_tbl())
		--count;
#endif

	return amdgpu_debugfs_add_files(adev, amdgpu_ttm_debugfs_list, count);
#else

	return 0;
#endif
}

static void amdgpu_ttm_debugfs_fini(struct amdgpu_device *adev)
{
#if defined(CONFIG_DEBUG_FS)

	debugfs_remove(adev->mman.vram);
	adev->mman.vram = NULL;

1585
#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS
A
Alex Deucher 已提交
1586 1587 1588
	debugfs_remove(adev->mman.gtt);
	adev->mman.gtt = NULL;
#endif
1589 1590

#endif
A
Alex Deucher 已提交
1591
}