amdgpu_vm.c 71.0 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright 2008 Advanced Micro Devices, Inc.
 * Copyright 2008 Red Hat Inc.
 * Copyright 2009 Jerome Glisse.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Dave Airlie
 *          Alex Deucher
 *          Jerome Glisse
 */
28
#include <linux/dma-fence-array.h>
29
#include <linux/interval_tree_generic.h>
30
#include <linux/idr.h>
A
Alex Deucher 已提交
31 32 33 34
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "amdgpu_trace.h"
35
#include "amdgpu_amdkfd.h"
A
Alex Deucher 已提交
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56

/*
 * GPUVM
 * GPUVM is similar to the legacy gart on older asics, however
 * rather than there being a single global gart table
 * for the entire GPU, there are multiple VM page tables active
 * at any given time.  The VM page tables can contain a mix
 * vram pages and system memory pages and system memory pages
 * can be mapped as snooped (cached system pages) or unsnooped
 * (uncached system pages).
 * Each VM has an ID associated with it and there is a page table
 * associated with each VMID.  When execting a command buffer,
 * the kernel tells the the ring what VMID to use for that command
 * buffer.  VMIDs are allocated dynamically as commands are submitted.
 * The userspace drivers maintain their own address space and the kernel
 * sets up their pages tables accordingly when they submit their
 * command buffers and a VMID is assigned.
 * Cayman/Trinity support up to 8 active VMs at any given time;
 * SI supports 16.
 */

57 58 59 60 61 62 63 64 65
#define START(node) ((node)->start)
#define LAST(node) ((node)->last)

INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
		     START, LAST, static, amdgpu_vm_it)

#undef START
#undef LAST

66 67 68
/* Local structure. Encapsulate some VM table update parameters to reduce
 * the number of function parameters
 */
69
struct amdgpu_pte_update_params {
70 71
	/* amdgpu device we do this update for */
	struct amdgpu_device *adev;
72 73
	/* optional amdgpu_vm we do this update for */
	struct amdgpu_vm *vm;
74 75 76 77
	/* address where to copy page table entries from */
	uint64_t src;
	/* indirect buffer to fill with commands */
	struct amdgpu_ib *ib;
78
	/* Function which actually does the update */
79 80
	void (*func)(struct amdgpu_pte_update_params *params,
		     struct amdgpu_bo *bo, uint64_t pe,
81
		     uint64_t addr, unsigned count, uint32_t incr,
82
		     uint64_t flags);
83 84 85 86 87 88
	/* The next two are used during VM update by CPU
	 *  DMA addresses to use for mapping
	 *  Kernel pointer of PD/PT BO that needs to be updated
	 */
	dma_addr_t *pages_addr;
	void *kptr;
89 90
};

91 92 93 94 95 96
/* Helper to disable partial resident texture feature from a fence callback */
struct amdgpu_prt_cb {
	struct amdgpu_device *adev;
	struct dma_fence_cb cb;
};

97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
				   struct amdgpu_vm *vm,
				   struct amdgpu_bo *bo)
{
	base->vm = vm;
	base->bo = bo;
	INIT_LIST_HEAD(&base->bo_list);
	INIT_LIST_HEAD(&base->vm_status);

	if (!bo)
		return;
	list_add_tail(&base->bo_list, &bo->va);

	if (bo->tbo.resv != vm->root.base.bo->tbo.resv)
		return;

	if (bo->preferred_domains &
	    amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type))
		return;

	/*
	 * we checked all the prerequisites, but it looks like this per vm bo
	 * is currently evicted. add the bo to the evicted list to make sure it
	 * is validated on next vm use to avoid fault.
	 * */
	spin_lock(&vm->status_lock);
	list_move_tail(&base->vm_status, &vm->evicted);
	spin_unlock(&vm->status_lock);
}

127 128 129 130 131 132 133 134 135 136
/**
 * amdgpu_vm_level_shift - return the addr shift for each level
 *
 * @adev: amdgpu_device pointer
 *
 * Returns the number of bits the pfn needs to be right shifted for a level.
 */
static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
				      unsigned level)
{
137 138 139 140 141 142 143
	unsigned shift = 0xff;

	switch (level) {
	case AMDGPU_VM_PDB2:
	case AMDGPU_VM_PDB1:
	case AMDGPU_VM_PDB0:
		shift = 9 * (AMDGPU_VM_PDB0 - level) +
144
			adev->vm_manager.block_size;
145 146 147 148 149 150 151 152 153
		break;
	case AMDGPU_VM_PTB:
		shift = 0;
		break;
	default:
		dev_err(adev->dev, "the level%d isn't supported.\n", level);
	}

	return shift;
154 155
}

A
Alex Deucher 已提交
156
/**
157
 * amdgpu_vm_num_entries - return the number of entries in a PD/PT
A
Alex Deucher 已提交
158 159 160
 *
 * @adev: amdgpu_device pointer
 *
161
 * Calculate the number of entries in a page directory or page table.
A
Alex Deucher 已提交
162
 */
163 164
static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
				      unsigned level)
A
Alex Deucher 已提交
165
{
166 167
	unsigned shift = amdgpu_vm_level_shift(adev,
					       adev->vm_manager.root_level);
168

169
	if (level == adev->vm_manager.root_level)
170
		/* For the root directory */
171
		return round_up(adev->vm_manager.max_pfn, 1 << shift) >> shift;
172
	else if (level != AMDGPU_VM_PTB)
173 174 175
		/* Everything in between */
		return 512;
	else
176
		/* For the page tables on the leaves */
177
		return AMDGPU_VM_PTE_COUNT(adev);
A
Alex Deucher 已提交
178 179 180
}

/**
181
 * amdgpu_vm_bo_size - returns the size of the BOs in bytes
A
Alex Deucher 已提交
182 183 184
 *
 * @adev: amdgpu_device pointer
 *
185
 * Calculate the size of the BO for a page directory or page table in bytes.
A
Alex Deucher 已提交
186
 */
187
static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
A
Alex Deucher 已提交
188
{
189
	return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
A
Alex Deucher 已提交
190 191 192
}

/**
193
 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
A
Alex Deucher 已提交
194 195
 *
 * @vm: vm providing the BOs
196
 * @validated: head of validation list
197
 * @entry: entry to add
A
Alex Deucher 已提交
198 199
 *
 * Add the page directory to the list of BOs to
200
 * validate for command submission.
A
Alex Deucher 已提交
201
 */
202 203 204
void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
			 struct list_head *validated,
			 struct amdgpu_bo_list_entry *entry)
A
Alex Deucher 已提交
205
{
206
	entry->robj = vm->root.base.bo;
207
	entry->priority = 0;
208
	entry->tv.bo = &entry->robj->tbo;
209
	entry->tv.shared = true;
210
	entry->user_pages = NULL;
211 212
	list_add(&entry->tv.head, validated);
}
A
Alex Deucher 已提交
213

214
/**
215
 * amdgpu_vm_validate_pt_bos - validate the page table BOs
216
 *
217
 * @adev: amdgpu device pointer
218
 * @vm: vm providing the BOs
219 220 221 222 223
 * @validate: callback to do the validation
 * @param: parameter for the validation callback
 *
 * Validate the page table BOs on command submission if neccessary.
 */
224 225 226
int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
			      int (*validate)(void *p, struct amdgpu_bo *bo),
			      void *param)
227
{
228
	struct ttm_bo_global *glob = adev->mman.bdev.glob;
229 230
	int r;

231 232 233 234
	spin_lock(&vm->status_lock);
	while (!list_empty(&vm->evicted)) {
		struct amdgpu_vm_bo_base *bo_base;
		struct amdgpu_bo *bo;
235

236 237 238 239
		bo_base = list_first_entry(&vm->evicted,
					   struct amdgpu_vm_bo_base,
					   vm_status);
		spin_unlock(&vm->status_lock);
240

241 242 243 244 245 246
		bo = bo_base->bo;
		BUG_ON(!bo);
		if (bo->parent) {
			r = validate(param, bo);
			if (r)
				return r;
247

248 249 250 251 252 253
			spin_lock(&glob->lru_lock);
			ttm_bo_move_to_lru_tail(&bo->tbo);
			if (bo->shadow)
				ttm_bo_move_to_lru_tail(&bo->shadow->tbo);
			spin_unlock(&glob->lru_lock);
		}
254

255 256
		if (bo->tbo.type == ttm_bo_type_kernel &&
		    vm->use_cpu_for_update) {
257 258 259 260
			r = amdgpu_bo_kmap(bo, NULL);
			if (r)
				return r;
		}
261

262
		spin_lock(&vm->status_lock);
263 264 265 266
		if (bo->tbo.type != ttm_bo_type_kernel)
			list_move(&bo_base->vm_status, &vm->moved);
		else
			list_move(&bo_base->vm_status, &vm->relocated);
267
	}
268
	spin_unlock(&vm->status_lock);
269

270
	return 0;
271 272
}

273
/**
274
 * amdgpu_vm_ready - check VM is ready for updates
275
 *
276
 * @vm: VM to check
A
Alex Deucher 已提交
277
 *
278
 * Check if all VM PDs/PTs are ready for updates
A
Alex Deucher 已提交
279
 */
280
bool amdgpu_vm_ready(struct amdgpu_vm *vm)
A
Alex Deucher 已提交
281
{
282
	bool ready;
A
Alex Deucher 已提交
283

284 285 286
	spin_lock(&vm->status_lock);
	ready = list_empty(&vm->evicted);
	spin_unlock(&vm->status_lock);
287

288
	return ready;
289 290
}

291 292 293 294 295 296 297 298 299 300
/**
 * amdgpu_vm_clear_bo - initially clear the PDs/PTs
 *
 * @adev: amdgpu_device pointer
 * @bo: BO to clear
 * @level: level this BO is at
 *
 * Root PD needs to be reserved when calling this.
 */
static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
301 302
			      struct amdgpu_vm *vm, struct amdgpu_bo *bo,
			      unsigned level, bool pte_support_ats)
303 304 305
{
	struct ttm_operation_ctx ctx = { true, false };
	struct dma_fence *fence = NULL;
306
	unsigned entries, ats_entries;
307 308
	struct amdgpu_ring *ring;
	struct amdgpu_job *job;
309
	uint64_t addr;
310 311
	int r;

312 313 314 315 316 317 318 319 320 321 322 323 324 325
	addr = amdgpu_bo_gpu_offset(bo);
	entries = amdgpu_bo_size(bo) / 8;

	if (pte_support_ats) {
		if (level == adev->vm_manager.root_level) {
			ats_entries = amdgpu_vm_level_shift(adev, level);
			ats_entries += AMDGPU_GPU_PAGE_SHIFT;
			ats_entries = AMDGPU_VA_HOLE_START >> ats_entries;
			ats_entries = min(ats_entries, entries);
			entries -= ats_entries;
		} else {
			ats_entries = entries;
			entries = 0;
		}
326
	} else {
327
		ats_entries = 0;
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
	}

	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);

	r = reservation_object_reserve_shared(bo->tbo.resv);
	if (r)
		return r;

	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
	if (r)
		goto error;

	r = amdgpu_job_alloc_with_ib(adev, 64, &job);
	if (r)
		goto error;

344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359
	if (ats_entries) {
		uint64_t ats_value;

		ats_value = AMDGPU_PTE_DEFAULT_ATC;
		if (level != AMDGPU_VM_PTB)
			ats_value |= AMDGPU_PDE_PTE;

		amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0,
				      ats_entries, 0, ats_value);
		addr += ats_entries * 8;
	}

	if (entries)
		amdgpu_vm_set_pte_pde(adev, &job->ibs[0], addr, 0,
				      entries, 0, 0);

360 361 362
	amdgpu_ring_pad_ib(ring, &job->ibs[0]);

	WARN_ON(job->ibs[0].length_dw > 64);
363 364 365 366 367
	r = amdgpu_sync_resv(adev, &job->sync, bo->tbo.resv,
			     AMDGPU_FENCE_OWNER_UNDEFINED, false);
	if (r)
		goto error_free;

368 369 370 371 372 373 374
	r = amdgpu_job_submit(job, ring, &vm->entity,
			      AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
	if (r)
		goto error_free;

	amdgpu_bo_fence(bo, fence, true);
	dma_fence_put(fence);
375 376 377 378 379

	if (bo->shadow)
		return amdgpu_vm_clear_bo(adev, vm, bo->shadow,
					  level, pte_support_ats);

380 381 382 383 384 385 386 387 388
	return 0;

error_free:
	amdgpu_job_free(job);

error:
	return r;
}

389
/**
390 391 392 393 394 395 396 397 398 399 400 401 402
 * amdgpu_vm_alloc_levels - allocate the PD/PT levels
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 * @saddr: start of the address range
 * @eaddr: end of the address range
 *
 * Make sure the page directories and page tables are allocated
 */
static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev,
				  struct amdgpu_vm *vm,
				  struct amdgpu_vm_pt *parent,
				  uint64_t saddr, uint64_t eaddr,
403
				  unsigned level, bool ats)
404
{
405
	unsigned shift = amdgpu_vm_level_shift(adev, level);
406
	unsigned pt_idx, from, to;
407
	u64 flags;
408
	int r;
409 410 411 412

	if (!parent->entries) {
		unsigned num_entries = amdgpu_vm_num_entries(adev, level);

M
Michal Hocko 已提交
413 414 415
		parent->entries = kvmalloc_array(num_entries,
						   sizeof(struct amdgpu_vm_pt),
						   GFP_KERNEL | __GFP_ZERO);
416 417 418 419 420
		if (!parent->entries)
			return -ENOMEM;
		memset(parent->entries, 0 , sizeof(struct amdgpu_vm_pt));
	}

421 422 423 424 425
	from = saddr >> shift;
	to = eaddr >> shift;
	if (from >= amdgpu_vm_num_entries(adev, level) ||
	    to >= amdgpu_vm_num_entries(adev, level))
		return -EINVAL;
426 427

	++level;
428 429
	saddr = saddr & ((1 << shift) - 1);
	eaddr = eaddr & ((1 << shift) - 1);
430

431
	flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
432 433 434 435 436 437
	if (vm->use_cpu_for_update)
		flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
	else
		flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
				AMDGPU_GEM_CREATE_SHADOW);

438 439
	/* walk over the address space and allocate the page tables */
	for (pt_idx = from; pt_idx <= to; ++pt_idx) {
440
		struct reservation_object *resv = vm->root.base.bo->tbo.resv;
441 442 443
		struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];
		struct amdgpu_bo *pt;

444
		if (!entry->base.bo) {
445 446 447 448 449 450 451 452 453 454
			struct amdgpu_bo_param bp;

			memset(&bp, 0, sizeof(bp));
			bp.size = amdgpu_vm_bo_size(adev, level);
			bp.byte_align = AMDGPU_GPU_PAGE_SIZE;
			bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
			bp.flags = flags;
			bp.type = ttm_bo_type_kernel;
			bp.resv = resv;
			r = amdgpu_bo_create(adev, &bp, &pt);
455 456 457
			if (r)
				return r;

458
			r = amdgpu_vm_clear_bo(adev, vm, pt, level, ats);
459
			if (r) {
460
				amdgpu_bo_unref(&pt->shadow);
461 462 463 464
				amdgpu_bo_unref(&pt);
				return r;
			}

465 466 467
			if (vm->use_cpu_for_update) {
				r = amdgpu_bo_kmap(pt, NULL);
				if (r) {
468
					amdgpu_bo_unref(&pt->shadow);
469 470 471 472 473
					amdgpu_bo_unref(&pt);
					return r;
				}
			}

474 475 476
			/* Keep a reference to the root directory to avoid
			* freeing them up in the wrong order.
			*/
477
			pt->parent = amdgpu_bo_ref(parent->base.bo);
478

479
			amdgpu_vm_bo_base_init(&entry->base, vm, pt);
480
			spin_lock(&vm->status_lock);
481
			list_move(&entry->base.vm_status, &vm->relocated);
482
			spin_unlock(&vm->status_lock);
483 484
		}

485
		if (level < AMDGPU_VM_PTB) {
486 487 488 489
			uint64_t sub_saddr = (pt_idx == from) ? saddr : 0;
			uint64_t sub_eaddr = (pt_idx == to) ? eaddr :
				((1 << shift) - 1);
			r = amdgpu_vm_alloc_levels(adev, vm, entry, sub_saddr,
490
						   sub_eaddr, level, ats);
491 492 493 494 495 496 497 498
			if (r)
				return r;
		}
	}

	return 0;
}

499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
/**
 * amdgpu_vm_alloc_pts - Allocate page tables.
 *
 * @adev: amdgpu_device pointer
 * @vm: VM to allocate page tables for
 * @saddr: Start address which needs to be allocated
 * @size: Size from start address we need.
 *
 * Make sure the page tables are allocated.
 */
int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
			struct amdgpu_vm *vm,
			uint64_t saddr, uint64_t size)
{
	uint64_t eaddr;
514
	bool ats = false;
515 516 517 518 519 520

	/* validate the parameters */
	if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
		return -EINVAL;

	eaddr = saddr + size - 1;
521 522 523

	if (vm->pte_support_ats)
		ats = saddr < AMDGPU_VA_HOLE_START;
524 525 526 527

	saddr /= AMDGPU_GPU_PAGE_SIZE;
	eaddr /= AMDGPU_GPU_PAGE_SIZE;

528 529 530 531 532 533
	if (eaddr >= adev->vm_manager.max_pfn) {
		dev_err(adev->dev, "va above limit (0x%08llX >= 0x%08llX)\n",
			eaddr, adev->vm_manager.max_pfn);
		return -EINVAL;
	}

534
	return amdgpu_vm_alloc_levels(adev, vm, &vm->root, saddr, eaddr,
535
				      adev->vm_manager.root_level, ats);
536 537
}

538 539 540 541 542 543
/**
 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
 *
 * @adev: amdgpu_device pointer
 */
void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
544
{
545
	const struct amdgpu_ip_block *ip_block;
546 547 548
	bool has_compute_vm_bug;
	struct amdgpu_ring *ring;
	int i;
549

550
	has_compute_vm_bug = false;
551

552
	ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
553 554 555 556 557 558 559 560 561
	if (ip_block) {
		/* Compute has a VM bug for GFX version < 7.
		   Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
		if (ip_block->version->major <= 7)
			has_compute_vm_bug = true;
		else if (ip_block->version->major == 8)
			if (adev->gfx.mec_fw_version < 673)
				has_compute_vm_bug = true;
	}
562

563 564 565 566 567
	for (i = 0; i < adev->num_rings; i++) {
		ring = adev->rings[i];
		if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
			/* only compute rings */
			ring->has_compute_vm_bug = has_compute_vm_bug;
568
		else
569
			ring->has_compute_vm_bug = false;
570 571 572
	}
}

573 574
bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
				  struct amdgpu_job *job)
A
Alex Xie 已提交
575
{
576 577
	struct amdgpu_device *adev = ring->adev;
	unsigned vmhub = ring->funcs->vmhub;
578 579
	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
	struct amdgpu_vmid *id;
580
	bool gds_switch_needed;
581
	bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
582

583
	if (job->vmid == 0)
584
		return false;
585
	id = &id_mgr->ids[job->vmid];
586 587 588 589 590 591 592
	gds_switch_needed = ring->funcs->emit_gds_switch && (
		id->gds_base != job->gds_base ||
		id->gds_size != job->gds_size ||
		id->gws_base != job->gws_base ||
		id->gws_size != job->gws_size ||
		id->oa_base != job->oa_base ||
		id->oa_size != job->oa_size);
A
Alex Xie 已提交
593

594
	if (amdgpu_vmid_had_gpu_reset(adev, id))
595
		return true;
A
Alex Xie 已提交
596

597
	return vm_flush_needed || gds_switch_needed;
598 599
}

600 601
static bool amdgpu_vm_is_large_bar(struct amdgpu_device *adev)
{
602
	return (adev->gmc.real_vram_size == adev->gmc.visible_vram_size);
A
Alex Xie 已提交
603 604
}

A
Alex Deucher 已提交
605 606 607 608
/**
 * amdgpu_vm_flush - hardware flush the vm
 *
 * @ring: ring to use for flush
609
 * @vmid: vmid number to use
610
 * @pd_addr: address of the page directory
A
Alex Deucher 已提交
611
 *
612
 * Emit a VM flush when it is necessary.
A
Alex Deucher 已提交
613
 */
M
Monk Liu 已提交
614
int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
A
Alex Deucher 已提交
615
{
616
	struct amdgpu_device *adev = ring->adev;
617
	unsigned vmhub = ring->funcs->vmhub;
618
	struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
619
	struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
620
	bool gds_switch_needed = ring->funcs->emit_gds_switch && (
621 622 623 624 625 626
		id->gds_base != job->gds_base ||
		id->gds_size != job->gds_size ||
		id->gws_base != job->gws_base ||
		id->gws_size != job->gws_size ||
		id->oa_base != job->oa_base ||
		id->oa_size != job->oa_size);
627
	bool vm_flush_needed = job->vm_needs_flush;
628 629 630 631
	bool pasid_mapping_needed = id->pasid != job->pasid ||
		!id->pasid_mapping ||
		!dma_fence_is_signaled(id->pasid_mapping);
	struct dma_fence *fence = NULL;
632
	unsigned patch_offset = 0;
633
	int r;
634

635
	if (amdgpu_vmid_had_gpu_reset(adev, id)) {
636 637
		gds_switch_needed = true;
		vm_flush_needed = true;
638
		pasid_mapping_needed = true;
639
	}
640

641 642 643 644 645
	gds_switch_needed &= !!ring->funcs->emit_gds_switch;
	vm_flush_needed &= !!ring->funcs->emit_vm_flush;
	pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
		ring->funcs->emit_wreg;

M
Monk Liu 已提交
646
	if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
647
		return 0;
648

649 650
	if (ring->funcs->init_cond_exec)
		patch_offset = amdgpu_ring_init_cond_exec(ring);
651

M
Monk Liu 已提交
652 653 654
	if (need_pipe_sync)
		amdgpu_ring_emit_pipeline_sync(ring);

655
	if (vm_flush_needed) {
656
		trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
657
		amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
658 659 660 661
	}

	if (pasid_mapping_needed)
		amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
662

663
	if (vm_flush_needed || pasid_mapping_needed) {
664
		r = amdgpu_fence_emit(ring, &fence, 0);
665 666
		if (r)
			return r;
667
	}
668

669
	if (vm_flush_needed) {
670
		mutex_lock(&id_mgr->lock);
671
		dma_fence_put(id->last_flush);
672 673 674
		id->last_flush = dma_fence_get(fence);
		id->current_gpu_reset_count =
			atomic_read(&adev->gpu_reset_counter);
675
		mutex_unlock(&id_mgr->lock);
676
	}
677

678 679 680 681 682 683 684
	if (pasid_mapping_needed) {
		id->pasid = job->pasid;
		dma_fence_put(id->pasid_mapping);
		id->pasid_mapping = dma_fence_get(fence);
	}
	dma_fence_put(fence);

685
	if (ring->funcs->emit_gds_switch && gds_switch_needed) {
686 687 688 689 690 691
		id->gds_base = job->gds_base;
		id->gds_size = job->gds_size;
		id->gws_base = job->gws_base;
		id->gws_size = job->gws_size;
		id->oa_base = job->oa_base;
		id->oa_size = job->oa_size;
692
		amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
693 694 695 696 697 698 699 700 701 702 703 704
					    job->gds_size, job->gws_base,
					    job->gws_size, job->oa_base,
					    job->oa_size);
	}

	if (ring->funcs->patch_cond_exec)
		amdgpu_ring_patch_cond_exec(ring, patch_offset);

	/* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
	if (ring->funcs->emit_switch_buffer) {
		amdgpu_ring_emit_switch_buffer(ring);
		amdgpu_ring_emit_switch_buffer(ring);
705
	}
706
	return 0;
707 708
}

A
Alex Deucher 已提交
709 710 711 712 713 714
/**
 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
 *
 * @vm: requested vm
 * @bo: requested buffer object
 *
715
 * Find @bo inside the requested vm.
A
Alex Deucher 已提交
716 717 718 719 720 721 722 723 724 725
 * Search inside the @bos vm list for the requested vm
 * Returns the found bo_va or NULL if none is found
 *
 * Object has to be reserved!
 */
struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
				       struct amdgpu_bo *bo)
{
	struct amdgpu_bo_va *bo_va;

726 727
	list_for_each_entry(bo_va, &bo->va, base.bo_list) {
		if (bo_va->base.vm == vm) {
A
Alex Deucher 已提交
728 729 730 731 732 733 734
			return bo_va;
		}
	}
	return NULL;
}

/**
735
 * amdgpu_vm_do_set_ptes - helper to call the right asic function
A
Alex Deucher 已提交
736
 *
737
 * @params: see amdgpu_pte_update_params definition
738
 * @bo: PD/PT to update
A
Alex Deucher 已提交
739 740 741 742 743 744 745 746 747
 * @pe: addr of the page entry
 * @addr: dst addr to write into pe
 * @count: number of page entries to update
 * @incr: increase next addr by incr bytes
 * @flags: hw access flags
 *
 * Traces the parameters and calls the right asic functions
 * to setup the page table using the DMA.
 */
748
static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
749
				  struct amdgpu_bo *bo,
750 751
				  uint64_t pe, uint64_t addr,
				  unsigned count, uint32_t incr,
752
				  uint64_t flags)
A
Alex Deucher 已提交
753
{
754
	pe += amdgpu_bo_gpu_offset(bo);
755
	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
A
Alex Deucher 已提交
756

757
	if (count < 3) {
758 759
		amdgpu_vm_write_pte(params->adev, params->ib, pe,
				    addr | flags, count, incr);
A
Alex Deucher 已提交
760 761

	} else {
762
		amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
A
Alex Deucher 已提交
763 764 765 766
				      count, incr, flags);
	}
}

767 768 769 770
/**
 * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
 *
 * @params: see amdgpu_pte_update_params definition
771
 * @bo: PD/PT to update
772 773 774 775 776 777 778 779 780
 * @pe: addr of the page entry
 * @addr: dst addr to write into pe
 * @count: number of page entries to update
 * @incr: increase next addr by incr bytes
 * @flags: hw access flags
 *
 * Traces the parameters and calls the DMA function to copy the PTEs.
 */
static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
781
				   struct amdgpu_bo *bo,
782 783
				   uint64_t pe, uint64_t addr,
				   unsigned count, uint32_t incr,
784
				   uint64_t flags)
785
{
786
	uint64_t src = (params->src + (addr >> 12) * 8);
787

788
	pe += amdgpu_bo_gpu_offset(bo);
789 790 791
	trace_amdgpu_vm_copy_ptes(pe, src, count);

	amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
792 793
}

A
Alex Deucher 已提交
794
/**
795
 * amdgpu_vm_map_gart - Resolve gart mapping of addr
A
Alex Deucher 已提交
796
 *
797
 * @pages_addr: optional DMA address to use for lookup
A
Alex Deucher 已提交
798 799 800
 * @addr: the unmapped addr
 *
 * Look up the physical address of the page that the pte resolves
801
 * to and return the pointer for the page table entry.
A
Alex Deucher 已提交
802
 */
803
static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
A
Alex Deucher 已提交
804 805 806
{
	uint64_t result;

807 808
	/* page table offset */
	result = pages_addr[addr >> PAGE_SHIFT];
809

810 811
	/* in case cpu page size != gpu page size*/
	result |= addr & (~PAGE_MASK);
A
Alex Deucher 已提交
812

813
	result &= 0xFFFFFFFFFFFFF000ULL;
A
Alex Deucher 已提交
814 815 816 817

	return result;
}

818 819 820 821
/**
 * amdgpu_vm_cpu_set_ptes - helper to update page tables via CPU
 *
 * @params: see amdgpu_pte_update_params definition
822
 * @bo: PD/PT to update
823 824 825 826 827 828 829 830 831
 * @pe: kmap addr of the page entry
 * @addr: dst addr to write into pe
 * @count: number of page entries to update
 * @incr: increase next addr by incr bytes
 * @flags: hw access flags
 *
 * Write count number of PT/PD entries directly.
 */
static void amdgpu_vm_cpu_set_ptes(struct amdgpu_pte_update_params *params,
832
				   struct amdgpu_bo *bo,
833 834 835 836 837
				   uint64_t pe, uint64_t addr,
				   unsigned count, uint32_t incr,
				   uint64_t flags)
{
	unsigned int i;
838
	uint64_t value;
839

840 841
	pe += (unsigned long)amdgpu_bo_kptr(bo);

842 843
	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);

844
	for (i = 0; i < count; i++) {
845 846 847
		value = params->pages_addr ?
			amdgpu_vm_map_gart(params->pages_addr, addr) :
			addr;
848 849
		amdgpu_gmc_set_pte_pde(params->adev, (void *)(uintptr_t)pe,
				       i, value, flags);
850 851 852 853
		addr += incr;
	}
}

854 855
static int amdgpu_vm_wait_pd(struct amdgpu_device *adev, struct amdgpu_vm *vm,
			     void *owner)
856 857 858 859 860
{
	struct amdgpu_sync sync;
	int r;

	amdgpu_sync_create(&sync);
861
	amdgpu_sync_resv(adev, &sync, vm->root.base.bo->tbo.resv, owner, false);
862 863 864 865 866 867
	r = amdgpu_sync_wait(&sync, true);
	amdgpu_sync_free(&sync);

	return r;
}

868
/*
869
 * amdgpu_vm_update_pde - update a single level in the hierarchy
870
 *
871
 * @param: parameters for the update
872
 * @vm: requested vm
873
 * @parent: parent directory
874
 * @entry: entry to update
875
 *
876
 * Makes sure the requested entry in parent is up to date.
877
 */
878 879 880 881
static void amdgpu_vm_update_pde(struct amdgpu_pte_update_params *params,
				 struct amdgpu_vm *vm,
				 struct amdgpu_vm_pt *parent,
				 struct amdgpu_vm_pt *entry)
A
Alex Deucher 已提交
882
{
883
	struct amdgpu_bo *bo = parent->base.bo, *pbo;
884 885
	uint64_t pde, pt, flags;
	unsigned level;
C
Chunming Zhou 已提交
886

887 888 889
	/* Don't update huge pages here */
	if (entry->huge)
		return;
A
Alex Deucher 已提交
890

891
	for (level = 0, pbo = bo->parent; pbo; ++level)
892 893
		pbo = pbo->parent;

894
	level += params->adev->vm_manager.root_level;
895
	pt = amdgpu_bo_gpu_offset(entry->base.bo);
896
	flags = AMDGPU_PTE_VALID;
897
	amdgpu_gmc_get_vm_pde(params->adev, level, &pt, &flags);
898 899 900 901
	pde = (entry - parent->entries) * 8;
	if (bo->shadow)
		params->func(params, bo->shadow, pde, pt, 1, 0, flags);
	params->func(params, bo, pde, pt, 1, 0, flags);
A
Alex Deucher 已提交
902 903
}

904 905 906 907 908 909 910
/*
 * amdgpu_vm_invalidate_level - mark all PD levels as invalid
 *
 * @parent: parent PD
 *
 * Mark all PD level as invalid after an error.
 */
911 912 913 914
static void amdgpu_vm_invalidate_level(struct amdgpu_device *adev,
				       struct amdgpu_vm *vm,
				       struct amdgpu_vm_pt *parent,
				       unsigned level)
915
{
916
	unsigned pt_idx, num_entries;
917 918 919 920 921

	/*
	 * Recurse into the subdirectories. This recursion is harmless because
	 * we only have a maximum of 5 layers.
	 */
922 923
	num_entries = amdgpu_vm_num_entries(adev, level);
	for (pt_idx = 0; pt_idx < num_entries; ++pt_idx) {
924 925
		struct amdgpu_vm_pt *entry = &parent->entries[pt_idx];

926
		if (!entry->base.bo)
927 928
			continue;

929
		spin_lock(&vm->status_lock);
930 931
		if (list_empty(&entry->base.vm_status))
			list_add(&entry->base.vm_status, &vm->relocated);
932
		spin_unlock(&vm->status_lock);
933
		amdgpu_vm_invalidate_level(adev, vm, entry, level + 1);
934 935 936
	}
}

937 938 939 940 941 942 943 944 945 946 947 948
/*
 * amdgpu_vm_update_directories - make sure that all directories are valid
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
 * Makes sure all directories are up to date.
 * Returns 0 for success, error for failure.
 */
int amdgpu_vm_update_directories(struct amdgpu_device *adev,
				 struct amdgpu_vm *vm)
{
949 950 951
	struct amdgpu_pte_update_params params;
	struct amdgpu_job *job;
	unsigned ndw = 0;
952
	int r = 0;
953

954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
	if (list_empty(&vm->relocated))
		return 0;

restart:
	memset(&params, 0, sizeof(params));
	params.adev = adev;

	if (vm->use_cpu_for_update) {
		r = amdgpu_vm_wait_pd(adev, vm, AMDGPU_FENCE_OWNER_VM);
		if (unlikely(r))
			return r;

		params.func = amdgpu_vm_cpu_set_ptes;
	} else {
		ndw = 512 * 8;
		r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
		if (r)
			return r;

		params.ib = &job->ibs[0];
		params.func = amdgpu_vm_do_set_ptes;
	}

977 978
	spin_lock(&vm->status_lock);
	while (!list_empty(&vm->relocated)) {
979 980
		struct amdgpu_vm_bo_base *bo_base, *parent;
		struct amdgpu_vm_pt *pt, *entry;
981 982 983 984 985
		struct amdgpu_bo *bo;

		bo_base = list_first_entry(&vm->relocated,
					   struct amdgpu_vm_bo_base,
					   vm_status);
986
		list_del_init(&bo_base->vm_status);
987 988 989
		spin_unlock(&vm->status_lock);

		bo = bo_base->bo->parent;
990
		if (!bo) {
991
			spin_lock(&vm->status_lock);
992
			continue;
993
		}
994 995 996 997 998 999 1000 1001 1002 1003 1004 1005

		parent = list_first_entry(&bo->va, struct amdgpu_vm_bo_base,
					  bo_list);
		pt = container_of(parent, struct amdgpu_vm_pt, base);
		entry = container_of(bo_base, struct amdgpu_vm_pt, base);

		amdgpu_vm_update_pde(&params, vm, pt, entry);

		spin_lock(&vm->status_lock);
		if (!vm->use_cpu_for_update &&
		    (ndw - params.ib->length_dw) < 32)
			break;
1006 1007
	}
	spin_unlock(&vm->status_lock);
1008

1009 1010 1011
	if (vm->use_cpu_for_update) {
		/* Flush HDP */
		mb();
1012
		amdgpu_asic_flush_hdp(adev, NULL);
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
	} else if (params.ib->length_dw == 0) {
		amdgpu_job_free(job);
	} else {
		struct amdgpu_bo *root = vm->root.base.bo;
		struct amdgpu_ring *ring;
		struct dma_fence *fence;

		ring = container_of(vm->entity.sched, struct amdgpu_ring,
				    sched);

		amdgpu_ring_pad_ib(ring, params.ib);
		amdgpu_sync_resv(adev, &job->sync, root->tbo.resv,
				 AMDGPU_FENCE_OWNER_VM, false);
		WARN_ON(params.ib->length_dw > ndw);
		r = amdgpu_job_submit(job, ring, &vm->entity,
				      AMDGPU_FENCE_OWNER_VM, &fence);
		if (r)
			goto error;

		amdgpu_bo_fence(root, fence, true);
		dma_fence_put(vm->last_update);
		vm->last_update = fence;
1035 1036
	}

1037 1038 1039 1040 1041 1042
	if (!list_empty(&vm->relocated))
		goto restart;

	return 0;

error:
1043 1044
	amdgpu_vm_invalidate_level(adev, vm, &vm->root,
				   adev->vm_manager.root_level);
1045
	amdgpu_job_free(job);
1046
	return r;
1047 1048
}

1049
/**
1050
 * amdgpu_vm_find_entry - find the entry for an address
1051 1052 1053
 *
 * @p: see amdgpu_pte_update_params definition
 * @addr: virtual address in question
1054 1055
 * @entry: resulting entry or NULL
 * @parent: parent entry
1056
 *
1057
 * Find the vm_pt entry and it's parent for the given address.
1058
 */
1059 1060 1061
void amdgpu_vm_get_entry(struct amdgpu_pte_update_params *p, uint64_t addr,
			 struct amdgpu_vm_pt **entry,
			 struct amdgpu_vm_pt **parent)
1062
{
1063
	unsigned level = p->adev->vm_manager.root_level;
1064

1065 1066 1067
	*parent = NULL;
	*entry = &p->vm->root;
	while ((*entry)->entries) {
1068
		unsigned shift = amdgpu_vm_level_shift(p->adev, level++);
1069

1070
		*parent = *entry;
1071 1072
		*entry = &(*entry)->entries[addr >> shift];
		addr &= (1ULL << shift) - 1;
1073 1074
	}

1075
	if (level != AMDGPU_VM_PTB)
1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
		*entry = NULL;
}

/**
 * amdgpu_vm_handle_huge_pages - handle updating the PD with huge pages
 *
 * @p: see amdgpu_pte_update_params definition
 * @entry: vm_pt entry to check
 * @parent: parent entry
 * @nptes: number of PTEs updated with this operation
 * @dst: destination address where the PTEs should point to
 * @flags: access flags fro the PTEs
 *
 * Check if we can update the PD with a huge page.
 */
1091 1092 1093 1094 1095
static void amdgpu_vm_handle_huge_pages(struct amdgpu_pte_update_params *p,
					struct amdgpu_vm_pt *entry,
					struct amdgpu_vm_pt *parent,
					unsigned nptes, uint64_t dst,
					uint64_t flags)
1096
{
1097
	uint64_t pde;
1098 1099

	/* In the case of a mixed PT the PDE must point to it*/
1100 1101
	if (p->adev->asic_type >= CHIP_VEGA10 && !p->src &&
	    nptes == AMDGPU_VM_PTE_COUNT(p->adev)) {
1102
		/* Set the huge page flag to stop scanning at this PDE */
1103 1104 1105
		flags |= AMDGPU_PDE_PTE;
	}

1106 1107 1108 1109 1110 1111 1112 1113
	if (!(flags & AMDGPU_PDE_PTE)) {
		if (entry->huge) {
			/* Add the entry to the relocated list to update it. */
			entry->huge = false;
			spin_lock(&p->vm->status_lock);
			list_move(&entry->base.vm_status, &p->vm->relocated);
			spin_unlock(&p->vm->status_lock);
		}
1114
		return;
1115
	}
1116

1117
	entry->huge = true;
1118
	amdgpu_gmc_get_vm_pde(p->adev, AMDGPU_VM_PDB0, &dst, &flags);
1119

1120 1121 1122 1123
	pde = (entry - parent->entries) * 8;
	if (parent->base.bo->shadow)
		p->func(p, parent->base.bo->shadow, pde, dst, 1, 0, flags);
	p->func(p, parent->base.bo, pde, dst, 1, 0, flags);
1124 1125
}

A
Alex Deucher 已提交
1126 1127 1128
/**
 * amdgpu_vm_update_ptes - make sure that page tables are valid
 *
1129
 * @params: see amdgpu_pte_update_params definition
A
Alex Deucher 已提交
1130 1131 1132
 * @vm: requested vm
 * @start: start of GPU address range
 * @end: end of GPU address range
1133
 * @dst: destination address to map to, the next dst inside the function
A
Alex Deucher 已提交
1134 1135
 * @flags: mapping flags
 *
1136
 * Update the page tables in the range @start - @end.
1137
 * Returns 0 for success, -EINVAL for failure.
A
Alex Deucher 已提交
1138
 */
1139
static int amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
1140
				  uint64_t start, uint64_t end,
1141
				  uint64_t dst, uint64_t flags)
A
Alex Deucher 已提交
1142
{
1143 1144
	struct amdgpu_device *adev = params->adev;
	const uint64_t mask = AMDGPU_VM_PTE_COUNT(adev) - 1;
1145

1146
	uint64_t addr, pe_start;
1147
	struct amdgpu_bo *pt;
1148
	unsigned nptes;
A
Alex Deucher 已提交
1149 1150

	/* walk over the address space and update the page tables */
1151 1152 1153 1154 1155 1156 1157
	for (addr = start; addr < end; addr += nptes,
	     dst += nptes * AMDGPU_GPU_PAGE_SIZE) {
		struct amdgpu_vm_pt *entry, *parent;

		amdgpu_vm_get_entry(params, addr, &entry, &parent);
		if (!entry)
			return -ENOENT;
1158

A
Alex Deucher 已提交
1159 1160 1161
		if ((addr & ~mask) == (end & ~mask))
			nptes = end - addr;
		else
1162
			nptes = AMDGPU_VM_PTE_COUNT(adev) - (addr & mask);
A
Alex Deucher 已提交
1163

1164 1165
		amdgpu_vm_handle_huge_pages(params, entry, parent,
					    nptes, dst, flags);
1166
		/* We don't need to update PTEs for huge pages */
1167
		if (entry->huge)
1168 1169
			continue;

1170
		pt = entry->base.bo;
1171 1172 1173 1174 1175
		pe_start = (addr & mask) * 8;
		if (pt->shadow)
			params->func(params, pt->shadow, pe_start, dst, nptes,
				     AMDGPU_GPU_PAGE_SIZE, flags);
		params->func(params, pt, pe_start, dst, nptes,
1176
			     AMDGPU_GPU_PAGE_SIZE, flags);
A
Alex Deucher 已提交
1177 1178
	}

1179
	return 0;
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
}

/*
 * amdgpu_vm_frag_ptes - add fragment information to PTEs
 *
 * @params: see amdgpu_pte_update_params definition
 * @vm: requested vm
 * @start: first PTE to handle
 * @end: last PTE to handle
 * @dst: addr those PTEs should point to
 * @flags: hw mapping flags
1191
 * Returns 0 for success, -EINVAL for failure.
1192
 */
1193
static int amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
1194
				uint64_t start, uint64_t end,
1195
				uint64_t dst, uint64_t flags)
1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214
{
	/**
	 * The MC L1 TLB supports variable sized pages, based on a fragment
	 * field in the PTE. When this field is set to a non-zero value, page
	 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
	 * flags are considered valid for all PTEs within the fragment range
	 * and corresponding mappings are assumed to be physically contiguous.
	 *
	 * The L1 TLB can store a single PTE for the whole fragment,
	 * significantly increasing the space available for translation
	 * caching. This leads to large improvements in throughput when the
	 * TLB is under pressure.
	 *
	 * The L2 TLB distributes small and large fragments into two
	 * asymmetric partitions. The large fragment cache is significantly
	 * larger. Thus, we try to use large fragments wherever possible.
	 * Userspace can support this by aligning virtual base address and
	 * allocation size to the fragment size.
	 */
1215 1216
	unsigned max_frag = params->adev->vm_manager.fragment_size;
	int r;
1217 1218

	/* system pages are non continuously */
1219
	if (params->src || !(flags & AMDGPU_PTE_VALID))
1220
		return amdgpu_vm_update_ptes(params, start, end, dst, flags);
1221

1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
	while (start != end) {
		uint64_t frag_flags, frag_end;
		unsigned frag;

		/* This intentionally wraps around if no bit is set */
		frag = min((unsigned)ffs(start) - 1,
			   (unsigned)fls64(end - start) - 1);
		if (frag >= max_frag) {
			frag_flags = AMDGPU_PTE_FRAG(max_frag);
			frag_end = end & ~((1ULL << max_frag) - 1);
		} else {
			frag_flags = AMDGPU_PTE_FRAG(frag);
			frag_end = start + (1 << frag);
		}

		r = amdgpu_vm_update_ptes(params, start, frag_end, dst,
					  flags | frag_flags);
1239 1240
		if (r)
			return r;
1241

1242 1243
		dst += (frag_end - start) * AMDGPU_GPU_PAGE_SIZE;
		start = frag_end;
1244
	}
1245 1246

	return 0;
A
Alex Deucher 已提交
1247 1248 1249 1250 1251 1252
}

/**
 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
 *
 * @adev: amdgpu_device pointer
1253
 * @exclusive: fence we need to sync to
1254
 * @pages_addr: DMA addresses to use for mapping
A
Alex Deucher 已提交
1255
 * @vm: requested vm
1256 1257 1258
 * @start: start of mapped range
 * @last: last mapped entry
 * @flags: flags for the entries
A
Alex Deucher 已提交
1259 1260 1261
 * @addr: addr to set the area to
 * @fence: optional resulting fence
 *
1262
 * Fill in the page table entries between @start and @last.
A
Alex Deucher 已提交
1263 1264 1265
 * Returns 0 for success, -EINVAL for failure.
 */
static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1266
				       struct dma_fence *exclusive,
1267
				       dma_addr_t *pages_addr,
A
Alex Deucher 已提交
1268
				       struct amdgpu_vm *vm,
1269
				       uint64_t start, uint64_t last,
1270
				       uint64_t flags, uint64_t addr,
1271
				       struct dma_fence **fence)
A
Alex Deucher 已提交
1272
{
1273
	struct amdgpu_ring *ring;
1274
	void *owner = AMDGPU_FENCE_OWNER_VM;
A
Alex Deucher 已提交
1275
	unsigned nptes, ncmds, ndw;
1276
	struct amdgpu_job *job;
1277
	struct amdgpu_pte_update_params params;
1278
	struct dma_fence *f = NULL;
A
Alex Deucher 已提交
1279 1280
	int r;

1281 1282
	memset(&params, 0, sizeof(params));
	params.adev = adev;
1283
	params.vm = vm;
1284

1285 1286 1287 1288
	/* sync to everything on unmapping */
	if (!(flags & AMDGPU_PTE_VALID))
		owner = AMDGPU_FENCE_OWNER_UNDEFINED;

1289 1290 1291 1292 1293 1294 1295 1296
	if (vm->use_cpu_for_update) {
		/* params.src is used as flag to indicate system Memory */
		if (pages_addr)
			params.src = ~0;

		/* Wait for PT BOs to be free. PTs share the same resv. object
		 * as the root PD BO
		 */
1297
		r = amdgpu_vm_wait_pd(adev, vm, owner);
1298 1299 1300 1301 1302 1303 1304 1305 1306
		if (unlikely(r))
			return r;

		params.func = amdgpu_vm_cpu_set_ptes;
		params.pages_addr = pages_addr;
		return amdgpu_vm_frag_ptes(&params, start, last + 1,
					   addr, flags);
	}

1307
	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
1308

1309
	nptes = last - start + 1;
A
Alex Deucher 已提交
1310 1311

	/*
1312
	 * reserve space for two commands every (1 << BLOCK_SIZE)
A
Alex Deucher 已提交
1313
	 *  entries or 2k dwords (whatever is smaller)
1314 1315
         *
         * The second command is for the shadow pagetables.
A
Alex Deucher 已提交
1316
	 */
1317 1318 1319 1320
	if (vm->root.base.bo->shadow)
		ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1) * 2;
	else
		ncmds = ((nptes >> min(adev->vm_manager.block_size, 11u)) + 1);
A
Alex Deucher 已提交
1321 1322 1323 1324

	/* padding, etc. */
	ndw = 64;

1325
	if (pages_addr) {
1326
		/* copy commands needed */
1327
		ndw += ncmds * adev->vm_manager.vm_pte_funcs->copy_pte_num_dw;
A
Alex Deucher 已提交
1328

1329
		/* and also PTEs */
A
Alex Deucher 已提交
1330 1331
		ndw += nptes * 2;

1332 1333
		params.func = amdgpu_vm_do_copy_ptes;

A
Alex Deucher 已提交
1334 1335
	} else {
		/* set page commands needed */
1336
		ndw += ncmds * 10;
A
Alex Deucher 已提交
1337

1338
		/* extra commands for begin/end fragments */
1339
		ndw += 2 * 10 * adev->vm_manager.fragment_size;
1340 1341

		params.func = amdgpu_vm_do_set_ptes;
A
Alex Deucher 已提交
1342 1343
	}

1344 1345
	r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
	if (r)
A
Alex Deucher 已提交
1346
		return r;
1347

1348
	params.ib = &job->ibs[0];
C
Chunming Zhou 已提交
1349

1350
	if (pages_addr) {
1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
		uint64_t *pte;
		unsigned i;

		/* Put the PTEs at the end of the IB. */
		i = ndw - nptes * 2;
		pte= (uint64_t *)&(job->ibs->ptr[i]);
		params.src = job->ibs->gpu_addr + i * 4;

		for (i = 0; i < nptes; ++i) {
			pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
						    AMDGPU_GPU_PAGE_SIZE);
			pte[i] |= flags;
		}
1364
		addr = 0;
1365 1366
	}

1367
	r = amdgpu_sync_fence(adev, &job->sync, exclusive, false);
1368 1369 1370
	if (r)
		goto error_free;

1371
	r = amdgpu_sync_resv(adev, &job->sync, vm->root.base.bo->tbo.resv,
1372
			     owner, false);
1373 1374
	if (r)
		goto error_free;
A
Alex Deucher 已提交
1375

1376
	r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);
1377 1378 1379
	if (r)
		goto error_free;

1380 1381 1382
	r = amdgpu_vm_frag_ptes(&params, start, last + 1, addr, flags);
	if (r)
		goto error_free;
A
Alex Deucher 已提交
1383

1384 1385
	amdgpu_ring_pad_ib(ring, params.ib);
	WARN_ON(params.ib->length_dw > ndw);
1386 1387
	r = amdgpu_job_submit(job, ring, &vm->entity,
			      AMDGPU_FENCE_OWNER_VM, &f);
1388 1389
	if (r)
		goto error_free;
A
Alex Deucher 已提交
1390

1391
	amdgpu_bo_fence(vm->root.base.bo, f, true);
1392 1393
	dma_fence_put(*fence);
	*fence = f;
A
Alex Deucher 已提交
1394
	return 0;
C
Chunming Zhou 已提交
1395 1396

error_free:
1397
	amdgpu_job_free(job);
1398
	return r;
A
Alex Deucher 已提交
1399 1400
}

1401 1402 1403 1404
/**
 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
 *
 * @adev: amdgpu_device pointer
1405
 * @exclusive: fence we need to sync to
1406
 * @pages_addr: DMA addresses to use for mapping
1407 1408
 * @vm: requested vm
 * @mapping: mapped range and flags to use for the update
1409
 * @flags: HW flags for the mapping
1410
 * @nodes: array of drm_mm_nodes with the MC addresses
1411 1412 1413 1414 1415 1416 1417
 * @fence: optional resulting fence
 *
 * Split the mapping into smaller chunks so that each update fits
 * into a SDMA IB.
 * Returns 0 for success, -EINVAL for failure.
 */
static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1418
				      struct dma_fence *exclusive,
1419
				      dma_addr_t *pages_addr,
1420 1421
				      struct amdgpu_vm *vm,
				      struct amdgpu_bo_va_mapping *mapping,
1422
				      uint64_t flags,
1423
				      struct drm_mm_node *nodes,
1424
				      struct dma_fence **fence)
1425
{
1426
	unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size;
1427
	uint64_t pfn, start = mapping->start;
1428 1429 1430 1431 1432 1433 1434 1435 1436 1437
	int r;

	/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
	 * but in case of something, we filter the flags in first place
	 */
	if (!(mapping->flags & AMDGPU_PTE_READABLE))
		flags &= ~AMDGPU_PTE_READABLE;
	if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
		flags &= ~AMDGPU_PTE_WRITEABLE;

1438 1439 1440
	flags &= ~AMDGPU_PTE_EXECUTABLE;
	flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;

1441 1442 1443
	flags &= ~AMDGPU_PTE_MTYPE_MASK;
	flags |= (mapping->flags & AMDGPU_PTE_MTYPE_MASK);

1444 1445 1446 1447 1448 1449
	if ((mapping->flags & AMDGPU_PTE_PRT) &&
	    (adev->asic_type >= CHIP_VEGA10)) {
		flags |= AMDGPU_PTE_PRT;
		flags &= ~AMDGPU_PTE_VALID;
	}

1450 1451
	trace_amdgpu_vm_bo_update(mapping);

1452 1453 1454 1455 1456 1457
	pfn = mapping->offset >> PAGE_SHIFT;
	if (nodes) {
		while (pfn >= nodes->size) {
			pfn -= nodes->size;
			++nodes;
		}
1458
	}
1459

1460
	do {
1461
		dma_addr_t *dma_addr = NULL;
1462 1463
		uint64_t max_entries;
		uint64_t addr, last;
1464

1465 1466 1467 1468 1469 1470 1471 1472
		if (nodes) {
			addr = nodes->start << PAGE_SHIFT;
			max_entries = (nodes->size - pfn) *
				(PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
		} else {
			addr = 0;
			max_entries = S64_MAX;
		}
1473

1474
		if (pages_addr) {
1475 1476
			uint64_t count;

1477
			max_entries = min(max_entries, 16ull * 1024ull);
1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
			for (count = 1; count < max_entries; ++count) {
				uint64_t idx = pfn + count;

				if (pages_addr[idx] !=
				    (pages_addr[idx - 1] + PAGE_SIZE))
					break;
			}

			if (count < min_linear_pages) {
				addr = pfn << PAGE_SHIFT;
				dma_addr = pages_addr;
			} else {
				addr = pages_addr[pfn];
				max_entries = count;
			}

1494 1495
		} else if (flags & AMDGPU_PTE_VALID) {
			addr += adev->vm_manager.vram_base_offset;
1496
			addr += pfn << PAGE_SHIFT;
1497 1498
		}

1499
		last = min((uint64_t)mapping->last, start + max_entries - 1);
1500
		r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,
1501 1502 1503 1504 1505
						start, last, flags, addr,
						fence);
		if (r)
			return r;

1506 1507 1508 1509 1510
		pfn += last - start + 1;
		if (nodes && nodes->size == pfn) {
			pfn = 0;
			++nodes;
		}
1511
		start = last + 1;
1512

1513
	} while (unlikely(start != mapping->last + 1));
1514 1515 1516 1517

	return 0;
}

A
Alex Deucher 已提交
1518 1519 1520 1521 1522
/**
 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
 *
 * @adev: amdgpu_device pointer
 * @bo_va: requested BO and VM object
1523
 * @clear: if true clear the entries
A
Alex Deucher 已提交
1524 1525 1526 1527 1528 1529
 *
 * Fill in the page table entries for @bo_va.
 * Returns 0 for success, -EINVAL for failure.
 */
int amdgpu_vm_bo_update(struct amdgpu_device *adev,
			struct amdgpu_bo_va *bo_va,
1530
			bool clear)
A
Alex Deucher 已提交
1531
{
1532 1533
	struct amdgpu_bo *bo = bo_va->base.bo;
	struct amdgpu_vm *vm = bo_va->base.vm;
A
Alex Deucher 已提交
1534
	struct amdgpu_bo_va_mapping *mapping;
1535
	dma_addr_t *pages_addr = NULL;
1536
	struct ttm_mem_reg *mem;
1537
	struct drm_mm_node *nodes;
1538
	struct dma_fence *exclusive, **last_update;
1539
	uint64_t flags;
A
Alex Deucher 已提交
1540 1541
	int r;

1542
	if (clear || !bo_va->base.bo) {
1543
		mem = NULL;
1544
		nodes = NULL;
1545 1546
		exclusive = NULL;
	} else {
1547 1548
		struct ttm_dma_tt *ttm;

1549
		mem = &bo_va->base.bo->tbo.mem;
1550 1551
		nodes = mem->mm_node;
		if (mem->mem_type == TTM_PL_TT) {
1552 1553
			ttm = container_of(bo_va->base.bo->tbo.ttm,
					   struct ttm_dma_tt, ttm);
1554
			pages_addr = ttm->dma_address;
1555
		}
1556
		exclusive = reservation_object_get_excl(bo->tbo.resv);
A
Alex Deucher 已提交
1557 1558
	}

1559
	if (bo)
1560
		flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1561
	else
1562
		flags = 0x0;
A
Alex Deucher 已提交
1563

1564 1565 1566 1567 1568
	if (clear || (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv))
		last_update = &vm->last_update;
	else
		last_update = &bo_va->last_pt_update;

1569 1570
	if (!clear && bo_va->base.moved) {
		bo_va->base.moved = false;
1571
		list_splice_init(&bo_va->valids, &bo_va->invalids);
1572

1573 1574
	} else if (bo_va->cleared != clear) {
		list_splice_init(&bo_va->valids, &bo_va->invalids);
1575
	}
1576 1577

	list_for_each_entry(mapping, &bo_va->invalids, list) {
1578
		r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
1579
					       mapping, flags, nodes,
1580
					       last_update);
A
Alex Deucher 已提交
1581 1582 1583 1584
		if (r)
			return r;
	}

1585 1586 1587
	if (vm->use_cpu_for_update) {
		/* Flush HDP */
		mb();
1588
		amdgpu_asic_flush_hdp(adev, NULL);
1589 1590
	}

A
Alex Deucher 已提交
1591
	spin_lock(&vm->status_lock);
1592
	list_del_init(&bo_va->base.vm_status);
1593

1594 1595 1596 1597 1598
	/* If the BO is not in its preferred location add it back to
	 * the evicted list so that it gets validated again on the
	 * next command submission.
	 */
	if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
1599 1600
	    !(bo->preferred_domains &
	    amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type)))
1601
		list_add_tail(&bo_va->base.vm_status, &vm->evicted);
A
Alex Deucher 已提交
1602 1603
	spin_unlock(&vm->status_lock);

1604 1605 1606 1607 1608 1609
	list_splice_init(&bo_va->invalids, &bo_va->valids);
	bo_va->cleared = clear;

	if (trace_amdgpu_vm_bo_mapping_enabled()) {
		list_for_each_entry(mapping, &bo_va->valids, list)
			trace_amdgpu_vm_bo_mapping(mapping);
1610 1611
	}

A
Alex Deucher 已提交
1612 1613 1614
	return 0;
}

1615 1616 1617 1618 1619 1620 1621 1622 1623
/**
 * amdgpu_vm_update_prt_state - update the global PRT state
 */
static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
{
	unsigned long flags;
	bool enable;

	spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1624
	enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1625
	adev->gmc.gmc_funcs->set_prt(adev, enable);
1626 1627 1628
	spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
}

1629
/**
1630
 * amdgpu_vm_prt_get - add a PRT user
1631 1632 1633
 */
static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
{
1634
	if (!adev->gmc.gmc_funcs->set_prt)
1635 1636
		return;

1637 1638 1639 1640
	if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
		amdgpu_vm_update_prt_state(adev);
}

1641 1642 1643 1644 1645
/**
 * amdgpu_vm_prt_put - drop a PRT user
 */
static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
{
1646
	if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1647 1648 1649
		amdgpu_vm_update_prt_state(adev);
}

1650
/**
1651
 * amdgpu_vm_prt_cb - callback for updating the PRT status
1652 1653 1654 1655 1656
 */
static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
{
	struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);

1657
	amdgpu_vm_prt_put(cb->adev);
1658 1659 1660
	kfree(cb);
}

1661 1662 1663 1664 1665 1666
/**
 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
 */
static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
				 struct dma_fence *fence)
{
1667
	struct amdgpu_prt_cb *cb;
1668

1669
	if (!adev->gmc.gmc_funcs->set_prt)
1670 1671 1672
		return;

	cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1673 1674 1675 1676 1677
	if (!cb) {
		/* Last resort when we are OOM */
		if (fence)
			dma_fence_wait(fence, false);

1678
		amdgpu_vm_prt_put(adev);
1679 1680 1681 1682 1683 1684 1685 1686
	} else {
		cb->adev = adev;
		if (!fence || dma_fence_add_callback(fence, &cb->cb,
						     amdgpu_vm_prt_cb))
			amdgpu_vm_prt_cb(fence, &cb->cb);
	}
}

1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701
/**
 * amdgpu_vm_free_mapping - free a mapping
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 * @mapping: mapping to be freed
 * @fence: fence of the unmap operation
 *
 * Free a mapping and make sure we decrease the PRT usage count if applicable.
 */
static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
				   struct amdgpu_vm *vm,
				   struct amdgpu_bo_va_mapping *mapping,
				   struct dma_fence *fence)
{
1702 1703 1704 1705
	if (mapping->flags & AMDGPU_PTE_PRT)
		amdgpu_vm_add_prt_cb(adev, fence);
	kfree(mapping);
}
1706

1707 1708 1709 1710 1711 1712 1713 1714 1715 1716
/**
 * amdgpu_vm_prt_fini - finish all prt mappings
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
 * Register a cleanup callback to disable PRT support after VM dies.
 */
static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
{
1717
	struct reservation_object *resv = vm->root.base.bo->tbo.resv;
1718 1719 1720
	struct dma_fence *excl, **shared;
	unsigned i, shared_count;
	int r;
1721

1722 1723 1724 1725 1726 1727 1728 1729 1730
	r = reservation_object_get_fences_rcu(resv, &excl,
					      &shared_count, &shared);
	if (r) {
		/* Not enough memory to grab the fence list, as last resort
		 * block for all the fences to complete.
		 */
		reservation_object_wait_timeout_rcu(resv, true, false,
						    MAX_SCHEDULE_TIMEOUT);
		return;
1731
	}
1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742

	/* Add a callback for each fence in the reservation object */
	amdgpu_vm_prt_get(adev);
	amdgpu_vm_add_prt_cb(adev, excl);

	for (i = 0; i < shared_count; ++i) {
		amdgpu_vm_prt_get(adev);
		amdgpu_vm_add_prt_cb(adev, shared[i]);
	}

	kfree(shared);
1743 1744
}

A
Alex Deucher 已提交
1745 1746 1747 1748 1749
/**
 * amdgpu_vm_clear_freed - clear freed BOs in the PT
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
1750 1751
 * @fence: optional resulting fence (unchanged if no work needed to be done
 * or if an error occurred)
A
Alex Deucher 已提交
1752 1753 1754 1755 1756 1757 1758
 *
 * Make sure all freed BOs are cleared in the PT.
 * Returns 0 for success.
 *
 * PTs have to be reserved and mutex must be locked!
 */
int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1759 1760
			  struct amdgpu_vm *vm,
			  struct dma_fence **fence)
A
Alex Deucher 已提交
1761 1762
{
	struct amdgpu_bo_va_mapping *mapping;
1763
	uint64_t init_pte_value = 0;
1764
	struct dma_fence *f = NULL;
A
Alex Deucher 已提交
1765 1766 1767 1768 1769 1770
	int r;

	while (!list_empty(&vm->freed)) {
		mapping = list_first_entry(&vm->freed,
			struct amdgpu_bo_va_mapping, list);
		list_del(&mapping->list);
1771

1772
		if (vm->pte_support_ats && mapping->start < AMDGPU_VA_HOLE_START)
1773
			init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
Y
Yong Zhao 已提交
1774

1775
		r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
1776
						mapping->start, mapping->last,
Y
Yong Zhao 已提交
1777
						init_pte_value, 0, &f);
1778
		amdgpu_vm_free_mapping(adev, vm, mapping, f);
1779
		if (r) {
1780
			dma_fence_put(f);
A
Alex Deucher 已提交
1781
			return r;
1782
		}
1783
	}
A
Alex Deucher 已提交
1784

1785 1786 1787 1788 1789
	if (fence && f) {
		dma_fence_put(*fence);
		*fence = f;
	} else {
		dma_fence_put(f);
A
Alex Deucher 已提交
1790
	}
1791

A
Alex Deucher 已提交
1792 1793 1794 1795 1796
	return 0;

}

/**
1797
 * amdgpu_vm_handle_moved - handle moved BOs in the PT
A
Alex Deucher 已提交
1798 1799 1800
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
1801
 * @sync: sync object to add fences to
A
Alex Deucher 已提交
1802
 *
1803
 * Make sure all BOs which are moved are updated in the PTs.
A
Alex Deucher 已提交
1804 1805
 * Returns 0 for success.
 *
1806
 * PTs have to be reserved!
A
Alex Deucher 已提交
1807
 */
1808
int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1809
			   struct amdgpu_vm *vm)
A
Alex Deucher 已提交
1810
{
1811
	bool clear;
1812
	int r = 0;
A
Alex Deucher 已提交
1813 1814

	spin_lock(&vm->status_lock);
1815
	while (!list_empty(&vm->moved)) {
1816
		struct amdgpu_bo_va *bo_va;
1817
		struct reservation_object *resv;
1818

1819
		bo_va = list_first_entry(&vm->moved,
1820
			struct amdgpu_bo_va, base.vm_status);
A
Alex Deucher 已提交
1821
		spin_unlock(&vm->status_lock);
1822

1823 1824
		resv = bo_va->base.bo->tbo.resv;

1825
		/* Per VM BOs never need to bo cleared in the page tables */
1826 1827 1828
		if (resv == vm->root.base.bo->tbo.resv)
			clear = false;
		/* Try to reserve the BO to avoid clearing its ptes */
1829
		else if (!amdgpu_vm_debug && reservation_object_trylock(resv))
1830 1831 1832 1833
			clear = false;
		/* Somebody else is using the BO right now */
		else
			clear = true;
1834 1835

		r = amdgpu_vm_bo_update(adev, bo_va, clear);
A
Alex Deucher 已提交
1836 1837 1838
		if (r)
			return r;

1839 1840 1841
		if (!clear && resv != vm->root.base.bo->tbo.resv)
			reservation_object_unlock(resv);

A
Alex Deucher 已提交
1842 1843 1844 1845
		spin_lock(&vm->status_lock);
	}
	spin_unlock(&vm->status_lock);

1846
	return r;
A
Alex Deucher 已提交
1847 1848 1849 1850 1851 1852 1853 1854 1855
}

/**
 * amdgpu_vm_bo_add - add a bo to a specific vm
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 * @bo: amdgpu buffer object
 *
1856
 * Add @bo into the requested vm.
A
Alex Deucher 已提交
1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871
 * Add @bo to the list of bos associated with the vm
 * Returns newly added bo_va or NULL for failure
 *
 * Object has to be reserved!
 */
struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
				      struct amdgpu_vm *vm,
				      struct amdgpu_bo *bo)
{
	struct amdgpu_bo_va *bo_va;

	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
	if (bo_va == NULL) {
		return NULL;
	}
1872
	amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
1873

A
Alex Deucher 已提交
1874
	bo_va->ref_count = 1;
1875 1876
	INIT_LIST_HEAD(&bo_va->valids);
	INIT_LIST_HEAD(&bo_va->invalids);
1877

A
Alex Deucher 已提交
1878 1879 1880
	return bo_va;
}

1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897

/**
 * amdgpu_vm_bo_insert_mapping - insert a new mapping
 *
 * @adev: amdgpu_device pointer
 * @bo_va: bo_va to store the address
 * @mapping: the mapping to insert
 *
 * Insert a new mapping into all structures.
 */
static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
				    struct amdgpu_bo_va *bo_va,
				    struct amdgpu_bo_va_mapping *mapping)
{
	struct amdgpu_vm *vm = bo_va->base.vm;
	struct amdgpu_bo *bo = bo_va->base.bo;

1898
	mapping->bo_va = bo_va;
1899 1900 1901 1902 1903 1904 1905 1906
	list_add(&mapping->list, &bo_va->invalids);
	amdgpu_vm_it_insert(mapping, &vm->va);

	if (mapping->flags & AMDGPU_PTE_PRT)
		amdgpu_vm_prt_get(adev);

	if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
		spin_lock(&vm->status_lock);
1907 1908
		if (list_empty(&bo_va->base.vm_status))
			list_add(&bo_va->base.vm_status, &vm->moved);
1909 1910 1911 1912 1913
		spin_unlock(&vm->status_lock);
	}
	trace_amdgpu_vm_bo_map(bo_va, mapping);
}

A
Alex Deucher 已提交
1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925
/**
 * amdgpu_vm_bo_map - map bo inside a vm
 *
 * @adev: amdgpu_device pointer
 * @bo_va: bo_va to store the address
 * @saddr: where to map the BO
 * @offset: requested offset in the BO
 * @flags: attributes of pages (read/write/valid/etc.)
 *
 * Add a mapping of the BO at the specefied addr into the VM.
 * Returns 0 for success, error for failure.
 *
1926
 * Object has to be reserved and unreserved outside!
A
Alex Deucher 已提交
1927 1928 1929 1930
 */
int amdgpu_vm_bo_map(struct amdgpu_device *adev,
		     struct amdgpu_bo_va *bo_va,
		     uint64_t saddr, uint64_t offset,
1931
		     uint64_t size, uint64_t flags)
A
Alex Deucher 已提交
1932
{
1933
	struct amdgpu_bo_va_mapping *mapping, *tmp;
1934 1935
	struct amdgpu_bo *bo = bo_va->base.bo;
	struct amdgpu_vm *vm = bo_va->base.vm;
A
Alex Deucher 已提交
1936 1937
	uint64_t eaddr;

1938 1939
	/* validate the parameters */
	if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1940
	    size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1941 1942
		return -EINVAL;

A
Alex Deucher 已提交
1943
	/* make sure object fit at this offset */
1944
	eaddr = saddr + size - 1;
1945
	if (saddr >= eaddr ||
1946
	    (bo && offset + size > amdgpu_bo_size(bo)))
A
Alex Deucher 已提交
1947 1948 1949 1950 1951
		return -EINVAL;

	saddr /= AMDGPU_GPU_PAGE_SIZE;
	eaddr /= AMDGPU_GPU_PAGE_SIZE;

1952 1953
	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
	if (tmp) {
A
Alex Deucher 已提交
1954 1955
		/* bo and tmp overlap, invalid addr */
		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
1956
			"0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
1957
			tmp->start, tmp->last + 1);
1958
		return -EINVAL;
A
Alex Deucher 已提交
1959 1960 1961
	}

	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1962 1963
	if (!mapping)
		return -ENOMEM;
A
Alex Deucher 已提交
1964

1965 1966
	mapping->start = saddr;
	mapping->last = eaddr;
A
Alex Deucher 已提交
1967 1968 1969
	mapping->offset = offset;
	mapping->flags = flags;

1970
	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995

	return 0;
}

/**
 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
 *
 * @adev: amdgpu_device pointer
 * @bo_va: bo_va to store the address
 * @saddr: where to map the BO
 * @offset: requested offset in the BO
 * @flags: attributes of pages (read/write/valid/etc.)
 *
 * Add a mapping of the BO at the specefied addr into the VM. Replace existing
 * mappings as we do so.
 * Returns 0 for success, error for failure.
 *
 * Object has to be reserved and unreserved outside!
 */
int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
			     struct amdgpu_bo_va *bo_va,
			     uint64_t saddr, uint64_t offset,
			     uint64_t size, uint64_t flags)
{
	struct amdgpu_bo_va_mapping *mapping;
1996
	struct amdgpu_bo *bo = bo_va->base.bo;
1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007
	uint64_t eaddr;
	int r;

	/* validate the parameters */
	if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
	    size == 0 || size & AMDGPU_GPU_PAGE_MASK)
		return -EINVAL;

	/* make sure object fit at this offset */
	eaddr = saddr + size - 1;
	if (saddr >= eaddr ||
2008
	    (bo && offset + size > amdgpu_bo_size(bo)))
2009 2010 2011 2012 2013 2014 2015
		return -EINVAL;

	/* Allocate all the needed memory */
	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
	if (!mapping)
		return -ENOMEM;

2016
	r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
2017 2018 2019 2020 2021 2022 2023 2024
	if (r) {
		kfree(mapping);
		return r;
	}

	saddr /= AMDGPU_GPU_PAGE_SIZE;
	eaddr /= AMDGPU_GPU_PAGE_SIZE;

2025 2026
	mapping->start = saddr;
	mapping->last = eaddr;
2027 2028 2029
	mapping->offset = offset;
	mapping->flags = flags;

2030
	amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2031

A
Alex Deucher 已提交
2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044
	return 0;
}

/**
 * amdgpu_vm_bo_unmap - remove bo mapping from vm
 *
 * @adev: amdgpu_device pointer
 * @bo_va: bo_va to remove the address from
 * @saddr: where to the BO is mapped
 *
 * Remove a mapping of the BO at the specefied addr from the VM.
 * Returns 0 for success, error for failure.
 *
2045
 * Object has to be reserved and unreserved outside!
A
Alex Deucher 已提交
2046 2047 2048 2049 2050 2051
 */
int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
		       struct amdgpu_bo_va *bo_va,
		       uint64_t saddr)
{
	struct amdgpu_bo_va_mapping *mapping;
2052
	struct amdgpu_vm *vm = bo_va->base.vm;
2053
	bool valid = true;
A
Alex Deucher 已提交
2054

2055
	saddr /= AMDGPU_GPU_PAGE_SIZE;
2056

2057
	list_for_each_entry(mapping, &bo_va->valids, list) {
2058
		if (mapping->start == saddr)
A
Alex Deucher 已提交
2059 2060 2061
			break;
	}

2062 2063 2064 2065
	if (&mapping->list == &bo_va->valids) {
		valid = false;

		list_for_each_entry(mapping, &bo_va->invalids, list) {
2066
			if (mapping->start == saddr)
2067 2068 2069
				break;
		}

2070
		if (&mapping->list == &bo_va->invalids)
2071
			return -ENOENT;
A
Alex Deucher 已提交
2072
	}
2073

A
Alex Deucher 已提交
2074
	list_del(&mapping->list);
2075
	amdgpu_vm_it_remove(mapping, &vm->va);
2076
	mapping->bo_va = NULL;
2077
	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
A
Alex Deucher 已提交
2078

2079
	if (valid)
A
Alex Deucher 已提交
2080
		list_add(&mapping->list, &vm->freed);
2081
	else
2082 2083
		amdgpu_vm_free_mapping(adev, vm, mapping,
				       bo_va->last_pt_update);
A
Alex Deucher 已提交
2084 2085 2086 2087

	return 0;
}

2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114
/**
 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
 *
 * @adev: amdgpu_device pointer
 * @vm: VM structure to use
 * @saddr: start of the range
 * @size: size of the range
 *
 * Remove all mappings in a range, split them as appropriate.
 * Returns 0 for success, error for failure.
 */
int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
				struct amdgpu_vm *vm,
				uint64_t saddr, uint64_t size)
{
	struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
	LIST_HEAD(removed);
	uint64_t eaddr;

	eaddr = saddr + size - 1;
	saddr /= AMDGPU_GPU_PAGE_SIZE;
	eaddr /= AMDGPU_GPU_PAGE_SIZE;

	/* Allocate all the needed memory */
	before = kzalloc(sizeof(*before), GFP_KERNEL);
	if (!before)
		return -ENOMEM;
2115
	INIT_LIST_HEAD(&before->list);
2116 2117 2118 2119 2120 2121

	after = kzalloc(sizeof(*after), GFP_KERNEL);
	if (!after) {
		kfree(before);
		return -ENOMEM;
	}
2122
	INIT_LIST_HEAD(&after->list);
2123 2124

	/* Now gather all removed mappings */
2125 2126
	tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
	while (tmp) {
2127
		/* Remember mapping split at the start */
2128 2129 2130
		if (tmp->start < saddr) {
			before->start = tmp->start;
			before->last = saddr - 1;
2131 2132 2133 2134 2135 2136
			before->offset = tmp->offset;
			before->flags = tmp->flags;
			list_add(&before->list, &tmp->list);
		}

		/* Remember mapping split at the end */
2137 2138 2139
		if (tmp->last > eaddr) {
			after->start = eaddr + 1;
			after->last = tmp->last;
2140
			after->offset = tmp->offset;
2141
			after->offset += after->start - tmp->start;
2142 2143 2144 2145 2146 2147
			after->flags = tmp->flags;
			list_add(&after->list, &tmp->list);
		}

		list_del(&tmp->list);
		list_add(&tmp->list, &removed);
2148 2149

		tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
2150 2151 2152 2153
	}

	/* And free them up */
	list_for_each_entry_safe(tmp, next, &removed, list) {
2154
		amdgpu_vm_it_remove(tmp, &vm->va);
2155 2156
		list_del(&tmp->list);

2157 2158 2159 2160
		if (tmp->start < saddr)
		    tmp->start = saddr;
		if (tmp->last > eaddr)
		    tmp->last = eaddr;
2161

2162
		tmp->bo_va = NULL;
2163 2164 2165 2166
		list_add(&tmp->list, &vm->freed);
		trace_amdgpu_vm_bo_unmap(NULL, tmp);
	}

2167 2168
	/* Insert partial mapping before the range */
	if (!list_empty(&before->list)) {
2169
		amdgpu_vm_it_insert(before, &vm->va);
2170 2171 2172 2173 2174 2175 2176
		if (before->flags & AMDGPU_PTE_PRT)
			amdgpu_vm_prt_get(adev);
	} else {
		kfree(before);
	}

	/* Insert partial mapping after the range */
2177
	if (!list_empty(&after->list)) {
2178
		amdgpu_vm_it_insert(after, &vm->va);
2179 2180 2181 2182 2183 2184 2185 2186 2187
		if (after->flags & AMDGPU_PTE_PRT)
			amdgpu_vm_prt_get(adev);
	} else {
		kfree(after);
	}

	return 0;
}

2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200
/**
 * amdgpu_vm_bo_lookup_mapping - find mapping by address
 *
 * @vm: the requested VM
 *
 * Find a mapping by it's address.
 */
struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
							 uint64_t addr)
{
	return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
}

A
Alex Deucher 已提交
2201 2202 2203 2204 2205 2206
/**
 * amdgpu_vm_bo_rmv - remove a bo to a specific vm
 *
 * @adev: amdgpu_device pointer
 * @bo_va: requested bo_va
 *
2207
 * Remove @bo_va->bo from the requested vm.
A
Alex Deucher 已提交
2208 2209 2210 2211 2212 2213 2214
 *
 * Object have to be reserved!
 */
void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
		      struct amdgpu_bo_va *bo_va)
{
	struct amdgpu_bo_va_mapping *mapping, *next;
2215
	struct amdgpu_vm *vm = bo_va->base.vm;
A
Alex Deucher 已提交
2216

2217
	list_del(&bo_va->base.bo_list);
A
Alex Deucher 已提交
2218 2219

	spin_lock(&vm->status_lock);
2220
	list_del(&bo_va->base.vm_status);
A
Alex Deucher 已提交
2221 2222
	spin_unlock(&vm->status_lock);

2223
	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
A
Alex Deucher 已提交
2224
		list_del(&mapping->list);
2225
		amdgpu_vm_it_remove(mapping, &vm->va);
2226
		mapping->bo_va = NULL;
2227
		trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2228 2229 2230 2231
		list_add(&mapping->list, &vm->freed);
	}
	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
		list_del(&mapping->list);
2232
		amdgpu_vm_it_remove(mapping, &vm->va);
2233 2234
		amdgpu_vm_free_mapping(adev, vm, mapping,
				       bo_va->last_pt_update);
A
Alex Deucher 已提交
2235
	}
2236

2237
	dma_fence_put(bo_va->last_pt_update);
A
Alex Deucher 已提交
2238 2239 2240 2241 2242 2243 2244 2245 2246 2247
	kfree(bo_va);
}

/**
 * amdgpu_vm_bo_invalidate - mark the bo as invalid
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 * @bo: amdgpu buffer object
 *
2248
 * Mark @bo as invalid.
A
Alex Deucher 已提交
2249 2250
 */
void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
2251
			     struct amdgpu_bo *bo, bool evicted)
A
Alex Deucher 已提交
2252
{
2253 2254
	struct amdgpu_vm_bo_base *bo_base;

2255 2256 2257 2258
	/* shadow bo doesn't have bo base, its validation needs its parent */
	if (bo->parent && bo->parent->shadow == bo)
		bo = bo->parent;

2259
	list_for_each_entry(bo_base, &bo->va, bo_list) {
2260 2261
		struct amdgpu_vm *vm = bo_base->vm;

2262
		bo_base->moved = true;
2263 2264
		if (evicted && bo->tbo.resv == vm->root.base.bo->tbo.resv) {
			spin_lock(&bo_base->vm->status_lock);
2265 2266 2267 2268 2269
			if (bo->tbo.type == ttm_bo_type_kernel)
				list_move(&bo_base->vm_status, &vm->evicted);
			else
				list_move_tail(&bo_base->vm_status,
					       &vm->evicted);
2270 2271 2272 2273
			spin_unlock(&bo_base->vm->status_lock);
			continue;
		}

2274 2275 2276 2277 2278
		if (bo->tbo.type == ttm_bo_type_kernel) {
			spin_lock(&bo_base->vm->status_lock);
			if (list_empty(&bo_base->vm_status))
				list_add(&bo_base->vm_status, &vm->relocated);
			spin_unlock(&bo_base->vm->status_lock);
2279
			continue;
2280
		}
2281

2282 2283
		spin_lock(&bo_base->vm->status_lock);
		if (list_empty(&bo_base->vm_status))
2284
			list_add(&bo_base->vm_status, &vm->moved);
2285
		spin_unlock(&bo_base->vm->status_lock);
A
Alex Deucher 已提交
2286 2287 2288
	}
}

2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301
static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
{
	/* Total bits covered by PD + PTs */
	unsigned bits = ilog2(vm_size) + 18;

	/* Make sure the PD is 4K in size up to 8GB address space.
	   Above that split equal between PD and PTs */
	if (vm_size <= 8)
		return (bits - 9);
	else
		return ((bits + 3) / 2);
}

2302 2303
/**
 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2304 2305 2306 2307
 *
 * @adev: amdgpu_device pointer
 * @vm_size: the default vm size if it's set auto
 */
2308
void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t vm_size,
2309 2310
			   uint32_t fragment_size_default, unsigned max_level,
			   unsigned max_bits)
2311
{
2312 2313 2314
	uint64_t tmp;

	/* adjust vm size first */
2315 2316 2317
	if (amdgpu_vm_size != -1) {
		unsigned max_size = 1 << (max_bits - 30);

2318
		vm_size = amdgpu_vm_size;
2319 2320 2321 2322 2323 2324
		if (vm_size > max_size) {
			dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
				 amdgpu_vm_size, max_size);
			vm_size = max_size;
		}
	}
2325 2326

	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2327 2328

	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2329 2330
	if (amdgpu_vm_block_size != -1)
		tmp >>= amdgpu_vm_block_size - 9;
2331 2332
	tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
	adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345
	switch (adev->vm_manager.num_level) {
	case 3:
		adev->vm_manager.root_level = AMDGPU_VM_PDB2;
		break;
	case 2:
		adev->vm_manager.root_level = AMDGPU_VM_PDB1;
		break;
	case 1:
		adev->vm_manager.root_level = AMDGPU_VM_PDB0;
		break;
	default:
		dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
	}
2346
	/* block size depends on vm size and hw setup*/
2347
	if (amdgpu_vm_block_size != -1)
2348
		adev->vm_manager.block_size =
2349 2350 2351 2352 2353
			min((unsigned)amdgpu_vm_block_size, max_bits
			    - AMDGPU_GPU_PAGE_SHIFT
			    - 9 * adev->vm_manager.num_level);
	else if (adev->vm_manager.num_level > 1)
		adev->vm_manager.block_size = 9;
2354
	else
2355
		adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2356

2357 2358 2359 2360
	if (amdgpu_vm_fragment_size == -1)
		adev->vm_manager.fragment_size = fragment_size_default;
	else
		adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2361

2362 2363 2364
	DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
		 vm_size, adev->vm_manager.num_level + 1,
		 adev->vm_manager.block_size,
2365
		 adev->vm_manager.fragment_size);
2366 2367
}

A
Alex Deucher 已提交
2368 2369 2370 2371 2372
/**
 * amdgpu_vm_init - initialize a vm instance
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
2373
 * @vm_context: Indicates if it GFX or Compute context
A
Alex Deucher 已提交
2374
 *
2375
 * Init @vm fields.
A
Alex Deucher 已提交
2376
 */
2377
int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2378
		   int vm_context, unsigned int pasid)
A
Alex Deucher 已提交
2379
{
2380
	struct amdgpu_bo_param bp;
2381
	struct amdgpu_bo *root;
A
Alex Deucher 已提交
2382
	const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
2383
		AMDGPU_VM_PTE_COUNT(adev) * 8);
2384 2385
	unsigned ring_instance;
	struct amdgpu_ring *ring;
2386
	struct drm_sched_rq *rq;
2387
	unsigned long size;
2388
	uint64_t flags;
2389
	int r, i;
A
Alex Deucher 已提交
2390

2391
	vm->va = RB_ROOT_CACHED;
2392 2393
	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
		vm->reserved_vmid[i] = NULL;
A
Alex Deucher 已提交
2394
	spin_lock_init(&vm->status_lock);
2395
	INIT_LIST_HEAD(&vm->evicted);
2396
	INIT_LIST_HEAD(&vm->relocated);
2397
	INIT_LIST_HEAD(&vm->moved);
A
Alex Deucher 已提交
2398
	INIT_LIST_HEAD(&vm->freed);
2399

2400
	/* create scheduler entity for page table updates */
2401 2402 2403 2404

	ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
	ring_instance %= adev->vm_manager.vm_pte_num_rings;
	ring = adev->vm_manager.vm_pte_rings[ring_instance];
2405 2406
	rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_KERNEL];
	r = drm_sched_entity_init(&ring->sched, &vm->entity,
2407
				  rq, NULL);
2408
	if (r)
2409
		return r;
2410

Y
Yong Zhao 已提交
2411 2412 2413
	vm->pte_support_ats = false;

	if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
2414 2415
		vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
						AMDGPU_VM_USE_CPU_FOR_COMPUTE);
Y
Yong Zhao 已提交
2416

2417
		if (adev->asic_type == CHIP_RAVEN)
Y
Yong Zhao 已提交
2418
			vm->pte_support_ats = true;
2419
	} else {
2420 2421
		vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
						AMDGPU_VM_USE_CPU_FOR_GFX);
2422
	}
2423 2424 2425 2426
	DRM_DEBUG_DRIVER("VM update mode is %s\n",
			 vm->use_cpu_for_update ? "CPU" : "SDMA");
	WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
		  "CPU update of VM recommended only for large BAR system\n");
2427
	vm->last_update = NULL;
2428

2429
	flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
2430 2431 2432
	if (vm->use_cpu_for_update)
		flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
	else
2433
		flags |= AMDGPU_GEM_CREATE_SHADOW;
2434

2435
	size = amdgpu_vm_bo_size(adev, adev->vm_manager.root_level);
2436 2437 2438 2439 2440 2441 2442
	memset(&bp, 0, sizeof(bp));
	bp.size = size;
	bp.byte_align = align;
	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
	bp.flags = flags;
	bp.type = ttm_bo_type_kernel;
	bp.resv = NULL;
2443
	r = amdgpu_bo_create(adev, &bp, &root);
A
Alex Deucher 已提交
2444
	if (r)
2445 2446
		goto error_free_sched_entity;

2447
	r = amdgpu_bo_reserve(root, true);
2448 2449 2450
	if (r)
		goto error_free_root;

2451
	r = amdgpu_vm_clear_bo(adev, vm, root,
2452 2453
			       adev->vm_manager.root_level,
			       vm->pte_support_ats);
2454 2455 2456
	if (r)
		goto error_unreserve;

2457
	amdgpu_vm_bo_base_init(&vm->root.base, vm, root);
2458
	amdgpu_bo_unreserve(vm->root.base.bo);
A
Alex Deucher 已提交
2459

2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470
	if (pasid) {
		unsigned long flags;

		spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
		r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
			      GFP_ATOMIC);
		spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
		if (r < 0)
			goto error_free_root;

		vm->pasid = pasid;
2471 2472
	}

2473
	INIT_KFIFO(vm->faults);
2474
	vm->fault_credit = 16;
A
Alex Deucher 已提交
2475 2476

	return 0;
2477

2478 2479 2480
error_unreserve:
	amdgpu_bo_unreserve(vm->root.base.bo);

2481
error_free_root:
2482 2483 2484
	amdgpu_bo_unref(&vm->root.base.bo->shadow);
	amdgpu_bo_unref(&vm->root.base.bo);
	vm->root.base.bo = NULL;
2485 2486

error_free_sched_entity:
2487
	drm_sched_entity_fini(&ring->sched, &vm->entity);
2488 2489

	return r;
A
Alex Deucher 已提交
2490 2491
}

2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558
/**
 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
 *
 * This only works on GFX VMs that don't have any BOs added and no
 * page tables allocated yet.
 *
 * Changes the following VM parameters:
 * - use_cpu_for_update
 * - pte_supports_ats
 * - pasid (old PASID is released, because compute manages its own PASIDs)
 *
 * Reinitializes the page directory to reflect the changed ATS
 * setting. May leave behind an unused shadow BO for the page
 * directory when switching from SDMA updates to CPU updates.
 *
 * Returns 0 for success, -errno for errors.
 */
int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
{
	bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
	int r;

	r = amdgpu_bo_reserve(vm->root.base.bo, true);
	if (r)
		return r;

	/* Sanity checks */
	if (!RB_EMPTY_ROOT(&vm->va.rb_root) || vm->root.entries) {
		r = -EINVAL;
		goto error;
	}

	/* Check if PD needs to be reinitialized and do it before
	 * changing any other state, in case it fails.
	 */
	if (pte_support_ats != vm->pte_support_ats) {
		r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo,
			       adev->vm_manager.root_level,
			       pte_support_ats);
		if (r)
			goto error;
	}

	/* Update VM state */
	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
				    AMDGPU_VM_USE_CPU_FOR_COMPUTE);
	vm->pte_support_ats = pte_support_ats;
	DRM_DEBUG_DRIVER("VM update mode is %s\n",
			 vm->use_cpu_for_update ? "CPU" : "SDMA");
	WARN_ONCE((vm->use_cpu_for_update & !amdgpu_vm_is_large_bar(adev)),
		  "CPU update of VM recommended only for large BAR system\n");

	if (vm->pasid) {
		unsigned long flags;

		spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
		idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
		spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);

		vm->pasid = 0;
	}

error:
	amdgpu_bo_unreserve(vm->root.base.bo);
	return r;
}

2559 2560 2561
/**
 * amdgpu_vm_free_levels - free PD/PT levels
 *
2562 2563 2564
 * @adev: amdgpu device structure
 * @parent: PD/PT starting level to free
 * @level: level of parent structure
2565 2566 2567
 *
 * Free the page directory or page table level and all sub levels.
 */
2568 2569 2570
static void amdgpu_vm_free_levels(struct amdgpu_device *adev,
				  struct amdgpu_vm_pt *parent,
				  unsigned level)
2571
{
2572
	unsigned i, num_entries = amdgpu_vm_num_entries(adev, level);
2573

2574 2575 2576 2577 2578
	if (parent->base.bo) {
		list_del(&parent->base.bo_list);
		list_del(&parent->base.vm_status);
		amdgpu_bo_unref(&parent->base.bo->shadow);
		amdgpu_bo_unref(&parent->base.bo);
2579 2580
	}

2581 2582 2583 2584
	if (parent->entries)
		for (i = 0; i < num_entries; i++)
			amdgpu_vm_free_levels(adev, &parent->entries[i],
					      level + 1);
2585

2586
	kvfree(parent->entries);
2587 2588
}

A
Alex Deucher 已提交
2589 2590 2591 2592 2593 2594
/**
 * amdgpu_vm_fini - tear down a vm instance
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
2595
 * Tear down @vm.
A
Alex Deucher 已提交
2596 2597 2598 2599 2600
 * Unbind the VM and remove all bos from the vm bo list
 */
void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
{
	struct amdgpu_bo_va_mapping *mapping, *tmp;
2601
	bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2602
	struct amdgpu_bo *root;
2603
	u64 fault;
2604
	int i, r;
A
Alex Deucher 已提交
2605

2606 2607
	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);

2608 2609 2610 2611
	/* Clear pending page faults from IH when the VM is destroyed */
	while (kfifo_get(&vm->faults, &fault))
		amdgpu_ih_clear_fault(adev, fault);

2612 2613 2614 2615 2616 2617 2618 2619
	if (vm->pasid) {
		unsigned long flags;

		spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
		idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
		spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
	}

2620
	drm_sched_entity_fini(vm->entity.sched, &vm->entity);
2621

2622
	if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
A
Alex Deucher 已提交
2623 2624
		dev_err(adev->dev, "still active bo inside vm\n");
	}
2625 2626
	rbtree_postorder_for_each_entry_safe(mapping, tmp,
					     &vm->va.rb_root, rb) {
A
Alex Deucher 已提交
2627
		list_del(&mapping->list);
2628
		amdgpu_vm_it_remove(mapping, &vm->va);
A
Alex Deucher 已提交
2629 2630 2631
		kfree(mapping);
	}
	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2632
		if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
2633
			amdgpu_vm_prt_fini(adev, vm);
2634
			prt_fini_needed = false;
2635
		}
2636

A
Alex Deucher 已提交
2637
		list_del(&mapping->list);
2638
		amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
A
Alex Deucher 已提交
2639 2640
	}

2641 2642 2643 2644 2645
	root = amdgpu_bo_ref(vm->root.base.bo);
	r = amdgpu_bo_reserve(root, true);
	if (r) {
		dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
	} else {
2646 2647
		amdgpu_vm_free_levels(adev, &vm->root,
				      adev->vm_manager.root_level);
2648 2649 2650
		amdgpu_bo_unreserve(root);
	}
	amdgpu_bo_unref(&root);
2651
	dma_fence_put(vm->last_update);
2652
	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2653
		amdgpu_vmid_free_reserved(adev, vm, i);
A
Alex Deucher 已提交
2654
}
2655

2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671
/**
 * amdgpu_vm_pasid_fault_credit - Check fault credit for given PASID
 *
 * @adev: amdgpu_device pointer
 * @pasid: PASID do identify the VM
 *
 * This function is expected to be called in interrupt context. Returns
 * true if there was fault credit, false otherwise
 */
bool amdgpu_vm_pasid_fault_credit(struct amdgpu_device *adev,
				  unsigned int pasid)
{
	struct amdgpu_vm *vm;

	spin_lock(&adev->vm_manager.pasid_lock);
	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
2672
	if (!vm) {
2673
		/* VM not found, can't track fault credit */
2674
		spin_unlock(&adev->vm_manager.pasid_lock);
2675
		return true;
2676
	}
2677 2678

	/* No lock needed. only accessed by IRQ handler */
2679
	if (!vm->fault_credit) {
2680
		/* Too many faults in this VM */
2681
		spin_unlock(&adev->vm_manager.pasid_lock);
2682
		return false;
2683
	}
2684 2685

	vm->fault_credit--;
2686
	spin_unlock(&adev->vm_manager.pasid_lock);
2687 2688 2689
	return true;
}

2690 2691 2692 2693 2694 2695 2696 2697 2698
/**
 * amdgpu_vm_manager_init - init the VM manager
 *
 * @adev: amdgpu_device pointer
 *
 * Initialize the VM manager structures
 */
void amdgpu_vm_manager_init(struct amdgpu_device *adev)
{
2699
	unsigned i;
2700

2701
	amdgpu_vmid_mgr_init(adev);
2702

2703 2704
	adev->vm_manager.fence_context =
		dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2705 2706 2707
	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
		adev->vm_manager.seqno[i] = 0;

2708
	atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
2709
	spin_lock_init(&adev->vm_manager.prt_lock);
2710
	atomic_set(&adev->vm_manager.num_prt_users, 0);
2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727

	/* If not overridden by the user, by default, only in large BAR systems
	 * Compute VM tables will be updated by CPU
	 */
#ifdef CONFIG_X86_64
	if (amdgpu_vm_update_mode == -1) {
		if (amdgpu_vm_is_large_bar(adev))
			adev->vm_manager.vm_update_mode =
				AMDGPU_VM_USE_CPU_FOR_COMPUTE;
		else
			adev->vm_manager.vm_update_mode = 0;
	} else
		adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
#else
	adev->vm_manager.vm_update_mode = 0;
#endif

2728 2729
	idr_init(&adev->vm_manager.pasid_idr);
	spin_lock_init(&adev->vm_manager.pasid_lock);
2730 2731
}

2732 2733 2734 2735 2736 2737 2738 2739 2740
/**
 * amdgpu_vm_manager_fini - cleanup VM manager
 *
 * @adev: amdgpu_device pointer
 *
 * Cleanup the VM manager and free resources.
 */
void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
{
2741 2742 2743
	WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
	idr_destroy(&adev->vm_manager.pasid_idr);

2744
	amdgpu_vmid_mgr_fini(adev);
2745
}
C
Chunming Zhou 已提交
2746 2747 2748 2749

int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
	union drm_amdgpu_vm *args = data;
2750 2751 2752
	struct amdgpu_device *adev = dev->dev_private;
	struct amdgpu_fpriv *fpriv = filp->driver_priv;
	int r;
C
Chunming Zhou 已提交
2753 2754 2755

	switch (args->in.op) {
	case AMDGPU_VM_OP_RESERVE_VMID:
2756
		/* current, we only have requirement to reserve vmid from gfxhub */
2757
		r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
2758 2759 2760
		if (r)
			return r;
		break;
C
Chunming Zhou 已提交
2761
	case AMDGPU_VM_OP_UNRESERVE_VMID:
2762
		amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB);
C
Chunming Zhou 已提交
2763 2764 2765 2766 2767 2768 2769
		break;
	default:
		return -EINVAL;
	}

	return 0;
}