amdgpu_vm.c 49.6 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright 2008 Advanced Micro Devices, Inc.
 * Copyright 2008 Red Hat Inc.
 * Copyright 2009 Jerome Glisse.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Dave Airlie
 *          Alex Deucher
 *          Jerome Glisse
 */
28
#include <linux/dma-fence-array.h>
A
Alex Deucher 已提交
29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "amdgpu_trace.h"

/*
 * GPUVM
 * GPUVM is similar to the legacy gart on older asics, however
 * rather than there being a single global gart table
 * for the entire GPU, there are multiple VM page tables active
 * at any given time.  The VM page tables can contain a mix
 * vram pages and system memory pages and system memory pages
 * can be mapped as snooped (cached system pages) or unsnooped
 * (uncached system pages).
 * Each VM has an ID associated with it and there is a page table
 * associated with each VMID.  When execting a command buffer,
 * the kernel tells the the ring what VMID to use for that command
 * buffer.  VMIDs are allocated dynamically as commands are submitted.
 * The userspace drivers maintain their own address space and the kernel
 * sets up their pages tables accordingly when they submit their
 * command buffers and a VMID is assigned.
 * Cayman/Trinity support up to 8 active VMs at any given time;
 * SI supports 16.
 */

54 55 56
/* Local structure. Encapsulate some VM table update parameters to reduce
 * the number of function parameters
 */
57
struct amdgpu_pte_update_params {
58 59
	/* amdgpu device we do this update for */
	struct amdgpu_device *adev;
60 61 62 63
	/* address where to copy page table entries from */
	uint64_t src;
	/* indirect buffer to fill with commands */
	struct amdgpu_ib *ib;
64 65 66
	/* Function which actually does the update */
	void (*func)(struct amdgpu_pte_update_params *params, uint64_t pe,
		     uint64_t addr, unsigned count, uint32_t incr,
67
		     uint64_t flags);
68 69
	/* indicate update pt or its shadow */
	bool shadow;
70 71
};

72 73 74 75 76 77
/* Helper to disable partial resident texture feature from a fence callback */
struct amdgpu_prt_cb {
	struct amdgpu_device *adev;
	struct dma_fence_cb cb;
};

A
Alex Deucher 已提交
78 79 80 81 82
/**
 * amdgpu_vm_num_pde - return the number of page directory entries
 *
 * @adev: amdgpu_device pointer
 *
83
 * Calculate the number of page directory entries.
A
Alex Deucher 已提交
84 85 86 87 88 89 90 91 92 93 94
 */
static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev)
{
	return adev->vm_manager.max_pfn >> amdgpu_vm_block_size;
}

/**
 * amdgpu_vm_directory_size - returns the size of the page directory in bytes
 *
 * @adev: amdgpu_device pointer
 *
95
 * Calculate the size of the page directory in bytes.
A
Alex Deucher 已提交
96 97 98 99 100 101 102
 */
static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev)
{
	return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8);
}

/**
103
 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
A
Alex Deucher 已提交
104 105
 *
 * @vm: vm providing the BOs
106
 * @validated: head of validation list
107
 * @entry: entry to add
A
Alex Deucher 已提交
108 109
 *
 * Add the page directory to the list of BOs to
110
 * validate for command submission.
A
Alex Deucher 已提交
111
 */
112 113 114
void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
			 struct list_head *validated,
			 struct amdgpu_bo_list_entry *entry)
A
Alex Deucher 已提交
115
{
116 117 118 119
	entry->robj = vm->page_directory;
	entry->priority = 0;
	entry->tv.bo = &vm->page_directory->tbo;
	entry->tv.shared = true;
120
	entry->user_pages = NULL;
121 122
	list_add(&entry->tv.head, validated);
}
A
Alex Deucher 已提交
123

124
/**
125
 * amdgpu_vm_validate_pt_bos - validate the page table BOs
126
 *
127
 * @adev: amdgpu device pointer
128
 * @vm: vm providing the BOs
129 130
 * @validate: callback to do the validation
 * @param: parameter for the validation callback
A
Alex Deucher 已提交
131
 *
132
 * Validate the page table BOs on command submission if neccessary.
A
Alex Deucher 已提交
133
 */
134 135 136
int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
			      int (*validate)(void *p, struct amdgpu_bo *bo),
			      void *param)
A
Alex Deucher 已提交
137
{
138
	uint64_t num_evictions;
139
	unsigned i;
140
	int r;
A
Alex Deucher 已提交
141

142 143 144 145 146
	/* We only need to validate the page tables
	 * if they aren't already valid.
	 */
	num_evictions = atomic64_read(&adev->num_evictions);
	if (num_evictions == vm->last_eviction_counter)
147
		return 0;
148

A
Alex Deucher 已提交
149
	/* add the vm page table to the list */
150
	for (i = 0; i <= vm->max_pde_used; ++i) {
151
		struct amdgpu_bo *bo = vm->page_tables[i].bo;
152

153
		if (!bo)
A
Alex Deucher 已提交
154 155
			continue;

156
		r = validate(param, bo);
157 158
		if (r)
			return r;
A
Alex Deucher 已提交
159
	}
160

161
	return 0;
162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
}

/**
 * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
 *
 * @adev: amdgpu device instance
 * @vm: vm providing the BOs
 *
 * Move the PT BOs to the tail of the LRU.
 */
void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
				  struct amdgpu_vm *vm)
{
	struct ttm_bo_global *glob = adev->mman.bdev.glob;
	unsigned i;

	spin_lock(&glob->lru_lock);
	for (i = 0; i <= vm->max_pde_used; ++i) {
180
		struct amdgpu_bo *bo = vm->page_tables[i].bo;
181

182
		if (!bo)
183 184
			continue;

185
		ttm_bo_move_to_lru_tail(&bo->tbo);
186 187
	}
	spin_unlock(&glob->lru_lock);
A
Alex Deucher 已提交
188 189
}

190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
/**
 * amdgpu_vm_alloc_pts - Allocate page tables.
 *
 * @adev: amdgpu_device pointer
 * @vm: VM to allocate page tables for
 * @saddr: Start address which needs to be allocated
 * @size: Size from start address we need.
 *
 * Make sure the page tables are allocated.
 */
int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
			struct amdgpu_vm *vm,
			uint64_t saddr, uint64_t size)
{
	unsigned last_pfn, pt_idx;
	uint64_t eaddr;
	int r;

	/* validate the parameters */
	if (saddr & AMDGPU_GPU_PAGE_MASK || size & AMDGPU_GPU_PAGE_MASK)
		return -EINVAL;

	eaddr = saddr + size - 1;
	last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
	if (last_pfn >= adev->vm_manager.max_pfn) {
		dev_err(adev->dev, "va above limit (0x%08X >= 0x%08X)\n",
			last_pfn, adev->vm_manager.max_pfn);
		return -EINVAL;
	}

	saddr /= AMDGPU_GPU_PAGE_SIZE;
	eaddr /= AMDGPU_GPU_PAGE_SIZE;

	saddr >>= amdgpu_vm_block_size;
	eaddr >>= amdgpu_vm_block_size;

	BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev));

	if (eaddr > vm->max_pde_used)
		vm->max_pde_used = eaddr;

	/* walk over the address space and allocate the page tables */
	for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) {
		struct reservation_object *resv = vm->page_directory->tbo.resv;
		struct amdgpu_bo *pt;

		if (vm->page_tables[pt_idx].bo)
			continue;

		r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
				     AMDGPU_GPU_PAGE_SIZE, true,
				     AMDGPU_GEM_DOMAIN_VRAM,
				     AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
				     AMDGPU_GEM_CREATE_SHADOW |
				     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
				     AMDGPU_GEM_CREATE_VRAM_CLEARED,
				     NULL, resv, &pt);
		if (r)
			return r;

		/* Keep a reference to the page table to avoid freeing
		 * them up in the wrong order.
		 */
		pt->parent = amdgpu_bo_ref(vm->page_directory);

		vm->page_tables[pt_idx].bo = pt;
		vm->page_tables[pt_idx].addr = 0;
	}

	return 0;
}

262 263 264 265 266 267 268
static bool amdgpu_vm_is_gpu_reset(struct amdgpu_device *adev,
			      struct amdgpu_vm_id *id)
{
	return id->current_gpu_reset_count !=
		atomic_read(&adev->gpu_reset_counter) ? true : false;
}

A
Alex Deucher 已提交
269 270 271 272
/**
 * amdgpu_vm_grab_id - allocate the next free VMID
 *
 * @vm: vm to allocate id for
273 274
 * @ring: ring we want to submit job to
 * @sync: sync object where we add dependencies
275
 * @fence: fence protecting ID from reuse
A
Alex Deucher 已提交
276
 *
277
 * Allocate an id for the vm, adding fences to the sync obj as necessary.
A
Alex Deucher 已提交
278
 */
279
int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
280
		      struct amdgpu_sync *sync, struct dma_fence *fence,
281
		      struct amdgpu_job *job)
A
Alex Deucher 已提交
282 283
{
	struct amdgpu_device *adev = ring->adev;
284
	uint64_t fence_context = adev->fence_context + ring->idx;
285
	struct dma_fence *updates = sync->last_vm_update;
286
	struct amdgpu_vm_id *id, *idle;
287
	struct dma_fence **fences;
288 289 290 291 292 293 294
	unsigned i;
	int r = 0;

	fences = kmalloc_array(sizeof(void *), adev->vm_manager.num_ids,
			       GFP_KERNEL);
	if (!fences)
		return -ENOMEM;
A
Alex Deucher 已提交
295

296 297
	mutex_lock(&adev->vm_manager.lock);

298
	/* Check if we have an idle VMID */
299
	i = 0;
300
	list_for_each_entry(idle, &adev->vm_manager.ids_lru, list) {
301 302
		fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
		if (!fences[i])
303
			break;
304
		++i;
305 306
	}

307
	/* If we can't find a idle VMID to use, wait till one becomes available */
308
	if (&idle->list == &adev->vm_manager.ids_lru) {
309 310
		u64 fence_context = adev->vm_manager.fence_context + ring->idx;
		unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
311
		struct dma_fence_array *array;
312 313 314
		unsigned j;

		for (j = 0; j < i; ++j)
315
			dma_fence_get(fences[j]);
316

317
		array = dma_fence_array_create(i, fences, fence_context,
318 319 320
					   seqno, true);
		if (!array) {
			for (j = 0; j < i; ++j)
321
				dma_fence_put(fences[j]);
322 323 324 325 326 327 328
			kfree(fences);
			r = -ENOMEM;
			goto error;
		}


		r = amdgpu_sync_fence(ring->adev, sync, &array->base);
329
		dma_fence_put(&array->base);
330 331 332 333 334 335 336 337 338
		if (r)
			goto error;

		mutex_unlock(&adev->vm_manager.lock);
		return 0;

	}
	kfree(fences);

339
	job->vm_needs_flush = true;
340 341 342
	/* Check if we can use a VMID already assigned to this VM */
	i = ring->idx;
	do {
343
		struct dma_fence *flushed;
344 345 346 347

		id = vm->ids[i++];
		if (i == AMDGPU_MAX_RINGS)
			i = 0;
348

349 350 351
		/* Check all the prerequisites to using this VMID */
		if (!id)
			continue;
352
		if (amdgpu_vm_is_gpu_reset(adev, id))
353
			continue;
354 355 356 357

		if (atomic64_read(&id->owner) != vm->client_id)
			continue;

358
		if (job->vm_pd_addr != id->pd_gpu_addr)
359 360
			continue;

361 362 363 364
		if (!id->last_flush)
			continue;

		if (id->last_flush->context != fence_context &&
365
		    !dma_fence_is_signaled(id->last_flush))
366 367 368 369
			continue;

		flushed  = id->flushed_updates;
		if (updates &&
370
		    (!flushed || dma_fence_is_later(updates, flushed)))
371 372
			continue;

373 374 375
		/* Good we can use this VMID. Remember this submission as
		 * user of the VMID.
		 */
376 377 378
		r = amdgpu_sync_fence(ring->adev, &id->active, fence);
		if (r)
			goto error;
379

380
		id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
381 382
		list_move_tail(&id->list, &adev->vm_manager.ids_lru);
		vm->ids[ring->idx] = id;
383

384 385
		job->vm_id = id - adev->vm_manager.ids;
		job->vm_needs_flush = false;
386
		trace_amdgpu_vm_grab_id(vm, ring->idx, job);
387

388 389
		mutex_unlock(&adev->vm_manager.lock);
		return 0;
390

391
	} while (i != ring->idx);
392

393 394
	/* Still no ID to use? Then use the idle one found earlier */
	id = idle;
395

396 397
	/* Remember this submission as user of the VMID */
	r = amdgpu_sync_fence(ring->adev, &id->active, fence);
398 399
	if (r)
		goto error;
400

401 402
	dma_fence_put(id->first);
	id->first = dma_fence_get(fence);
403

404
	dma_fence_put(id->last_flush);
405 406
	id->last_flush = NULL;

407 408
	dma_fence_put(id->flushed_updates);
	id->flushed_updates = dma_fence_get(updates);
409

410
	id->pd_gpu_addr = job->vm_pd_addr;
411
	id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
412
	list_move_tail(&id->list, &adev->vm_manager.ids_lru);
413
	atomic64_set(&id->owner, vm->client_id);
414
	vm->ids[ring->idx] = id;
A
Alex Deucher 已提交
415

416
	job->vm_id = id - adev->vm_manager.ids;
417
	trace_amdgpu_vm_grab_id(vm, ring->idx, job);
418 419

error:
420
	mutex_unlock(&adev->vm_manager.lock);
421
	return r;
A
Alex Deucher 已提交
422 423
}

424 425 426
static bool amdgpu_vm_ring_has_compute_vm_bug(struct amdgpu_ring *ring)
{
	struct amdgpu_device *adev = ring->adev;
427
	const struct amdgpu_ip_block *ip_block;
428

429
	if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
430 431 432 433 434 435 436
		/* only compute rings */
		return false;

	ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
	if (!ip_block)
		return false;

437
	if (ip_block->version->major <= 7) {
438 439
		/* gfx7 has no workaround */
		return true;
440
	} else if (ip_block->version->major == 8) {
441 442 443 444 445 446 447 448 449
		if (adev->gfx.mec_fw_version >= 673)
			/* gfx8 is fixed in MEC firmware 673 */
			return false;
		else
			return true;
	}
	return false;
}

A
Alex Deucher 已提交
450 451 452 453
/**
 * amdgpu_vm_flush - hardware flush the vm
 *
 * @ring: ring to use for flush
454
 * @vm_id: vmid number to use
455
 * @pd_addr: address of the page directory
A
Alex Deucher 已提交
456
 *
457
 * Emit a VM flush when it is necessary.
A
Alex Deucher 已提交
458
 */
459
int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
A
Alex Deucher 已提交
460
{
461
	struct amdgpu_device *adev = ring->adev;
462
	struct amdgpu_vm_id *id = &adev->vm_manager.ids[job->vm_id];
463
	bool gds_switch_needed = ring->funcs->emit_gds_switch && (
464 465 466 467 468 469
		id->gds_base != job->gds_base ||
		id->gds_size != job->gds_size ||
		id->gws_base != job->gws_base ||
		id->gws_size != job->gws_size ||
		id->oa_base != job->oa_base ||
		id->oa_size != job->oa_size);
470
	int r;
471 472

	if (ring->funcs->emit_pipeline_sync && (
473
	    job->vm_needs_flush || gds_switch_needed ||
474
	    amdgpu_vm_ring_has_compute_vm_bug(ring)))
475
		amdgpu_ring_emit_pipeline_sync(ring);
476

477 478
	if (ring->funcs->emit_vm_flush && (job->vm_needs_flush ||
	    amdgpu_vm_is_gpu_reset(adev, id))) {
479
		struct dma_fence *fence;
480

481 482
		trace_amdgpu_vm_flush(job->vm_pd_addr, ring->idx, job->vm_id);
		amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
483

484 485 486 487
		r = amdgpu_fence_emit(ring, &fence);
		if (r)
			return r;

488
		mutex_lock(&adev->vm_manager.lock);
489
		dma_fence_put(id->last_flush);
490
		id->last_flush = fence;
491
		mutex_unlock(&adev->vm_manager.lock);
A
Alex Deucher 已提交
492
	}
493

494
	if (gds_switch_needed) {
495 496 497 498 499 500 501 502 503 504
		id->gds_base = job->gds_base;
		id->gds_size = job->gds_size;
		id->gws_base = job->gws_base;
		id->gws_size = job->gws_size;
		id->oa_base = job->oa_base;
		id->oa_size = job->oa_size;
		amdgpu_ring_emit_gds_switch(ring, job->vm_id,
					    job->gds_base, job->gds_size,
					    job->gws_base, job->gws_size,
					    job->oa_base, job->oa_size);
505
	}
506 507

	return 0;
508 509 510 511 512 513 514 515 516 517 518 519
}

/**
 * amdgpu_vm_reset_id - reset VMID to zero
 *
 * @adev: amdgpu device structure
 * @vm_id: vmid number to use
 *
 * Reset saved GDW, GWS and OA to force switch on next flush.
 */
void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id)
{
520 521 522 523 524 525 526 527
	struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id];

	id->gds_base = 0;
	id->gds_size = 0;
	id->gws_base = 0;
	id->gws_size = 0;
	id->oa_base = 0;
	id->oa_size = 0;
A
Alex Deucher 已提交
528 529 530 531 532 533 534 535
}

/**
 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
 *
 * @vm: requested vm
 * @bo: requested buffer object
 *
536
 * Find @bo inside the requested vm.
A
Alex Deucher 已提交
537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
 * Search inside the @bos vm list for the requested vm
 * Returns the found bo_va or NULL if none is found
 *
 * Object has to be reserved!
 */
struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
				       struct amdgpu_bo *bo)
{
	struct amdgpu_bo_va *bo_va;

	list_for_each_entry(bo_va, &bo->va, bo_list) {
		if (bo_va->vm == vm) {
			return bo_va;
		}
	}
	return NULL;
}

/**
556
 * amdgpu_vm_do_set_ptes - helper to call the right asic function
A
Alex Deucher 已提交
557
 *
558
 * @params: see amdgpu_pte_update_params definition
A
Alex Deucher 已提交
559 560 561 562 563 564 565 566 567
 * @pe: addr of the page entry
 * @addr: dst addr to write into pe
 * @count: number of page entries to update
 * @incr: increase next addr by incr bytes
 * @flags: hw access flags
 *
 * Traces the parameters and calls the right asic functions
 * to setup the page table using the DMA.
 */
568 569 570
static void amdgpu_vm_do_set_ptes(struct amdgpu_pte_update_params *params,
				  uint64_t pe, uint64_t addr,
				  unsigned count, uint32_t incr,
571
				  uint64_t flags)
A
Alex Deucher 已提交
572
{
573
	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags);
A
Alex Deucher 已提交
574

575
	if (count < 3) {
576 577
		amdgpu_vm_write_pte(params->adev, params->ib, pe,
				    addr | flags, count, incr);
A
Alex Deucher 已提交
578 579

	} else {
580
		amdgpu_vm_set_pte_pde(params->adev, params->ib, pe, addr,
A
Alex Deucher 已提交
581 582 583 584
				      count, incr, flags);
	}
}

585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
/**
 * amdgpu_vm_do_copy_ptes - copy the PTEs from the GART
 *
 * @params: see amdgpu_pte_update_params definition
 * @pe: addr of the page entry
 * @addr: dst addr to write into pe
 * @count: number of page entries to update
 * @incr: increase next addr by incr bytes
 * @flags: hw access flags
 *
 * Traces the parameters and calls the DMA function to copy the PTEs.
 */
static void amdgpu_vm_do_copy_ptes(struct amdgpu_pte_update_params *params,
				   uint64_t pe, uint64_t addr,
				   unsigned count, uint32_t incr,
600
				   uint64_t flags)
601
{
602
	uint64_t src = (params->src + (addr >> 12) * 8);
603

604 605 606 607

	trace_amdgpu_vm_copy_ptes(pe, src, count);

	amdgpu_vm_copy_pte(params->adev, params->ib, pe, src, count);
608 609
}

A
Alex Deucher 已提交
610
/**
611
 * amdgpu_vm_map_gart - Resolve gart mapping of addr
A
Alex Deucher 已提交
612
 *
613
 * @pages_addr: optional DMA address to use for lookup
A
Alex Deucher 已提交
614 615 616
 * @addr: the unmapped addr
 *
 * Look up the physical address of the page that the pte resolves
617
 * to and return the pointer for the page table entry.
A
Alex Deucher 已提交
618
 */
619
static uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
A
Alex Deucher 已提交
620 621 622
{
	uint64_t result;

623 624
	/* page table offset */
	result = pages_addr[addr >> PAGE_SHIFT];
625

626 627
	/* in case cpu page size != gpu page size*/
	result |= addr & (~PAGE_MASK);
A
Alex Deucher 已提交
628

629
	result &= 0xFFFFFFFFFFFFF000ULL;
A
Alex Deucher 已提交
630 631 632 633

	return result;
}

634 635 636 637 638 639 640 641 642 643 644 645 646 647
/*
 * amdgpu_vm_update_pdes - make sure that page directory is valid
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 * @start: start of GPU address range
 * @end: end of GPU address range
 *
 * Allocates new page tables if necessary
 * and updates the page directory.
 * Returns 0 for success, error for failure.
 */
int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
				    struct amdgpu_vm *vm)
A
Alex Deucher 已提交
648
{
649
	struct amdgpu_bo *shadow;
650
	struct amdgpu_ring *ring;
651
	uint64_t pd_addr, shadow_addr;
A
Alex Deucher 已提交
652
	uint32_t incr = AMDGPU_VM_PTE_COUNT * 8;
653
	uint64_t last_pde = ~0, last_pt = ~0, last_shadow = ~0;
A
Alex Deucher 已提交
654
	unsigned count = 0, pt_idx, ndw;
655
	struct amdgpu_job *job;
656
	struct amdgpu_pte_update_params params;
657
	struct dma_fence *fence = NULL;
C
Chunming Zhou 已提交
658

A
Alex Deucher 已提交
659 660
	int r;

661
	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
662
	shadow = vm->page_directory->shadow;
663

A
Alex Deucher 已提交
664 665 666 667 668 669
	/* padding, etc. */
	ndw = 64;

	/* assume the worst case */
	ndw += vm->max_pde_used * 6;

670 671 672 673 674 675 676 677 678 679 680
	pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
	if (shadow) {
		r = amdgpu_ttm_bind(&shadow->tbo, &shadow->tbo.mem);
		if (r)
			return r;
		shadow_addr = amdgpu_bo_gpu_offset(shadow);
		ndw *= 2;
	} else {
		shadow_addr = 0;
	}

681 682
	r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
	if (r)
A
Alex Deucher 已提交
683
		return r;
684

685 686
	memset(&params, 0, sizeof(params));
	params.adev = adev;
687
	params.ib = &job->ibs[0];
A
Alex Deucher 已提交
688 689 690

	/* walk over the address space and update the page directory */
	for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
691
		struct amdgpu_bo *bo = vm->page_tables[pt_idx].bo;
A
Alex Deucher 已提交
692 693 694 695 696
		uint64_t pde, pt;

		if (bo == NULL)
			continue;

697
		if (bo->shadow) {
698
			struct amdgpu_bo *pt_shadow = bo->shadow;
699

700 701
			r = amdgpu_ttm_bind(&pt_shadow->tbo,
					    &pt_shadow->tbo.mem);
702 703 704 705
			if (r)
				return r;
		}

A
Alex Deucher 已提交
706
		pt = amdgpu_bo_gpu_offset(bo);
707 708 709 710
		if (vm->page_tables[pt_idx].addr == pt)
			continue;

		vm->page_tables[pt_idx].addr = pt;
A
Alex Deucher 已提交
711 712 713

		pde = pd_addr + pt_idx * 8;
		if (((last_pde + 8 * count) != pde) ||
714 715
		    ((last_pt + incr * count) != pt) ||
		    (count == AMDGPU_VM_MAX_UPDATE_SIZE)) {
A
Alex Deucher 已提交
716 717

			if (count) {
718 719 720 721 722 723 724
				if (shadow)
					amdgpu_vm_do_set_ptes(&params,
							      last_shadow,
							      last_pt, count,
							      incr,
							      AMDGPU_PTE_VALID);

725 726 727
				amdgpu_vm_do_set_ptes(&params, last_pde,
						      last_pt, count, incr,
						      AMDGPU_PTE_VALID);
A
Alex Deucher 已提交
728 729 730 731
			}

			count = 1;
			last_pde = pde;
732
			last_shadow = shadow_addr + pt_idx * 8;
A
Alex Deucher 已提交
733 734 735 736 737 738
			last_pt = pt;
		} else {
			++count;
		}
	}

739 740 741 742 743
	if (count) {
		if (vm->page_directory->shadow)
			amdgpu_vm_do_set_ptes(&params, last_shadow, last_pt,
					      count, incr, AMDGPU_PTE_VALID);

744 745
		amdgpu_vm_do_set_ptes(&params, last_pde, last_pt,
				      count, incr, AMDGPU_PTE_VALID);
746
	}
A
Alex Deucher 已提交
747

748 749 750 751 752 753 754 755 756 757
	if (params.ib->length_dw == 0) {
		amdgpu_job_free(job);
		return 0;
	}

	amdgpu_ring_pad_ib(ring, params.ib);
	amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
			 AMDGPU_FENCE_OWNER_VM);
	if (shadow)
		amdgpu_sync_resv(adev, &job->sync, shadow->tbo.resv,
758
				 AMDGPU_FENCE_OWNER_VM);
759

760 761 762 763 764
	WARN_ON(params.ib->length_dw > ndw);
	r = amdgpu_job_submit(job, ring, &vm->entity,
			      AMDGPU_FENCE_OWNER_VM, &fence);
	if (r)
		goto error_free;
C
Chunming Zhou 已提交
765

766
	amdgpu_bo_fence(vm->page_directory, fence, true);
767 768 769
	dma_fence_put(vm->page_directory_fence);
	vm->page_directory_fence = dma_fence_get(fence);
	dma_fence_put(fence);
A
Alex Deucher 已提交
770 771

	return 0;
C
Chunming Zhou 已提交
772 773

error_free:
774
	amdgpu_job_free(job);
775
	return r;
A
Alex Deucher 已提交
776 777 778 779 780
}

/**
 * amdgpu_vm_update_ptes - make sure that page tables are valid
 *
781
 * @params: see amdgpu_pte_update_params definition
A
Alex Deucher 已提交
782 783 784
 * @vm: requested vm
 * @start: start of GPU address range
 * @end: end of GPU address range
785
 * @dst: destination address to map to, the next dst inside the function
A
Alex Deucher 已提交
786 787
 * @flags: mapping flags
 *
788
 * Update the page tables in the range @start - @end.
A
Alex Deucher 已提交
789
 */
790
static void amdgpu_vm_update_ptes(struct amdgpu_pte_update_params *params,
791 792
				  struct amdgpu_vm *vm,
				  uint64_t start, uint64_t end,
793
				  uint64_t dst, uint64_t flags)
A
Alex Deucher 已提交
794
{
795 796
	const uint64_t mask = AMDGPU_VM_PTE_COUNT - 1;

797
	uint64_t cur_pe_start, cur_nptes, cur_dst;
798
	uint64_t addr; /* next GPU address to be updated */
799 800 801 802 803 804 805 806
	uint64_t pt_idx;
	struct amdgpu_bo *pt;
	unsigned nptes; /* next number of ptes to be updated */
	uint64_t next_pe_start;

	/* initialize the variables */
	addr = start;
	pt_idx = addr >> amdgpu_vm_block_size;
807
	pt = vm->page_tables[pt_idx].bo;
808 809 810
	if (params->shadow) {
		if (!pt->shadow)
			return;
811
		pt = pt->shadow;
812
	}
813 814 815 816 817 818 819
	if ((addr & ~mask) == (end & ~mask))
		nptes = end - addr;
	else
		nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);

	cur_pe_start = amdgpu_bo_gpu_offset(pt);
	cur_pe_start += (addr & mask) * 8;
820
	cur_nptes = nptes;
821 822 823 824 825
	cur_dst = dst;

	/* for next ptb*/
	addr += nptes;
	dst += nptes * AMDGPU_GPU_PAGE_SIZE;
A
Alex Deucher 已提交
826 827

	/* walk over the address space and update the page tables */
828 829
	while (addr < end) {
		pt_idx = addr >> amdgpu_vm_block_size;
830
		pt = vm->page_tables[pt_idx].bo;
831 832 833
		if (params->shadow) {
			if (!pt->shadow)
				return;
834
			pt = pt->shadow;
835
		}
A
Alex Deucher 已提交
836 837 838 839 840 841

		if ((addr & ~mask) == (end & ~mask))
			nptes = end - addr;
		else
			nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);

842 843
		next_pe_start = amdgpu_bo_gpu_offset(pt);
		next_pe_start += (addr & mask) * 8;
A
Alex Deucher 已提交
844

845 846
		if ((cur_pe_start + 8 * cur_nptes) == next_pe_start &&
		    ((cur_nptes + nptes) <= AMDGPU_VM_MAX_UPDATE_SIZE)) {
847
			/* The next ptb is consecutive to current ptb.
848
			 * Don't call the update function now.
849 850
			 * Will update two ptbs together in future.
			*/
851
			cur_nptes += nptes;
852
		} else {
853 854
			params->func(params, cur_pe_start, cur_dst, cur_nptes,
				     AMDGPU_GPU_PAGE_SIZE, flags);
A
Alex Deucher 已提交
855

856
			cur_pe_start = next_pe_start;
857
			cur_nptes = nptes;
858
			cur_dst = dst;
A
Alex Deucher 已提交
859 860
		}

861
		/* for next ptb*/
A
Alex Deucher 已提交
862 863 864 865
		addr += nptes;
		dst += nptes * AMDGPU_GPU_PAGE_SIZE;
	}

866 867
	params->func(params, cur_pe_start, cur_dst, cur_nptes,
		     AMDGPU_GPU_PAGE_SIZE, flags);
868 869 870 871 872 873 874 875 876 877 878 879 880 881 882
}

/*
 * amdgpu_vm_frag_ptes - add fragment information to PTEs
 *
 * @params: see amdgpu_pte_update_params definition
 * @vm: requested vm
 * @start: first PTE to handle
 * @end: last PTE to handle
 * @dst: addr those PTEs should point to
 * @flags: hw mapping flags
 */
static void amdgpu_vm_frag_ptes(struct amdgpu_pte_update_params	*params,
				struct amdgpu_vm *vm,
				uint64_t start, uint64_t end,
883
				uint64_t dst, uint64_t flags)
884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903
{
	/**
	 * The MC L1 TLB supports variable sized pages, based on a fragment
	 * field in the PTE. When this field is set to a non-zero value, page
	 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
	 * flags are considered valid for all PTEs within the fragment range
	 * and corresponding mappings are assumed to be physically contiguous.
	 *
	 * The L1 TLB can store a single PTE for the whole fragment,
	 * significantly increasing the space available for translation
	 * caching. This leads to large improvements in throughput when the
	 * TLB is under pressure.
	 *
	 * The L2 TLB distributes small and large fragments into two
	 * asymmetric partitions. The large fragment cache is significantly
	 * larger. Thus, we try to use large fragments wherever possible.
	 * Userspace can support this by aligning virtual base address and
	 * allocation size to the fragment size.
	 */

904 905 906
	/* SI and newer are optimized for 64KB */
	uint64_t frag_flags = AMDGPU_PTE_FRAG(AMDGPU_LOG2_PAGES_PER_FRAG);
	uint64_t frag_align = 1 << AMDGPU_LOG2_PAGES_PER_FRAG;
907 908 909 910 911

	uint64_t frag_start = ALIGN(start, frag_align);
	uint64_t frag_end = end & ~(frag_align - 1);

	/* system pages are non continuously */
912
	if (params->src || !(flags & AMDGPU_PTE_VALID) ||
913 914 915 916 917 918 919 920 921 922 923 924 925 926 927
	    (frag_start >= frag_end)) {

		amdgpu_vm_update_ptes(params, vm, start, end, dst, flags);
		return;
	}

	/* handle the 4K area at the beginning */
	if (start != frag_start) {
		amdgpu_vm_update_ptes(params, vm, start, frag_start,
				      dst, flags);
		dst += (frag_start - start) * AMDGPU_GPU_PAGE_SIZE;
	}

	/* handle the area in the middle */
	amdgpu_vm_update_ptes(params, vm, frag_start, frag_end, dst,
928
			      flags | frag_flags);
929 930 931 932 933 934

	/* handle the 4K area at the end */
	if (frag_end != end) {
		dst += (frag_end - frag_start) * AMDGPU_GPU_PAGE_SIZE;
		amdgpu_vm_update_ptes(params, vm, frag_end, end, dst, flags);
	}
A
Alex Deucher 已提交
935 936 937 938 939 940
}

/**
 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
 *
 * @adev: amdgpu_device pointer
941
 * @exclusive: fence we need to sync to
942 943
 * @src: address where to copy page table entries from
 * @pages_addr: DMA addresses to use for mapping
A
Alex Deucher 已提交
944
 * @vm: requested vm
945 946 947
 * @start: start of mapped range
 * @last: last mapped entry
 * @flags: flags for the entries
A
Alex Deucher 已提交
948 949 950
 * @addr: addr to set the area to
 * @fence: optional resulting fence
 *
951
 * Fill in the page table entries between @start and @last.
A
Alex Deucher 已提交
952 953 954
 * Returns 0 for success, -EINVAL for failure.
 */
static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
955
				       struct dma_fence *exclusive,
956 957
				       uint64_t src,
				       dma_addr_t *pages_addr,
A
Alex Deucher 已提交
958
				       struct amdgpu_vm *vm,
959
				       uint64_t start, uint64_t last,
960
				       uint64_t flags, uint64_t addr,
961
				       struct dma_fence **fence)
A
Alex Deucher 已提交
962
{
963
	struct amdgpu_ring *ring;
964
	void *owner = AMDGPU_FENCE_OWNER_VM;
A
Alex Deucher 已提交
965
	unsigned nptes, ncmds, ndw;
966
	struct amdgpu_job *job;
967
	struct amdgpu_pte_update_params params;
968
	struct dma_fence *f = NULL;
A
Alex Deucher 已提交
969 970
	int r;

971 972 973 974
	memset(&params, 0, sizeof(params));
	params.adev = adev;
	params.src = src;

975
	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);
976

977
	memset(&params, 0, sizeof(params));
978
	params.adev = adev;
979
	params.src = src;
980

981 982 983 984
	/* sync to everything on unmapping */
	if (!(flags & AMDGPU_PTE_VALID))
		owner = AMDGPU_FENCE_OWNER_UNDEFINED;

985
	nptes = last - start + 1;
A
Alex Deucher 已提交
986 987 988 989 990 991 992 993 994 995

	/*
	 * reserve space for one command every (1 << BLOCK_SIZE)
	 *  entries or 2k dwords (whatever is smaller)
	 */
	ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1;

	/* padding, etc. */
	ndw = 64;

996
	if (src) {
A
Alex Deucher 已提交
997 998 999
		/* only copy commands needed */
		ndw += ncmds * 7;

1000 1001
		params.func = amdgpu_vm_do_copy_ptes;

1002 1003 1004
	} else if (pages_addr) {
		/* copy commands needed */
		ndw += ncmds * 7;
A
Alex Deucher 已提交
1005

1006
		/* and also PTEs */
A
Alex Deucher 已提交
1007 1008
		ndw += nptes * 2;

1009 1010
		params.func = amdgpu_vm_do_copy_ptes;

A
Alex Deucher 已提交
1011 1012 1013 1014 1015 1016
	} else {
		/* set page commands needed */
		ndw += ncmds * 10;

		/* two extra commands for begin/end of fragment */
		ndw += 2 * 10;
1017 1018

		params.func = amdgpu_vm_do_set_ptes;
A
Alex Deucher 已提交
1019 1020
	}

1021 1022
	r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
	if (r)
A
Alex Deucher 已提交
1023
		return r;
1024

1025
	params.ib = &job->ibs[0];
C
Chunming Zhou 已提交
1026

1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
	if (!src && pages_addr) {
		uint64_t *pte;
		unsigned i;

		/* Put the PTEs at the end of the IB. */
		i = ndw - nptes * 2;
		pte= (uint64_t *)&(job->ibs->ptr[i]);
		params.src = job->ibs->gpu_addr + i * 4;

		for (i = 0; i < nptes; ++i) {
			pte[i] = amdgpu_vm_map_gart(pages_addr, addr + i *
						    AMDGPU_GPU_PAGE_SIZE);
			pte[i] |= flags;
		}
1041
		addr = 0;
1042 1043
	}

1044 1045 1046 1047
	r = amdgpu_sync_fence(adev, &job->sync, exclusive);
	if (r)
		goto error_free;

1048
	r = amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
1049 1050 1051
			     owner);
	if (r)
		goto error_free;
A
Alex Deucher 已提交
1052

1053 1054 1055 1056
	r = reservation_object_reserve_shared(vm->page_directory->tbo.resv);
	if (r)
		goto error_free;

1057 1058 1059
	params.shadow = true;
	amdgpu_vm_frag_ptes(&params, vm, start, last + 1, addr, flags);
	params.shadow = false;
1060
	amdgpu_vm_frag_ptes(&params, vm, start, last + 1, addr, flags);
A
Alex Deucher 已提交
1061

1062 1063
	amdgpu_ring_pad_ib(ring, params.ib);
	WARN_ON(params.ib->length_dw > ndw);
1064 1065
	r = amdgpu_job_submit(job, ring, &vm->entity,
			      AMDGPU_FENCE_OWNER_VM, &f);
1066 1067
	if (r)
		goto error_free;
A
Alex Deucher 已提交
1068

1069
	amdgpu_bo_fence(vm->page_directory, f, true);
1070 1071
	dma_fence_put(*fence);
	*fence = f;
A
Alex Deucher 已提交
1072
	return 0;
C
Chunming Zhou 已提交
1073 1074

error_free:
1075
	amdgpu_job_free(job);
1076
	return r;
A
Alex Deucher 已提交
1077 1078
}

1079 1080 1081 1082
/**
 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
 *
 * @adev: amdgpu_device pointer
1083
 * @exclusive: fence we need to sync to
1084 1085
 * @gtt_flags: flags as they are used for GTT
 * @pages_addr: DMA addresses to use for mapping
1086 1087
 * @vm: requested vm
 * @mapping: mapped range and flags to use for the update
1088
 * @flags: HW flags for the mapping
1089
 * @nodes: array of drm_mm_nodes with the MC addresses
1090 1091 1092 1093 1094 1095 1096
 * @fence: optional resulting fence
 *
 * Split the mapping into smaller chunks so that each update fits
 * into a SDMA IB.
 * Returns 0 for success, -EINVAL for failure.
 */
static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1097
				      struct dma_fence *exclusive,
1098
				      uint64_t gtt_flags,
1099
				      dma_addr_t *pages_addr,
1100 1101
				      struct amdgpu_vm *vm,
				      struct amdgpu_bo_va_mapping *mapping,
1102
				      uint64_t flags,
1103
				      struct drm_mm_node *nodes,
1104
				      struct dma_fence **fence)
1105
{
1106
	uint64_t pfn, src = 0, start = mapping->it.start;
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118
	int r;

	/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
	 * but in case of something, we filter the flags in first place
	 */
	if (!(mapping->flags & AMDGPU_PTE_READABLE))
		flags &= ~AMDGPU_PTE_READABLE;
	if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
		flags &= ~AMDGPU_PTE_WRITEABLE;

	trace_amdgpu_vm_bo_update(mapping);

1119 1120 1121 1122 1123 1124
	pfn = mapping->offset >> PAGE_SHIFT;
	if (nodes) {
		while (pfn >= nodes->size) {
			pfn -= nodes->size;
			++nodes;
		}
1125
	}
1126

1127 1128 1129
	do {
		uint64_t max_entries;
		uint64_t addr, last;
1130

1131 1132 1133 1134 1135 1136 1137 1138
		if (nodes) {
			addr = nodes->start << PAGE_SHIFT;
			max_entries = (nodes->size - pfn) *
				(PAGE_SIZE / AMDGPU_GPU_PAGE_SIZE);
		} else {
			addr = 0;
			max_entries = S64_MAX;
		}
1139

1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
		if (pages_addr) {
			if (flags == gtt_flags)
				src = adev->gart.table_addr +
					(addr >> AMDGPU_GPU_PAGE_SHIFT) * 8;
			else
				max_entries = min(max_entries, 16ull * 1024ull);
			addr = 0;
		} else if (flags & AMDGPU_PTE_VALID) {
			addr += adev->vm_manager.vram_base_offset;
		}
		addr += pfn << PAGE_SHIFT;

		last = min((uint64_t)mapping->it.last, start + max_entries - 1);
1153 1154
		r = amdgpu_vm_bo_update_mapping(adev, exclusive,
						src, pages_addr, vm,
1155 1156 1157 1158 1159
						start, last, flags, addr,
						fence);
		if (r)
			return r;

1160 1161 1162 1163 1164
		pfn += last - start + 1;
		if (nodes && nodes->size == pfn) {
			pfn = 0;
			++nodes;
		}
1165
		start = last + 1;
1166 1167

	} while (unlikely(start != mapping->it.last + 1));
1168 1169 1170 1171

	return 0;
}

A
Alex Deucher 已提交
1172 1173 1174 1175 1176
/**
 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
 *
 * @adev: amdgpu_device pointer
 * @bo_va: requested BO and VM object
1177
 * @clear: if true clear the entries
A
Alex Deucher 已提交
1178 1179 1180 1181 1182 1183
 *
 * Fill in the page table entries for @bo_va.
 * Returns 0 for success, -EINVAL for failure.
 */
int amdgpu_vm_bo_update(struct amdgpu_device *adev,
			struct amdgpu_bo_va *bo_va,
1184
			bool clear)
A
Alex Deucher 已提交
1185 1186 1187
{
	struct amdgpu_vm *vm = bo_va->vm;
	struct amdgpu_bo_va_mapping *mapping;
1188
	dma_addr_t *pages_addr = NULL;
1189
	uint64_t gtt_flags, flags;
1190
	struct ttm_mem_reg *mem;
1191
	struct drm_mm_node *nodes;
1192
	struct dma_fence *exclusive;
A
Alex Deucher 已提交
1193 1194
	int r;

1195
	if (clear || !bo_va->bo) {
1196
		mem = NULL;
1197
		nodes = NULL;
1198 1199
		exclusive = NULL;
	} else {
1200 1201
		struct ttm_dma_tt *ttm;

1202
		mem = &bo_va->bo->tbo.mem;
1203 1204
		nodes = mem->mm_node;
		if (mem->mem_type == TTM_PL_TT) {
1205 1206 1207
			ttm = container_of(bo_va->bo->tbo.ttm, struct
					   ttm_dma_tt, ttm);
			pages_addr = ttm->dma_address;
1208
		}
1209
		exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
A
Alex Deucher 已提交
1210 1211
	}

1212 1213 1214 1215 1216 1217 1218 1219 1220
	if (bo_va->bo) {
		flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
		gtt_flags = (amdgpu_ttm_is_bound(bo_va->bo->tbo.ttm) &&
			adev == amdgpu_ttm_adev(bo_va->bo->tbo.bdev)) ?
			flags : 0;
	} else {
		flags = 0x0;
		gtt_flags = ~0x0;
	}
A
Alex Deucher 已提交
1221

1222 1223 1224 1225 1226 1227
	spin_lock(&vm->status_lock);
	if (!list_empty(&bo_va->vm_status))
		list_splice_init(&bo_va->valids, &bo_va->invalids);
	spin_unlock(&vm->status_lock);

	list_for_each_entry(mapping, &bo_va->invalids, list) {
1228 1229
		r = amdgpu_vm_bo_split_mapping(adev, exclusive,
					       gtt_flags, pages_addr, vm,
1230
					       mapping, flags, nodes,
1231
					       &bo_va->last_pt_update);
A
Alex Deucher 已提交
1232 1233 1234 1235
		if (r)
			return r;
	}

1236 1237 1238 1239 1240 1241 1242 1243
	if (trace_amdgpu_vm_bo_mapping_enabled()) {
		list_for_each_entry(mapping, &bo_va->valids, list)
			trace_amdgpu_vm_bo_mapping(mapping);

		list_for_each_entry(mapping, &bo_va->invalids, list)
			trace_amdgpu_vm_bo_mapping(mapping);
	}

A
Alex Deucher 已提交
1244
	spin_lock(&vm->status_lock);
1245
	list_splice_init(&bo_va->invalids, &bo_va->valids);
A
Alex Deucher 已提交
1246
	list_del_init(&bo_va->vm_status);
1247
	if (clear)
1248
		list_add(&bo_va->vm_status, &vm->cleared);
A
Alex Deucher 已提交
1249 1250 1251 1252 1253
	spin_unlock(&vm->status_lock);

	return 0;
}

1254 1255 1256 1257 1258 1259 1260 1261 1262
/**
 * amdgpu_vm_update_prt_state - update the global PRT state
 */
static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
{
	unsigned long flags;
	bool enable;

	spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1263
	enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1264 1265 1266 1267
	adev->gart.gart_funcs->set_prt(adev, enable);
	spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
}

1268
/**
1269
 * amdgpu_vm_prt_get - add a PRT user
1270 1271 1272
 */
static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
{
1273 1274 1275
	if (!adev->gart.gart_funcs->set_prt)
		return;

1276 1277 1278 1279
	if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
		amdgpu_vm_update_prt_state(adev);
}

1280 1281 1282 1283 1284
/**
 * amdgpu_vm_prt_put - drop a PRT user
 */
static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
{
1285
	if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1286 1287 1288
		amdgpu_vm_update_prt_state(adev);
}

1289
/**
1290
 * amdgpu_vm_prt_cb - callback for updating the PRT status
1291 1292 1293 1294 1295
 */
static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
{
	struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);

1296
	amdgpu_vm_prt_put(cb->adev);
1297 1298 1299
	kfree(cb);
}

1300 1301 1302 1303 1304 1305
/**
 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
 */
static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
				 struct dma_fence *fence)
{
1306
	struct amdgpu_prt_cb *cb;
1307

1308 1309 1310 1311
	if (!adev->gart.gart_funcs->set_prt)
		return;

	cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325
	if (!cb) {
		/* Last resort when we are OOM */
		if (fence)
			dma_fence_wait(fence, false);

		amdgpu_vm_prt_put(cb->adev);
	} else {
		cb->adev = adev;
		if (!fence || dma_fence_add_callback(fence, &cb->cb,
						     amdgpu_vm_prt_cb))
			amdgpu_vm_prt_cb(fence, &cb->cb);
	}
}

1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340
/**
 * amdgpu_vm_free_mapping - free a mapping
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 * @mapping: mapping to be freed
 * @fence: fence of the unmap operation
 *
 * Free a mapping and make sure we decrease the PRT usage count if applicable.
 */
static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
				   struct amdgpu_vm *vm,
				   struct amdgpu_bo_va_mapping *mapping,
				   struct dma_fence *fence)
{
1341 1342 1343 1344
	if (mapping->flags & AMDGPU_PTE_PRT)
		amdgpu_vm_add_prt_cb(adev, fence);
	kfree(mapping);
}
1345

1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
/**
 * amdgpu_vm_prt_fini - finish all prt mappings
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
 * Register a cleanup callback to disable PRT support after VM dies.
 */
static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
{
	struct reservation_object *resv = vm->page_directory->tbo.resv;
	struct dma_fence *excl, **shared;
	unsigned i, shared_count;
	int r;
1360

1361 1362 1363 1364 1365 1366 1367 1368 1369
	r = reservation_object_get_fences_rcu(resv, &excl,
					      &shared_count, &shared);
	if (r) {
		/* Not enough memory to grab the fence list, as last resort
		 * block for all the fences to complete.
		 */
		reservation_object_wait_timeout_rcu(resv, true, false,
						    MAX_SCHEDULE_TIMEOUT);
		return;
1370
	}
1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381

	/* Add a callback for each fence in the reservation object */
	amdgpu_vm_prt_get(adev);
	amdgpu_vm_add_prt_cb(adev, excl);

	for (i = 0; i < shared_count; ++i) {
		amdgpu_vm_prt_get(adev);
		amdgpu_vm_add_prt_cb(adev, shared[i]);
	}

	kfree(shared);
1382 1383
}

A
Alex Deucher 已提交
1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
/**
 * amdgpu_vm_clear_freed - clear freed BOs in the PT
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
 * Make sure all freed BOs are cleared in the PT.
 * Returns 0 for success.
 *
 * PTs have to be reserved and mutex must be locked!
 */
int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
			  struct amdgpu_vm *vm)
{
	struct amdgpu_bo_va_mapping *mapping;
1399
	struct dma_fence *fence = NULL;
A
Alex Deucher 已提交
1400 1401 1402 1403 1404 1405
	int r;

	while (!list_empty(&vm->freed)) {
		mapping = list_first_entry(&vm->freed,
			struct amdgpu_bo_va_mapping, list);
		list_del(&mapping->list);
1406

1407
		r = amdgpu_vm_bo_split_mapping(adev, NULL, 0, NULL, vm, mapping,
1408 1409 1410 1411
					       0, 0, &fence);
		amdgpu_vm_free_mapping(adev, vm, mapping, fence);
		if (r) {
			dma_fence_put(fence);
A
Alex Deucher 已提交
1412
			return r;
1413
		}
A
Alex Deucher 已提交
1414 1415

	}
1416
	dma_fence_put(fence);
A
Alex Deucher 已提交
1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
	return 0;

}

/**
 * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
 * Make sure all invalidated BOs are cleared in the PT.
 * Returns 0 for success.
 *
 * PTs have to be reserved and mutex must be locked!
 */
int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
1433
			     struct amdgpu_vm *vm, struct amdgpu_sync *sync)
A
Alex Deucher 已提交
1434
{
1435
	struct amdgpu_bo_va *bo_va = NULL;
1436
	int r = 0;
A
Alex Deucher 已提交
1437 1438 1439 1440 1441 1442

	spin_lock(&vm->status_lock);
	while (!list_empty(&vm->invalidated)) {
		bo_va = list_first_entry(&vm->invalidated,
			struct amdgpu_bo_va, vm_status);
		spin_unlock(&vm->status_lock);
1443

1444
		r = amdgpu_vm_bo_update(adev, bo_va, true);
A
Alex Deucher 已提交
1445 1446 1447 1448 1449 1450 1451
		if (r)
			return r;

		spin_lock(&vm->status_lock);
	}
	spin_unlock(&vm->status_lock);

1452
	if (bo_va)
1453
		r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
1454 1455

	return r;
A
Alex Deucher 已提交
1456 1457 1458 1459 1460 1461 1462 1463 1464
}

/**
 * amdgpu_vm_bo_add - add a bo to a specific vm
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 * @bo: amdgpu buffer object
 *
1465
 * Add @bo into the requested vm.
A
Alex Deucher 已提交
1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484
 * Add @bo to the list of bos associated with the vm
 * Returns newly added bo_va or NULL for failure
 *
 * Object has to be reserved!
 */
struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
				      struct amdgpu_vm *vm,
				      struct amdgpu_bo *bo)
{
	struct amdgpu_bo_va *bo_va;

	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
	if (bo_va == NULL) {
		return NULL;
	}
	bo_va->vm = vm;
	bo_va->bo = bo;
	bo_va->ref_count = 1;
	INIT_LIST_HEAD(&bo_va->bo_list);
1485 1486
	INIT_LIST_HEAD(&bo_va->valids);
	INIT_LIST_HEAD(&bo_va->invalids);
A
Alex Deucher 已提交
1487
	INIT_LIST_HEAD(&bo_va->vm_status);
1488

1489 1490
	if (bo)
		list_add_tail(&bo_va->bo_list, &bo->va);
A
Alex Deucher 已提交
1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506

	return bo_va;
}

/**
 * amdgpu_vm_bo_map - map bo inside a vm
 *
 * @adev: amdgpu_device pointer
 * @bo_va: bo_va to store the address
 * @saddr: where to map the BO
 * @offset: requested offset in the BO
 * @flags: attributes of pages (read/write/valid/etc.)
 *
 * Add a mapping of the BO at the specefied addr into the VM.
 * Returns 0 for success, error for failure.
 *
1507
 * Object has to be reserved and unreserved outside!
A
Alex Deucher 已提交
1508 1509 1510 1511
 */
int amdgpu_vm_bo_map(struct amdgpu_device *adev,
		     struct amdgpu_bo_va *bo_va,
		     uint64_t saddr, uint64_t offset,
1512
		     uint64_t size, uint64_t flags)
A
Alex Deucher 已提交
1513 1514 1515 1516 1517 1518
{
	struct amdgpu_bo_va_mapping *mapping;
	struct amdgpu_vm *vm = bo_va->vm;
	struct interval_tree_node *it;
	uint64_t eaddr;

1519 1520
	/* validate the parameters */
	if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1521
	    size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1522 1523
		return -EINVAL;

A
Alex Deucher 已提交
1524
	/* make sure object fit at this offset */
1525
	eaddr = saddr + size - 1;
1526 1527
	if (saddr >= eaddr ||
	    (bo_va->bo && offset + size > amdgpu_bo_size(bo_va->bo)))
A
Alex Deucher 已提交
1528 1529 1530 1531 1532
		return -EINVAL;

	saddr /= AMDGPU_GPU_PAGE_SIZE;
	eaddr /= AMDGPU_GPU_PAGE_SIZE;

1533
	it = interval_tree_iter_first(&vm->va, saddr, eaddr);
A
Alex Deucher 已提交
1534 1535 1536 1537 1538 1539 1540
	if (it) {
		struct amdgpu_bo_va_mapping *tmp;
		tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
		/* bo and tmp overlap, invalid addr */
		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
			"0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr,
			tmp->it.start, tmp->it.last + 1);
1541
		return -EINVAL;
A
Alex Deucher 已提交
1542 1543 1544
	}

	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
1545 1546
	if (!mapping)
		return -ENOMEM;
A
Alex Deucher 已提交
1547 1548 1549

	INIT_LIST_HEAD(&mapping->list);
	mapping->it.start = saddr;
1550
	mapping->it.last = eaddr;
A
Alex Deucher 已提交
1551 1552 1553
	mapping->offset = offset;
	mapping->flags = flags;

1554
	list_add(&mapping->list, &bo_va->invalids);
A
Alex Deucher 已提交
1555 1556
	interval_tree_insert(&mapping->it, &vm->va);

1557 1558 1559
	if (flags & AMDGPU_PTE_PRT)
		amdgpu_vm_prt_get(adev);

A
Alex Deucher 已提交
1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572
	return 0;
}

/**
 * amdgpu_vm_bo_unmap - remove bo mapping from vm
 *
 * @adev: amdgpu_device pointer
 * @bo_va: bo_va to remove the address from
 * @saddr: where to the BO is mapped
 *
 * Remove a mapping of the BO at the specefied addr from the VM.
 * Returns 0 for success, error for failure.
 *
1573
 * Object has to be reserved and unreserved outside!
A
Alex Deucher 已提交
1574 1575 1576 1577 1578 1579 1580
 */
int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
		       struct amdgpu_bo_va *bo_va,
		       uint64_t saddr)
{
	struct amdgpu_bo_va_mapping *mapping;
	struct amdgpu_vm *vm = bo_va->vm;
1581
	bool valid = true;
A
Alex Deucher 已提交
1582

1583
	saddr /= AMDGPU_GPU_PAGE_SIZE;
1584

1585
	list_for_each_entry(mapping, &bo_va->valids, list) {
A
Alex Deucher 已提交
1586 1587 1588 1589
		if (mapping->it.start == saddr)
			break;
	}

1590 1591 1592 1593 1594 1595 1596 1597
	if (&mapping->list == &bo_va->valids) {
		valid = false;

		list_for_each_entry(mapping, &bo_va->invalids, list) {
			if (mapping->it.start == saddr)
				break;
		}

1598
		if (&mapping->list == &bo_va->invalids)
1599
			return -ENOENT;
A
Alex Deucher 已提交
1600
	}
1601

A
Alex Deucher 已提交
1602 1603
	list_del(&mapping->list);
	interval_tree_remove(&mapping->it, &vm->va);
1604
	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
A
Alex Deucher 已提交
1605

1606
	if (valid)
A
Alex Deucher 已提交
1607
		list_add(&mapping->list, &vm->freed);
1608
	else
1609 1610
		amdgpu_vm_free_mapping(adev, vm, mapping,
				       bo_va->last_pt_update);
A
Alex Deucher 已提交
1611 1612 1613 1614

	return 0;
}

1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713
/**
 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
 *
 * @adev: amdgpu_device pointer
 * @vm: VM structure to use
 * @saddr: start of the range
 * @size: size of the range
 *
 * Remove all mappings in a range, split them as appropriate.
 * Returns 0 for success, error for failure.
 */
int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
				struct amdgpu_vm *vm,
				uint64_t saddr, uint64_t size)
{
	struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
	struct interval_tree_node *it;
	LIST_HEAD(removed);
	uint64_t eaddr;

	eaddr = saddr + size - 1;
	saddr /= AMDGPU_GPU_PAGE_SIZE;
	eaddr /= AMDGPU_GPU_PAGE_SIZE;

	/* Allocate all the needed memory */
	before = kzalloc(sizeof(*before), GFP_KERNEL);
	if (!before)
		return -ENOMEM;

	after = kzalloc(sizeof(*after), GFP_KERNEL);
	if (!after) {
		kfree(before);
		return -ENOMEM;
	}

	/* Now gather all removed mappings */
	it = interval_tree_iter_first(&vm->va, saddr, eaddr);
	while (it) {
		tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
		it = interval_tree_iter_next(it, saddr, eaddr);

		/* Remember mapping split at the start */
		if (tmp->it.start < saddr) {
			before->it.start = tmp->it.start;;
			before->it.last = saddr - 1;
			before->offset = tmp->offset;
			before->flags = tmp->flags;
			list_add(&before->list, &tmp->list);
		}

		/* Remember mapping split at the end */
		if (tmp->it.last > eaddr) {
			after->it.start = eaddr + 1;
			after->it.last = tmp->it.last;
			after->offset = tmp->offset;
			after->offset += after->it.start - tmp->it.start;
			after->flags = tmp->flags;
			list_add(&after->list, &tmp->list);
		}

		list_del(&tmp->list);
		list_add(&tmp->list, &removed);
	}

	/* And free them up */
	list_for_each_entry_safe(tmp, next, &removed, list) {
		interval_tree_remove(&tmp->it, &vm->va);
		list_del(&tmp->list);

		if (tmp->it.start < saddr)
		    tmp->it.start = saddr;
		if (tmp->it.last > eaddr)
		    tmp->it.last = eaddr;

		list_add(&tmp->list, &vm->freed);
		trace_amdgpu_vm_bo_unmap(NULL, tmp);
	}

	/* Insert partial mapping before the range*/
	if (before->it.start != before->it.last) {
		interval_tree_insert(&before->it, &vm->va);
		if (before->flags & AMDGPU_PTE_PRT)
			amdgpu_vm_prt_get(adev);
	} else {
		kfree(before);
	}

	/* Insert partial mapping after the range */
	if (after->it.start != after->it.last) {
		interval_tree_insert(&after->it, &vm->va);
		if (after->flags & AMDGPU_PTE_PRT)
			amdgpu_vm_prt_get(adev);
	} else {
		kfree(after);
	}

	return 0;
}

A
Alex Deucher 已提交
1714 1715 1716 1717 1718 1719
/**
 * amdgpu_vm_bo_rmv - remove a bo to a specific vm
 *
 * @adev: amdgpu_device pointer
 * @bo_va: requested bo_va
 *
1720
 * Remove @bo_va->bo from the requested vm.
A
Alex Deucher 已提交
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735
 *
 * Object have to be reserved!
 */
void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
		      struct amdgpu_bo_va *bo_va)
{
	struct amdgpu_bo_va_mapping *mapping, *next;
	struct amdgpu_vm *vm = bo_va->vm;

	list_del(&bo_va->bo_list);

	spin_lock(&vm->status_lock);
	list_del(&bo_va->vm_status);
	spin_unlock(&vm->status_lock);

1736
	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
A
Alex Deucher 已提交
1737 1738
		list_del(&mapping->list);
		interval_tree_remove(&mapping->it, &vm->va);
1739
		trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1740 1741 1742 1743 1744
		list_add(&mapping->list, &vm->freed);
	}
	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
		list_del(&mapping->list);
		interval_tree_remove(&mapping->it, &vm->va);
1745 1746
		amdgpu_vm_free_mapping(adev, vm, mapping,
				       bo_va->last_pt_update);
A
Alex Deucher 已提交
1747
	}
1748

1749
	dma_fence_put(bo_va->last_pt_update);
A
Alex Deucher 已提交
1750 1751 1752 1753 1754 1755 1756 1757 1758 1759
	kfree(bo_va);
}

/**
 * amdgpu_vm_bo_invalidate - mark the bo as invalid
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 * @bo: amdgpu buffer object
 *
1760
 * Mark @bo as invalid.
A
Alex Deucher 已提交
1761 1762 1763 1764 1765 1766 1767
 */
void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
			     struct amdgpu_bo *bo)
{
	struct amdgpu_bo_va *bo_va;

	list_for_each_entry(bo_va, &bo->va, bo_list) {
1768 1769
		spin_lock(&bo_va->vm->status_lock);
		if (list_empty(&bo_va->vm_status))
A
Alex Deucher 已提交
1770
			list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
1771
		spin_unlock(&bo_va->vm->status_lock);
A
Alex Deucher 已提交
1772 1773 1774 1775 1776 1777 1778 1779 1780
	}
}

/**
 * amdgpu_vm_init - initialize a vm instance
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
1781
 * Init @vm fields.
A
Alex Deucher 已提交
1782 1783 1784 1785 1786
 */
int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
{
	const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
		AMDGPU_VM_PTE_COUNT * 8);
1787
	unsigned pd_size, pd_entries;
1788 1789
	unsigned ring_instance;
	struct amdgpu_ring *ring;
1790
	struct amd_sched_rq *rq;
A
Alex Deucher 已提交
1791 1792
	int i, r;

1793 1794
	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
		vm->ids[i] = NULL;
A
Alex Deucher 已提交
1795
	vm->va = RB_ROOT;
1796
	vm->client_id = atomic64_inc_return(&adev->vm_manager.client_counter);
A
Alex Deucher 已提交
1797 1798
	spin_lock_init(&vm->status_lock);
	INIT_LIST_HEAD(&vm->invalidated);
1799
	INIT_LIST_HEAD(&vm->cleared);
A
Alex Deucher 已提交
1800
	INIT_LIST_HEAD(&vm->freed);
1801

A
Alex Deucher 已提交
1802 1803 1804 1805
	pd_size = amdgpu_vm_directory_size(adev);
	pd_entries = amdgpu_vm_num_pdes(adev);

	/* allocate page table array */
1806
	vm->page_tables = drm_calloc_large(pd_entries, sizeof(struct amdgpu_vm_pt));
A
Alex Deucher 已提交
1807 1808 1809 1810 1811
	if (vm->page_tables == NULL) {
		DRM_ERROR("Cannot allocate memory for page table array\n");
		return -ENOMEM;
	}

1812
	/* create scheduler entity for page table updates */
1813 1814 1815 1816

	ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
	ring_instance %= adev->vm_manager.vm_pte_num_rings;
	ring = adev->vm_manager.vm_pte_rings[ring_instance];
1817 1818 1819 1820
	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
	r = amd_sched_entity_init(&ring->sched, &vm->entity,
				  rq, amdgpu_sched_jobs);
	if (r)
1821
		goto err;
1822

1823 1824
	vm->page_directory_fence = NULL;

A
Alex Deucher 已提交
1825
	r = amdgpu_bo_create(adev, pd_size, align, true,
1826
			     AMDGPU_GEM_DOMAIN_VRAM,
1827
			     AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
1828
			     AMDGPU_GEM_CREATE_SHADOW |
1829 1830
			     AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
			     AMDGPU_GEM_CREATE_VRAM_CLEARED,
1831
			     NULL, NULL, &vm->page_directory);
A
Alex Deucher 已提交
1832
	if (r)
1833 1834
		goto error_free_sched_entity;

1835
	r = amdgpu_bo_reserve(vm->page_directory, false);
1836 1837 1838
	if (r)
		goto error_free_page_directory;

1839
	vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
1840
	amdgpu_bo_unreserve(vm->page_directory);
A
Alex Deucher 已提交
1841 1842

	return 0;
1843 1844

error_free_page_directory:
1845
	amdgpu_bo_unref(&vm->page_directory->shadow);
1846 1847 1848 1849 1850 1851
	amdgpu_bo_unref(&vm->page_directory);
	vm->page_directory = NULL;

error_free_sched_entity:
	amd_sched_entity_fini(&ring->sched, &vm->entity);

1852 1853 1854
err:
	drm_free_large(vm->page_tables);

1855
	return r;
A
Alex Deucher 已提交
1856 1857 1858 1859 1860 1861 1862 1863
}

/**
 * amdgpu_vm_fini - tear down a vm instance
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
1864
 * Tear down @vm.
A
Alex Deucher 已提交
1865 1866 1867 1868 1869
 * Unbind the VM and remove all bos from the vm bo list
 */
void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
{
	struct amdgpu_bo_va_mapping *mapping, *tmp;
1870
	bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
A
Alex Deucher 已提交
1871 1872
	int i;

1873
	amd_sched_entity_fini(vm->entity.sched, &vm->entity);
1874

A
Alex Deucher 已提交
1875 1876 1877 1878 1879 1880 1881 1882 1883
	if (!RB_EMPTY_ROOT(&vm->va)) {
		dev_err(adev->dev, "still active bo inside vm\n");
	}
	rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) {
		list_del(&mapping->list);
		interval_tree_remove(&mapping->it, &vm->va);
		kfree(mapping);
	}
	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
1884
		if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
1885
			amdgpu_vm_prt_fini(adev, vm);
1886
			prt_fini_needed = false;
1887
		}
1888

A
Alex Deucher 已提交
1889
		list_del(&mapping->list);
1890
		amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
A
Alex Deucher 已提交
1891 1892
	}

1893
	for (i = 0; i < amdgpu_vm_num_pdes(adev); i++) {
1894
		struct amdgpu_bo *pt = vm->page_tables[i].bo;
1895 1896 1897 1898 1899 1900

		if (!pt)
			continue;

		amdgpu_bo_unref(&pt->shadow);
		amdgpu_bo_unref(&pt);
1901
	}
1902
	drm_free_large(vm->page_tables);
A
Alex Deucher 已提交
1903

1904
	amdgpu_bo_unref(&vm->page_directory->shadow);
A
Alex Deucher 已提交
1905
	amdgpu_bo_unref(&vm->page_directory);
1906
	dma_fence_put(vm->page_directory_fence);
A
Alex Deucher 已提交
1907
}
1908

1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922
/**
 * amdgpu_vm_manager_init - init the VM manager
 *
 * @adev: amdgpu_device pointer
 *
 * Initialize the VM manager structures
 */
void amdgpu_vm_manager_init(struct amdgpu_device *adev)
{
	unsigned i;

	INIT_LIST_HEAD(&adev->vm_manager.ids_lru);

	/* skip over VMID 0, since it is the system VM */
1923 1924
	for (i = 1; i < adev->vm_manager.num_ids; ++i) {
		amdgpu_vm_reset_id(adev, i);
1925
		amdgpu_sync_create(&adev->vm_manager.ids[i].active);
1926 1927
		list_add_tail(&adev->vm_manager.ids[i].list,
			      &adev->vm_manager.ids_lru);
1928
	}
1929

1930 1931
	adev->vm_manager.fence_context =
		dma_fence_context_alloc(AMDGPU_MAX_RINGS);
1932 1933 1934
	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
		adev->vm_manager.seqno[i] = 0;

1935
	atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
1936
	atomic64_set(&adev->vm_manager.client_counter, 0);
1937
	spin_lock_init(&adev->vm_manager.prt_lock);
1938
	atomic_set(&adev->vm_manager.num_prt_users, 0);
1939 1940
}

1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951
/**
 * amdgpu_vm_manager_fini - cleanup VM manager
 *
 * @adev: amdgpu_device pointer
 *
 * Cleanup the VM manager and free resources.
 */
void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
{
	unsigned i;

1952 1953 1954
	for (i = 0; i < AMDGPU_NUM_VM; ++i) {
		struct amdgpu_vm_id *id = &adev->vm_manager.ids[i];

1955
		dma_fence_put(adev->vm_manager.ids[i].first);
1956
		amdgpu_sync_free(&adev->vm_manager.ids[i].active);
1957
		dma_fence_put(id->flushed_updates);
1958
		dma_fence_put(id->last_flush);
1959
	}
1960
}