amdgpu_vm.c 37.8 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
/*
 * Copyright 2008 Advanced Micro Devices, Inc.
 * Copyright 2008 Red Hat Inc.
 * Copyright 2009 Jerome Glisse.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Dave Airlie
 *          Alex Deucher
 *          Jerome Glisse
 */
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "amdgpu_trace.h"

/*
 * GPUVM
 * GPUVM is similar to the legacy gart on older asics, however
 * rather than there being a single global gart table
 * for the entire GPU, there are multiple VM page tables active
 * at any given time.  The VM page tables can contain a mix
 * vram pages and system memory pages and system memory pages
 * can be mapped as snooped (cached system pages) or unsnooped
 * (uncached system pages).
 * Each VM has an ID associated with it and there is a page table
 * associated with each VMID.  When execting a command buffer,
 * the kernel tells the the ring what VMID to use for that command
 * buffer.  VMIDs are allocated dynamically as commands are submitted.
 * The userspace drivers maintain their own address space and the kernel
 * sets up their pages tables accordingly when they submit their
 * command buffers and a VMID is assigned.
 * Cayman/Trinity support up to 8 active VMs at any given time;
 * SI supports 16.
 */

53 54 55
/* Special value that no flush is necessary */
#define AMDGPU_VM_NO_FLUSH (~0ll)

A
Alex Deucher 已提交
56 57 58 59 60
/**
 * amdgpu_vm_num_pde - return the number of page directory entries
 *
 * @adev: amdgpu_device pointer
 *
61
 * Calculate the number of page directory entries.
A
Alex Deucher 已提交
62 63 64 65 66 67 68 69 70 71 72
 */
static unsigned amdgpu_vm_num_pdes(struct amdgpu_device *adev)
{
	return adev->vm_manager.max_pfn >> amdgpu_vm_block_size;
}

/**
 * amdgpu_vm_directory_size - returns the size of the page directory in bytes
 *
 * @adev: amdgpu_device pointer
 *
73
 * Calculate the size of the page directory in bytes.
A
Alex Deucher 已提交
74 75 76 77 78 79 80
 */
static unsigned amdgpu_vm_directory_size(struct amdgpu_device *adev)
{
	return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_pdes(adev) * 8);
}

/**
81
 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
A
Alex Deucher 已提交
82 83
 *
 * @vm: vm providing the BOs
84
 * @validated: head of validation list
85
 * @entry: entry to add
A
Alex Deucher 已提交
86 87
 *
 * Add the page directory to the list of BOs to
88
 * validate for command submission.
A
Alex Deucher 已提交
89
 */
90 91 92
void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
			 struct list_head *validated,
			 struct amdgpu_bo_list_entry *entry)
A
Alex Deucher 已提交
93
{
94 95 96 97
	entry->robj = vm->page_directory;
	entry->priority = 0;
	entry->tv.bo = &vm->page_directory->tbo;
	entry->tv.shared = true;
98
	entry->user_pages = NULL;
99 100
	list_add(&entry->tv.head, validated);
}
A
Alex Deucher 已提交
101

102
/**
103
 * amdgpu_vm_get_bos - add the vm BOs to a duplicates list
104 105
 *
 * @vm: vm providing the BOs
106
 * @duplicates: head of duplicates list
A
Alex Deucher 已提交
107
 *
108 109
 * Add the page directory to the BO duplicates list
 * for command submission.
A
Alex Deucher 已提交
110
 */
111
void amdgpu_vm_get_pt_bos(struct amdgpu_vm *vm, struct list_head *duplicates)
A
Alex Deucher 已提交
112
{
113
	unsigned i;
A
Alex Deucher 已提交
114 115

	/* add the vm page table to the list */
116 117 118 119
	for (i = 0; i <= vm->max_pde_used; ++i) {
		struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;

		if (!entry->robj)
A
Alex Deucher 已提交
120 121
			continue;

122
		list_add(&entry->tv.head, duplicates);
A
Alex Deucher 已提交
123
	}
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150

}

/**
 * amdgpu_vm_move_pt_bos_in_lru - move the PT BOs to the LRU tail
 *
 * @adev: amdgpu device instance
 * @vm: vm providing the BOs
 *
 * Move the PT BOs to the tail of the LRU.
 */
void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
				  struct amdgpu_vm *vm)
{
	struct ttm_bo_global *glob = adev->mman.bdev.glob;
	unsigned i;

	spin_lock(&glob->lru_lock);
	for (i = 0; i <= vm->max_pde_used; ++i) {
		struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;

		if (!entry->robj)
			continue;

		ttm_bo_move_to_lru_tail(&entry->robj->tbo);
	}
	spin_unlock(&glob->lru_lock);
A
Alex Deucher 已提交
151 152 153 154 155 156
}

/**
 * amdgpu_vm_grab_id - allocate the next free VMID
 *
 * @vm: vm to allocate id for
157 158
 * @ring: ring we want to submit job to
 * @sync: sync object where we add dependencies
159
 * @fence: fence protecting ID from reuse
A
Alex Deucher 已提交
160
 *
161
 * Allocate an id for the vm, adding fences to the sync obj as necessary.
A
Alex Deucher 已提交
162
 */
163
int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
164 165
		      struct amdgpu_sync *sync, struct fence *fence,
		      unsigned *vm_id, uint64_t *vm_pd_addr)
A
Alex Deucher 已提交
166
{
167
	uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
A
Alex Deucher 已提交
168
	struct amdgpu_device *adev = ring->adev;
169
	struct amdgpu_vm_id *id = vm->ids[ring->idx];
170
	struct fence *updates = sync->last_vm_update;
171
	int r;
A
Alex Deucher 已提交
172

173 174
	mutex_lock(&adev->vm_manager.lock);

A
Alex Deucher 已提交
175
	/* check if the id is still valid */
176
	if (id) {
177
		struct fence *flushed = id->flushed_updates;
178 179
		long owner = atomic_long_read(&id->owner);
		bool usable = pd_addr == id->pd_gpu_addr;
180

181 182 183 184
		if (owner != (long)&vm->ids[ring->idx])
			usable = false;
		else if (!flushed)
			usable = false;
185
		else if (!updates)
186
			usable = true;
187
		else
188
			usable = !fence_is_later(updates, flushed);
189

190
		if (usable) {
191

192 193 194
			r = amdgpu_sync_fence(ring->adev, sync, id->first);
			if (r)
				goto error;
195

196 197 198
			r = amdgpu_sync_fence(ring->adev, &id->active, fence);
			if (r)
				goto error;
199

200
			list_move_tail(&id->list, &adev->vm_manager.ids_lru);
A
Alex Deucher 已提交
201

202
			*vm_id = id - adev->vm_manager.ids;
203
			*vm_pd_addr = AMDGPU_VM_NO_FLUSH;
204 205
			trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id,
						*vm_pd_addr);
A
Alex Deucher 已提交
206

207
			mutex_unlock(&adev->vm_manager.lock);
208
			return 0;
A
Alex Deucher 已提交
209 210 211
		}
	}

212 213 214
	id = list_first_entry(&adev->vm_manager.ids_lru,
			      struct amdgpu_vm_id,
			      list);
215

216
	if (!amdgpu_sync_is_idle(&id->active)) {
217
		struct list_head *head = &adev->vm_manager.ids_lru;
218
		struct amdgpu_vm_id *tmp;
219 220 221

		list_for_each_entry_safe(id, tmp, &adev->vm_manager.ids_lru,
					 list) {
222
			if (amdgpu_sync_is_idle(&id->active)) {
223 224
				list_move(&id->list, head);
				head = &id->list;
225 226
			}
		}
227 228 229
		id = list_first_entry(&adev->vm_manager.ids_lru,
				      struct amdgpu_vm_id,
				      list);
230 231
	}

232 233 234
	r = amdgpu_sync_cycle_fences(sync, &id->active, fence);
	if (r)
		goto error;
235

236 237
	fence_put(id->first);
	id->first = fence_get(fence);
238

239 240 241
	fence_put(id->last_flush);
	id->last_flush = NULL;

242 243
	fence_put(id->flushed_updates);
	id->flushed_updates = fence_get(updates);
244

245
	id->pd_gpu_addr = pd_addr;
246

247 248 249
	list_move_tail(&id->list, &adev->vm_manager.ids_lru);
	atomic_long_set(&id->owner, (long)id);
	vm->ids[ring->idx] = id;
A
Alex Deucher 已提交
250

251 252 253 254 255
	*vm_id = id - adev->vm_manager.ids;
	*vm_pd_addr = pd_addr;
	trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);

error:
256
	mutex_unlock(&adev->vm_manager.lock);
257
	return r;
A
Alex Deucher 已提交
258 259 260 261 262 263
}

/**
 * amdgpu_vm_flush - hardware flush the vm
 *
 * @ring: ring to use for flush
264
 * @vm_id: vmid number to use
265
 * @pd_addr: address of the page directory
A
Alex Deucher 已提交
266
 *
267
 * Emit a VM flush when it is necessary.
A
Alex Deucher 已提交
268
 */
269 270 271 272 273
int amdgpu_vm_flush(struct amdgpu_ring *ring,
		    unsigned vm_id, uint64_t pd_addr,
		    uint32_t gds_base, uint32_t gds_size,
		    uint32_t gws_base, uint32_t gws_size,
		    uint32_t oa_base, uint32_t oa_size)
A
Alex Deucher 已提交
274
{
275
	struct amdgpu_device *adev = ring->adev;
276
	struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id];
277
	bool gds_switch_needed = ring->funcs->emit_gds_switch && (
278 279 280 281 282 283
		id->gds_base != gds_base ||
		id->gds_size != gds_size ||
		id->gws_base != gws_base ||
		id->gws_size != gws_size ||
		id->oa_base != oa_base ||
		id->oa_size != oa_size);
284
	int r;
285 286 287 288

	if (ring->funcs->emit_pipeline_sync && (
	    pd_addr != AMDGPU_VM_NO_FLUSH || gds_switch_needed))
		amdgpu_ring_emit_pipeline_sync(ring);
289

290
	if (pd_addr != AMDGPU_VM_NO_FLUSH) {
291 292
		struct fence *fence;

293 294
		trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id);
		amdgpu_ring_emit_vm_flush(ring, vm_id, pd_addr);
295 296 297 298 299 300 301 302
		r = amdgpu_fence_emit(ring, &fence);
		if (r)
			return r;

		mutex_lock(&adev->vm_manager.lock);
		fence_put(id->last_flush);
		id->last_flush = fence;
		mutex_unlock(&adev->vm_manager.lock);
A
Alex Deucher 已提交
303
	}
304

305
	if (gds_switch_needed) {
306 307 308 309 310 311
		id->gds_base = gds_base;
		id->gds_size = gds_size;
		id->gws_base = gws_base;
		id->gws_size = gws_size;
		id->oa_base = oa_base;
		id->oa_size = oa_size;
312 313 314 315
		amdgpu_ring_emit_gds_switch(ring, vm_id,
					    gds_base, gds_size,
					    gws_base, gws_size,
					    oa_base, oa_size);
316
	}
317 318

	return 0;
319 320 321 322 323 324 325 326 327 328 329 330
}

/**
 * amdgpu_vm_reset_id - reset VMID to zero
 *
 * @adev: amdgpu device structure
 * @vm_id: vmid number to use
 *
 * Reset saved GDW, GWS and OA to force switch on next flush.
 */
void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id)
{
331 332 333 334 335 336 337 338
	struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id];

	id->gds_base = 0;
	id->gds_size = 0;
	id->gws_base = 0;
	id->gws_size = 0;
	id->oa_base = 0;
	id->oa_size = 0;
A
Alex Deucher 已提交
339 340 341 342 343 344 345 346
}

/**
 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
 *
 * @vm: requested vm
 * @bo: requested buffer object
 *
347
 * Find @bo inside the requested vm.
A
Alex Deucher 已提交
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
 * Search inside the @bos vm list for the requested vm
 * Returns the found bo_va or NULL if none is found
 *
 * Object has to be reserved!
 */
struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
				       struct amdgpu_bo *bo)
{
	struct amdgpu_bo_va *bo_va;

	list_for_each_entry(bo_va, &bo->va, bo_list) {
		if (bo_va->vm == vm) {
			return bo_va;
		}
	}
	return NULL;
}

/**
 * amdgpu_vm_update_pages - helper to call the right asic function
 *
 * @adev: amdgpu_device pointer
370 371
 * @src: address where to copy page table entries from
 * @pages_addr: DMA addresses to use for mapping
A
Alex Deucher 已提交
372 373 374 375 376 377 378 379 380 381 382
 * @ib: indirect buffer to fill with commands
 * @pe: addr of the page entry
 * @addr: dst addr to write into pe
 * @count: number of page entries to update
 * @incr: increase next addr by incr bytes
 * @flags: hw access flags
 *
 * Traces the parameters and calls the right asic functions
 * to setup the page table using the DMA.
 */
static void amdgpu_vm_update_pages(struct amdgpu_device *adev,
383 384
				   uint64_t src,
				   dma_addr_t *pages_addr,
A
Alex Deucher 已提交
385 386 387
				   struct amdgpu_ib *ib,
				   uint64_t pe, uint64_t addr,
				   unsigned count, uint32_t incr,
388
				   uint32_t flags)
A
Alex Deucher 已提交
389 390 391
{
	trace_amdgpu_vm_set_page(pe, addr, count, incr, flags);

392 393
	if (src) {
		src += (addr >> 12) * 8;
A
Alex Deucher 已提交
394 395
		amdgpu_vm_copy_pte(adev, ib, pe, src, count);

396
	} else if (pages_addr) {
397 398 399 400 401 402
		amdgpu_vm_write_pte(adev, ib, pages_addr, pe, addr,
				    count, incr, flags);

	} else if (count < 3) {
		amdgpu_vm_write_pte(adev, ib, NULL, pe, addr,
				    count, incr, flags);
A
Alex Deucher 已提交
403 404 405 406 407 408 409 410 411 412 413 414

	} else {
		amdgpu_vm_set_pte_pde(adev, ib, pe, addr,
				      count, incr, flags);
	}
}

/**
 * amdgpu_vm_clear_bo - initially clear the page dir/table
 *
 * @adev: amdgpu_device pointer
 * @bo: bo to clear
415 416
 *
 * need to reserve bo first before calling it.
A
Alex Deucher 已提交
417 418
 */
static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
419
			      struct amdgpu_vm *vm,
A
Alex Deucher 已提交
420 421
			      struct amdgpu_bo *bo)
{
422
	struct amdgpu_ring *ring;
423
	struct fence *fence = NULL;
424
	struct amdgpu_job *job;
A
Alex Deucher 已提交
425 426 427 428
	unsigned entries;
	uint64_t addr;
	int r;

429 430
	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);

M
monk.liu 已提交
431 432 433 434
	r = reservation_object_reserve_shared(bo->tbo.resv);
	if (r)
		return r;

A
Alex Deucher 已提交
435 436
	r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
	if (r)
437
		goto error;
A
Alex Deucher 已提交
438 439 440 441

	addr = amdgpu_bo_gpu_offset(bo);
	entries = amdgpu_bo_size(bo) / 8;

442 443
	r = amdgpu_job_alloc_with_ib(adev, 64, &job);
	if (r)
444
		goto error;
A
Alex Deucher 已提交
445

446
	amdgpu_vm_update_pages(adev, 0, NULL, &job->ibs[0], addr, 0, entries,
447 448 449 450
			       0, 0);
	amdgpu_ring_pad_ib(ring, &job->ibs[0]);

	WARN_ON(job->ibs[0].length_dw > 64);
451 452
	r = amdgpu_job_submit(job, ring, &vm->entity,
			      AMDGPU_FENCE_OWNER_VM, &fence);
A
Alex Deucher 已提交
453 454 455
	if (r)
		goto error_free;

456
	amdgpu_bo_fence(bo, fence, true);
457
	fence_put(fence);
458
	return 0;
459

A
Alex Deucher 已提交
460
error_free:
461
	amdgpu_job_free(job);
A
Alex Deucher 已提交
462

463
error:
A
Alex Deucher 已提交
464 465 466 467
	return r;
}

/**
468
 * amdgpu_vm_map_gart - Resolve gart mapping of addr
A
Alex Deucher 已提交
469
 *
470
 * @pages_addr: optional DMA address to use for lookup
A
Alex Deucher 已提交
471 472 473
 * @addr: the unmapped addr
 *
 * Look up the physical address of the page that the pte resolves
474
 * to and return the pointer for the page table entry.
A
Alex Deucher 已提交
475
 */
476
uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
A
Alex Deucher 已提交
477 478 479
{
	uint64_t result;

480 481 482 483 484 485 486 487 488 489 490
	if (pages_addr) {
		/* page table offset */
		result = pages_addr[addr >> PAGE_SHIFT];

		/* in case cpu page size != gpu page size*/
		result |= addr & (~PAGE_MASK);

	} else {
		/* No mapping required */
		result = addr;
	}
A
Alex Deucher 已提交
491

492
	result &= 0xFFFFFFFFFFFFF000ULL;
A
Alex Deucher 已提交
493 494 495 496 497 498 499 500 501 502 503 504 505

	return result;
}

/**
 * amdgpu_vm_update_pdes - make sure that page directory is valid
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 * @start: start of GPU address range
 * @end: end of GPU address range
 *
 * Allocates new page tables if necessary
506
 * and updates the page directory.
A
Alex Deucher 已提交
507 508 509 510 511
 * Returns 0 for success, error for failure.
 */
int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
				    struct amdgpu_vm *vm)
{
512
	struct amdgpu_ring *ring;
A
Alex Deucher 已提交
513 514 515 516 517
	struct amdgpu_bo *pd = vm->page_directory;
	uint64_t pd_addr = amdgpu_bo_gpu_offset(pd);
	uint32_t incr = AMDGPU_VM_PTE_COUNT * 8;
	uint64_t last_pde = ~0, last_pt = ~0;
	unsigned count = 0, pt_idx, ndw;
518
	struct amdgpu_job *job;
C
Chunming Zhou 已提交
519
	struct amdgpu_ib *ib;
520
	struct fence *fence = NULL;
C
Chunming Zhou 已提交
521

A
Alex Deucher 已提交
522 523
	int r;

524 525
	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);

A
Alex Deucher 已提交
526 527 528 529 530 531
	/* padding, etc. */
	ndw = 64;

	/* assume the worst case */
	ndw += vm->max_pde_used * 6;

532 533
	r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
	if (r)
A
Alex Deucher 已提交
534
		return r;
535 536

	ib = &job->ibs[0];
A
Alex Deucher 已提交
537 538 539

	/* walk over the address space and update the page directory */
	for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
540
		struct amdgpu_bo *bo = vm->page_tables[pt_idx].entry.robj;
A
Alex Deucher 已提交
541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
		uint64_t pde, pt;

		if (bo == NULL)
			continue;

		pt = amdgpu_bo_gpu_offset(bo);
		if (vm->page_tables[pt_idx].addr == pt)
			continue;
		vm->page_tables[pt_idx].addr = pt;

		pde = pd_addr + pt_idx * 8;
		if (((last_pde + 8 * count) != pde) ||
		    ((last_pt + incr * count) != pt)) {

			if (count) {
556
				amdgpu_vm_update_pages(adev, 0, NULL, ib,
557 558 559
						       last_pde, last_pt,
						       count, incr,
						       AMDGPU_PTE_VALID);
A
Alex Deucher 已提交
560 561 562 563 564 565 566 567 568 569 570
			}

			count = 1;
			last_pde = pde;
			last_pt = pt;
		} else {
			++count;
		}
	}

	if (count)
571
		amdgpu_vm_update_pages(adev, 0, NULL, ib, last_pde, last_pt,
572
				       count, incr, AMDGPU_PTE_VALID);
A
Alex Deucher 已提交
573

C
Chunming Zhou 已提交
574
	if (ib->length_dw != 0) {
575
		amdgpu_ring_pad_ib(ring, ib);
576 577
		amdgpu_sync_resv(adev, &job->sync, pd->tbo.resv,
				 AMDGPU_FENCE_OWNER_VM);
C
Chunming Zhou 已提交
578
		WARN_ON(ib->length_dw > ndw);
579 580
		r = amdgpu_job_submit(job, ring, &vm->entity,
				      AMDGPU_FENCE_OWNER_VM, &fence);
581 582
		if (r)
			goto error_free;
583

584
		amdgpu_bo_fence(pd, fence, true);
585 586
		fence_put(vm->page_directory_fence);
		vm->page_directory_fence = fence_get(fence);
587
		fence_put(fence);
C
Chunming Zhou 已提交
588

589 590
	} else {
		amdgpu_job_free(job);
C
Chunming Zhou 已提交
591
	}
A
Alex Deucher 已提交
592 593

	return 0;
C
Chunming Zhou 已提交
594 595

error_free:
596
	amdgpu_job_free(job);
597
	return r;
A
Alex Deucher 已提交
598 599 600 601 602 603
}

/**
 * amdgpu_vm_frag_ptes - add fragment information to PTEs
 *
 * @adev: amdgpu_device pointer
604 605
 * @src: address where to copy page table entries from
 * @pages_addr: DMA addresses to use for mapping
A
Alex Deucher 已提交
606 607 608 609 610 611 612
 * @ib: IB for the update
 * @pe_start: first PTE to handle
 * @pe_end: last PTE to handle
 * @addr: addr those PTEs should point to
 * @flags: hw mapping flags
 */
static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
613 614
				uint64_t src,
				dma_addr_t *pages_addr,
A
Alex Deucher 已提交
615 616
				struct amdgpu_ib *ib,
				uint64_t pe_start, uint64_t pe_end,
617
				uint64_t addr, uint32_t flags)
A
Alex Deucher 已提交
618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646
{
	/**
	 * The MC L1 TLB supports variable sized pages, based on a fragment
	 * field in the PTE. When this field is set to a non-zero value, page
	 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
	 * flags are considered valid for all PTEs within the fragment range
	 * and corresponding mappings are assumed to be physically contiguous.
	 *
	 * The L1 TLB can store a single PTE for the whole fragment,
	 * significantly increasing the space available for translation
	 * caching. This leads to large improvements in throughput when the
	 * TLB is under pressure.
	 *
	 * The L2 TLB distributes small and large fragments into two
	 * asymmetric partitions. The large fragment cache is significantly
	 * larger. Thus, we try to use large fragments wherever possible.
	 * Userspace can support this by aligning virtual base address and
	 * allocation size to the fragment size.
	 */

	/* SI and newer are optimized for 64KB */
	uint64_t frag_flags = AMDGPU_PTE_FRAG_64KB;
	uint64_t frag_align = 0x80;

	uint64_t frag_start = ALIGN(pe_start, frag_align);
	uint64_t frag_end = pe_end & ~(frag_align - 1);

	unsigned count;

647 648 649 650
	/* Abort early if there isn't anything to do */
	if (pe_start == pe_end)
		return;

A
Alex Deucher 已提交
651
	/* system pages are non continuously */
652 653
	if (src || pages_addr || !(flags & AMDGPU_PTE_VALID) ||
	    (frag_start >= frag_end)) {
A
Alex Deucher 已提交
654 655

		count = (pe_end - pe_start) / 8;
656
		amdgpu_vm_update_pages(adev, src, pages_addr, ib, pe_start,
657 658
				       addr, count, AMDGPU_GPU_PAGE_SIZE,
				       flags);
A
Alex Deucher 已提交
659 660 661 662 663 664
		return;
	}

	/* handle the 4K area at the beginning */
	if (pe_start != frag_start) {
		count = (frag_start - pe_start) / 8;
665
		amdgpu_vm_update_pages(adev, 0, NULL, ib, pe_start, addr,
666
				       count, AMDGPU_GPU_PAGE_SIZE, flags);
A
Alex Deucher 已提交
667 668 669 670 671
		addr += AMDGPU_GPU_PAGE_SIZE * count;
	}

	/* handle the area in the middle */
	count = (frag_end - frag_start) / 8;
672
	amdgpu_vm_update_pages(adev, 0, NULL, ib, frag_start, addr, count,
673
			       AMDGPU_GPU_PAGE_SIZE, flags | frag_flags);
A
Alex Deucher 已提交
674 675 676 677 678

	/* handle the 4K area at the end */
	if (frag_end != pe_end) {
		addr += AMDGPU_GPU_PAGE_SIZE * count;
		count = (pe_end - frag_end) / 8;
679
		amdgpu_vm_update_pages(adev, 0, NULL, ib, frag_end, addr,
680
				       count, AMDGPU_GPU_PAGE_SIZE, flags);
A
Alex Deucher 已提交
681 682 683 684 685 686 687
	}
}

/**
 * amdgpu_vm_update_ptes - make sure that page tables are valid
 *
 * @adev: amdgpu_device pointer
688 689
 * @src: address where to copy page table entries from
 * @pages_addr: DMA addresses to use for mapping
A
Alex Deucher 已提交
690 691 692 693 694 695
 * @vm: requested vm
 * @start: start of GPU address range
 * @end: end of GPU address range
 * @dst: destination address to map to
 * @flags: mapping flags
 *
696
 * Update the page tables in the range @start - @end.
A
Alex Deucher 已提交
697
 */
698
static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
699 700
				  uint64_t src,
				  dma_addr_t *pages_addr,
701 702 703 704
				  struct amdgpu_vm *vm,
				  struct amdgpu_ib *ib,
				  uint64_t start, uint64_t end,
				  uint64_t dst, uint32_t flags)
A
Alex Deucher 已提交
705
{
706 707 708
	const uint64_t mask = AMDGPU_VM_PTE_COUNT - 1;

	uint64_t last_pe_start = ~0, last_pe_end = ~0, last_dst = ~0;
A
Alex Deucher 已提交
709 710 711 712 713
	uint64_t addr;

	/* walk over the address space and update the page tables */
	for (addr = start; addr < end; ) {
		uint64_t pt_idx = addr >> amdgpu_vm_block_size;
714
		struct amdgpu_bo *pt = vm->page_tables[pt_idx].entry.robj;
A
Alex Deucher 已提交
715
		unsigned nptes;
716
		uint64_t pe_start;
A
Alex Deucher 已提交
717 718 719 720 721 722

		if ((addr & ~mask) == (end & ~mask))
			nptes = end - addr;
		else
			nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);

723 724
		pe_start = amdgpu_bo_gpu_offset(pt);
		pe_start += (addr & mask) * 8;
A
Alex Deucher 已提交
725

726
		if (last_pe_end != pe_start) {
A
Alex Deucher 已提交
727

728
			amdgpu_vm_frag_ptes(adev, src, pages_addr, ib,
729 730
					    last_pe_start, last_pe_end,
					    last_dst, flags);
A
Alex Deucher 已提交
731

732 733
			last_pe_start = pe_start;
			last_pe_end = pe_start + 8 * nptes;
A
Alex Deucher 已提交
734 735
			last_dst = dst;
		} else {
736
			last_pe_end += 8 * nptes;
A
Alex Deucher 已提交
737 738 739 740 741 742
		}

		addr += nptes;
		dst += nptes * AMDGPU_GPU_PAGE_SIZE;
	}

743 744
	amdgpu_vm_frag_ptes(adev, src, pages_addr, ib, last_pe_start,
			    last_pe_end, last_dst, flags);
A
Alex Deucher 已提交
745 746 747 748 749 750
}

/**
 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
 *
 * @adev: amdgpu_device pointer
751 752
 * @src: address where to copy page table entries from
 * @pages_addr: DMA addresses to use for mapping
A
Alex Deucher 已提交
753
 * @vm: requested vm
754 755 756
 * @start: start of mapped range
 * @last: last mapped entry
 * @flags: flags for the entries
A
Alex Deucher 已提交
757 758 759
 * @addr: addr to set the area to
 * @fence: optional resulting fence
 *
760
 * Fill in the page table entries between @start and @last.
A
Alex Deucher 已提交
761 762 763
 * Returns 0 for success, -EINVAL for failure.
 */
static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
764 765
				       uint64_t src,
				       dma_addr_t *pages_addr,
A
Alex Deucher 已提交
766
				       struct amdgpu_vm *vm,
767 768 769
				       uint64_t start, uint64_t last,
				       uint32_t flags, uint64_t addr,
				       struct fence **fence)
A
Alex Deucher 已提交
770
{
771
	struct amdgpu_ring *ring;
772
	void *owner = AMDGPU_FENCE_OWNER_VM;
A
Alex Deucher 已提交
773
	unsigned nptes, ncmds, ndw;
774
	struct amdgpu_job *job;
C
Chunming Zhou 已提交
775
	struct amdgpu_ib *ib;
776
	struct fence *f = NULL;
A
Alex Deucher 已提交
777 778
	int r;

779 780
	ring = container_of(vm->entity.sched, struct amdgpu_ring, sched);

781 782 783 784
	/* sync to everything on unmapping */
	if (!(flags & AMDGPU_PTE_VALID))
		owner = AMDGPU_FENCE_OWNER_UNDEFINED;

785
	nptes = last - start + 1;
A
Alex Deucher 已提交
786 787 788 789 790 791 792 793 794 795

	/*
	 * reserve space for one command every (1 << BLOCK_SIZE)
	 *  entries or 2k dwords (whatever is smaller)
	 */
	ncmds = (nptes >> min(amdgpu_vm_block_size, 11)) + 1;

	/* padding, etc. */
	ndw = 64;

796
	if (src) {
A
Alex Deucher 已提交
797 798 799
		/* only copy commands needed */
		ndw += ncmds * 7;

800
	} else if (pages_addr) {
A
Alex Deucher 已提交
801 802 803 804 805 806 807 808 809 810 811 812 813 814
		/* header for write data commands */
		ndw += ncmds * 4;

		/* body of write data command */
		ndw += nptes * 2;

	} else {
		/* set page commands needed */
		ndw += ncmds * 10;

		/* two extra commands for begin/end of fragment */
		ndw += 2 * 10;
	}

815 816
	r = amdgpu_job_alloc_with_ib(adev, ndw * 4, &job);
	if (r)
A
Alex Deucher 已提交
817
		return r;
818 819

	ib = &job->ibs[0];
C
Chunming Zhou 已提交
820

821
	r = amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
822 823 824
			     owner);
	if (r)
		goto error_free;
A
Alex Deucher 已提交
825

826 827 828 829
	r = reservation_object_reserve_shared(vm->page_directory->tbo.resv);
	if (r)
		goto error_free;

830 831
	amdgpu_vm_update_ptes(adev, src, pages_addr, vm, ib, start,
			      last + 1, addr, flags);
A
Alex Deucher 已提交
832

833
	amdgpu_ring_pad_ib(ring, ib);
C
Chunming Zhou 已提交
834
	WARN_ON(ib->length_dw > ndw);
835 836
	r = amdgpu_job_submit(job, ring, &vm->entity,
			      AMDGPU_FENCE_OWNER_VM, &f);
837 838
	if (r)
		goto error_free;
A
Alex Deucher 已提交
839

840
	amdgpu_bo_fence(vm->page_directory, f, true);
841 842 843 844
	if (fence) {
		fence_put(*fence);
		*fence = fence_get(f);
	}
845
	fence_put(f);
A
Alex Deucher 已提交
846
	return 0;
C
Chunming Zhou 已提交
847 848

error_free:
849
	amdgpu_job_free(job);
850
	return r;
A
Alex Deucher 已提交
851 852
}

853 854 855 856
/**
 * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
 *
 * @adev: amdgpu_device pointer
857 858
 * @gtt_flags: flags as they are used for GTT
 * @pages_addr: DMA addresses to use for mapping
859 860 861
 * @vm: requested vm
 * @mapping: mapped range and flags to use for the update
 * @addr: addr to set the area to
862
 * @flags: HW flags for the mapping
863 864 865 866 867 868 869 870
 * @fence: optional resulting fence
 *
 * Split the mapping into smaller chunks so that each update fits
 * into a SDMA IB.
 * Returns 0 for success, -EINVAL for failure.
 */
static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
				      uint32_t gtt_flags,
871
				      dma_addr_t *pages_addr,
872 873
				      struct amdgpu_vm *vm,
				      struct amdgpu_bo_va_mapping *mapping,
874 875
				      uint32_t flags, uint64_t addr,
				      struct fence **fence)
876 877 878
{
	const uint64_t max_size = 64ULL * 1024ULL * 1024ULL / AMDGPU_GPU_PAGE_SIZE;

879
	uint64_t src = 0, start = mapping->it.start;
880 881 882 883 884 885 886 887 888 889 890 891
	int r;

	/* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
	 * but in case of something, we filter the flags in first place
	 */
	if (!(mapping->flags & AMDGPU_PTE_READABLE))
		flags &= ~AMDGPU_PTE_READABLE;
	if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
		flags &= ~AMDGPU_PTE_WRITEABLE;

	trace_amdgpu_vm_bo_update(mapping);

892
	if (pages_addr) {
893 894 895 896
		if (flags == gtt_flags)
			src = adev->gart.table_addr + (addr >> 12) * 8;
		addr = 0;
	}
897 898
	addr += mapping->offset;

899
	if (!pages_addr || src)
900
		return amdgpu_vm_bo_update_mapping(adev, src, pages_addr, vm,
901 902 903 904 905 906
						   start, mapping->it.last,
						   flags, addr, fence);

	while (start != mapping->it.last + 1) {
		uint64_t last;

907
		last = min((uint64_t)mapping->it.last, start + max_size - 1);
908
		r = amdgpu_vm_bo_update_mapping(adev, src, pages_addr, vm,
909 910 911 912 913 914
						start, last, flags, addr,
						fence);
		if (r)
			return r;

		start = last + 1;
915
		addr += max_size * AMDGPU_GPU_PAGE_SIZE;
916 917 918 919 920
	}

	return 0;
}

A
Alex Deucher 已提交
921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938
/**
 * amdgpu_vm_bo_update - update all BO mappings in the vm page table
 *
 * @adev: amdgpu_device pointer
 * @bo_va: requested BO and VM object
 * @mem: ttm mem
 *
 * Fill in the page table entries for @bo_va.
 * Returns 0 for success, -EINVAL for failure.
 *
 * Object have to be reserved and mutex must be locked!
 */
int amdgpu_vm_bo_update(struct amdgpu_device *adev,
			struct amdgpu_bo_va *bo_va,
			struct ttm_mem_reg *mem)
{
	struct amdgpu_vm *vm = bo_va->vm;
	struct amdgpu_bo_va_mapping *mapping;
939
	dma_addr_t *pages_addr = NULL;
940
	uint32_t gtt_flags, flags;
A
Alex Deucher 已提交
941 942 943 944
	uint64_t addr;
	int r;

	if (mem) {
945 946
		struct ttm_dma_tt *ttm;

947
		addr = (u64)mem->start << PAGE_SHIFT;
948 949
		switch (mem->mem_type) {
		case TTM_PL_TT:
950 951 952
			ttm = container_of(bo_va->bo->tbo.ttm, struct
					   ttm_dma_tt, ttm);
			pages_addr = ttm->dma_address;
953 954 955
			break;

		case TTM_PL_VRAM:
A
Alex Deucher 已提交
956
			addr += adev->vm_manager.vram_base_offset;
957 958 959 960 961
			break;

		default:
			break;
		}
A
Alex Deucher 已提交
962 963 964 965 966
	} else {
		addr = 0;
	}

	flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
967
	gtt_flags = (adev == bo_va->bo->adev) ? flags : 0;
A
Alex Deucher 已提交
968

969 970 971 972 973 974
	spin_lock(&vm->status_lock);
	if (!list_empty(&bo_va->vm_status))
		list_splice_init(&bo_va->valids, &bo_va->invalids);
	spin_unlock(&vm->status_lock);

	list_for_each_entry(mapping, &bo_va->invalids, list) {
975 976 977
		r = amdgpu_vm_bo_split_mapping(adev, gtt_flags, pages_addr, vm,
					       mapping, flags, addr,
					       &bo_va->last_pt_update);
A
Alex Deucher 已提交
978 979 980 981
		if (r)
			return r;
	}

982 983 984 985 986 987 988 989
	if (trace_amdgpu_vm_bo_mapping_enabled()) {
		list_for_each_entry(mapping, &bo_va->valids, list)
			trace_amdgpu_vm_bo_mapping(mapping);

		list_for_each_entry(mapping, &bo_va->invalids, list)
			trace_amdgpu_vm_bo_mapping(mapping);
	}

A
Alex Deucher 已提交
990
	spin_lock(&vm->status_lock);
991
	list_splice_init(&bo_va->invalids, &bo_va->valids);
A
Alex Deucher 已提交
992
	list_del_init(&bo_va->vm_status);
993 994
	if (!mem)
		list_add(&bo_va->vm_status, &vm->cleared);
A
Alex Deucher 已提交
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
	spin_unlock(&vm->status_lock);

	return 0;
}

/**
 * amdgpu_vm_clear_freed - clear freed BOs in the PT
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
 * Make sure all freed BOs are cleared in the PT.
 * Returns 0 for success.
 *
 * PTs have to be reserved and mutex must be locked!
 */
int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
			  struct amdgpu_vm *vm)
{
	struct amdgpu_bo_va_mapping *mapping;
	int r;

	while (!list_empty(&vm->freed)) {
		mapping = list_first_entry(&vm->freed,
			struct amdgpu_bo_va_mapping, list);
		list_del(&mapping->list);
1021

1022
		r = amdgpu_vm_bo_split_mapping(adev, 0, NULL, vm, mapping,
1023
					       0, 0, NULL);
A
Alex Deucher 已提交
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
		kfree(mapping);
		if (r)
			return r;

	}
	return 0;

}

/**
 * amdgpu_vm_clear_invalids - clear invalidated BOs in the PT
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
 * Make sure all invalidated BOs are cleared in the PT.
 * Returns 0 for success.
 *
 * PTs have to be reserved and mutex must be locked!
 */
int amdgpu_vm_clear_invalids(struct amdgpu_device *adev,
1045
			     struct amdgpu_vm *vm, struct amdgpu_sync *sync)
A
Alex Deucher 已提交
1046
{
1047
	struct amdgpu_bo_va *bo_va = NULL;
1048
	int r = 0;
A
Alex Deucher 已提交
1049 1050 1051 1052 1053 1054

	spin_lock(&vm->status_lock);
	while (!list_empty(&vm->invalidated)) {
		bo_va = list_first_entry(&vm->invalidated,
			struct amdgpu_bo_va, vm_status);
		spin_unlock(&vm->status_lock);
1055

A
Alex Deucher 已提交
1056 1057 1058 1059 1060 1061 1062 1063
		r = amdgpu_vm_bo_update(adev, bo_va, NULL);
		if (r)
			return r;

		spin_lock(&vm->status_lock);
	}
	spin_unlock(&vm->status_lock);

1064
	if (bo_va)
1065
		r = amdgpu_sync_fence(adev, sync, bo_va->last_pt_update);
1066 1067

	return r;
A
Alex Deucher 已提交
1068 1069 1070 1071 1072 1073 1074 1075 1076
}

/**
 * amdgpu_vm_bo_add - add a bo to a specific vm
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 * @bo: amdgpu buffer object
 *
1077
 * Add @bo into the requested vm.
A
Alex Deucher 已提交
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
 * Add @bo to the list of bos associated with the vm
 * Returns newly added bo_va or NULL for failure
 *
 * Object has to be reserved!
 */
struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
				      struct amdgpu_vm *vm,
				      struct amdgpu_bo *bo)
{
	struct amdgpu_bo_va *bo_va;

	bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
	if (bo_va == NULL) {
		return NULL;
	}
	bo_va->vm = vm;
	bo_va->bo = bo;
	bo_va->ref_count = 1;
	INIT_LIST_HEAD(&bo_va->bo_list);
1097 1098
	INIT_LIST_HEAD(&bo_va->valids);
	INIT_LIST_HEAD(&bo_va->invalids);
A
Alex Deucher 已提交
1099
	INIT_LIST_HEAD(&bo_va->vm_status);
1100

A
Alex Deucher 已提交
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117
	list_add_tail(&bo_va->bo_list, &bo->va);

	return bo_va;
}

/**
 * amdgpu_vm_bo_map - map bo inside a vm
 *
 * @adev: amdgpu_device pointer
 * @bo_va: bo_va to store the address
 * @saddr: where to map the BO
 * @offset: requested offset in the BO
 * @flags: attributes of pages (read/write/valid/etc.)
 *
 * Add a mapping of the BO at the specefied addr into the VM.
 * Returns 0 for success, error for failure.
 *
1118
 * Object has to be reserved and unreserved outside!
A
Alex Deucher 已提交
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
 */
int amdgpu_vm_bo_map(struct amdgpu_device *adev,
		     struct amdgpu_bo_va *bo_va,
		     uint64_t saddr, uint64_t offset,
		     uint64_t size, uint32_t flags)
{
	struct amdgpu_bo_va_mapping *mapping;
	struct amdgpu_vm *vm = bo_va->vm;
	struct interval_tree_node *it;
	unsigned last_pfn, pt_idx;
	uint64_t eaddr;
	int r;

1132 1133
	/* validate the parameters */
	if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
1134
	    size == 0 || size & AMDGPU_GPU_PAGE_MASK)
1135 1136
		return -EINVAL;

A
Alex Deucher 已提交
1137
	/* make sure object fit at this offset */
1138
	eaddr = saddr + size - 1;
1139
	if ((saddr >= eaddr) || (offset + size > amdgpu_bo_size(bo_va->bo)))
A
Alex Deucher 已提交
1140 1141 1142
		return -EINVAL;

	last_pfn = eaddr / AMDGPU_GPU_PAGE_SIZE;
1143 1144
	if (last_pfn >= adev->vm_manager.max_pfn) {
		dev_err(adev->dev, "va above limit (0x%08X >= 0x%08X)\n",
A
Alex Deucher 已提交
1145 1146 1147 1148 1149 1150 1151
			last_pfn, adev->vm_manager.max_pfn);
		return -EINVAL;
	}

	saddr /= AMDGPU_GPU_PAGE_SIZE;
	eaddr /= AMDGPU_GPU_PAGE_SIZE;

1152
	it = interval_tree_iter_first(&vm->va, saddr, eaddr);
A
Alex Deucher 已提交
1153 1154 1155 1156 1157 1158 1159 1160
	if (it) {
		struct amdgpu_bo_va_mapping *tmp;
		tmp = container_of(it, struct amdgpu_bo_va_mapping, it);
		/* bo and tmp overlap, invalid addr */
		dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
			"0x%010lx-0x%010lx\n", bo_va->bo, saddr, eaddr,
			tmp->it.start, tmp->it.last + 1);
		r = -EINVAL;
1161
		goto error;
A
Alex Deucher 已提交
1162 1163 1164 1165 1166
	}

	mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
	if (!mapping) {
		r = -ENOMEM;
1167
		goto error;
A
Alex Deucher 已提交
1168 1169 1170 1171
	}

	INIT_LIST_HEAD(&mapping->list);
	mapping->it.start = saddr;
1172
	mapping->it.last = eaddr;
A
Alex Deucher 已提交
1173 1174 1175
	mapping->offset = offset;
	mapping->flags = flags;

1176
	list_add(&mapping->list, &bo_va->invalids);
A
Alex Deucher 已提交
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189
	interval_tree_insert(&mapping->it, &vm->va);

	/* Make sure the page tables are allocated */
	saddr >>= amdgpu_vm_block_size;
	eaddr >>= amdgpu_vm_block_size;

	BUG_ON(eaddr >= amdgpu_vm_num_pdes(adev));

	if (eaddr > vm->max_pde_used)
		vm->max_pde_used = eaddr;

	/* walk over the address space and allocate the page tables */
	for (pt_idx = saddr; pt_idx <= eaddr; ++pt_idx) {
1190
		struct reservation_object *resv = vm->page_directory->tbo.resv;
1191
		struct amdgpu_bo_list_entry *entry;
A
Alex Deucher 已提交
1192 1193
		struct amdgpu_bo *pt;

1194 1195
		entry = &vm->page_tables[pt_idx].entry;
		if (entry->robj)
A
Alex Deucher 已提交
1196 1197 1198 1199
			continue;

		r = amdgpu_bo_create(adev, AMDGPU_VM_PTE_COUNT * 8,
				     AMDGPU_GPU_PAGE_SIZE, true,
1200 1201
				     AMDGPU_GEM_DOMAIN_VRAM,
				     AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
1202
				     NULL, resv, &pt);
1203
		if (r)
A
Alex Deucher 已提交
1204
			goto error_free;
1205

1206 1207 1208 1209 1210
		/* Keep a reference to the page table to avoid freeing
		 * them up in the wrong order.
		 */
		pt->parent = amdgpu_bo_ref(vm->page_directory);

1211
		r = amdgpu_vm_clear_bo(adev, vm, pt);
A
Alex Deucher 已提交
1212 1213 1214 1215 1216
		if (r) {
			amdgpu_bo_unref(&pt);
			goto error_free;
		}

1217 1218 1219 1220
		entry->robj = pt;
		entry->priority = 0;
		entry->tv.bo = &entry->robj->tbo;
		entry->tv.shared = true;
1221
		entry->user_pages = NULL;
A
Alex Deucher 已提交
1222 1223 1224 1225 1226 1227 1228 1229
		vm->page_tables[pt_idx].addr = 0;
	}

	return 0;

error_free:
	list_del(&mapping->list);
	interval_tree_remove(&mapping->it, &vm->va);
1230
	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
A
Alex Deucher 已提交
1231 1232
	kfree(mapping);

1233
error:
A
Alex Deucher 已提交
1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246
	return r;
}

/**
 * amdgpu_vm_bo_unmap - remove bo mapping from vm
 *
 * @adev: amdgpu_device pointer
 * @bo_va: bo_va to remove the address from
 * @saddr: where to the BO is mapped
 *
 * Remove a mapping of the BO at the specefied addr from the VM.
 * Returns 0 for success, error for failure.
 *
1247
 * Object has to be reserved and unreserved outside!
A
Alex Deucher 已提交
1248 1249 1250 1251 1252 1253 1254
 */
int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
		       struct amdgpu_bo_va *bo_va,
		       uint64_t saddr)
{
	struct amdgpu_bo_va_mapping *mapping;
	struct amdgpu_vm *vm = bo_va->vm;
1255
	bool valid = true;
A
Alex Deucher 已提交
1256

1257
	saddr /= AMDGPU_GPU_PAGE_SIZE;
1258

1259
	list_for_each_entry(mapping, &bo_va->valids, list) {
A
Alex Deucher 已提交
1260 1261 1262 1263
		if (mapping->it.start == saddr)
			break;
	}

1264 1265 1266 1267 1268 1269 1270 1271
	if (&mapping->list == &bo_va->valids) {
		valid = false;

		list_for_each_entry(mapping, &bo_va->invalids, list) {
			if (mapping->it.start == saddr)
				break;
		}

1272
		if (&mapping->list == &bo_va->invalids)
1273
			return -ENOENT;
A
Alex Deucher 已提交
1274
	}
1275

A
Alex Deucher 已提交
1276 1277
	list_del(&mapping->list);
	interval_tree_remove(&mapping->it, &vm->va);
1278
	trace_amdgpu_vm_bo_unmap(bo_va, mapping);
A
Alex Deucher 已提交
1279

1280
	if (valid)
A
Alex Deucher 已提交
1281
		list_add(&mapping->list, &vm->freed);
1282
	else
A
Alex Deucher 已提交
1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293
		kfree(mapping);

	return 0;
}

/**
 * amdgpu_vm_bo_rmv - remove a bo to a specific vm
 *
 * @adev: amdgpu_device pointer
 * @bo_va: requested bo_va
 *
1294
 * Remove @bo_va->bo from the requested vm.
A
Alex Deucher 已提交
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309
 *
 * Object have to be reserved!
 */
void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
		      struct amdgpu_bo_va *bo_va)
{
	struct amdgpu_bo_va_mapping *mapping, *next;
	struct amdgpu_vm *vm = bo_va->vm;

	list_del(&bo_va->bo_list);

	spin_lock(&vm->status_lock);
	list_del(&bo_va->vm_status);
	spin_unlock(&vm->status_lock);

1310
	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
A
Alex Deucher 已提交
1311 1312
		list_del(&mapping->list);
		interval_tree_remove(&mapping->it, &vm->va);
1313
		trace_amdgpu_vm_bo_unmap(bo_va, mapping);
1314 1315 1316 1317 1318 1319
		list_add(&mapping->list, &vm->freed);
	}
	list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
		list_del(&mapping->list);
		interval_tree_remove(&mapping->it, &vm->va);
		kfree(mapping);
A
Alex Deucher 已提交
1320
	}
1321

1322
	fence_put(bo_va->last_pt_update);
A
Alex Deucher 已提交
1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
	kfree(bo_va);
}

/**
 * amdgpu_vm_bo_invalidate - mark the bo as invalid
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 * @bo: amdgpu buffer object
 *
1333
 * Mark @bo as invalid.
A
Alex Deucher 已提交
1334 1335 1336 1337 1338 1339 1340
 */
void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
			     struct amdgpu_bo *bo)
{
	struct amdgpu_bo_va *bo_va;

	list_for_each_entry(bo_va, &bo->va, bo_list) {
1341 1342
		spin_lock(&bo_va->vm->status_lock);
		if (list_empty(&bo_va->vm_status))
A
Alex Deucher 已提交
1343
			list_add(&bo_va->vm_status, &bo_va->vm->invalidated);
1344
		spin_unlock(&bo_va->vm->status_lock);
A
Alex Deucher 已提交
1345 1346 1347 1348 1349 1350 1351 1352 1353
	}
}

/**
 * amdgpu_vm_init - initialize a vm instance
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
1354
 * Init @vm fields.
A
Alex Deucher 已提交
1355 1356 1357 1358 1359
 */
int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
{
	const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
		AMDGPU_VM_PTE_COUNT * 8);
1360
	unsigned pd_size, pd_entries;
1361 1362
	unsigned ring_instance;
	struct amdgpu_ring *ring;
1363
	struct amd_sched_rq *rq;
A
Alex Deucher 已提交
1364 1365
	int i, r;

1366 1367
	for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
		vm->ids[i] = NULL;
A
Alex Deucher 已提交
1368 1369 1370
	vm->va = RB_ROOT;
	spin_lock_init(&vm->status_lock);
	INIT_LIST_HEAD(&vm->invalidated);
1371
	INIT_LIST_HEAD(&vm->cleared);
A
Alex Deucher 已提交
1372
	INIT_LIST_HEAD(&vm->freed);
1373

A
Alex Deucher 已提交
1374 1375 1376 1377
	pd_size = amdgpu_vm_directory_size(adev);
	pd_entries = amdgpu_vm_num_pdes(adev);

	/* allocate page table array */
1378
	vm->page_tables = drm_calloc_large(pd_entries, sizeof(struct amdgpu_vm_pt));
A
Alex Deucher 已提交
1379 1380 1381 1382 1383
	if (vm->page_tables == NULL) {
		DRM_ERROR("Cannot allocate memory for page table array\n");
		return -ENOMEM;
	}

1384
	/* create scheduler entity for page table updates */
1385 1386 1387 1388

	ring_instance = atomic_inc_return(&adev->vm_manager.vm_pte_next_ring);
	ring_instance %= adev->vm_manager.vm_pte_num_rings;
	ring = adev->vm_manager.vm_pte_rings[ring_instance];
1389 1390 1391 1392 1393 1394
	rq = &ring->sched.sched_rq[AMD_SCHED_PRIORITY_KERNEL];
	r = amd_sched_entity_init(&ring->sched, &vm->entity,
				  rq, amdgpu_sched_jobs);
	if (r)
		return r;

1395 1396
	vm->page_directory_fence = NULL;

A
Alex Deucher 已提交
1397
	r = amdgpu_bo_create(adev, pd_size, align, true,
1398 1399
			     AMDGPU_GEM_DOMAIN_VRAM,
			     AMDGPU_GEM_CREATE_NO_CPU_ACCESS,
1400
			     NULL, NULL, &vm->page_directory);
A
Alex Deucher 已提交
1401
	if (r)
1402 1403
		goto error_free_sched_entity;

1404
	r = amdgpu_bo_reserve(vm->page_directory, false);
1405 1406 1407 1408
	if (r)
		goto error_free_page_directory;

	r = amdgpu_vm_clear_bo(adev, vm, vm->page_directory);
1409
	amdgpu_bo_unreserve(vm->page_directory);
1410 1411
	if (r)
		goto error_free_page_directory;
A
Alex Deucher 已提交
1412 1413

	return 0;
1414 1415 1416 1417 1418 1419 1420 1421 1422

error_free_page_directory:
	amdgpu_bo_unref(&vm->page_directory);
	vm->page_directory = NULL;

error_free_sched_entity:
	amd_sched_entity_fini(&ring->sched, &vm->entity);

	return r;
A
Alex Deucher 已提交
1423 1424 1425 1426 1427 1428 1429 1430
}

/**
 * amdgpu_vm_fini - tear down a vm instance
 *
 * @adev: amdgpu_device pointer
 * @vm: requested vm
 *
1431
 * Tear down @vm.
A
Alex Deucher 已提交
1432 1433 1434 1435 1436 1437 1438
 * Unbind the VM and remove all bos from the vm bo list
 */
void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
{
	struct amdgpu_bo_va_mapping *mapping, *tmp;
	int i;

1439
	amd_sched_entity_fini(vm->entity.sched, &vm->entity);
1440

A
Alex Deucher 已提交
1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
	if (!RB_EMPTY_ROOT(&vm->va)) {
		dev_err(adev->dev, "still active bo inside vm\n");
	}
	rbtree_postorder_for_each_entry_safe(mapping, tmp, &vm->va, it.rb) {
		list_del(&mapping->list);
		interval_tree_remove(&mapping->it, &vm->va);
		kfree(mapping);
	}
	list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
		list_del(&mapping->list);
		kfree(mapping);
	}

	for (i = 0; i < amdgpu_vm_num_pdes(adev); i++)
1455
		amdgpu_bo_unref(&vm->page_tables[i].entry.robj);
1456
	drm_free_large(vm->page_tables);
A
Alex Deucher 已提交
1457 1458

	amdgpu_bo_unref(&vm->page_directory);
1459
	fence_put(vm->page_directory_fence);
1460

A
Alex Deucher 已提交
1461
	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1462
		struct amdgpu_vm_id *id = vm->ids[i];
1463

1464 1465 1466 1467
		if (!id)
			continue;

		atomic_long_cmpxchg(&id->owner, (long)&vm->ids[i], 0);
A
Alex Deucher 已提交
1468 1469
	}
}
1470

1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484
/**
 * amdgpu_vm_manager_init - init the VM manager
 *
 * @adev: amdgpu_device pointer
 *
 * Initialize the VM manager structures
 */
void amdgpu_vm_manager_init(struct amdgpu_device *adev)
{
	unsigned i;

	INIT_LIST_HEAD(&adev->vm_manager.ids_lru);

	/* skip over VMID 0, since it is the system VM */
1485 1486
	for (i = 1; i < adev->vm_manager.num_ids; ++i) {
		amdgpu_vm_reset_id(adev, i);
1487
		amdgpu_sync_create(&adev->vm_manager.ids[i].active);
1488 1489
		list_add_tail(&adev->vm_manager.ids[i].list,
			      &adev->vm_manager.ids_lru);
1490
	}
1491 1492

	atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
1493 1494
}

1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505
/**
 * amdgpu_vm_manager_fini - cleanup VM manager
 *
 * @adev: amdgpu_device pointer
 *
 * Cleanup the VM manager and free resources.
 */
void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
{
	unsigned i;

1506 1507 1508
	for (i = 0; i < AMDGPU_NUM_VM; ++i) {
		struct amdgpu_vm_id *id = &adev->vm_manager.ids[i];

1509 1510
		fence_put(adev->vm_manager.ids[i].first);
		amdgpu_sync_free(&adev->vm_manager.ids[i].active);
1511 1512
		fence_put(id->flushed_updates);
	}
1513
}