amdgpu_cs.c 23.6 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
/*
 * Copyright 2008 Jerome Glisse.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Authors:
 *    Jerome Glisse <glisse@freedesktop.org>
 */
#include <linux/list_sort.h>
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "amdgpu_trace.h"

int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
		       u32 ip_instance, u32 ring,
		       struct amdgpu_ring **out_ring)
{
	/* Right now all IPs have only one instance - multiple rings. */
	if (ip_instance != 0) {
		DRM_ERROR("invalid ip instance: %d\n", ip_instance);
		return -EINVAL;
	}

	switch (ip_type) {
	default:
		DRM_ERROR("unknown ip type: %d\n", ip_type);
		return -EINVAL;
	case AMDGPU_HW_IP_GFX:
		if (ring < adev->gfx.num_gfx_rings) {
			*out_ring = &adev->gfx.gfx_ring[ring];
		} else {
			DRM_ERROR("only %d gfx rings are supported now\n",
				  adev->gfx.num_gfx_rings);
			return -EINVAL;
		}
		break;
	case AMDGPU_HW_IP_COMPUTE:
		if (ring < adev->gfx.num_compute_rings) {
			*out_ring = &adev->gfx.compute_ring[ring];
		} else {
			DRM_ERROR("only %d compute rings are supported now\n",
				  adev->gfx.num_compute_rings);
			return -EINVAL;
		}
		break;
	case AMDGPU_HW_IP_DMA:
A
Alex Deucher 已提交
66 67
		if (ring < adev->sdma.num_instances) {
			*out_ring = &adev->sdma.instance[ring].ring;
A
Alex Deucher 已提交
68
		} else {
A
Alex Deucher 已提交
69 70
			DRM_ERROR("only %d SDMA rings are supported\n",
				  adev->sdma.num_instances);
A
Alex Deucher 已提交
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
			return -EINVAL;
		}
		break;
	case AMDGPU_HW_IP_UVD:
		*out_ring = &adev->uvd.ring;
		break;
	case AMDGPU_HW_IP_VCE:
		if (ring < 2){
			*out_ring = &adev->vce.ring[ring];
		} else {
			DRM_ERROR("only two VCE rings are supported\n");
			return -EINVAL;
		}
		break;
	}
	return 0;
}

89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117
static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
				      struct drm_amdgpu_cs_chunk_fence *fence_data)
{
	struct drm_gem_object *gobj;
	uint32_t handle;

	handle = fence_data->handle;
	gobj = drm_gem_object_lookup(p->adev->ddev, p->filp,
				     fence_data->handle);
	if (gobj == NULL)
		return -EINVAL;

	p->uf.bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
	p->uf.offset = fence_data->offset;

	if (amdgpu_ttm_tt_has_userptr(p->uf.bo->tbo.ttm)) {
		drm_gem_object_unreference_unlocked(gobj);
		return -EINVAL;
	}

	p->uf_entry.robj = amdgpu_bo_ref(p->uf.bo);
	p->uf_entry.priority = 0;
	p->uf_entry.tv.bo = &p->uf_entry.robj->tbo;
	p->uf_entry.tv.shared = true;

	drm_gem_object_unreference_unlocked(gobj);
	return 0;
}

A
Alex Deucher 已提交
118 119 120 121
int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
{
	union drm_amdgpu_cs *cs = data;
	uint64_t *chunk_array_user;
122
	uint64_t *chunk_array;
A
Alex Deucher 已提交
123
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
124 125
	unsigned size;
	int i;
126
	int ret;
A
Alex Deucher 已提交
127

128 129 130 131 132 133
	if (cs->in.num_chunks == 0)
		return 0;

	chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
	if (!chunk_array)
		return -ENOMEM;
A
Alex Deucher 已提交
134

135 136
	p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
	if (!p->ctx) {
137 138
		ret = -EINVAL;
		goto free_chunk;
139
	}
140

A
Alex Deucher 已提交
141
	/* get chunks */
142
	chunk_array_user = (uint64_t __user *)(unsigned long)(cs->in.chunks);
A
Alex Deucher 已提交
143 144
	if (copy_from_user(chunk_array, chunk_array_user,
			   sizeof(uint64_t)*cs->in.num_chunks)) {
145
		ret = -EFAULT;
146
		goto put_ctx;
A
Alex Deucher 已提交
147 148 149
	}

	p->nchunks = cs->in.num_chunks;
150
	p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
A
Alex Deucher 已提交
151
			    GFP_KERNEL);
152 153
	if (!p->chunks) {
		ret = -ENOMEM;
154
		goto put_ctx;
A
Alex Deucher 已提交
155 156 157 158 159 160 161
	}

	for (i = 0; i < p->nchunks; i++) {
		struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL;
		struct drm_amdgpu_cs_chunk user_chunk;
		uint32_t __user *cdata;

162
		chunk_ptr = (void __user *)(unsigned long)chunk_array[i];
A
Alex Deucher 已提交
163 164
		if (copy_from_user(&user_chunk, chunk_ptr,
				       sizeof(struct drm_amdgpu_cs_chunk))) {
165 166 167
			ret = -EFAULT;
			i--;
			goto free_partial_kdata;
A
Alex Deucher 已提交
168 169 170 171 172
		}
		p->chunks[i].chunk_id = user_chunk.chunk_id;
		p->chunks[i].length_dw = user_chunk.length_dw;

		size = p->chunks[i].length_dw;
173
		cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
A
Alex Deucher 已提交
174 175 176

		p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
		if (p->chunks[i].kdata == NULL) {
177 178 179
			ret = -ENOMEM;
			i--;
			goto free_partial_kdata;
A
Alex Deucher 已提交
180 181 182
		}
		size *= sizeof(uint32_t);
		if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
183 184
			ret = -EFAULT;
			goto free_partial_kdata;
A
Alex Deucher 已提交
185 186
		}

187 188 189 190 191 192
		switch (p->chunks[i].chunk_id) {
		case AMDGPU_CHUNK_ID_IB:
			p->num_ibs++;
			break;

		case AMDGPU_CHUNK_ID_FENCE:
A
Alex Deucher 已提交
193
			size = sizeof(struct drm_amdgpu_cs_chunk_fence);
194
			if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
195 196
				ret = -EINVAL;
				goto free_partial_kdata;
A
Alex Deucher 已提交
197
			}
198 199 200 201 202

			ret = amdgpu_cs_user_fence_chunk(p, (void *)p->chunks[i].kdata);
			if (ret)
				goto free_partial_kdata;

203 204
			break;

205 206 207
		case AMDGPU_CHUNK_ID_DEPENDENCIES:
			break;

208
		default:
209 210
			ret = -EINVAL;
			goto free_partial_kdata;
A
Alex Deucher 已提交
211 212 213
		}
	}

214

215
	p->ibs = kcalloc(p->num_ibs, sizeof(struct amdgpu_ib), GFP_KERNEL);
216 217 218 219
	if (!p->ibs) {
		ret = -ENOMEM;
		goto free_all_kdata;
	}
A
Alex Deucher 已提交
220 221

	kfree(chunk_array);
222 223 224 225 226 227 228 229
	return 0;

free_all_kdata:
	i = p->nchunks - 1;
free_partial_kdata:
	for (; i >= 0; i--)
		drm_free_large(p->chunks[i].kdata);
	kfree(p->chunks);
230
put_ctx:
231 232 233 234 235
	amdgpu_ctx_put(p->ctx);
free_chunk:
	kfree(chunk_array);

	return ret;
A
Alex Deucher 已提交
236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289
}

/* Returns how many bytes TTM can move per IB.
 */
static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev)
{
	u64 real_vram_size = adev->mc.real_vram_size;
	u64 vram_usage = atomic64_read(&adev->vram_usage);

	/* This function is based on the current VRAM usage.
	 *
	 * - If all of VRAM is free, allow relocating the number of bytes that
	 *   is equal to 1/4 of the size of VRAM for this IB.

	 * - If more than one half of VRAM is occupied, only allow relocating
	 *   1 MB of data for this IB.
	 *
	 * - From 0 to one half of used VRAM, the threshold decreases
	 *   linearly.
	 *         __________________
	 * 1/4 of -|\               |
	 * VRAM    | \              |
	 *         |  \             |
	 *         |   \            |
	 *         |    \           |
	 *         |     \          |
	 *         |      \         |
	 *         |       \________|1 MB
	 *         |----------------|
	 *    VRAM 0 %             100 %
	 *         used            used
	 *
	 * Note: It's a threshold, not a limit. The threshold must be crossed
	 * for buffer relocations to stop, so any buffer of an arbitrary size
	 * can be moved as long as the threshold isn't crossed before
	 * the relocation takes place. We don't want to disable buffer
	 * relocations completely.
	 *
	 * The idea is that buffers should be placed in VRAM at creation time
	 * and TTM should only do a minimum number of relocations during
	 * command submission. In practice, you need to submit at least
	 * a dozen IBs to move all buffers to VRAM if they are in GTT.
	 *
	 * Also, things can get pretty crazy under memory pressure and actual
	 * VRAM usage can change a lot, so playing safe even at 50% does
	 * consistently increase performance.
	 */

	u64 half_vram = real_vram_size >> 1;
	u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage;
	u64 bytes_moved_threshold = half_free_vram >> 1;
	return max(bytes_moved_threshold, 1024*1024ull);
}

290
int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
291
			    struct list_head *validated)
A
Alex Deucher 已提交
292 293
{
	struct amdgpu_bo_list_entry *lobj;
294
	u64 initial_bytes_moved;
A
Alex Deucher 已提交
295 296
	int r;

297
	list_for_each_entry(lobj, validated, tv.head) {
298 299
		struct amdgpu_bo *bo = lobj->robj;
		uint32_t domain;
A
Alex Deucher 已提交
300

301 302 303 304 305 306 307 308 309 310 311 312
		if (bo->pin_count)
			continue;

		/* Avoid moving this one if we have moved too many buffers
		 * for this IB already.
		 *
		 * Note that this allows moving at least one buffer of
		 * any size, because it doesn't take the current "bo"
		 * into account. We don't want to disallow buffer moves
		 * completely.
		 */
		if (p->bytes_moved <= p->bytes_moved_threshold)
313
			domain = bo->prefered_domains;
314
		else
315
			domain = bo->allowed_domains;
316 317 318 319 320 321 322 323 324

	retry:
		amdgpu_ttm_placement_from_domain(bo, domain);
		initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
		r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
		p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
			       initial_bytes_moved;

		if (unlikely(r)) {
325 326
			if (r != -ERESTARTSYS && domain != bo->allowed_domains) {
				domain = bo->allowed_domains;
327
				goto retry;
A
Alex Deucher 已提交
328
			}
329
			return r;
A
Alex Deucher 已提交
330 331 332 333 334
		}
	}
	return 0;
}

335 336
static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
				union drm_amdgpu_cs *cs)
A
Alex Deucher 已提交
337 338
{
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
339
	struct list_head duplicates;
340
	bool need_mmap_lock = false;
341
	int r;
A
Alex Deucher 已提交
342

343 344 345
	INIT_LIST_HEAD(&p->validated);

	p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
346 347
	if (p->bo_list) {
		need_mmap_lock = p->bo_list->has_userptr;
348
		amdgpu_bo_list_get_list(p->bo_list, &p->validated);
349
	}
A
Alex Deucher 已提交
350

351
	INIT_LIST_HEAD(&duplicates);
352
	amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
A
Alex Deucher 已提交
353

354 355 356
	if (p->uf.bo)
		list_add(&p->uf_entry.tv.head, &p->validated);

A
Alex Deucher 已提交
357 358 359
	if (need_mmap_lock)
		down_read(&current->mm->mmap_sem);

360 361 362 363
	r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true, &duplicates);
	if (unlikely(r != 0))
		goto error_reserve;

364
	amdgpu_vm_get_pt_bos(&fpriv->vm, &duplicates);
365

366 367 368 369
	p->bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(p->adev);
	p->bytes_moved = 0;

	r = amdgpu_cs_list_validate(p, &duplicates);
370 371 372
	if (r)
		goto error_validate;

373
	r = amdgpu_cs_list_validate(p, &p->validated);
374 375 376 377 378 379 380 381 382 383 384 385 386
	if (r)
		goto error_validate;

	if (p->bo_list) {
		struct amdgpu_vm *vm = &fpriv->vm;
		unsigned i;

		for (i = 0; i < p->bo_list->num_entries; i++) {
			struct amdgpu_bo *bo = p->bo_list->array[i].robj;

			p->bo_list->array[i].bo_va = amdgpu_vm_bo_find(vm, bo);
		}
	}
387 388

error_validate:
389 390
	if (r) {
		amdgpu_vm_move_pt_bos_in_lru(p->adev, &fpriv->vm);
391
		ttm_eu_backoff_reservation(&p->ticket, &p->validated);
392
	}
A
Alex Deucher 已提交
393

394
error_reserve:
A
Alex Deucher 已提交
395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425
	if (need_mmap_lock)
		up_read(&current->mm->mmap_sem);

	return r;
}

static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
{
	struct amdgpu_bo_list_entry *e;
	int r;

	list_for_each_entry(e, &p->validated, tv.head) {
		struct reservation_object *resv = e->robj->tbo.resv;
		r = amdgpu_sync_resv(p->adev, &p->ibs[0].sync, resv, p->filp);

		if (r)
			return r;
	}
	return 0;
}

static int cmp_size_smaller_first(void *priv, struct list_head *a,
				  struct list_head *b)
{
	struct amdgpu_bo_list_entry *la = list_entry(a, struct amdgpu_bo_list_entry, tv.head);
	struct amdgpu_bo_list_entry *lb = list_entry(b, struct amdgpu_bo_list_entry, tv.head);

	/* Sort A before B if A is smaller. */
	return (int)la->robj->tbo.num_pages - (int)lb->robj->tbo.num_pages;
}

426 427 428 429 430 431 432 433 434
/**
 * cs_parser_fini() - clean parser states
 * @parser:	parser structure holding parsing context.
 * @error:	error number
 *
 * If error is set than unvalidate buffer, otherwise just free memory
 * used by parsing context.
 **/
static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)
C
Chunming Zhou 已提交
435
{
436
	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
437 438
	unsigned i;

A
Alex Deucher 已提交
439
	if (!error) {
440 441
		amdgpu_vm_move_pt_bos_in_lru(parser->adev, &fpriv->vm);

A
Alex Deucher 已提交
442 443 444 445 446 447 448 449 450 451 452 453 454
		/* Sort the buffer list from the smallest to largest buffer,
		 * which affects the order of buffers in the LRU list.
		 * This assures that the smallest buffers are added first
		 * to the LRU list, so they are likely to be later evicted
		 * first, instead of large buffers whose eviction is more
		 * expensive.
		 *
		 * This slightly lowers the number of bytes moved by TTM
		 * per frame under memory pressure.
		 */
		list_sort(NULL, &parser->validated, cmp_size_smaller_first);

		ttm_eu_fence_buffer_objects(&parser->ticket,
455 456
					    &parser->validated,
					    parser->fence);
A
Alex Deucher 已提交
457 458 459 460
	} else if (backoff) {
		ttm_eu_backoff_reservation(&parser->ticket,
					   &parser->validated);
	}
461
	fence_put(parser->fence);
462

463 464
	if (parser->ctx)
		amdgpu_ctx_put(parser->ctx);
465 466 467
	if (parser->bo_list)
		amdgpu_bo_list_put(parser->bo_list);

A
Alex Deucher 已提交
468 469 470
	for (i = 0; i < parser->nchunks; i++)
		drm_free_large(parser->chunks[i].kdata);
	kfree(parser->chunks);
471 472 473 474
	if (parser->ibs)
		for (i = 0; i < parser->num_ibs; i++)
			amdgpu_ib_free(parser->adev, &parser->ibs[i]);
	kfree(parser->ibs);
475 476
	amdgpu_bo_unref(&parser->uf.bo);
	amdgpu_bo_unref(&parser->uf_entry.robj);
A
Alex Deucher 已提交
477 478 479 480 481 482 483 484 485 486 487 488 489 490
}

static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
				   struct amdgpu_vm *vm)
{
	struct amdgpu_device *adev = p->adev;
	struct amdgpu_bo_va *bo_va;
	struct amdgpu_bo *bo;
	int i, r;

	r = amdgpu_vm_update_page_directory(adev, vm);
	if (r)
		return r;

491 492 493 494
	r = amdgpu_sync_fence(adev, &p->ibs[0].sync, vm->page_directory_fence);
	if (r)
		return r;

A
Alex Deucher 已提交
495 496 497 498 499 500
	r = amdgpu_vm_clear_freed(adev, vm);
	if (r)
		return r;

	if (p->bo_list) {
		for (i = 0; i < p->bo_list->num_entries; i++) {
501 502
			struct fence *f;

A
Alex Deucher 已提交
503 504 505 506 507 508 509 510 511 512 513 514 515
			/* ignore duplicates */
			bo = p->bo_list->array[i].robj;
			if (!bo)
				continue;

			bo_va = p->bo_list->array[i].bo_va;
			if (bo_va == NULL)
				continue;

			r = amdgpu_vm_bo_update(adev, bo_va, &bo->tbo.mem);
			if (r)
				return r;

516
			f = bo_va->last_pt_update;
517 518 519
			r = amdgpu_sync_fence(adev, &p->ibs[0].sync, f);
			if (r)
				return r;
A
Alex Deucher 已提交
520
		}
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535

	}

	r = amdgpu_vm_clear_invalids(adev, vm, &p->ibs[0].sync);

	if (amdgpu_vm_debug && p->bo_list) {
		/* Invalidate all BOs to test for userspace bugs */
		for (i = 0; i < p->bo_list->num_entries; i++) {
			/* ignore duplicates */
			bo = p->bo_list->array[i].robj;
			if (!bo)
				continue;

			amdgpu_vm_bo_invalidate(adev, bo);
		}
A
Alex Deucher 已提交
536 537
	}

538
	return r;
A
Alex Deucher 已提交
539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562
}

static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
				 struct amdgpu_cs_parser *parser)
{
	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
	struct amdgpu_vm *vm = &fpriv->vm;
	struct amdgpu_ring *ring;
	int i, r;

	if (parser->num_ibs == 0)
		return 0;

	/* Only for UVD/VCE VM emulation */
	for (i = 0; i < parser->num_ibs; i++) {
		ring = parser->ibs[i].ring;
		if (ring->funcs->parse_cs) {
			r = amdgpu_ring_parse_cs(ring, parser, i);
			if (r)
				return r;
		}
	}

	r = amdgpu_bo_vm_update_pte(parser, vm);
563 564
	if (!r)
		amdgpu_cs_sync_rings(parser);
A
Alex Deucher 已提交
565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602

	return r;
}

static int amdgpu_cs_handle_lockup(struct amdgpu_device *adev, int r)
{
	if (r == -EDEADLK) {
		r = amdgpu_gpu_reset(adev);
		if (!r)
			r = -EAGAIN;
	}
	return r;
}

static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
			     struct amdgpu_cs_parser *parser)
{
	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
	struct amdgpu_vm *vm = &fpriv->vm;
	int i, j;
	int r;

	for (i = 0, j = 0; i < parser->nchunks && j < parser->num_ibs; i++) {
		struct amdgpu_cs_chunk *chunk;
		struct amdgpu_ib *ib;
		struct drm_amdgpu_cs_chunk_ib *chunk_ib;
		struct amdgpu_ring *ring;

		chunk = &parser->chunks[i];
		ib = &parser->ibs[j];
		chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;

		if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
			continue;

		r = amdgpu_cs_get_ring(adev, chunk_ib->ip_type,
				       chunk_ib->ip_instance, chunk_ib->ring,
				       &ring);
603
		if (r)
A
Alex Deucher 已提交
604 605 606
			return r;

		if (ring->funcs->parse_cs) {
607
			struct amdgpu_bo_va_mapping *m;
608
			struct amdgpu_bo *aobj = NULL;
609 610
			uint64_t offset;
			uint8_t *kptr;
611

612 613
			m = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
						   &aobj);
614 615 616
			if (!aobj) {
				DRM_ERROR("IB va_start is invalid\n");
				return -EINVAL;
A
Alex Deucher 已提交
617 618
			}

619 620 621 622 623 624
			if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
			    (m->it.last + 1) * AMDGPU_GPU_PAGE_SIZE) {
				DRM_ERROR("IB va_start+ib_bytes is invalid\n");
				return -EINVAL;
			}

625
			/* the IB should be reserved at this point */
626
			r = amdgpu_bo_kmap(aobj, (void **)&kptr);
A
Alex Deucher 已提交
627 628 629 630
			if (r) {
				return r;
			}

631 632 633
			offset = ((uint64_t)m->it.start) * AMDGPU_GPU_PAGE_SIZE;
			kptr += chunk_ib->va_start - offset;

A
Alex Deucher 已提交
634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
			r =  amdgpu_ib_get(ring, NULL, chunk_ib->ib_bytes, ib);
			if (r) {
				DRM_ERROR("Failed to get ib !\n");
				return r;
			}

			memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
			amdgpu_bo_kunmap(aobj);
		} else {
			r =  amdgpu_ib_get(ring, vm, 0, ib);
			if (r) {
				DRM_ERROR("Failed to get ib !\n");
				return r;
			}

			ib->gpu_addr = chunk_ib->va_start;
		}

652
		ib->length_dw = chunk_ib->ib_bytes / 4;
653
		ib->flags = chunk_ib->flags;
654
		ib->ctx = parser->ctx;
A
Alex Deucher 已提交
655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695
		j++;
	}

	if (!parser->num_ibs)
		return 0;

	/* add GDS resources to first IB */
	if (parser->bo_list) {
		struct amdgpu_bo *gds = parser->bo_list->gds_obj;
		struct amdgpu_bo *gws = parser->bo_list->gws_obj;
		struct amdgpu_bo *oa = parser->bo_list->oa_obj;
		struct amdgpu_ib *ib = &parser->ibs[0];

		if (gds) {
			ib->gds_base = amdgpu_bo_gpu_offset(gds);
			ib->gds_size = amdgpu_bo_size(gds);
		}
		if (gws) {
			ib->gws_base = amdgpu_bo_gpu_offset(gws);
			ib->gws_size = amdgpu_bo_size(gws);
		}
		if (oa) {
			ib->oa_base = amdgpu_bo_gpu_offset(oa);
			ib->oa_size = amdgpu_bo_size(oa);
		}
	}
	/* wrap the last IB with user fence */
	if (parser->uf.bo) {
		struct amdgpu_ib *ib = &parser->ibs[parser->num_ibs - 1];

		/* UVD & VCE fw doesn't support user fences */
		if (ib->ring->type == AMDGPU_RING_TYPE_UVD ||
		    ib->ring->type == AMDGPU_RING_TYPE_VCE)
			return -EINVAL;

		ib->user = &parser->uf;
	}

	return 0;
}

696 697 698
static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
				  struct amdgpu_cs_parser *p)
{
699
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723
	struct amdgpu_ib *ib;
	int i, j, r;

	if (!p->num_ibs)
		return 0;

	/* Add dependencies to first IB */
	ib = &p->ibs[0];
	for (i = 0; i < p->nchunks; ++i) {
		struct drm_amdgpu_cs_chunk_dep *deps;
		struct amdgpu_cs_chunk *chunk;
		unsigned num_deps;

		chunk = &p->chunks[i];

		if (chunk->chunk_id != AMDGPU_CHUNK_ID_DEPENDENCIES)
			continue;

		deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata;
		num_deps = chunk->length_dw * 4 /
			sizeof(struct drm_amdgpu_cs_chunk_dep);

		for (j = 0; j < num_deps; ++j) {
			struct amdgpu_ring *ring;
724
			struct amdgpu_ctx *ctx;
725
			struct fence *fence;
726 727 728 729 730 731 732

			r = amdgpu_cs_get_ring(adev, deps[j].ip_type,
					       deps[j].ip_instance,
					       deps[j].ring, &ring);
			if (r)
				return r;

733 734 735 736
			ctx = amdgpu_ctx_get(fpriv, deps[j].ctx_id);
			if (ctx == NULL)
				return -EINVAL;

737 738 739 740
			fence = amdgpu_ctx_get_fence(ctx, ring,
						     deps[j].handle);
			if (IS_ERR(fence)) {
				r = PTR_ERR(fence);
741
				amdgpu_ctx_put(ctx);
742
				return r;
743

744 745 746 747 748 749 750
			} else if (fence) {
				r = amdgpu_sync_fence(adev, &ib->sync, fence);
				fence_put(fence);
				amdgpu_ctx_put(ctx);
				if (r)
					return r;
			}
751 752 753 754 755 756
		}
	}

	return 0;
}

757
static int amdgpu_cs_free_job(struct amdgpu_job *job)
758 759
{
	int i;
760 761 762 763 764
	if (job->ibs)
		for (i = 0; i < job->num_ibs; i++)
			amdgpu_ib_free(job->adev, &job->ibs[i]);
	kfree(job->ibs);
	if (job->uf.bo)
765
		amdgpu_bo_unref(&job->uf.bo);
766 767 768
	return 0;
}

C
Chunming Zhou 已提交
769 770 771 772
int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
	struct amdgpu_device *adev = dev->dev_private;
	union drm_amdgpu_cs *cs = data;
773
	struct amdgpu_cs_parser parser = {};
774 775
	bool reserved_buffers = false;
	int i, r;
C
Chunming Zhou 已提交
776

777
	if (!adev->accel_working)
C
Chunming Zhou 已提交
778
		return -EBUSY;
779

780 781 782 783
	parser.adev = adev;
	parser.filp = filp;

	r = amdgpu_cs_parser_init(&parser, data);
A
Alex Deucher 已提交
784
	if (r) {
C
Chunming Zhou 已提交
785
		DRM_ERROR("Failed to initialize parser !\n");
786
		amdgpu_cs_parser_fini(&parser, r, false);
A
Alex Deucher 已提交
787 788 789
		r = amdgpu_cs_handle_lockup(adev, r);
		return r;
	}
790
	r = amdgpu_cs_parser_bos(&parser, data);
791 792 793 794 795 796
	if (r == -ENOMEM)
		DRM_ERROR("Not enough memory for command submission!\n");
	else if (r && r != -ERESTARTSYS)
		DRM_ERROR("Failed to process the buffer list %d!\n", r);
	else if (!r) {
		reserved_buffers = true;
797
		r = amdgpu_cs_ib_fill(adev, &parser);
798 799 800
	}

	if (!r) {
801
		r = amdgpu_cs_dependencies(adev, &parser);
802 803 804 805 806 807 808
		if (r)
			DRM_ERROR("Failed in the dependencies handling %d!\n", r);
	}

	if (r)
		goto out;

809 810
	for (i = 0; i < parser.num_ibs; i++)
		trace_amdgpu_cs(&parser, i);
811

812
	r = amdgpu_cs_ib_vm_chunk(adev, &parser);
813 814 815
	if (r)
		goto out;

816
	if (parser.num_ibs) {
817
		struct amdgpu_ring * ring = parser.ibs->ring;
818 819
		struct amd_sched_fence *fence;
		struct amdgpu_job *job;
820

821
		job = kzalloc(sizeof(struct amdgpu_job), GFP_KERNEL);
822 823 824 825
		if (!job) {
			r = -ENOMEM;
			goto out;
		}
826

827
		job->base.sched = &ring->sched;
828 829
		job->base.s_entity = &parser.ctx->rings[ring->idx].entity;
		job->adev = parser.adev;
830 831 832
		job->owner = parser.filp;
		job->free_job = amdgpu_cs_free_job;

833 834 835 836 837
		job->ibs = parser.ibs;
		job->num_ibs = parser.num_ibs;
		parser.ibs = NULL;
		parser.num_ibs = 0;

838
		if (job->ibs[job->num_ibs - 1].user) {
839
			job->uf = parser.uf;
840
			job->ibs[job->num_ibs - 1].user = &job->uf;
841
			parser.uf.bo = NULL;
842 843
		}

844 845 846 847
		fence = amd_sched_fence_create(job->base.s_entity,
					       parser.filp);
		if (!fence) {
			r = -ENOMEM;
848 849
			amdgpu_cs_free_job(job);
			kfree(job);
850 851
			goto out;
		}
852
		job->base.s_fence = fence;
853
		parser.fence = fence_get(&fence->base);
854 855 856

		cs->out.handle = amdgpu_ctx_add_fence(parser.ctx, ring,
						      &fence->base);
857
		job->ibs[job->num_ibs - 1].sequence = cs->out.handle;
858

859
		trace_amdgpu_cs_ioctl(job);
860
		amd_sched_entity_push_job(&job->base);
A
Alex Deucher 已提交
861 862 863
	}

out:
864
	amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
A
Alex Deucher 已提交
865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883
	r = amdgpu_cs_handle_lockup(adev, r);
	return r;
}

/**
 * amdgpu_cs_wait_ioctl - wait for a command submission to finish
 *
 * @dev: drm device
 * @data: data from userspace
 * @filp: file private
 *
 * Wait for the command submission identified by handle to finish.
 */
int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
			 struct drm_file *filp)
{
	union drm_amdgpu_wait_cs *wait = data;
	struct amdgpu_device *adev = dev->dev_private;
	unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
884
	struct amdgpu_ring *ring = NULL;
885
	struct amdgpu_ctx *ctx;
886
	struct fence *fence;
A
Alex Deucher 已提交
887 888
	long r;

889 890 891 892 893
	r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance,
			       wait->in.ring, &ring);
	if (r)
		return r;

894 895 896
	ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
	if (ctx == NULL)
		return -EINVAL;
A
Alex Deucher 已提交
897

898 899 900 901 902 903 904 905
	fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle);
	if (IS_ERR(fence))
		r = PTR_ERR(fence);
	else if (fence) {
		r = fence_wait_timeout(fence, true, timeout);
		fence_put(fence);
	} else
		r = 1;
C
Chunming Zhou 已提交
906

907
	amdgpu_ctx_put(ctx);
A
Alex Deucher 已提交
908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932
	if (r < 0)
		return r;

	memset(wait, 0, sizeof(*wait));
	wait->out.status = (r == 0);

	return 0;
}

/**
 * amdgpu_cs_find_bo_va - find bo_va for VM address
 *
 * @parser: command submission parser context
 * @addr: VM address
 * @bo: resulting BO of the mapping found
 *
 * Search the buffer objects in the command submission context for a certain
 * virtual memory address. Returns allocation structure when found, NULL
 * otherwise.
 */
struct amdgpu_bo_va_mapping *
amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
		       uint64_t addr, struct amdgpu_bo **bo)
{
	struct amdgpu_bo_va_mapping *mapping;
933 934 935 936
	unsigned i;

	if (!parser->bo_list)
		return NULL;
A
Alex Deucher 已提交
937 938 939

	addr /= AMDGPU_GPU_PAGE_SIZE;

940 941 942 943 944
	for (i = 0; i < parser->bo_list->num_entries; i++) {
		struct amdgpu_bo_list_entry *lobj;

		lobj = &parser->bo_list->array[i];
		if (!lobj->bo_va)
A
Alex Deucher 已提交
945 946
			continue;

947
		list_for_each_entry(mapping, &lobj->bo_va->valids, list) {
948 949 950 951
			if (mapping->it.start > addr ||
			    addr > mapping->it.last)
				continue;

952
			*bo = lobj->bo_va->bo;
953 954 955
			return mapping;
		}

956
		list_for_each_entry(mapping, &lobj->bo_va->invalids, list) {
A
Alex Deucher 已提交
957 958 959 960
			if (mapping->it.start > addr ||
			    addr > mapping->it.last)
				continue;

961
			*bo = lobj->bo_va->bo;
A
Alex Deucher 已提交
962 963 964 965 966 967
			return mapping;
		}
	}

	return NULL;
}