amdgpu_cs.c 29.0 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
/*
 * Copyright 2008 Jerome Glisse.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Authors:
 *    Jerome Glisse <glisse@freedesktop.org>
 */
27
#include <linux/pagemap.h>
A
Alex Deucher 已提交
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
#include "amdgpu.h"
#include "amdgpu_trace.h"

int amdgpu_cs_get_ring(struct amdgpu_device *adev, u32 ip_type,
		       u32 ip_instance, u32 ring,
		       struct amdgpu_ring **out_ring)
{
	/* Right now all IPs have only one instance - multiple rings. */
	if (ip_instance != 0) {
		DRM_ERROR("invalid ip instance: %d\n", ip_instance);
		return -EINVAL;
	}

	switch (ip_type) {
	default:
		DRM_ERROR("unknown ip type: %d\n", ip_type);
		return -EINVAL;
	case AMDGPU_HW_IP_GFX:
		if (ring < adev->gfx.num_gfx_rings) {
			*out_ring = &adev->gfx.gfx_ring[ring];
		} else {
			DRM_ERROR("only %d gfx rings are supported now\n",
				  adev->gfx.num_gfx_rings);
			return -EINVAL;
		}
		break;
	case AMDGPU_HW_IP_COMPUTE:
		if (ring < adev->gfx.num_compute_rings) {
			*out_ring = &adev->gfx.compute_ring[ring];
		} else {
			DRM_ERROR("only %d compute rings are supported now\n",
				  adev->gfx.num_compute_rings);
			return -EINVAL;
		}
		break;
	case AMDGPU_HW_IP_DMA:
A
Alex Deucher 已提交
66 67
		if (ring < adev->sdma.num_instances) {
			*out_ring = &adev->sdma.instance[ring].ring;
A
Alex Deucher 已提交
68
		} else {
A
Alex Deucher 已提交
69 70
			DRM_ERROR("only %d SDMA rings are supported\n",
				  adev->sdma.num_instances);
A
Alex Deucher 已提交
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
			return -EINVAL;
		}
		break;
	case AMDGPU_HW_IP_UVD:
		*out_ring = &adev->uvd.ring;
		break;
	case AMDGPU_HW_IP_VCE:
		if (ring < 2){
			*out_ring = &adev->vce.ring[ring];
		} else {
			DRM_ERROR("only two VCE rings are supported\n");
			return -EINVAL;
		}
		break;
	}
	return 0;
}

89
static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
90 91
				      struct drm_amdgpu_cs_chunk_fence *data,
				      uint32_t *offset)
92 93 94
{
	struct drm_gem_object *gobj;

95
	gobj = drm_gem_object_lookup(p->filp, data->handle);
96 97 98
	if (gobj == NULL)
		return -EINVAL;

99
	p->uf_entry.robj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
100 101 102
	p->uf_entry.priority = 0;
	p->uf_entry.tv.bo = &p->uf_entry.robj->tbo;
	p->uf_entry.tv.shared = true;
103
	p->uf_entry.user_pages = NULL;
104
	*offset = data->offset;
105 106

	drm_gem_object_unreference_unlocked(gobj);
107 108 109 110 111 112

	if (amdgpu_ttm_tt_get_usermm(p->uf_entry.robj->tbo.ttm)) {
		amdgpu_bo_unref(&p->uf_entry.robj);
		return -EINVAL;
	}

113 114 115
	return 0;
}

A
Alex Deucher 已提交
116 117
int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
{
118
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
119
	struct amdgpu_vm *vm = &fpriv->vm;
A
Alex Deucher 已提交
120 121
	union drm_amdgpu_cs *cs = data;
	uint64_t *chunk_array_user;
122
	uint64_t *chunk_array;
123
	unsigned size, num_ibs = 0;
124
	uint32_t uf_offset = 0;
125
	int i;
126
	int ret;
A
Alex Deucher 已提交
127

128 129 130 131 132 133
	if (cs->in.num_chunks == 0)
		return 0;

	chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
	if (!chunk_array)
		return -ENOMEM;
A
Alex Deucher 已提交
134

135 136
	p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
	if (!p->ctx) {
137 138
		ret = -EINVAL;
		goto free_chunk;
139
	}
140

A
Alex Deucher 已提交
141
	/* get chunks */
142
	chunk_array_user = (uint64_t __user *)(unsigned long)(cs->in.chunks);
A
Alex Deucher 已提交
143 144
	if (copy_from_user(chunk_array, chunk_array_user,
			   sizeof(uint64_t)*cs->in.num_chunks)) {
145
		ret = -EFAULT;
146
		goto put_ctx;
A
Alex Deucher 已提交
147 148 149
	}

	p->nchunks = cs->in.num_chunks;
150
	p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
A
Alex Deucher 已提交
151
			    GFP_KERNEL);
152 153
	if (!p->chunks) {
		ret = -ENOMEM;
154
		goto put_ctx;
A
Alex Deucher 已提交
155 156 157 158 159 160 161
	}

	for (i = 0; i < p->nchunks; i++) {
		struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL;
		struct drm_amdgpu_cs_chunk user_chunk;
		uint32_t __user *cdata;

162
		chunk_ptr = (void __user *)(unsigned long)chunk_array[i];
A
Alex Deucher 已提交
163 164
		if (copy_from_user(&user_chunk, chunk_ptr,
				       sizeof(struct drm_amdgpu_cs_chunk))) {
165 166 167
			ret = -EFAULT;
			i--;
			goto free_partial_kdata;
A
Alex Deucher 已提交
168 169 170 171 172
		}
		p->chunks[i].chunk_id = user_chunk.chunk_id;
		p->chunks[i].length_dw = user_chunk.length_dw;

		size = p->chunks[i].length_dw;
173
		cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
A
Alex Deucher 已提交
174 175 176

		p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
		if (p->chunks[i].kdata == NULL) {
177 178 179
			ret = -ENOMEM;
			i--;
			goto free_partial_kdata;
A
Alex Deucher 已提交
180 181 182
		}
		size *= sizeof(uint32_t);
		if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
183 184
			ret = -EFAULT;
			goto free_partial_kdata;
A
Alex Deucher 已提交
185 186
		}

187 188
		switch (p->chunks[i].chunk_id) {
		case AMDGPU_CHUNK_ID_IB:
189
			++num_ibs;
190 191 192
			break;

		case AMDGPU_CHUNK_ID_FENCE:
A
Alex Deucher 已提交
193
			size = sizeof(struct drm_amdgpu_cs_chunk_fence);
194
			if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
195 196
				ret = -EINVAL;
				goto free_partial_kdata;
A
Alex Deucher 已提交
197
			}
198

199 200
			ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata,
							 &uf_offset);
201 202 203
			if (ret)
				goto free_partial_kdata;

204 205
			break;

206 207 208
		case AMDGPU_CHUNK_ID_DEPENDENCIES:
			break;

209
		default:
210 211
			ret = -EINVAL;
			goto free_partial_kdata;
A
Alex Deucher 已提交
212 213 214
		}
	}

215
	ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm);
216
	if (ret)
C
Christian König 已提交
217
		goto free_all_kdata;
A
Alex Deucher 已提交
218

219 220
	if (p->uf_entry.robj)
		p->job->uf_addr = uf_offset;
A
Alex Deucher 已提交
221
	kfree(chunk_array);
222 223 224 225 226 227 228 229
	return 0;

free_all_kdata:
	i = p->nchunks - 1;
free_partial_kdata:
	for (; i >= 0; i--)
		drm_free_large(p->chunks[i].kdata);
	kfree(p->chunks);
230
put_ctx:
231 232 233 234 235
	amdgpu_ctx_put(p->ctx);
free_chunk:
	kfree(chunk_array);

	return ret;
A
Alex Deucher 已提交
236 237
}

238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269
/* Convert microseconds to bytes. */
static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)
{
	if (us <= 0 || !adev->mm_stats.log2_max_MBps)
		return 0;

	/* Since accum_us is incremented by a million per second, just
	 * multiply it by the number of MB/s to get the number of bytes.
	 */
	return us << adev->mm_stats.log2_max_MBps;
}

static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)
{
	if (!adev->mm_stats.log2_max_MBps)
		return 0;

	return bytes >> adev->mm_stats.log2_max_MBps;
}

/* Returns how many bytes TTM can move right now. If no bytes can be moved,
 * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
 * which means it can go over the threshold once. If that happens, the driver
 * will be in debt and no other buffer migrations can be done until that debt
 * is repaid.
 *
 * This approach allows moving a buffer of any size (it's important to allow
 * that).
 *
 * The currency is simply time in microseconds and it increases as the clock
 * ticks. The accumulated microseconds (us) are converted to bytes and
 * returned.
A
Alex Deucher 已提交
270 271 272
 */
static u64 amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev)
{
273 274 275
	s64 time_us, increment_us;
	u64 max_bytes;
	u64 free_vram, total_vram, used_vram;
A
Alex Deucher 已提交
276

277 278
	/* Allow a maximum of 200 accumulated ms. This is basically per-IB
	 * throttling.
A
Alex Deucher 已提交
279
	 *
280 281 282 283 284
	 * It means that in order to get full max MBps, at least 5 IBs per
	 * second must be submitted and not more than 200ms apart from each
	 * other.
	 */
	const s64 us_upper_bound = 200000;
A
Alex Deucher 已提交
285

286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
	if (!adev->mm_stats.log2_max_MBps)
		return 0;

	total_vram = adev->mc.real_vram_size - adev->vram_pin_size;
	used_vram = atomic64_read(&adev->vram_usage);
	free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;

	spin_lock(&adev->mm_stats.lock);

	/* Increase the amount of accumulated us. */
	time_us = ktime_to_us(ktime_get());
	increment_us = time_us - adev->mm_stats.last_update_us;
	adev->mm_stats.last_update_us = time_us;
	adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us,
                                      us_upper_bound);

	/* This prevents the short period of low performance when the VRAM
	 * usage is low and the driver is in debt or doesn't have enough
	 * accumulated us to fill VRAM quickly.
A
Alex Deucher 已提交
305
	 *
306 307 308 309
	 * The situation can occur in these cases:
	 * - a lot of VRAM is freed by userspace
	 * - the presence of a big buffer causes a lot of evictions
	 *   (solution: split buffers into smaller ones)
A
Alex Deucher 已提交
310
	 *
311 312
	 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
	 * accum_us to a positive number.
A
Alex Deucher 已提交
313
	 */
314 315 316 317 318 319 320 321 322 323 324 325 326
	if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) {
		s64 min_us;

		/* Be more aggresive on dGPUs. Try to fill a portion of free
		 * VRAM now.
		 */
		if (!(adev->flags & AMD_IS_APU))
			min_us = bytes_to_us(adev, free_vram / 4);
		else
			min_us = 0; /* Reset accum_us on APUs. */

		adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);
	}
A
Alex Deucher 已提交
327

328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
	/* This returns 0 if the driver is in debt to disallow (optional)
	 * buffer moves.
	 */
	max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);

	spin_unlock(&adev->mm_stats.lock);
	return max_bytes;
}

/* Report how many bytes have really been moved for the last command
 * submission. This can result in a debt that can stop buffer migrations
 * temporarily.
 */
static void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev,
					 u64 num_bytes)
{
	spin_lock(&adev->mm_stats.lock);
	adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);
	spin_unlock(&adev->mm_stats.lock);
A
Alex Deucher 已提交
347 348
}

349 350 351 352 353 354 355 356 357 358
static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
				 struct amdgpu_bo *bo)
{
	u64 initial_bytes_moved;
	uint32_t domain;
	int r;

	if (bo->pin_count)
		return 0;

359 360
	/* Don't move this buffer if we have depleted our allowance
	 * to move it. Don't move anything if the threshold is zero.
361
	 */
362
	if (p->bytes_moved < p->bytes_moved_threshold)
363 364 365 366 367 368 369 370 371 372 373
		domain = bo->prefered_domains;
	else
		domain = bo->allowed_domains;

retry:
	amdgpu_ttm_placement_from_domain(bo, domain);
	initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
	r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
	p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
		initial_bytes_moved;

374 375 376
	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
		domain = bo->allowed_domains;
		goto retry;
377 378 379 380 381
	}

	return r;
}

382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433
/* Last resort, try to evict something from the current working set */
static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
				struct amdgpu_bo_list_entry *lobj)
{
	uint32_t domain = lobj->robj->allowed_domains;
	int r;

	if (!p->evictable)
		return false;

	for (;&p->evictable->tv.head != &p->validated;
	     p->evictable = list_prev_entry(p->evictable, tv.head)) {

		struct amdgpu_bo_list_entry *candidate = p->evictable;
		struct amdgpu_bo *bo = candidate->robj;
		u64 initial_bytes_moved;
		uint32_t other;

		/* If we reached our current BO we can forget it */
		if (candidate == lobj)
			break;

		other = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);

		/* Check if this BO is in one of the domains we need space for */
		if (!(other & domain))
			continue;

		/* Check if we can move this BO somewhere else */
		other = bo->allowed_domains & ~domain;
		if (!other)
			continue;

		/* Good we can try to move this BO somewhere else */
		amdgpu_ttm_placement_from_domain(bo, other);
		initial_bytes_moved = atomic64_read(&bo->adev->num_bytes_moved);
		r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
		p->bytes_moved += atomic64_read(&bo->adev->num_bytes_moved) -
			initial_bytes_moved;

		if (unlikely(r))
			break;

		p->evictable = list_prev_entry(p->evictable, tv.head);
		list_move(&candidate->tv.head, &p->validated);

		return true;
	}

	return false;
}

434
static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
435
			    struct list_head *validated)
A
Alex Deucher 已提交
436 437 438 439
{
	struct amdgpu_bo_list_entry *lobj;
	int r;

440
	list_for_each_entry(lobj, validated, tv.head) {
441
		struct amdgpu_bo *bo = lobj->robj;
442
		bool binding_userptr = false;
443
		struct mm_struct *usermm;
A
Alex Deucher 已提交
444

445 446 447 448
		usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
		if (usermm && usermm != current->mm)
			return -EPERM;

449 450 451 452 453 454 455 456 457
		/* Check if we have user pages and nobody bound the BO already */
		if (lobj->user_pages && bo->tbo.ttm->state != tt_bound) {
			size_t size = sizeof(struct page *);

			size *= bo->tbo.ttm->num_pages;
			memcpy(bo->tbo.ttm->pages, lobj->user_pages, size);
			binding_userptr = true;
		}

458 459 460 461 462 463
		if (p->evictable == lobj)
			p->evictable = NULL;

		do {
			r = amdgpu_cs_bo_validate(p, bo);
		} while (r == -ENOMEM && amdgpu_cs_try_evict(p, lobj));
464
		if (r)
465
			return r;
466

467 468 469 470
		if (bo->shadow) {
			r = amdgpu_cs_bo_validate(p, bo);
			if (r)
				return r;
A
Alex Deucher 已提交
471
		}
472 473 474 475 476

		if (binding_userptr) {
			drm_free_large(lobj->user_pages);
			lobj->user_pages = NULL;
		}
A
Alex Deucher 已提交
477 478 479 480
	}
	return 0;
}

481 482
static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
				union drm_amdgpu_cs *cs)
A
Alex Deucher 已提交
483 484
{
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
485
	struct amdgpu_bo_list_entry *e;
486
	struct list_head duplicates;
487
	bool need_mmap_lock = false;
488
	unsigned i, tries = 10;
489
	int r;
A
Alex Deucher 已提交
490

491 492 493
	INIT_LIST_HEAD(&p->validated);

	p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
494
	if (p->bo_list) {
495 496
		need_mmap_lock = p->bo_list->first_userptr !=
			p->bo_list->num_entries;
497
		amdgpu_bo_list_get_list(p->bo_list, &p->validated);
498
	}
A
Alex Deucher 已提交
499

500
	INIT_LIST_HEAD(&duplicates);
501
	amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
A
Alex Deucher 已提交
502

503
	if (p->uf_entry.robj)
504 505
		list_add(&p->uf_entry.tv.head, &p->validated);

A
Alex Deucher 已提交
506 507 508
	if (need_mmap_lock)
		down_read(&current->mm->mmap_sem);

509 510 511 512 513 514
	while (1) {
		struct list_head need_pages;
		unsigned i;

		r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
					   &duplicates);
515 516
		if (unlikely(r != 0)) {
			DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
517
			goto error_free_pages;
518
		}
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557

		/* Without a BO list we don't have userptr BOs */
		if (!p->bo_list)
			break;

		INIT_LIST_HEAD(&need_pages);
		for (i = p->bo_list->first_userptr;
		     i < p->bo_list->num_entries; ++i) {

			e = &p->bo_list->array[i];

			if (amdgpu_ttm_tt_userptr_invalidated(e->robj->tbo.ttm,
				 &e->user_invalidated) && e->user_pages) {

				/* We acquired a page array, but somebody
				 * invalidated it. Free it an try again
				 */
				release_pages(e->user_pages,
					      e->robj->tbo.ttm->num_pages,
					      false);
				drm_free_large(e->user_pages);
				e->user_pages = NULL;
			}

			if (e->robj->tbo.ttm->state != tt_bound &&
			    !e->user_pages) {
				list_del(&e->tv.head);
				list_add(&e->tv.head, &need_pages);

				amdgpu_bo_unreserve(e->robj);
			}
		}

		if (list_empty(&need_pages))
			break;

		/* Unreserve everything again. */
		ttm_eu_backoff_reservation(&p->ticket, &p->validated);

558
		/* We tried too many times, just abort */
559 560
		if (!--tries) {
			r = -EDEADLK;
561
			DRM_ERROR("deadlock in %s\n", __func__);
562 563 564 565 566 567 568 569 570 571 572
			goto error_free_pages;
		}

		/* Fill the page arrays for all useptrs. */
		list_for_each_entry(e, &need_pages, tv.head) {
			struct ttm_tt *ttm = e->robj->tbo.ttm;

			e->user_pages = drm_calloc_large(ttm->num_pages,
							 sizeof(struct page*));
			if (!e->user_pages) {
				r = -ENOMEM;
573
				DRM_ERROR("calloc failure in %s\n", __func__);
574 575 576 577 578
				goto error_free_pages;
			}

			r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages);
			if (r) {
579
				DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n");
580 581 582 583 584 585 586 587 588
				drm_free_large(e->user_pages);
				e->user_pages = NULL;
				goto error_free_pages;
			}
		}

		/* And try again. */
		list_splice(&need_pages, &p->validated);
	}
589

590
	amdgpu_vm_get_pt_bos(p->adev, &fpriv->vm, &duplicates);
591

592 593
	p->bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(p->adev);
	p->bytes_moved = 0;
594 595 596
	p->evictable = list_last_entry(&p->validated,
				       struct amdgpu_bo_list_entry,
				       tv.head);
597 598

	r = amdgpu_cs_list_validate(p, &duplicates);
599 600
	if (r) {
		DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n");
601
		goto error_validate;
602
	}
603

604
	r = amdgpu_cs_list_validate(p, &p->validated);
605 606
	if (r) {
		DRM_ERROR("amdgpu_cs_list_validate(validated) failed.\n");
607
		goto error_validate;
608
	}
609

610 611
	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved);

612 613 614
	fpriv->vm.last_eviction_counter =
		atomic64_read(&p->adev->num_evictions);

615
	if (p->bo_list) {
616 617 618
		struct amdgpu_bo *gds = p->bo_list->gds_obj;
		struct amdgpu_bo *gws = p->bo_list->gws_obj;
		struct amdgpu_bo *oa = p->bo_list->oa_obj;
619 620 621 622 623 624 625 626
		struct amdgpu_vm *vm = &fpriv->vm;
		unsigned i;

		for (i = 0; i < p->bo_list->num_entries; i++) {
			struct amdgpu_bo *bo = p->bo_list->array[i].robj;

			p->bo_list->array[i].bo_va = amdgpu_vm_bo_find(vm, bo);
		}
627 628 629 630 631 632 633 634 635 636 637 638 639

		if (gds) {
			p->job->gds_base = amdgpu_bo_gpu_offset(gds);
			p->job->gds_size = amdgpu_bo_size(gds);
		}
		if (gws) {
			p->job->gws_base = amdgpu_bo_gpu_offset(gws);
			p->job->gws_size = amdgpu_bo_size(gws);
		}
		if (oa) {
			p->job->oa_base = amdgpu_bo_gpu_offset(oa);
			p->job->oa_size = amdgpu_bo_size(oa);
		}
640
	}
641

642 643 644 645 646 647
	if (!r && p->uf_entry.robj) {
		struct amdgpu_bo *uf = p->uf_entry.robj;

		r = amdgpu_ttm_bind(uf->tbo.ttm, &uf->tbo.mem);
		p->job->uf_addr += amdgpu_bo_gpu_offset(uf);
	}
648

649
error_validate:
650 651
	if (r) {
		amdgpu_vm_move_pt_bos_in_lru(p->adev, &fpriv->vm);
652
		ttm_eu_backoff_reservation(&p->ticket, &p->validated);
653
	}
A
Alex Deucher 已提交
654

655 656
error_free_pages:

A
Alex Deucher 已提交
657 658 659
	if (need_mmap_lock)
		up_read(&current->mm->mmap_sem);

660 661 662 663 664 665 666 667 668 669 670 671 672 673 674
	if (p->bo_list) {
		for (i = p->bo_list->first_userptr;
		     i < p->bo_list->num_entries; ++i) {
			e = &p->bo_list->array[i];

			if (!e->user_pages)
				continue;

			release_pages(e->user_pages,
				      e->robj->tbo.ttm->num_pages,
				      false);
			drm_free_large(e->user_pages);
		}
	}

A
Alex Deucher 已提交
675 676 677 678 679 680 681 682 683 684
	return r;
}

static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
{
	struct amdgpu_bo_list_entry *e;
	int r;

	list_for_each_entry(e, &p->validated, tv.head) {
		struct reservation_object *resv = e->robj->tbo.resv;
685
		r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp);
A
Alex Deucher 已提交
686 687 688 689 690 691 692

		if (r)
			return r;
	}
	return 0;
}

693 694 695 696 697 698 699 700 701
/**
 * cs_parser_fini() - clean parser states
 * @parser:	parser structure holding parsing context.
 * @error:	error number
 *
 * If error is set than unvalidate buffer, otherwise just free memory
 * used by parsing context.
 **/
static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error, bool backoff)
C
Chunming Zhou 已提交
702
{
703
	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
704 705
	unsigned i;

A
Alex Deucher 已提交
706
	if (!error) {
707 708
		amdgpu_vm_move_pt_bos_in_lru(parser->adev, &fpriv->vm);

A
Alex Deucher 已提交
709
		ttm_eu_fence_buffer_objects(&parser->ticket,
710 711
					    &parser->validated,
					    parser->fence);
A
Alex Deucher 已提交
712 713 714 715
	} else if (backoff) {
		ttm_eu_backoff_reservation(&parser->ticket,
					   &parser->validated);
	}
716
	fence_put(parser->fence);
717

718 719
	if (parser->ctx)
		amdgpu_ctx_put(parser->ctx);
720 721 722
	if (parser->bo_list)
		amdgpu_bo_list_put(parser->bo_list);

A
Alex Deucher 已提交
723 724 725
	for (i = 0; i < parser->nchunks; i++)
		drm_free_large(parser->chunks[i].kdata);
	kfree(parser->chunks);
726 727
	if (parser->job)
		amdgpu_job_free(parser->job);
728
	amdgpu_bo_unref(&parser->uf_entry.robj);
A
Alex Deucher 已提交
729 730 731 732 733 734 735 736 737 738 739 740 741 742
}

static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p,
				   struct amdgpu_vm *vm)
{
	struct amdgpu_device *adev = p->adev;
	struct amdgpu_bo_va *bo_va;
	struct amdgpu_bo *bo;
	int i, r;

	r = amdgpu_vm_update_page_directory(adev, vm);
	if (r)
		return r;

743
	r = amdgpu_sync_fence(adev, &p->job->sync, vm->page_directory_fence);
744 745 746
	if (r)
		return r;

A
Alex Deucher 已提交
747 748 749 750 751 752
	r = amdgpu_vm_clear_freed(adev, vm);
	if (r)
		return r;

	if (p->bo_list) {
		for (i = 0; i < p->bo_list->num_entries; i++) {
753 754
			struct fence *f;

A
Alex Deucher 已提交
755 756 757 758 759 760 761 762 763
			/* ignore duplicates */
			bo = p->bo_list->array[i].robj;
			if (!bo)
				continue;

			bo_va = p->bo_list->array[i].bo_va;
			if (bo_va == NULL)
				continue;

764
			r = amdgpu_vm_bo_update(adev, bo_va, false);
A
Alex Deucher 已提交
765 766 767
			if (r)
				return r;

768
			f = bo_va->last_pt_update;
769
			r = amdgpu_sync_fence(adev, &p->job->sync, f);
770 771
			if (r)
				return r;
A
Alex Deucher 已提交
772
		}
773 774 775

	}

776
	r = amdgpu_vm_clear_invalids(adev, vm, &p->job->sync);
777 778 779 780 781 782 783 784 785 786 787

	if (amdgpu_vm_debug && p->bo_list) {
		/* Invalidate all BOs to test for userspace bugs */
		for (i = 0; i < p->bo_list->num_entries; i++) {
			/* ignore duplicates */
			bo = p->bo_list->array[i].robj;
			if (!bo)
				continue;

			amdgpu_vm_bo_invalidate(adev, bo);
		}
A
Alex Deucher 已提交
788 789
	}

790
	return r;
A
Alex Deucher 已提交
791 792 793
}

static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
794
				 struct amdgpu_cs_parser *p)
A
Alex Deucher 已提交
795
{
796
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
A
Alex Deucher 已提交
797
	struct amdgpu_vm *vm = &fpriv->vm;
798
	struct amdgpu_ring *ring = p->job->ring;
A
Alex Deucher 已提交
799 800 801
	int i, r;

	/* Only for UVD/VCE VM emulation */
802
	if (ring->funcs->parse_cs) {
803
		p->job->vm = NULL;
804 805
		for (i = 0; i < p->job->num_ibs; i++) {
			r = amdgpu_ring_parse_cs(ring, p, i);
A
Alex Deucher 已提交
806 807 808
			if (r)
				return r;
		}
809 810
	} else {
		p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
811

812 813 814 815
		r = amdgpu_bo_vm_update_pte(p, vm);
		if (r)
			return r;
	}
A
Alex Deucher 已提交
816

817
	return amdgpu_cs_sync_rings(p);
A
Alex Deucher 已提交
818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837
}

static int amdgpu_cs_handle_lockup(struct amdgpu_device *adev, int r)
{
	if (r == -EDEADLK) {
		r = amdgpu_gpu_reset(adev);
		if (!r)
			r = -EAGAIN;
	}
	return r;
}

static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
			     struct amdgpu_cs_parser *parser)
{
	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
	struct amdgpu_vm *vm = &fpriv->vm;
	int i, j;
	int r;

838
	for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) {
A
Alex Deucher 已提交
839 840 841 842 843 844
		struct amdgpu_cs_chunk *chunk;
		struct amdgpu_ib *ib;
		struct drm_amdgpu_cs_chunk_ib *chunk_ib;
		struct amdgpu_ring *ring;

		chunk = &parser->chunks[i];
845
		ib = &parser->job->ibs[j];
A
Alex Deucher 已提交
846 847 848 849 850 851 852 853
		chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;

		if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
			continue;

		r = amdgpu_cs_get_ring(adev, chunk_ib->ip_type,
				       chunk_ib->ip_instance, chunk_ib->ring,
				       &ring);
854
		if (r)
A
Alex Deucher 已提交
855 856
			return r;

857 858 859 860 861 862 863 864
		if (ib->flags & AMDGPU_IB_FLAG_PREAMBLE) {
			parser->job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT;
			if (!parser->ctx->preamble_presented) {
				parser->job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;
				parser->ctx->preamble_presented = true;
			}
		}

865 866 867 868 869
		if (parser->job->ring && parser->job->ring != ring)
			return -EINVAL;

		parser->job->ring = ring;

A
Alex Deucher 已提交
870
		if (ring->funcs->parse_cs) {
871
			struct amdgpu_bo_va_mapping *m;
872
			struct amdgpu_bo *aobj = NULL;
873 874
			uint64_t offset;
			uint8_t *kptr;
875

876 877
			m = amdgpu_cs_find_mapping(parser, chunk_ib->va_start,
						   &aobj);
878 879 880
			if (!aobj) {
				DRM_ERROR("IB va_start is invalid\n");
				return -EINVAL;
A
Alex Deucher 已提交
881 882
			}

883 884 885 886 887 888
			if ((chunk_ib->va_start + chunk_ib->ib_bytes) >
			    (m->it.last + 1) * AMDGPU_GPU_PAGE_SIZE) {
				DRM_ERROR("IB va_start+ib_bytes is invalid\n");
				return -EINVAL;
			}

889
			/* the IB should be reserved at this point */
890
			r = amdgpu_bo_kmap(aobj, (void **)&kptr);
A
Alex Deucher 已提交
891 892 893 894
			if (r) {
				return r;
			}

895 896 897
			offset = ((uint64_t)m->it.start) * AMDGPU_GPU_PAGE_SIZE;
			kptr += chunk_ib->va_start - offset;

898
			r =  amdgpu_ib_get(adev, NULL, chunk_ib->ib_bytes, ib);
A
Alex Deucher 已提交
899 900 901 902 903 904 905 906
			if (r) {
				DRM_ERROR("Failed to get ib !\n");
				return r;
			}

			memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
			amdgpu_bo_kunmap(aobj);
		} else {
907
			r =  amdgpu_ib_get(adev, vm, 0, ib);
A
Alex Deucher 已提交
908 909 910 911 912 913 914 915
			if (r) {
				DRM_ERROR("Failed to get ib !\n");
				return r;
			}

			ib->gpu_addr = chunk_ib->va_start;
		}

916
		ib->length_dw = chunk_ib->ib_bytes / 4;
917
		ib->flags = chunk_ib->flags;
A
Alex Deucher 已提交
918 919 920
		j++;
	}

921
	/* UVD & VCE fw doesn't support user fences */
922
	if (parser->job->uf_addr && (
923 924 925
	    parser->job->ring->type == AMDGPU_RING_TYPE_UVD ||
	    parser->job->ring->type == AMDGPU_RING_TYPE_VCE))
		return -EINVAL;
A
Alex Deucher 已提交
926 927 928 929

	return 0;
}

930 931 932
static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
				  struct amdgpu_cs_parser *p)
{
933
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951
	int i, j, r;

	for (i = 0; i < p->nchunks; ++i) {
		struct drm_amdgpu_cs_chunk_dep *deps;
		struct amdgpu_cs_chunk *chunk;
		unsigned num_deps;

		chunk = &p->chunks[i];

		if (chunk->chunk_id != AMDGPU_CHUNK_ID_DEPENDENCIES)
			continue;

		deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata;
		num_deps = chunk->length_dw * 4 /
			sizeof(struct drm_amdgpu_cs_chunk_dep);

		for (j = 0; j < num_deps; ++j) {
			struct amdgpu_ring *ring;
952
			struct amdgpu_ctx *ctx;
953
			struct fence *fence;
954 955 956 957 958 959 960

			r = amdgpu_cs_get_ring(adev, deps[j].ip_type,
					       deps[j].ip_instance,
					       deps[j].ring, &ring);
			if (r)
				return r;

961 962 963 964
			ctx = amdgpu_ctx_get(fpriv, deps[j].ctx_id);
			if (ctx == NULL)
				return -EINVAL;

965 966 967 968
			fence = amdgpu_ctx_get_fence(ctx, ring,
						     deps[j].handle);
			if (IS_ERR(fence)) {
				r = PTR_ERR(fence);
969
				amdgpu_ctx_put(ctx);
970
				return r;
971

972
			} else if (fence) {
973 974
				r = amdgpu_sync_fence(adev, &p->job->sync,
						      fence);
975 976 977 978 979
				fence_put(fence);
				amdgpu_ctx_put(ctx);
				if (r)
					return r;
			}
980 981 982 983 984 985
		}
	}

	return 0;
}

986 987 988
static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
			    union drm_amdgpu_cs *cs)
{
989
	struct amdgpu_ring *ring = p->job->ring;
990
	struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
991
	struct amdgpu_job *job;
992
	int r;
993

994 995
	job = p->job;
	p->job = NULL;
996

997
	r = amd_sched_job_init(&job->base, &ring->sched, entity, p->filp);
998
	if (r) {
999
		amdgpu_job_free(job);
1000
		return r;
1001 1002
	}

1003
	job->owner = p->filp;
1004
	job->fence_ctx = entity->fence_context;
1005 1006
	p->fence = fence_get(&job->base.s_fence->finished);
	cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence);
1007
	job->uf_sequence = cs->out.handle;
1008
	amdgpu_job_free_resources(job);
1009 1010 1011 1012 1013 1014 1015

	trace_amdgpu_cs_ioctl(job);
	amd_sched_entity_push_job(&job->base);

	return 0;
}

C
Chunming Zhou 已提交
1016 1017 1018 1019
int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
	struct amdgpu_device *adev = dev->dev_private;
	union drm_amdgpu_cs *cs = data;
1020
	struct amdgpu_cs_parser parser = {};
1021 1022
	bool reserved_buffers = false;
	int i, r;
C
Chunming Zhou 已提交
1023

1024
	if (!adev->accel_working)
C
Chunming Zhou 已提交
1025
		return -EBUSY;
1026

1027 1028 1029 1030
	parser.adev = adev;
	parser.filp = filp;

	r = amdgpu_cs_parser_init(&parser, data);
A
Alex Deucher 已提交
1031
	if (r) {
C
Chunming Zhou 已提交
1032
		DRM_ERROR("Failed to initialize parser !\n");
1033
		amdgpu_cs_parser_fini(&parser, r, false);
A
Alex Deucher 已提交
1034 1035 1036
		r = amdgpu_cs_handle_lockup(adev, r);
		return r;
	}
1037
	r = amdgpu_cs_parser_bos(&parser, data);
1038 1039 1040 1041 1042 1043
	if (r == -ENOMEM)
		DRM_ERROR("Not enough memory for command submission!\n");
	else if (r && r != -ERESTARTSYS)
		DRM_ERROR("Failed to process the buffer list %d!\n", r);
	else if (!r) {
		reserved_buffers = true;
1044
		r = amdgpu_cs_ib_fill(adev, &parser);
1045 1046 1047
	}

	if (!r) {
1048
		r = amdgpu_cs_dependencies(adev, &parser);
1049 1050 1051 1052 1053 1054 1055
		if (r)
			DRM_ERROR("Failed in the dependencies handling %d!\n", r);
	}

	if (r)
		goto out;

1056
	for (i = 0; i < parser.job->num_ibs; i++)
1057
		trace_amdgpu_cs(&parser, i);
1058

1059
	r = amdgpu_cs_ib_vm_chunk(adev, &parser);
1060 1061 1062
	if (r)
		goto out;

C
Christian König 已提交
1063
	r = amdgpu_cs_submit(&parser, cs);
A
Alex Deucher 已提交
1064 1065

out:
1066
	amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
A
Alex Deucher 已提交
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085
	r = amdgpu_cs_handle_lockup(adev, r);
	return r;
}

/**
 * amdgpu_cs_wait_ioctl - wait for a command submission to finish
 *
 * @dev: drm device
 * @data: data from userspace
 * @filp: file private
 *
 * Wait for the command submission identified by handle to finish.
 */
int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
			 struct drm_file *filp)
{
	union drm_amdgpu_wait_cs *wait = data;
	struct amdgpu_device *adev = dev->dev_private;
	unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
1086
	struct amdgpu_ring *ring = NULL;
1087
	struct amdgpu_ctx *ctx;
1088
	struct fence *fence;
A
Alex Deucher 已提交
1089 1090
	long r;

1091 1092 1093 1094 1095
	r = amdgpu_cs_get_ring(adev, wait->in.ip_type, wait->in.ip_instance,
			       wait->in.ring, &ring);
	if (r)
		return r;

1096 1097 1098
	ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
	if (ctx == NULL)
		return -EINVAL;
A
Alex Deucher 已提交
1099

1100 1101 1102 1103 1104 1105 1106 1107
	fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle);
	if (IS_ERR(fence))
		r = PTR_ERR(fence);
	else if (fence) {
		r = fence_wait_timeout(fence, true, timeout);
		fence_put(fence);
	} else
		r = 1;
C
Chunming Zhou 已提交
1108

1109
	amdgpu_ctx_put(ctx);
A
Alex Deucher 已提交
1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134
	if (r < 0)
		return r;

	memset(wait, 0, sizeof(*wait));
	wait->out.status = (r == 0);

	return 0;
}

/**
 * amdgpu_cs_find_bo_va - find bo_va for VM address
 *
 * @parser: command submission parser context
 * @addr: VM address
 * @bo: resulting BO of the mapping found
 *
 * Search the buffer objects in the command submission context for a certain
 * virtual memory address. Returns allocation structure when found, NULL
 * otherwise.
 */
struct amdgpu_bo_va_mapping *
amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
		       uint64_t addr, struct amdgpu_bo **bo)
{
	struct amdgpu_bo_va_mapping *mapping;
1135 1136 1137 1138
	unsigned i;

	if (!parser->bo_list)
		return NULL;
A
Alex Deucher 已提交
1139 1140 1141

	addr /= AMDGPU_GPU_PAGE_SIZE;

1142 1143 1144 1145 1146
	for (i = 0; i < parser->bo_list->num_entries; i++) {
		struct amdgpu_bo_list_entry *lobj;

		lobj = &parser->bo_list->array[i];
		if (!lobj->bo_va)
A
Alex Deucher 已提交
1147 1148
			continue;

1149
		list_for_each_entry(mapping, &lobj->bo_va->valids, list) {
1150 1151 1152 1153
			if (mapping->it.start > addr ||
			    addr > mapping->it.last)
				continue;

1154
			*bo = lobj->bo_va->bo;
1155 1156 1157
			return mapping;
		}

1158
		list_for_each_entry(mapping, &lobj->bo_va->invalids, list) {
A
Alex Deucher 已提交
1159 1160 1161 1162
			if (mapping->it.start > addr ||
			    addr > mapping->it.last)
				continue;

1163
			*bo = lobj->bo_va->bo;
A
Alex Deucher 已提交
1164 1165 1166 1167 1168 1169
			return mapping;
		}
	}

	return NULL;
}
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195

/**
 * amdgpu_cs_sysvm_access_required - make BOs accessible by the system VM
 *
 * @parser: command submission parser context
 *
 * Helper for UVD/VCE VM emulation, make sure BOs are accessible by the system VM.
 */
int amdgpu_cs_sysvm_access_required(struct amdgpu_cs_parser *parser)
{
	unsigned i;
	int r;

	if (!parser->bo_list)
		return 0;

	for (i = 0; i < parser->bo_list->num_entries; i++) {
		struct amdgpu_bo *bo = parser->bo_list->array[i].robj;

		r = amdgpu_ttm_bind(bo->tbo.ttm, &bo->tbo.mem);
		if (unlikely(r))
			return r;
	}

	return 0;
}