amdgpu_cs.c 40.2 KB
Newer Older
A
Alex Deucher 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
/*
 * Copyright 2008 Jerome Glisse.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Authors:
 *    Jerome Glisse <glisse@freedesktop.org>
 */
27
#include <linux/pagemap.h>
28
#include <linux/sync_file.h>
A
Alex Deucher 已提交
29 30
#include <drm/drmP.h>
#include <drm/amdgpu_drm.h>
31
#include <drm/drm_syncobj.h>
A
Alex Deucher 已提交
32 33
#include "amdgpu.h"
#include "amdgpu_trace.h"
34
#include "amdgpu_gmc.h"
A
Alex Deucher 已提交
35

36
static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
37 38
				      struct drm_amdgpu_cs_chunk_fence *data,
				      uint32_t *offset)
39 40
{
	struct drm_gem_object *gobj;
41
	unsigned long size;
42

43
	gobj = drm_gem_object_lookup(p->filp, data->handle);
44 45 46
	if (gobj == NULL)
		return -EINVAL;

47
	p->uf_entry.robj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
48 49 50
	p->uf_entry.priority = 0;
	p->uf_entry.tv.bo = &p->uf_entry.robj->tbo;
	p->uf_entry.tv.shared = true;
51
	p->uf_entry.user_pages = NULL;
52 53 54 55 56

	size = amdgpu_bo_size(p->uf_entry.robj);
	if (size != PAGE_SIZE || (data->offset + 8) > size)
		return -EINVAL;

57
	*offset = data->offset;
58

59
	drm_gem_object_put_unlocked(gobj);
60 61 62 63 64 65

	if (amdgpu_ttm_tt_get_usermm(p->uf_entry.robj->tbo.ttm)) {
		amdgpu_bo_unref(&p->uf_entry.robj);
		return -EINVAL;
	}

66 67 68
	return 0;
}

69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p,
				      struct drm_amdgpu_bo_list_in *data)
{
	int r;
	struct drm_amdgpu_bo_list_entry *info = NULL;

	r = amdgpu_bo_create_list_entry_array(data, &info);
	if (r)
		return r;

	r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number,
				  &p->bo_list);
	if (r)
		goto error_free;

	kvfree(info);
	return 0;

error_free:
	if (info)
		kvfree(info);

	return r;
}

static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs)
A
Alex Deucher 已提交
95
{
96
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
97
	struct amdgpu_vm *vm = &fpriv->vm;
A
Alex Deucher 已提交
98
	uint64_t *chunk_array_user;
99
	uint64_t *chunk_array;
100
	unsigned size, num_ibs = 0;
101
	uint32_t uf_offset = 0;
102
	int i;
103
	int ret;
A
Alex Deucher 已提交
104

105 106 107 108 109 110
	if (cs->in.num_chunks == 0)
		return 0;

	chunk_array = kmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
	if (!chunk_array)
		return -ENOMEM;
A
Alex Deucher 已提交
111

112 113
	p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
	if (!p->ctx) {
114 115
		ret = -EINVAL;
		goto free_chunk;
116
	}
117

118 119 120 121 122 123
	/* skip guilty context job */
	if (atomic_read(&p->ctx->guilty) == 1) {
		ret = -ECANCELED;
		goto free_chunk;
	}

124 125
	mutex_lock(&p->ctx->lock);

A
Alex Deucher 已提交
126
	/* get chunks */
127
	chunk_array_user = u64_to_user_ptr(cs->in.chunks);
A
Alex Deucher 已提交
128 129
	if (copy_from_user(chunk_array, chunk_array_user,
			   sizeof(uint64_t)*cs->in.num_chunks)) {
130
		ret = -EFAULT;
131
		goto free_chunk;
A
Alex Deucher 已提交
132 133 134
	}

	p->nchunks = cs->in.num_chunks;
135
	p->chunks = kmalloc_array(p->nchunks, sizeof(struct amdgpu_cs_chunk),
A
Alex Deucher 已提交
136
			    GFP_KERNEL);
137 138
	if (!p->chunks) {
		ret = -ENOMEM;
139
		goto free_chunk;
A
Alex Deucher 已提交
140 141 142 143 144 145 146
	}

	for (i = 0; i < p->nchunks; i++) {
		struct drm_amdgpu_cs_chunk __user **chunk_ptr = NULL;
		struct drm_amdgpu_cs_chunk user_chunk;
		uint32_t __user *cdata;

147
		chunk_ptr = u64_to_user_ptr(chunk_array[i]);
A
Alex Deucher 已提交
148 149
		if (copy_from_user(&user_chunk, chunk_ptr,
				       sizeof(struct drm_amdgpu_cs_chunk))) {
150 151 152
			ret = -EFAULT;
			i--;
			goto free_partial_kdata;
A
Alex Deucher 已提交
153 154 155 156 157
		}
		p->chunks[i].chunk_id = user_chunk.chunk_id;
		p->chunks[i].length_dw = user_chunk.length_dw;

		size = p->chunks[i].length_dw;
158
		cdata = u64_to_user_ptr(user_chunk.chunk_data);
A
Alex Deucher 已提交
159

M
Michal Hocko 已提交
160
		p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
A
Alex Deucher 已提交
161
		if (p->chunks[i].kdata == NULL) {
162 163 164
			ret = -ENOMEM;
			i--;
			goto free_partial_kdata;
A
Alex Deucher 已提交
165 166 167
		}
		size *= sizeof(uint32_t);
		if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
168 169
			ret = -EFAULT;
			goto free_partial_kdata;
A
Alex Deucher 已提交
170 171
		}

172 173
		switch (p->chunks[i].chunk_id) {
		case AMDGPU_CHUNK_ID_IB:
174
			++num_ibs;
175 176 177
			break;

		case AMDGPU_CHUNK_ID_FENCE:
A
Alex Deucher 已提交
178
			size = sizeof(struct drm_amdgpu_cs_chunk_fence);
179
			if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
180 181
				ret = -EINVAL;
				goto free_partial_kdata;
A
Alex Deucher 已提交
182
			}
183

184 185
			ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata,
							 &uf_offset);
186 187 188
			if (ret)
				goto free_partial_kdata;

189 190
			break;

191 192 193 194 195 196 197 198 199 200 201 202 203
		case AMDGPU_CHUNK_ID_BO_HANDLES:
			size = sizeof(struct drm_amdgpu_bo_list_in);
			if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
				ret = -EINVAL;
				goto free_partial_kdata;
			}

			ret = amdgpu_cs_bo_handles_chunk(p, p->chunks[i].kdata);
			if (ret)
				goto free_partial_kdata;

			break;

204
		case AMDGPU_CHUNK_ID_DEPENDENCIES:
205 206
		case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
		case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
207 208
			break;

209
		default:
210 211
			ret = -EINVAL;
			goto free_partial_kdata;
A
Alex Deucher 已提交
212 213 214
		}
	}

215
	ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm);
216
	if (ret)
C
Christian König 已提交
217
		goto free_all_kdata;
A
Alex Deucher 已提交
218

219 220 221 222
	if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) {
		ret = -ECANCELED;
		goto free_all_kdata;
	}
223

224 225
	if (p->uf_entry.robj)
		p->job->uf_addr = uf_offset;
A
Alex Deucher 已提交
226
	kfree(chunk_array);
227 228 229 230

	/* Use this opportunity to fill in task info for the vm */
	amdgpu_vm_set_task_info(vm);

231 232 233 234 235 236
	return 0;

free_all_kdata:
	i = p->nchunks - 1;
free_partial_kdata:
	for (; i >= 0; i--)
M
Michal Hocko 已提交
237
		kvfree(p->chunks[i].kdata);
238
	kfree(p->chunks);
239 240
	p->chunks = NULL;
	p->nchunks = 0;
241 242 243 244
free_chunk:
	kfree(chunk_array);

	return ret;
A
Alex Deucher 已提交
245 246
}

247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
/* Convert microseconds to bytes. */
static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)
{
	if (us <= 0 || !adev->mm_stats.log2_max_MBps)
		return 0;

	/* Since accum_us is incremented by a million per second, just
	 * multiply it by the number of MB/s to get the number of bytes.
	 */
	return us << adev->mm_stats.log2_max_MBps;
}

static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes)
{
	if (!adev->mm_stats.log2_max_MBps)
		return 0;

	return bytes >> adev->mm_stats.log2_max_MBps;
}

/* Returns how many bytes TTM can move right now. If no bytes can be moved,
 * it returns 0. If it returns non-zero, it's OK to move at least one buffer,
 * which means it can go over the threshold once. If that happens, the driver
 * will be in debt and no other buffer migrations can be done until that debt
 * is repaid.
 *
 * This approach allows moving a buffer of any size (it's important to allow
 * that).
 *
 * The currency is simply time in microseconds and it increases as the clock
 * ticks. The accumulated microseconds (us) are converted to bytes and
 * returned.
A
Alex Deucher 已提交
279
 */
280 281 282
static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev,
					      u64 *max_bytes,
					      u64 *max_vis_bytes)
A
Alex Deucher 已提交
283
{
284 285
	s64 time_us, increment_us;
	u64 free_vram, total_vram, used_vram;
A
Alex Deucher 已提交
286

287 288
	/* Allow a maximum of 200 accumulated ms. This is basically per-IB
	 * throttling.
A
Alex Deucher 已提交
289
	 *
290 291 292 293 294
	 * It means that in order to get full max MBps, at least 5 IBs per
	 * second must be submitted and not more than 200ms apart from each
	 * other.
	 */
	const s64 us_upper_bound = 200000;
A
Alex Deucher 已提交
295

296 297 298 299 300
	if (!adev->mm_stats.log2_max_MBps) {
		*max_bytes = 0;
		*max_vis_bytes = 0;
		return;
	}
301

302
	total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size);
303
	used_vram = amdgpu_vram_mgr_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
304 305 306 307 308 309 310 311 312 313 314 315 316 317
	free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram;

	spin_lock(&adev->mm_stats.lock);

	/* Increase the amount of accumulated us. */
	time_us = ktime_to_us(ktime_get());
	increment_us = time_us - adev->mm_stats.last_update_us;
	adev->mm_stats.last_update_us = time_us;
	adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us,
                                      us_upper_bound);

	/* This prevents the short period of low performance when the VRAM
	 * usage is low and the driver is in debt or doesn't have enough
	 * accumulated us to fill VRAM quickly.
A
Alex Deucher 已提交
318
	 *
319 320 321 322
	 * The situation can occur in these cases:
	 * - a lot of VRAM is freed by userspace
	 * - the presence of a big buffer causes a lot of evictions
	 *   (solution: split buffers into smaller ones)
A
Alex Deucher 已提交
323
	 *
324 325
	 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting
	 * accum_us to a positive number.
A
Alex Deucher 已提交
326
	 */
327 328 329 330 331 332 333 334 335 336 337 338 339
	if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) {
		s64 min_us;

		/* Be more aggresive on dGPUs. Try to fill a portion of free
		 * VRAM now.
		 */
		if (!(adev->flags & AMD_IS_APU))
			min_us = bytes_to_us(adev, free_vram / 4);
		else
			min_us = 0; /* Reset accum_us on APUs. */

		adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us);
	}
A
Alex Deucher 已提交
340

341
	/* This is set to 0 if the driver is in debt to disallow (optional)
342 343
	 * buffer moves.
	 */
344 345 346
	*max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us);

	/* Do the same for visible VRAM if half of it is free */
347
	if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) {
348
		u64 total_vis_vram = adev->gmc.visible_vram_size;
349 350
		u64 used_vis_vram =
			amdgpu_vram_mgr_vis_usage(&adev->mman.bdev.man[TTM_PL_VRAM]);
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366

		if (used_vis_vram < total_vis_vram) {
			u64 free_vis_vram = total_vis_vram - used_vis_vram;
			adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis +
							  increment_us, us_upper_bound);

			if (free_vis_vram >= total_vis_vram / 2)
				adev->mm_stats.accum_us_vis =
					max(bytes_to_us(adev, free_vis_vram / 2),
					    adev->mm_stats.accum_us_vis);
		}

		*max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis);
	} else {
		*max_vis_bytes = 0;
	}
367 368 369 370 371 372 373 374

	spin_unlock(&adev->mm_stats.lock);
}

/* Report how many bytes have really been moved for the last command
 * submission. This can result in a debt that can stop buffer migrations
 * temporarily.
 */
375 376
void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes,
				  u64 num_vis_bytes)
377 378 379
{
	spin_lock(&adev->mm_stats.lock);
	adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes);
380
	adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes);
381
	spin_unlock(&adev->mm_stats.lock);
A
Alex Deucher 已提交
382 383
}

384 385 386
static int amdgpu_cs_bo_validate(struct amdgpu_cs_parser *p,
				 struct amdgpu_bo *bo)
{
387
	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
388 389 390
	struct ttm_operation_ctx ctx = {
		.interruptible = true,
		.no_wait_gpu = false,
391 392
		.resv = bo->tbo.resv,
		.flags = 0
393
	};
394 395 396 397 398 399
	uint32_t domain;
	int r;

	if (bo->pin_count)
		return 0;

400 401
	/* Don't move this buffer if we have depleted our allowance
	 * to move it. Don't move anything if the threshold is zero.
402
	 */
403
	if (p->bytes_moved < p->bytes_moved_threshold) {
404
		if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
405 406 407 408 409 410
		    (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) {
			/* And don't move a CPU_ACCESS_REQUIRED BO to limited
			 * visible VRAM if we've depleted our allowance to do
			 * that.
			 */
			if (p->bytes_moved_vis < p->bytes_moved_vis_threshold)
K
Kent Russell 已提交
411
				domain = bo->preferred_domains;
412 413 414
			else
				domain = bo->allowed_domains;
		} else {
K
Kent Russell 已提交
415
			domain = bo->preferred_domains;
416 417
		}
	} else {
418
		domain = bo->allowed_domains;
419
	}
420 421

retry:
422
	amdgpu_bo_placement_from_domain(bo, domain);
423
	r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
424 425

	p->bytes_moved += ctx.bytes_moved;
426
	if (!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
427
	    amdgpu_bo_in_cpu_visible_vram(bo))
428
		p->bytes_moved_vis += ctx.bytes_moved;
429

430 431 432
	if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) {
		domain = bo->allowed_domains;
		goto retry;
433 434 435 436 437
	}

	return r;
}

438 439
/* Last resort, try to evict something from the current working set */
static bool amdgpu_cs_try_evict(struct amdgpu_cs_parser *p,
440
				struct amdgpu_bo *validated)
441
{
442
	uint32_t domain = validated->allowed_domains;
443
	struct ttm_operation_ctx ctx = { true, false };
444 445 446 447 448 449 450 451 452 453
	int r;

	if (!p->evictable)
		return false;

	for (;&p->evictable->tv.head != &p->validated;
	     p->evictable = list_prev_entry(p->evictable, tv.head)) {

		struct amdgpu_bo_list_entry *candidate = p->evictable;
		struct amdgpu_bo *bo = candidate->robj;
454
		struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
455
		bool update_bytes_moved_vis;
456 457 458
		uint32_t other;

		/* If we reached our current BO we can forget it */
459
		if (candidate->robj == validated)
460 461
			break;

462 463 464 465
		/* We can't move pinned BOs here */
		if (bo->pin_count)
			continue;

466 467 468 469 470 471 472 473 474 475 476 477
		other = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);

		/* Check if this BO is in one of the domains we need space for */
		if (!(other & domain))
			continue;

		/* Check if we can move this BO somewhere else */
		other = bo->allowed_domains & ~domain;
		if (!other)
			continue;

		/* Good we can try to move this BO somewhere else */
478
		update_bytes_moved_vis =
479 480
				!amdgpu_gmc_vram_full_visible(&adev->gmc) &&
				amdgpu_bo_in_cpu_visible_vram(bo);
481
		amdgpu_bo_placement_from_domain(bo, other);
482
		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
483
		p->bytes_moved += ctx.bytes_moved;
484
		if (update_bytes_moved_vis)
485
			p->bytes_moved_vis += ctx.bytes_moved;
486 487 488 489 490 491 492 493 494 495 496 497 498

		if (unlikely(r))
			break;

		p->evictable = list_prev_entry(p->evictable, tv.head);
		list_move(&candidate->tv.head, &p->validated);

		return true;
	}

	return false;
}

499 500 501 502 503 504 505 506 507 508 509 510
static int amdgpu_cs_validate(void *param, struct amdgpu_bo *bo)
{
	struct amdgpu_cs_parser *p = param;
	int r;

	do {
		r = amdgpu_cs_bo_validate(p, bo);
	} while (r == -ENOMEM && amdgpu_cs_try_evict(p, bo));
	if (r)
		return r;

	if (bo->shadow)
A
Alex Xie 已提交
511
		r = amdgpu_cs_bo_validate(p, bo->shadow);
512 513 514 515

	return r;
}

516
static int amdgpu_cs_list_validate(struct amdgpu_cs_parser *p,
517
			    struct list_head *validated)
A
Alex Deucher 已提交
518
{
519
	struct ttm_operation_ctx ctx = { true, false };
A
Alex Deucher 已提交
520 521 522
	struct amdgpu_bo_list_entry *lobj;
	int r;

523
	list_for_each_entry(lobj, validated, tv.head) {
524
		struct amdgpu_bo *bo = lobj->robj;
525
		bool binding_userptr = false;
526
		struct mm_struct *usermm;
A
Alex Deucher 已提交
527

528 529 530 531
		usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
		if (usermm && usermm != current->mm)
			return -EPERM;

532
		/* Check if we have user pages and nobody bound the BO already */
533 534
		if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
		    lobj->user_pages) {
535 536
			amdgpu_bo_placement_from_domain(bo,
							AMDGPU_GEM_DOMAIN_CPU);
537
			r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
538 539
			if (r)
				return r;
540 541
			amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm,
						     lobj->user_pages);
542 543 544
			binding_userptr = true;
		}

545 546 547
		if (p->evictable == lobj)
			p->evictable = NULL;

548
		r = amdgpu_cs_validate(p, bo);
549
		if (r)
550
			return r;
551

552
		if (binding_userptr) {
M
Michal Hocko 已提交
553
			kvfree(lobj->user_pages);
554 555
			lobj->user_pages = NULL;
		}
A
Alex Deucher 已提交
556 557 558 559
	}
	return 0;
}

560 561
static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
				union drm_amdgpu_cs *cs)
A
Alex Deucher 已提交
562 563
{
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
564
	struct amdgpu_vm *vm = &fpriv->vm;
565
	struct amdgpu_bo_list_entry *e;
566
	struct list_head duplicates;
567 568 569
	struct amdgpu_bo *gds;
	struct amdgpu_bo *gws;
	struct amdgpu_bo *oa;
570
	unsigned tries = 10;
571
	int r;
A
Alex Deucher 已提交
572

573 574
	INIT_LIST_HEAD(&p->validated);

575
	/* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */
576 577 578
	if (cs->in.bo_list_handle) {
		if (p->bo_list)
			return -EINVAL;
579

580 581 582 583
		r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle,
				       &p->bo_list);
		if (r)
			return r;
584 585 586 587 588 589
	} else if (!p->bo_list) {
		/* Create a empty bo_list when no handle is provided */
		r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0,
					  &p->bo_list);
		if (r)
			return r;
590 591
	}

592 593 594
	amdgpu_bo_list_get_list(p->bo_list, &p->validated);
	if (p->bo_list->first_userptr != p->bo_list->num_entries)
		p->mn = amdgpu_mn_get(p->adev, AMDGPU_MN_TYPE_GFX);
A
Alex Deucher 已提交
595

596
	INIT_LIST_HEAD(&duplicates);
597
	amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
A
Alex Deucher 已提交
598

599
	if (p->uf_entry.robj && !p->uf_entry.robj->parent)
600 601
		list_add(&p->uf_entry.tv.head, &p->validated);

602 603 604 605 606
	while (1) {
		struct list_head need_pages;

		r = ttm_eu_reserve_buffers(&p->ticket, &p->validated, true,
					   &duplicates);
607
		if (unlikely(r != 0)) {
608 609
			if (r != -ERESTARTSYS)
				DRM_ERROR("ttm_eu_reserve_buffers failed.\n");
610
			goto error_free_pages;
611
		}
612 613

		INIT_LIST_HEAD(&need_pages);
614 615
		amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
			struct amdgpu_bo *bo = e->robj;
616

617
			if (amdgpu_ttm_tt_userptr_invalidated(bo->tbo.ttm,
618 619 620
				 &e->user_invalidated) && e->user_pages) {

				/* We acquired a page array, but somebody
621
				 * invalidated it. Free it and try again
622 623
				 */
				release_pages(e->user_pages,
624
					      bo->tbo.ttm->num_pages);
M
Michal Hocko 已提交
625
				kvfree(e->user_pages);
626 627 628
				e->user_pages = NULL;
			}

629
			if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm) &&
630 631 632 633 634 635 636 637 638 639 640 641 642 643
			    !e->user_pages) {
				list_del(&e->tv.head);
				list_add(&e->tv.head, &need_pages);

				amdgpu_bo_unreserve(e->robj);
			}
		}

		if (list_empty(&need_pages))
			break;

		/* Unreserve everything again. */
		ttm_eu_backoff_reservation(&p->ticket, &p->validated);

644
		/* We tried too many times, just abort */
645 646
		if (!--tries) {
			r = -EDEADLK;
647
			DRM_ERROR("deadlock in %s\n", __func__);
648 649 650
			goto error_free_pages;
		}

A
Alex Xie 已提交
651
		/* Fill the page arrays for all userptrs. */
652 653 654
		list_for_each_entry(e, &need_pages, tv.head) {
			struct ttm_tt *ttm = e->robj->tbo.ttm;

M
Michal Hocko 已提交
655 656 657
			e->user_pages = kvmalloc_array(ttm->num_pages,
							 sizeof(struct page*),
							 GFP_KERNEL | __GFP_ZERO);
658 659
			if (!e->user_pages) {
				r = -ENOMEM;
660
				DRM_ERROR("calloc failure in %s\n", __func__);
661 662 663 664 665
				goto error_free_pages;
			}

			r = amdgpu_ttm_tt_get_user_pages(ttm, e->user_pages);
			if (r) {
666
				DRM_ERROR("amdgpu_ttm_tt_get_user_pages failed.\n");
M
Michal Hocko 已提交
667
				kvfree(e->user_pages);
668 669 670 671 672 673 674 675
				e->user_pages = NULL;
				goto error_free_pages;
			}
		}

		/* And try again. */
		list_splice(&need_pages, &p->validated);
	}
676

677 678
	amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,
					  &p->bytes_moved_vis_threshold);
679
	p->bytes_moved = 0;
680
	p->bytes_moved_vis = 0;
681 682 683
	p->evictable = list_last_entry(&p->validated,
				       struct amdgpu_bo_list_entry,
				       tv.head);
684

685 686 687 688 689 690 691
	r = amdgpu_vm_validate_pt_bos(p->adev, &fpriv->vm,
				      amdgpu_cs_validate, p);
	if (r) {
		DRM_ERROR("amdgpu_vm_validate_pt_bos() failed.\n");
		goto error_validate;
	}

692
	r = amdgpu_cs_list_validate(p, &duplicates);
693 694
	if (r) {
		DRM_ERROR("amdgpu_cs_list_validate(duplicates) failed.\n");
695
		goto error_validate;
696
	}
697

698
	r = amdgpu_cs_list_validate(p, &p->validated);
699 700
	if (r) {
		DRM_ERROR("amdgpu_cs_list_validate(validated) failed.\n");
701
		goto error_validate;
702
	}
703

704 705
	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
				     p->bytes_moved_vis);
706

707 708 709
	gds = p->bo_list->gds_obj;
	gws = p->bo_list->gws_obj;
	oa = p->bo_list->oa_obj;
710

711 712
	amdgpu_bo_list_for_each_entry(e, p->bo_list)
		e->bo_va = amdgpu_vm_bo_find(vm, e->robj);
713

714 715 716 717 718 719 720 721 722 723 724
	if (gds) {
		p->job->gds_base = amdgpu_bo_gpu_offset(gds);
		p->job->gds_size = amdgpu_bo_size(gds);
	}
	if (gws) {
		p->job->gws_base = amdgpu_bo_gpu_offset(gws);
		p->job->gws_size = amdgpu_bo_size(gws);
	}
	if (oa) {
		p->job->oa_base = amdgpu_bo_gpu_offset(oa);
		p->job->oa_size = amdgpu_bo_size(oa);
725
	}
726

727 728 729
	if (!r && p->uf_entry.robj) {
		struct amdgpu_bo *uf = p->uf_entry.robj;

730
		r = amdgpu_ttm_alloc_gart(&uf->tbo);
731 732
		p->job->uf_addr += amdgpu_bo_gpu_offset(uf);
	}
733

734
error_validate:
735
	if (r)
736
		ttm_eu_backoff_reservation(&p->ticket, &p->validated);
A
Alex Deucher 已提交
737

738 739
error_free_pages:

740 741 742
	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
		if (!e->user_pages)
			continue;
743

744 745 746
		release_pages(e->user_pages,
			      e->robj->tbo.ttm->num_pages);
		kvfree(e->user_pages);
747 748
	}

A
Alex Deucher 已提交
749 750 751 752 753 754 755 756 757 758
	return r;
}

static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
{
	struct amdgpu_bo_list_entry *e;
	int r;

	list_for_each_entry(e, &p->validated, tv.head) {
		struct reservation_object *resv = e->robj->tbo.resv;
759 760
		r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, p->filp,
				     amdgpu_bo_explicit_sync(e->robj));
A
Alex Deucher 已提交
761 762 763 764 765 766 767

		if (r)
			return r;
	}
	return 0;
}

768 769 770 771 772 773 774 775
/**
 * cs_parser_fini() - clean parser states
 * @parser:	parser structure holding parsing context.
 * @error:	error number
 *
 * If error is set than unvalidate buffer, otherwise just free memory
 * used by parsing context.
 **/
776 777
static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
				  bool backoff)
C
Chunming Zhou 已提交
778
{
779 780
	unsigned i;

781
	if (error && backoff)
A
Alex Deucher 已提交
782 783
		ttm_eu_backoff_reservation(&parser->ticket,
					   &parser->validated);
784 785 786 787 788

	for (i = 0; i < parser->num_post_dep_syncobjs; i++)
		drm_syncobj_put(parser->post_dep_syncobjs[i]);
	kfree(parser->post_dep_syncobjs);

789
	dma_fence_put(parser->fence);
790

791 792
	if (parser->ctx) {
		mutex_unlock(&parser->ctx->lock);
793
		amdgpu_ctx_put(parser->ctx);
794
	}
795 796 797
	if (parser->bo_list)
		amdgpu_bo_list_put(parser->bo_list);

A
Alex Deucher 已提交
798
	for (i = 0; i < parser->nchunks; i++)
M
Michal Hocko 已提交
799
		kvfree(parser->chunks[i].kdata);
A
Alex Deucher 已提交
800
	kfree(parser->chunks);
801 802
	if (parser->job)
		amdgpu_job_free(parser->job);
803
	amdgpu_bo_unref(&parser->uf_entry.robj);
A
Alex Deucher 已提交
804 805
}

806
static int amdgpu_bo_vm_update_pte(struct amdgpu_cs_parser *p)
A
Alex Deucher 已提交
807
{
808
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
809
	struct amdgpu_device *adev = p->adev;
810
	struct amdgpu_vm *vm = &fpriv->vm;
811
	struct amdgpu_bo_list_entry *e;
A
Alex Deucher 已提交
812 813
	struct amdgpu_bo_va *bo_va;
	struct amdgpu_bo *bo;
814
	int r;
A
Alex Deucher 已提交
815

816
	r = amdgpu_vm_clear_freed(adev, vm, NULL);
A
Alex Deucher 已提交
817 818 819
	if (r)
		return r;

820 821 822 823 824
	r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false);
	if (r)
		return r;

	r = amdgpu_sync_fence(adev, &p->job->sync,
825
			      fpriv->prt_va->last_pt_update, false);
826 827 828
	if (r)
		return r;

M
Monk Liu 已提交
829 830
	if (amdgpu_sriov_vf(adev)) {
		struct dma_fence *f;
831 832

		bo_va = fpriv->csa_va;
M
Monk Liu 已提交
833 834 835 836 837 838
		BUG_ON(!bo_va);
		r = amdgpu_vm_bo_update(adev, bo_va, false);
		if (r)
			return r;

		f = bo_va->last_pt_update;
839
		r = amdgpu_sync_fence(adev, &p->job->sync, f, false);
M
Monk Liu 已提交
840 841 842 843
		if (r)
			return r;
	}

844 845
	amdgpu_bo_list_for_each_entry(e, p->bo_list) {
		struct dma_fence *f;
A
Alex Deucher 已提交
846

847 848 849 850
		/* ignore duplicates */
		bo = e->robj;
		if (!bo)
			continue;
A
Alex Deucher 已提交
851

852 853 854
		bo_va = e->bo_va;
		if (bo_va == NULL)
			continue;
A
Alex Deucher 已提交
855

856 857 858
		r = amdgpu_vm_bo_update(adev, bo_va, false);
		if (r)
			return r;
859

860 861 862 863
		f = bo_va->last_pt_update;
		r = amdgpu_sync_fence(adev, &p->job->sync, f, false);
		if (r)
			return r;
864 865
	}

866
	r = amdgpu_vm_handle_moved(adev, vm);
867 868 869
	if (r)
		return r;

870 871 872 873
	r = amdgpu_vm_update_directories(adev, vm);
	if (r)
		return r;

874
	r = amdgpu_sync_fence(adev, &p->job->sync, vm->last_update, false);
875 876
	if (r)
		return r;
877

878
	if (amdgpu_vm_debug) {
879
		/* Invalidate all BOs to test for userspace bugs */
880
		amdgpu_bo_list_for_each_entry(e, p->bo_list) {
881
			/* ignore duplicates */
882
			if (!e->robj)
883 884
				continue;

885
			amdgpu_vm_bo_invalidate(adev, e->robj, false);
886
		}
A
Alex Deucher 已提交
887 888
	}

889
	return r;
A
Alex Deucher 已提交
890 891 892
}

static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
893
				 struct amdgpu_cs_parser *p)
A
Alex Deucher 已提交
894
{
895
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
A
Alex Deucher 已提交
896
	struct amdgpu_vm *vm = &fpriv->vm;
C
Christian König 已提交
897
	struct amdgpu_ring *ring = p->ring;
898
	int r;
A
Alex Deucher 已提交
899 900

	/* Only for UVD/VCE VM emulation */
901
	if (p->ring->funcs->parse_cs || p->ring->funcs->patch_cs_in_place) {
902
		unsigned i, j;
903

904 905
		for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) {
			struct drm_amdgpu_cs_chunk_ib *chunk_ib;
906 907
			struct amdgpu_bo_va_mapping *m;
			struct amdgpu_bo *aobj = NULL;
908
			struct amdgpu_cs_chunk *chunk;
909
			uint64_t offset, va_start;
910
			struct amdgpu_ib *ib;
911 912
			uint8_t *kptr;

913 914 915 916 917 918 919
			chunk = &p->chunks[i];
			ib = &p->job->ibs[j];
			chunk_ib = chunk->kdata;

			if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
				continue;

920 921
			va_start = chunk_ib->va_start & AMDGPU_VA_HOLE_MASK;
			r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);
922 923 924 925
			if (r) {
				DRM_ERROR("IB va_start is invalid\n");
				return r;
			}
A
Alex Deucher 已提交
926

927
			if ((va_start + chunk_ib->ib_bytes) >
928
			    (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
929 930 931 932 933 934 935 936 937 938 939
				DRM_ERROR("IB va_start+ib_bytes is invalid\n");
				return -EINVAL;
			}

			/* the IB should be reserved at this point */
			r = amdgpu_bo_kmap(aobj, (void **)&kptr);
			if (r) {
				return r;
			}

			offset = m->start * AMDGPU_GPU_PAGE_SIZE;
940
			kptr += va_start - offset;
941

942 943 944 945 946 947 948 949 950 951 952 953 954 955
			if (p->ring->funcs->parse_cs) {
				memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
				amdgpu_bo_kunmap(aobj);

				r = amdgpu_ring_parse_cs(ring, p, j);
				if (r)
					return r;
			} else {
				ib->ptr = (uint32_t *)kptr;
				r = amdgpu_ring_patch_cs_in_place(ring, p, j);
				amdgpu_bo_kunmap(aobj);
				if (r)
					return r;
			}
956 957

			j++;
A
Alex Deucher 已提交
958
		}
959 960 961
	}

	if (p->job->vm) {
962
		p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->root.base.bo);
963

964
		r = amdgpu_bo_vm_update_pte(p);
965 966
		if (r)
			return r;
967 968 969 970

		r = reservation_object_reserve_shared(vm->root.base.bo->tbo.resv);
		if (r)
			return r;
971
	}
A
Alex Deucher 已提交
972

973
	return amdgpu_cs_sync_rings(p);
A
Alex Deucher 已提交
974 975 976 977 978 979 980 981
}

static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
			     struct amdgpu_cs_parser *parser)
{
	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
	struct amdgpu_vm *vm = &fpriv->vm;
	int i, j;
M
Monk Liu 已提交
982
	int r, ce_preempt = 0, de_preempt = 0;
A
Alex Deucher 已提交
983

984
	for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) {
A
Alex Deucher 已提交
985 986 987 988 989 990
		struct amdgpu_cs_chunk *chunk;
		struct amdgpu_ib *ib;
		struct drm_amdgpu_cs_chunk_ib *chunk_ib;
		struct amdgpu_ring *ring;

		chunk = &parser->chunks[i];
991
		ib = &parser->job->ibs[j];
A
Alex Deucher 已提交
992 993 994 995 996
		chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;

		if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
			continue;

997
		if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && amdgpu_sriov_vf(adev)) {
998
			if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
999 1000 1001 1002
				if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
					ce_preempt++;
				else
					de_preempt++;
1003
			}
1004 1005 1006

			/* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
			if (ce_preempt > 1 || de_preempt > 1)
1007
				return -EINVAL;
M
Monk Liu 已提交
1008 1009
		}

1010 1011
		r = amdgpu_queue_mgr_map(adev, &parser->ctx->queue_mgr, chunk_ib->ip_type,
					 chunk_ib->ip_instance, chunk_ib->ring, &ring);
1012
		if (r)
A
Alex Deucher 已提交
1013 1014
			return r;

1015 1016 1017
		if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE)
			parser->job->preamble_status |=
				AMDGPU_PREAMBLE_IB_PRESENT;
1018

C
Christian König 已提交
1019
		if (parser->ring && parser->ring != ring)
1020 1021
			return -EINVAL;

C
Christian König 已提交
1022
		parser->ring = ring;
1023

1024 1025 1026 1027 1028 1029
		r =  amdgpu_ib_get(adev, vm,
					ring->funcs->parse_cs ? chunk_ib->ib_bytes : 0,
					ib);
		if (r) {
			DRM_ERROR("Failed to get ib !\n");
			return r;
A
Alex Deucher 已提交
1030 1031
		}

1032
		ib->gpu_addr = chunk_ib->va_start;
1033
		ib->length_dw = chunk_ib->ib_bytes / 4;
1034
		ib->flags = chunk_ib->flags;
1035

A
Alex Deucher 已提交
1036 1037 1038
		j++;
	}

1039
	/* UVD & VCE fw doesn't support user fences */
1040
	if (parser->job->uf_addr && (
C
Christian König 已提交
1041 1042
	    parser->ring->funcs->type == AMDGPU_RING_TYPE_UVD ||
	    parser->ring->funcs->type == AMDGPU_RING_TYPE_VCE))
1043
		return -EINVAL;
A
Alex Deucher 已提交
1044

C
Christian König 已提交
1045
	return amdgpu_ctx_wait_prev_fence(parser->ctx, parser->ring->idx);
A
Alex Deucher 已提交
1046 1047
}

1048 1049
static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
				       struct amdgpu_cs_chunk *chunk)
1050
{
1051
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
1052 1053 1054
	unsigned num_deps;
	int i, r;
	struct drm_amdgpu_cs_chunk_dep *deps;
1055

1056 1057 1058
	deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata;
	num_deps = chunk->length_dw * 4 /
		sizeof(struct drm_amdgpu_cs_chunk_dep);
1059

1060 1061 1062 1063
	for (i = 0; i < num_deps; ++i) {
		struct amdgpu_ring *ring;
		struct amdgpu_ctx *ctx;
		struct dma_fence *fence;
1064

1065 1066 1067
		ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id);
		if (ctx == NULL)
			return -EINVAL;
1068

1069 1070 1071 1072 1073 1074 1075 1076
		r = amdgpu_queue_mgr_map(p->adev, &ctx->queue_mgr,
					 deps[i].ip_type,
					 deps[i].ip_instance,
					 deps[i].ring, &ring);
		if (r) {
			amdgpu_ctx_put(ctx);
			return r;
		}
1077

1078 1079 1080 1081 1082 1083 1084
		fence = amdgpu_ctx_get_fence(ctx, ring,
					     deps[i].handle);
		if (IS_ERR(fence)) {
			r = PTR_ERR(fence);
			amdgpu_ctx_put(ctx);
			return r;
		} else if (fence) {
1085 1086
			r = amdgpu_sync_fence(p->adev, &p->job->sync, fence,
					true);
1087 1088 1089 1090 1091 1092 1093 1094
			dma_fence_put(fence);
			amdgpu_ctx_put(ctx);
			if (r)
				return r;
		}
	}
	return 0;
}
1095

1096 1097 1098 1099 1100
static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p,
						 uint32_t handle)
{
	int r;
	struct dma_fence *fence;
1101
	r = drm_syncobj_find_fence(p->filp, handle, &fence);
1102 1103 1104
	if (r)
		return r;

1105
	r = amdgpu_sync_fence(p->adev, &p->job->sync, fence, true);
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144
	dma_fence_put(fence);

	return r;
}

static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p,
					    struct amdgpu_cs_chunk *chunk)
{
	unsigned num_deps;
	int i, r;
	struct drm_amdgpu_cs_chunk_sem *deps;

	deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata;
	num_deps = chunk->length_dw * 4 /
		sizeof(struct drm_amdgpu_cs_chunk_sem);

	for (i = 0; i < num_deps; ++i) {
		r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle);
		if (r)
			return r;
	}
	return 0;
}

static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p,
					     struct amdgpu_cs_chunk *chunk)
{
	unsigned num_deps;
	int i;
	struct drm_amdgpu_cs_chunk_sem *deps;
	deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata;
	num_deps = chunk->length_dw * 4 /
		sizeof(struct drm_amdgpu_cs_chunk_sem);

	p->post_dep_syncobjs = kmalloc_array(num_deps,
					     sizeof(struct drm_syncobj *),
					     GFP_KERNEL);
	p->num_post_dep_syncobjs = 0;

1145 1146 1147
	if (!p->post_dep_syncobjs)
		return -ENOMEM;

1148 1149 1150 1151 1152 1153 1154 1155 1156
	for (i = 0; i < num_deps; ++i) {
		p->post_dep_syncobjs[i] = drm_syncobj_find(p->filp, deps[i].handle);
		if (!p->post_dep_syncobjs[i])
			return -EINVAL;
		p->num_post_dep_syncobjs++;
	}
	return 0;
}

1157 1158 1159 1160
static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
				  struct amdgpu_cs_parser *p)
{
	int i, r;
1161

1162 1163
	for (i = 0; i < p->nchunks; ++i) {
		struct amdgpu_cs_chunk *chunk;
1164

1165
		chunk = &p->chunks[i];
1166

1167 1168 1169 1170
		if (chunk->chunk_id == AMDGPU_CHUNK_ID_DEPENDENCIES) {
			r = amdgpu_cs_process_fence_dep(p, chunk);
			if (r)
				return r;
1171 1172 1173 1174 1175 1176 1177 1178
		} else if (chunk->chunk_id == AMDGPU_CHUNK_ID_SYNCOBJ_IN) {
			r = amdgpu_cs_process_syncobj_in_dep(p, chunk);
			if (r)
				return r;
		} else if (chunk->chunk_id == AMDGPU_CHUNK_ID_SYNCOBJ_OUT) {
			r = amdgpu_cs_process_syncobj_out_dep(p, chunk);
			if (r)
				return r;
1179 1180 1181 1182 1183 1184
		}
	}

	return 0;
}

1185 1186 1187 1188
static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p)
{
	int i;

1189 1190
	for (i = 0; i < p->num_post_dep_syncobjs; ++i)
		drm_syncobj_replace_fence(p->post_dep_syncobjs[i], p->fence);
1191 1192
}

1193 1194 1195
static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
			    union drm_amdgpu_cs *cs)
{
1196
	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
C
Christian König 已提交
1197
	struct amdgpu_ring *ring = p->ring;
1198
	struct drm_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
1199
	enum drm_sched_priority priority;
1200
	struct amdgpu_bo_list_entry *e;
1201
	struct amdgpu_job *job;
1202 1203
	uint64_t seq;

1204
	int r;
1205

1206 1207 1208 1209 1210 1211 1212 1213
	job = p->job;
	p->job = NULL;

	r = drm_sched_job_init(&job->base, entity, p->filp);
	if (r)
		goto error_unlock;

	/* No memory allocation is allowed while holding the mn lock */
1214
	amdgpu_mn_lock(p->mn);
1215 1216
	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
		struct amdgpu_bo *bo = e->robj;
1217

1218
		if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm)) {
1219 1220
			r = -ERESTARTSYS;
			goto error_abort;
1221 1222 1223
		}
	}

1224
	job->owner = p->filp;
1225
	p->fence = dma_fence_get(&job->base.s_fence->finished);
1226

1227 1228 1229 1230 1231 1232 1233 1234 1235
	r = amdgpu_ctx_add_fence(p->ctx, ring, p->fence, &seq);
	if (r) {
		dma_fence_put(p->fence);
		dma_fence_put(&job->base.s_fence->finished);
		amdgpu_job_free(job);
		amdgpu_mn_unlock(p->mn);
		return r;
	}

1236 1237
	amdgpu_cs_post_dependencies(p);

1238 1239 1240 1241 1242 1243
	if ((job->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) &&
	    !p->ctx->preamble_presented) {
		job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;
		p->ctx->preamble_presented = true;
	}

1244 1245 1246
	cs->out.handle = seq;
	job->uf_sequence = seq;

1247
	amdgpu_job_free_resources(job);
1248 1249

	trace_amdgpu_cs_ioctl(job);
1250
	amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->ticket);
1251
	priority = job->base.s_priority;
1252
	drm_sched_entity_push_job(&job->base, entity);
1253

1254
	ring = to_amdgpu_ring(entity->rq->sched);
1255 1256
	amdgpu_ring_priority_get(ring, priority);

1257 1258 1259
	ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
	amdgpu_mn_unlock(p->mn);

1260
	return 0;
1261 1262 1263 1264 1265 1266 1267 1268 1269

error_abort:
	dma_fence_put(&job->base.s_fence->finished);
	job->base.s_fence = NULL;

error_unlock:
	amdgpu_job_free(job);
	amdgpu_mn_unlock(p->mn);
	return r;
1270 1271
}

C
Chunming Zhou 已提交
1272 1273 1274 1275
int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
{
	struct amdgpu_device *adev = dev->dev_private;
	union drm_amdgpu_cs *cs = data;
1276
	struct amdgpu_cs_parser parser = {};
1277 1278
	bool reserved_buffers = false;
	int i, r;
C
Chunming Zhou 已提交
1279

1280
	if (!adev->accel_working)
C
Chunming Zhou 已提交
1281
		return -EBUSY;
1282

1283 1284 1285 1286
	parser.adev = adev;
	parser.filp = filp;

	r = amdgpu_cs_parser_init(&parser, data);
A
Alex Deucher 已提交
1287
	if (r) {
C
Chunming Zhou 已提交
1288
		DRM_ERROR("Failed to initialize parser !\n");
1289
		goto out;
1290 1291
	}

1292 1293 1294 1295
	r = amdgpu_cs_ib_fill(adev, &parser);
	if (r)
		goto out;

1296 1297 1298 1299 1300 1301 1302
	r = amdgpu_cs_parser_bos(&parser, data);
	if (r) {
		if (r == -ENOMEM)
			DRM_ERROR("Not enough memory for command submission!\n");
		else if (r != -ERESTARTSYS)
			DRM_ERROR("Failed to process the buffer list %d!\n", r);
		goto out;
1303 1304
	}

1305
	reserved_buffers = true;
1306

1307 1308 1309 1310 1311 1312
	r = amdgpu_cs_dependencies(adev, &parser);
	if (r) {
		DRM_ERROR("Failed in the dependencies handling %d!\n", r);
		goto out;
	}

1313
	for (i = 0; i < parser.job->num_ibs; i++)
1314
		trace_amdgpu_cs(&parser, i);
1315

1316
	r = amdgpu_cs_ib_vm_chunk(adev, &parser);
1317 1318 1319
	if (r)
		goto out;

C
Christian König 已提交
1320
	r = amdgpu_cs_submit(&parser, cs);
A
Alex Deucher 已提交
1321 1322

out:
1323
	amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
A
Alex Deucher 已提交
1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341
	return r;
}

/**
 * amdgpu_cs_wait_ioctl - wait for a command submission to finish
 *
 * @dev: drm device
 * @data: data from userspace
 * @filp: file private
 *
 * Wait for the command submission identified by handle to finish.
 */
int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data,
			 struct drm_file *filp)
{
	union drm_amdgpu_wait_cs *wait = data;
	struct amdgpu_device *adev = dev->dev_private;
	unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout);
1342
	struct amdgpu_ring *ring = NULL;
1343
	struct amdgpu_ctx *ctx;
1344
	struct dma_fence *fence;
A
Alex Deucher 已提交
1345 1346
	long r;

1347 1348 1349
	ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id);
	if (ctx == NULL)
		return -EINVAL;
A
Alex Deucher 已提交
1350

1351 1352 1353 1354 1355 1356 1357 1358
	r = amdgpu_queue_mgr_map(adev, &ctx->queue_mgr,
				 wait->in.ip_type, wait->in.ip_instance,
				 wait->in.ring, &ring);
	if (r) {
		amdgpu_ctx_put(ctx);
		return r;
	}

1359 1360 1361 1362
	fence = amdgpu_ctx_get_fence(ctx, ring, wait->in.handle);
	if (IS_ERR(fence))
		r = PTR_ERR(fence);
	else if (fence) {
1363
		r = dma_fence_wait_timeout(fence, true, timeout);
1364 1365
		if (r > 0 && fence->error)
			r = fence->error;
1366
		dma_fence_put(fence);
1367 1368
	} else
		r = 1;
C
Chunming Zhou 已提交
1369

1370
	amdgpu_ctx_put(ctx);
A
Alex Deucher 已提交
1371 1372 1373 1374 1375 1376 1377 1378 1379
	if (r < 0)
		return r;

	memset(wait, 0, sizeof(*wait));
	wait->out.status = (r == 0);

	return 0;
}

1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399
/**
 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence
 *
 * @adev: amdgpu device
 * @filp: file private
 * @user: drm_amdgpu_fence copied from user space
 */
static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev,
					     struct drm_file *filp,
					     struct drm_amdgpu_fence *user)
{
	struct amdgpu_ring *ring;
	struct amdgpu_ctx *ctx;
	struct dma_fence *fence;
	int r;

	ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id);
	if (ctx == NULL)
		return ERR_PTR(-EINVAL);

1400 1401 1402 1403 1404 1405 1406
	r = amdgpu_queue_mgr_map(adev, &ctx->queue_mgr, user->ip_type,
				 user->ip_instance, user->ring, &ring);
	if (r) {
		amdgpu_ctx_put(ctx);
		return ERR_PTR(r);
	}

1407 1408 1409 1410 1411 1412
	fence = amdgpu_ctx_get_fence(ctx, ring, user->seq_no);
	amdgpu_ctx_put(ctx);

	return fence;
}

1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468
int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data,
				    struct drm_file *filp)
{
	struct amdgpu_device *adev = dev->dev_private;
	union drm_amdgpu_fence_to_handle *info = data;
	struct dma_fence *fence;
	struct drm_syncobj *syncobj;
	struct sync_file *sync_file;
	int fd, r;

	fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence);
	if (IS_ERR(fence))
		return PTR_ERR(fence);

	switch (info->in.what) {
	case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ:
		r = drm_syncobj_create(&syncobj, 0, fence);
		dma_fence_put(fence);
		if (r)
			return r;
		r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle);
		drm_syncobj_put(syncobj);
		return r;

	case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD:
		r = drm_syncobj_create(&syncobj, 0, fence);
		dma_fence_put(fence);
		if (r)
			return r;
		r = drm_syncobj_get_fd(syncobj, (int*)&info->out.handle);
		drm_syncobj_put(syncobj);
		return r;

	case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD:
		fd = get_unused_fd_flags(O_CLOEXEC);
		if (fd < 0) {
			dma_fence_put(fence);
			return fd;
		}

		sync_file = sync_file_create(fence);
		dma_fence_put(fence);
		if (!sync_file) {
			put_unused_fd(fd);
			return -ENOMEM;
		}

		fd_install(fd, sync_file->file);
		info->out.handle = fd;
		return 0;

	default:
		return -EINVAL;
	}
}

1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496
/**
 * amdgpu_cs_wait_all_fence - wait on all fences to signal
 *
 * @adev: amdgpu device
 * @filp: file private
 * @wait: wait parameters
 * @fences: array of drm_amdgpu_fence
 */
static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev,
				     struct drm_file *filp,
				     union drm_amdgpu_wait_fences *wait,
				     struct drm_amdgpu_fence *fences)
{
	uint32_t fence_count = wait->in.fence_count;
	unsigned int i;
	long r = 1;

	for (i = 0; i < fence_count; i++) {
		struct dma_fence *fence;
		unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);

		fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
		if (IS_ERR(fence))
			return PTR_ERR(fence);
		else if (!fence)
			continue;

		r = dma_fence_wait_timeout(fence, true, timeout);
1497
		dma_fence_put(fence);
1498 1499 1500 1501 1502
		if (r < 0)
			return r;

		if (r == 0)
			break;
1503 1504 1505

		if (fence->error)
			return fence->error;
1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550
	}

	memset(wait, 0, sizeof(*wait));
	wait->out.status = (r > 0);

	return 0;
}

/**
 * amdgpu_cs_wait_any_fence - wait on any fence to signal
 *
 * @adev: amdgpu device
 * @filp: file private
 * @wait: wait parameters
 * @fences: array of drm_amdgpu_fence
 */
static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev,
				    struct drm_file *filp,
				    union drm_amdgpu_wait_fences *wait,
				    struct drm_amdgpu_fence *fences)
{
	unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns);
	uint32_t fence_count = wait->in.fence_count;
	uint32_t first = ~0;
	struct dma_fence **array;
	unsigned int i;
	long r;

	/* Prepare the fence array */
	array = kcalloc(fence_count, sizeof(struct dma_fence *), GFP_KERNEL);

	if (array == NULL)
		return -ENOMEM;

	for (i = 0; i < fence_count; i++) {
		struct dma_fence *fence;

		fence = amdgpu_cs_get_fence(adev, filp, &fences[i]);
		if (IS_ERR(fence)) {
			r = PTR_ERR(fence);
			goto err_free_fence_array;
		} else if (fence) {
			array[i] = fence;
		} else { /* NULL, the fence has been already signaled */
			r = 1;
M
Monk Liu 已提交
1551
			first = i;
1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564
			goto out;
		}
	}

	r = dma_fence_wait_any_timeout(array, fence_count, true, timeout,
				       &first);
	if (r < 0)
		goto err_free_fence_array;

out:
	memset(wait, 0, sizeof(*wait));
	wait->out.status = (r > 0);
	wait->out.first_signaled = first;
1565

1566
	if (first < fence_count && array[first])
1567 1568 1569
		r = array[first]->error;
	else
		r = 0;
1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601

err_free_fence_array:
	for (i = 0; i < fence_count; i++)
		dma_fence_put(array[i]);
	kfree(array);

	return r;
}

/**
 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish
 *
 * @dev: drm device
 * @data: data from userspace
 * @filp: file private
 */
int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data,
				struct drm_file *filp)
{
	struct amdgpu_device *adev = dev->dev_private;
	union drm_amdgpu_wait_fences *wait = data;
	uint32_t fence_count = wait->in.fence_count;
	struct drm_amdgpu_fence *fences_user;
	struct drm_amdgpu_fence *fences;
	int r;

	/* Get the fences from userspace */
	fences = kmalloc_array(fence_count, sizeof(struct drm_amdgpu_fence),
			GFP_KERNEL);
	if (fences == NULL)
		return -ENOMEM;

1602
	fences_user = u64_to_user_ptr(wait->in.fences);
1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619
	if (copy_from_user(fences, fences_user,
		sizeof(struct drm_amdgpu_fence) * fence_count)) {
		r = -EFAULT;
		goto err_free_fences;
	}

	if (wait->in.wait_all)
		r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences);
	else
		r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences);

err_free_fences:
	kfree(fences);

	return r;
}

A
Alex Deucher 已提交
1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630
/**
 * amdgpu_cs_find_bo_va - find bo_va for VM address
 *
 * @parser: command submission parser context
 * @addr: VM address
 * @bo: resulting BO of the mapping found
 *
 * Search the buffer objects in the command submission context for a certain
 * virtual memory address. Returns allocation structure when found, NULL
 * otherwise.
 */
1631 1632 1633
int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser,
			   uint64_t addr, struct amdgpu_bo **bo,
			   struct amdgpu_bo_va_mapping **map)
A
Alex Deucher 已提交
1634
{
1635
	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
1636
	struct ttm_operation_ctx ctx = { false, false };
1637
	struct amdgpu_vm *vm = &fpriv->vm;
A
Alex Deucher 已提交
1638
	struct amdgpu_bo_va_mapping *mapping;
1639 1640
	int r;

A
Alex Deucher 已提交
1641
	addr /= AMDGPU_GPU_PAGE_SIZE;
1642

1643 1644 1645
	mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
	if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo)
		return -EINVAL;
1646

1647 1648
	*bo = mapping->bo_va->base.bo;
	*map = mapping;
1649

1650 1651 1652
	/* Double check that the BO is reserved by this CS */
	if (READ_ONCE((*bo)->tbo.resv->lock.ctx) != &parser->ticket)
		return -EINVAL;
1653

1654 1655
	if (!((*bo)->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)) {
		(*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1656
		amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains);
1657
		r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx);
1658
		if (r)
1659
			return r;
1660 1661
	}

1662
	return amdgpu_ttm_alloc_gart(&(*bo)->tbo);
1663
}