nouveau_gem.c 20.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
/*
 * Copyright (C) 2008 Ben Skeggs.
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial
 * portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */
#include "drmP.h"
#include "drm.h"

#include "nouveau_drv.h"
#include "nouveau_drm.h"
#include "nouveau_dma.h"

#define nouveau_gem_pushbuf_sync(chan) 0

int
nouveau_gem_object_new(struct drm_gem_object *gem)
{
	return 0;
}

void
nouveau_gem_object_del(struct drm_gem_object *gem)
{
	struct nouveau_bo *nvbo = gem->driver_private;
	struct ttm_buffer_object *bo = &nvbo->bo;

	if (!nvbo)
		return;
	nvbo->gem = NULL;

	if (unlikely(nvbo->cpu_filp))
		ttm_bo_synccpu_write_release(bo);

	if (unlikely(nvbo->pin_refcnt)) {
		nvbo->pin_refcnt = 1;
		nouveau_bo_unpin(nvbo);
	}

	ttm_bo_unref(&bo);
60 61 62

	drm_gem_object_release(gem);
	kfree(gem);
63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
}

int
nouveau_gem_new(struct drm_device *dev, struct nouveau_channel *chan,
		int size, int align, uint32_t flags, uint32_t tile_mode,
		uint32_t tile_flags, bool no_vm, bool mappable,
		struct nouveau_bo **pnvbo)
{
	struct nouveau_bo *nvbo;
	int ret;

	ret = nouveau_bo_new(dev, chan, size, align, flags, tile_mode,
			     tile_flags, no_vm, mappable, pnvbo);
	if (ret)
		return ret;
	nvbo = *pnvbo;

	nvbo->gem = drm_gem_object_alloc(dev, nvbo->bo.mem.size);
	if (!nvbo->gem) {
		nouveau_bo_ref(NULL, pnvbo);
		return -ENOMEM;
	}

	nvbo->bo.persistant_swap_storage = nvbo->gem->filp;
	nvbo->gem->driver_private = nvbo;
	return 0;
}

static int
nouveau_gem_info(struct drm_gem_object *gem, struct drm_nouveau_gem_info *rep)
{
	struct nouveau_bo *nvbo = nouveau_gem_object(gem);

	if (nvbo->bo.mem.mem_type == TTM_PL_TT)
		rep->domain = NOUVEAU_GEM_DOMAIN_GART;
	else
		rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;

	rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
	rep->offset = nvbo->bo.offset;
	rep->map_handle = nvbo->mappable ? nvbo->bo.addr_space_offset : 0;
	rep->tile_mode = nvbo->tile_mode;
	rep->tile_flags = nvbo->tile_flags;
	return 0;
}

static bool
nouveau_gem_tile_flags_valid(struct drm_device *dev, uint32_t tile_flags) {
	switch (tile_flags) {
	case 0x0000:
	case 0x1800:
	case 0x2800:
	case 0x4800:
	case 0x7000:
	case 0x7400:
	case 0x7a00:
	case 0xe000:
		break;
	default:
		NV_ERROR(dev, "bad page flags: 0x%08x\n", tile_flags);
		return false;
	}

	return true;
}

int
nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
		      struct drm_file *file_priv)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct drm_nouveau_gem_new *req = data;
	struct nouveau_bo *nvbo = NULL;
	struct nouveau_channel *chan = NULL;
	uint32_t flags = 0;
	int ret = 0;

	if (unlikely(dev_priv->ttm.bdev.dev_mapping == NULL))
		dev_priv->ttm.bdev.dev_mapping = dev_priv->dev->dev_mapping;

	if (req->channel_hint) {
		NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(req->channel_hint,
						     file_priv, chan);
	}

	if (req->info.domain & NOUVEAU_GEM_DOMAIN_VRAM)
		flags |= TTM_PL_FLAG_VRAM;
	if (req->info.domain & NOUVEAU_GEM_DOMAIN_GART)
		flags |= TTM_PL_FLAG_TT;
	if (!flags || req->info.domain & NOUVEAU_GEM_DOMAIN_CPU)
		flags |= TTM_PL_FLAG_SYSTEM;

	if (!nouveau_gem_tile_flags_valid(dev, req->info.tile_flags))
		return -EINVAL;

	ret = nouveau_gem_new(dev, chan, req->info.size, req->align, flags,
			      req->info.tile_mode, req->info.tile_flags, false,
			      (req->info.domain & NOUVEAU_GEM_DOMAIN_MAPPABLE),
			      &nvbo);
	if (ret)
		return ret;

	ret = nouveau_gem_info(nvbo->gem, &req->info);
	if (ret)
		goto out;

	ret = drm_gem_handle_create(file_priv, nvbo->gem, &req->info.handle);
out:
171
	drm_gem_object_handle_unreference_unlocked(nvbo->gem);
172 173

	if (ret)
174
		drm_gem_object_unreference_unlocked(nvbo->gem);
175 176 177 178 179 180 181 182 183
	return ret;
}

static int
nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
		       uint32_t write_domains, uint32_t valid_domains)
{
	struct nouveau_bo *nvbo = gem->driver_private;
	struct ttm_buffer_object *bo = &nvbo->bo;
184 185 186
	uint32_t domains = valid_domains &
		(write_domains ? write_domains : read_domains);
	uint32_t pref_flags = 0, valid_flags = 0;
187

188
	if (!domains)
189 190
		return -EINVAL;

191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211
	if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
		valid_flags |= TTM_PL_FLAG_VRAM;

	if (valid_domains & NOUVEAU_GEM_DOMAIN_GART)
		valid_flags |= TTM_PL_FLAG_TT;

	if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
	    bo->mem.mem_type == TTM_PL_VRAM)
		pref_flags |= TTM_PL_FLAG_VRAM;

	else if ((domains & NOUVEAU_GEM_DOMAIN_GART) &&
		 bo->mem.mem_type == TTM_PL_TT)
		pref_flags |= TTM_PL_FLAG_TT;

	else if (domains & NOUVEAU_GEM_DOMAIN_VRAM)
		pref_flags |= TTM_PL_FLAG_VRAM;

	else
		pref_flags |= TTM_PL_FLAG_TT;

	nouveau_bo_placement_set(nvbo, pref_flags, valid_flags);
212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239

	return 0;
}

struct validate_op {
	struct list_head vram_list;
	struct list_head gart_list;
	struct list_head both_list;
};

static void
validate_fini_list(struct list_head *list, struct nouveau_fence *fence)
{
	struct list_head *entry, *tmp;
	struct nouveau_bo *nvbo;

	list_for_each_safe(entry, tmp, list) {
		nvbo = list_entry(entry, struct nouveau_bo, entry);
		if (likely(fence)) {
			struct nouveau_fence *prev_fence;

			spin_lock(&nvbo->bo.lock);
			prev_fence = nvbo->bo.sync_obj;
			nvbo->bo.sync_obj = nouveau_fence_ref(fence);
			spin_unlock(&nvbo->bo.lock);
			nouveau_fence_unref((void *)&prev_fence);
		}

240 241 242 243 244
		if (unlikely(nvbo->validate_mapped)) {
			ttm_bo_kunmap(&nvbo->kmap);
			nvbo->validate_mapped = false;
		}

245 246 247 248 249 250 251 252
		list_del(&nvbo->entry);
		nvbo->reserved_by = NULL;
		ttm_bo_unreserve(&nvbo->bo);
		drm_gem_object_unreference(nvbo->gem);
	}
}

static void
253
validate_fini(struct validate_op *op, struct nouveau_fence* fence)
254
{
255 256 257
	validate_fini_list(&op->vram_list, fence);
	validate_fini_list(&op->gart_list, fence);
	validate_fini_list(&op->both_list, fence);
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
}

static int
validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
	      struct drm_nouveau_gem_pushbuf_bo *pbbo,
	      int nr_buffers, struct validate_op *op)
{
	struct drm_device *dev = chan->dev;
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	uint32_t sequence;
	int trycnt = 0;
	int ret, i;

	sequence = atomic_add_return(1, &dev_priv->ttm.validate_sequence);
retry:
	if (++trycnt > 100000) {
		NV_ERROR(dev, "%s failed and gave up.\n", __func__);
		return -EINVAL;
	}

	for (i = 0; i < nr_buffers; i++) {
		struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i];
		struct drm_gem_object *gem;
		struct nouveau_bo *nvbo;

		gem = drm_gem_object_lookup(dev, file_priv, b->handle);
		if (!gem) {
			NV_ERROR(dev, "Unknown handle 0x%08x\n", b->handle);
			validate_fini(op, NULL);
			return -EINVAL;
		}
		nvbo = gem->driver_private;

		if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
			NV_ERROR(dev, "multiple instances of buffer %d on "
				      "validation list\n", b->handle);
			validate_fini(op, NULL);
			return -EINVAL;
		}

		ret = ttm_bo_reserve(&nvbo->bo, false, false, true, sequence);
		if (ret) {
			validate_fini(op, NULL);
			if (ret == -EAGAIN)
				ret = ttm_bo_wait_unreserved(&nvbo->bo, false);
			drm_gem_object_unreference(gem);
304 305
			if (ret) {
				NV_ERROR(dev, "fail reserve\n");
306
				return ret;
307
			}
308 309 310
			goto retry;
		}

311
		b->user_priv = (uint64_t)(unsigned long)nvbo;
312 313 314 315 316 317 318 319 320 321 322 323 324 325
		nvbo->reserved_by = file_priv;
		nvbo->pbbo_index = i;
		if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
		    (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
			list_add_tail(&nvbo->entry, &op->both_list);
		else
		if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
			list_add_tail(&nvbo->entry, &op->vram_list);
		else
		if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
			list_add_tail(&nvbo->entry, &op->gart_list);
		else {
			NV_ERROR(dev, "invalid valid domains: 0x%08x\n",
				 b->valid_domains);
326
			list_add_tail(&nvbo->entry, &op->both_list);
327 328 329 330 331 332 333 334 335 336 337 338 339 340
			validate_fini(op, NULL);
			return -EINVAL;
		}

		if (unlikely(atomic_read(&nvbo->bo.cpu_writers) > 0)) {
			validate_fini(op, NULL);

			if (nvbo->cpu_filp == file_priv) {
				NV_ERROR(dev, "bo %p mapped by process trying "
					      "to validate it!\n", nvbo);
				return -EINVAL;
			}

			ret = ttm_bo_wait_cpu(&nvbo->bo, false);
341 342
			if (ret) {
				NV_ERROR(dev, "fail wait_cpu\n");
343
				return ret;
344
			}
345 346 347 348 349 350 351 352 353 354 355 356 357
			goto retry;
		}
	}

	return 0;
}

static int
validate_list(struct nouveau_channel *chan, struct list_head *list,
	      struct drm_nouveau_gem_pushbuf_bo *pbbo, uint64_t user_pbbo_ptr)
{
	struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
				(void __force __user *)(uintptr_t)user_pbbo_ptr;
358
	struct drm_device *dev = chan->dev;
359 360 361 362 363 364
	struct nouveau_bo *nvbo;
	int ret, relocs = 0;

	list_for_each_entry(nvbo, list, entry) {
		struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];

365 366 367 368
		ret = nouveau_bo_sync_gpu(nvbo, chan);
		if (unlikely(ret)) {
			NV_ERROR(dev, "fail pre-validate sync\n");
			return ret;
369 370 371 372 373
		}

		ret = nouveau_gem_set_domain(nvbo->gem, b->read_domains,
					     b->write_domains,
					     b->valid_domains);
374 375
		if (unlikely(ret)) {
			NV_ERROR(dev, "fail set_domain\n");
376
			return ret;
377
		}
378

379
		nvbo->channel = (b->read_domains & (1 << 31)) ? NULL : chan;
380
		ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement,
381
				      false, false, false);
382
		nvbo->channel = NULL;
383 384
		if (unlikely(ret)) {
			NV_ERROR(dev, "fail ttm_validate\n");
385
			return ret;
386
		}
387

388 389 390 391 392 393
		ret = nouveau_bo_sync_gpu(nvbo, chan);
		if (unlikely(ret)) {
			NV_ERROR(dev, "fail post-validate sync\n");
			return ret;
		}

394
		if (nvbo->bo.offset == b->presumed.offset &&
395
		    ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
396
		      b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
397
		     (nvbo->bo.mem.mem_type == TTM_PL_TT &&
398
		      b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
399 400 401
			continue;

		if (nvbo->bo.mem.mem_type == TTM_PL_TT)
402
			b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
403
		else
404 405 406
			b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
		b->presumed.offset = nvbo->bo.offset;
		b->presumed.valid = 0;
407 408
		relocs++;

409 410
		if (DRM_COPY_TO_USER(&upbbo[nvbo->pbbo_index].presumed,
				     &b->presumed, sizeof(b->presumed)))
411 412 413 414 415 416 417 418 419 420 421 422 423
			return -EFAULT;
	}

	return relocs;
}

static int
nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
			     struct drm_file *file_priv,
			     struct drm_nouveau_gem_pushbuf_bo *pbbo,
			     uint64_t user_buffers, int nr_buffers,
			     struct validate_op *op, int *apply_relocs)
{
424
	struct drm_device *dev = chan->dev;
425 426 427 428 429 430 431 432 433 434
	int ret, relocs = 0;

	INIT_LIST_HEAD(&op->vram_list);
	INIT_LIST_HEAD(&op->gart_list);
	INIT_LIST_HEAD(&op->both_list);

	if (nr_buffers == 0)
		return 0;

	ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
435 436
	if (unlikely(ret)) {
		NV_ERROR(dev, "validate_init\n");
437
		return ret;
438
	}
439 440 441

	ret = validate_list(chan, &op->vram_list, pbbo, user_buffers);
	if (unlikely(ret < 0)) {
442
		NV_ERROR(dev, "validate vram_list\n");
443 444 445 446 447 448 449
		validate_fini(op, NULL);
		return ret;
	}
	relocs += ret;

	ret = validate_list(chan, &op->gart_list, pbbo, user_buffers);
	if (unlikely(ret < 0)) {
450
		NV_ERROR(dev, "validate gart_list\n");
451 452 453 454 455 456 457
		validate_fini(op, NULL);
		return ret;
	}
	relocs += ret;

	ret = validate_list(chan, &op->both_list, pbbo, user_buffers);
	if (unlikely(ret < 0)) {
458
		NV_ERROR(dev, "validate both_list\n");
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
		validate_fini(op, NULL);
		return ret;
	}
	relocs += ret;

	*apply_relocs = relocs;
	return 0;
}

static inline void *
u_memcpya(uint64_t user, unsigned nmemb, unsigned size)
{
	void *mem;
	void __user *userptr = (void __force __user *)(uintptr_t)user;

	mem = kmalloc(nmemb * size, GFP_KERNEL);
	if (!mem)
		return ERR_PTR(-ENOMEM);

	if (DRM_COPY_FROM_USER(mem, userptr, nmemb * size)) {
		kfree(mem);
		return ERR_PTR(-EFAULT);
	}

	return mem;
}

static int
487 488 489
nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev,
				struct drm_nouveau_gem_pushbuf *req,
				struct drm_nouveau_gem_pushbuf_bo *bo)
490 491
{
	struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
492 493
	int ret = 0;
	unsigned i;
494

495
	reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc));
496 497 498
	if (IS_ERR(reloc))
		return PTR_ERR(reloc);

499
	for (i = 0; i < req->nr_relocs; i++) {
500 501
		struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i];
		struct drm_nouveau_gem_pushbuf_bo *b;
502
		struct nouveau_bo *nvbo;
503 504
		uint32_t data;

505 506
		if (unlikely(r->bo_index > req->nr_buffers)) {
			NV_ERROR(dev, "reloc bo index invalid\n");
507 508 509 510 511
			ret = -EINVAL;
			break;
		}

		b = &bo[r->bo_index];
512
		if (b->presumed.valid)
513 514
			continue;

515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538
		if (unlikely(r->reloc_bo_index > req->nr_buffers)) {
			NV_ERROR(dev, "reloc container bo index invalid\n");
			ret = -EINVAL;
			break;
		}
		nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv;

		if (unlikely(r->reloc_bo_offset + 4 >
			     nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
			NV_ERROR(dev, "reloc outside of bo\n");
			ret = -EINVAL;
			break;
		}

		if (!nvbo->kmap.virtual) {
			ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
					  &nvbo->kmap);
			if (ret) {
				NV_ERROR(dev, "failed kmap for reloc\n");
				break;
			}
			nvbo->validate_mapped = true;
		}

539
		if (r->flags & NOUVEAU_GEM_RELOC_LOW)
540
			data = b->presumed.offset + r->data;
541 542
		else
		if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
543
			data = (b->presumed.offset + r->data) >> 32;
544 545 546 547
		else
			data = r->data;

		if (r->flags & NOUVEAU_GEM_RELOC_OR) {
548
			if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART)
549 550 551 552 553
				data |= r->tor;
			else
				data |= r->vor;
		}

554 555
		spin_lock(&nvbo->bo.lock);
		ret = ttm_bo_wait(&nvbo->bo, false, false, false);
556
		spin_unlock(&nvbo->bo.lock);
557 558 559 560 561 562
		if (ret) {
			NV_ERROR(dev, "reloc wait_idle failed: %d\n", ret);
			break;
		}

		nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data);
563 564 565 566 567 568 569 570 571 572
	}

	kfree(reloc);
	return ret;
}

int
nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
			  struct drm_file *file_priv)
{
573
	struct drm_nouveau_private *dev_priv = dev->dev_private;
574
	struct drm_nouveau_gem_pushbuf *req = data;
575 576
	struct drm_nouveau_gem_pushbuf_push *push;
	struct drm_nouveau_gem_pushbuf_bo *bo;
577 578
	struct nouveau_channel *chan;
	struct validate_op op;
579
	struct nouveau_fence *fence = NULL;
580
	int i, j, ret = 0, do_reloc = 0;
581 582 583

	NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(req->channel, file_priv, chan);

584 585 586 587
	req->vram_available = dev_priv->fb_aper_free;
	req->gart_available = dev_priv->gart_info.aper_free;
	if (unlikely(req->nr_push == 0))
		goto out_next;
588

589 590 591 592
	if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
		NV_ERROR(dev, "pushbuf push count exceeds limit: %d max %d\n",
			 req->nr_push, NOUVEAU_GEM_MAX_PUSH);
		return -EINVAL;
593 594
	}

595 596 597 598
	if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
		NV_ERROR(dev, "pushbuf bo count exceeds limit: %d max %d\n",
			 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
		return -EINVAL;
599 600
	}

601 602 603
	if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
		NV_ERROR(dev, "pushbuf reloc count exceeds limit: %d max %d\n",
			 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
604 605 606
		return -EINVAL;
	}

607 608 609 610
	push = u_memcpya(req->push, req->nr_push, sizeof(*push));
	if (IS_ERR(push))
		return PTR_ERR(push);

611
	bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
612 613
	if (IS_ERR(bo)) {
		kfree(push);
614
		return PTR_ERR(bo);
615
	}
616 617 618

	mutex_lock(&dev->struct_mutex);

619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
	/* Mark push buffers as being used on PFIFO, the validation code
	 * will then make sure that if the pushbuf bo moves, that they
	 * happen on the kernel channel, which will in turn cause a sync
	 * to happen before we try and submit the push buffer.
	 */
	for (i = 0; i < req->nr_push; i++) {
		if (push[i].bo_index >= req->nr_buffers) {
			NV_ERROR(dev, "push %d buffer not in list\n", i);
			ret = -EINVAL;
			goto out;
		}

		bo[push[i].bo_index].read_domains |= (1 << 31);
	}

634 635 636 637 638 639 640 641 642 643
	/* Validate buffer list */
	ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers,
					   req->nr_buffers, &op, &do_reloc);
	if (ret) {
		NV_ERROR(dev, "validate: %d\n", ret);
		goto out;
	}

	/* Apply any relocations that are required */
	if (do_reloc) {
644
		ret = nouveau_gem_pushbuf_reloc_apply(dev, req, bo);
645 646 647 648 649 650
		if (ret) {
			NV_ERROR(dev, "reloc apply: %d\n", ret);
			goto out;
		}
	}

651
	if (chan->dma.ib_max) {
652
		ret = nouveau_dma_wait(chan, req->nr_push + 1, 6);
653
		if (ret) {
654
			NV_INFO(dev, "nv50cal_space: %d\n", ret);
655 656 657
			goto out;
		}

658 659 660 661 662 663 664
		for (i = 0; i < req->nr_push; i++) {
			struct nouveau_bo *nvbo = (void *)(unsigned long)
				bo[push[i].bo_index].user_priv;

			nv50_dma_push(chan, nvbo, push[i].offset,
				      push[i].length);
		}
665
	} else
666
	if (dev_priv->chipset >= 0x25) {
667
		ret = RING_SPACE(chan, req->nr_push * 2);
668 669 670 671
		if (ret) {
			NV_ERROR(dev, "cal_space: %d\n", ret);
			goto out;
		}
672 673 674 675 676 677 678 679 680 681

		for (i = 0; i < req->nr_push; i++) {
			struct nouveau_bo *nvbo = (void *)(unsigned long)
				bo[push[i].bo_index].user_priv;
			struct drm_mm_node *mem = nvbo->bo.mem.mm_node;

			OUT_RING(chan, ((mem->start << PAGE_SHIFT) +
					push[i].offset) | 2);
			OUT_RING(chan, 0);
		}
682
	} else {
683
		ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
684 685 686 687 688
		if (ret) {
			NV_ERROR(dev, "jmp_space: %d\n", ret);
			goto out;
		}

689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715
		for (i = 0; i < req->nr_push; i++) {
			struct nouveau_bo *nvbo = (void *)(unsigned long)
				bo[push[i].bo_index].user_priv;
			struct drm_mm_node *mem = nvbo->bo.mem.mm_node;
			uint32_t cmd;

			cmd = chan->pushbuf_base + ((chan->dma.cur + 2) << 2);
			cmd |= 0x20000000;
			if (unlikely(cmd != req->suffix0)) {
				if (!nvbo->kmap.virtual) {
					ret = ttm_bo_kmap(&nvbo->bo, 0,
							  nvbo->bo.mem.
							  num_pages,
							  &nvbo->kmap);
					if (ret) {
						WIND_RING(chan);
						goto out;
					}
					nvbo->validate_mapped = true;
				}

				nouveau_bo_wr32(nvbo, (push[i].offset +
						push[i].length - 8) / 4, cmd);
			}

			OUT_RING(chan, ((mem->start << PAGE_SHIFT) +
					push[i].offset) | 0x20000000);
716
			OUT_RING(chan, 0);
717 718 719
			for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
				OUT_RING(chan, 0);
		}
720 721
	}

722
	ret = nouveau_fence_new(chan, &fence, true);
723 724 725 726 727 728 729
	if (ret) {
		NV_ERROR(dev, "error fencing pushbuf: %d\n", ret);
		WIND_RING(chan);
		goto out;
	}

out:
730 731
	validate_fini(&op, fence);
	nouveau_fence_unref((void**)&fence);
732 733
	mutex_unlock(&dev->struct_mutex);
	kfree(bo);
734
	kfree(push);
735 736

out_next:
737 738 739 740
	if (chan->dma.ib_max) {
		req->suffix0 = 0x00000000;
		req->suffix1 = 0x00000000;
	} else
741
	if (dev_priv->chipset >= 0x25) {
742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790
		req->suffix0 = 0x00020000;
		req->suffix1 = 0x00000000;
	} else {
		req->suffix0 = 0x20000000 |
			      (chan->pushbuf_base + ((chan->dma.cur + 2) << 2));
		req->suffix1 = 0x00000000;
	}

	return ret;
}

static inline uint32_t
domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain)
{
	uint32_t flags = 0;

	if (domain & NOUVEAU_GEM_DOMAIN_VRAM)
		flags |= TTM_PL_FLAG_VRAM;
	if (domain & NOUVEAU_GEM_DOMAIN_GART)
		flags |= TTM_PL_FLAG_TT;

	return flags;
}

int
nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
			   struct drm_file *file_priv)
{
	struct drm_nouveau_gem_cpu_prep *req = data;
	struct drm_gem_object *gem;
	struct nouveau_bo *nvbo;
	bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
	int ret = -EINVAL;

	gem = drm_gem_object_lookup(dev, file_priv, req->handle);
	if (!gem)
		return ret;
	nvbo = nouveau_gem_object(gem);

	if (nvbo->cpu_filp) {
		if (nvbo->cpu_filp == file_priv)
			goto out;

		ret = ttm_bo_wait_cpu(&nvbo->bo, no_wait);
		if (ret)
			goto out;
	}

	if (req->flags & NOUVEAU_GEM_CPU_PREP_NOBLOCK) {
791
		spin_lock(&nvbo->bo.lock);
792
		ret = ttm_bo_wait(&nvbo->bo, false, false, no_wait);
793
		spin_unlock(&nvbo->bo.lock);
794 795 796 797 798 799 800
	} else {
		ret = ttm_bo_synccpu_write_grab(&nvbo->bo, no_wait);
		if (ret == 0)
			nvbo->cpu_filp = file_priv;
	}

out:
801
	drm_gem_object_unreference_unlocked(gem);
802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
	return ret;
}

int
nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
			   struct drm_file *file_priv)
{
	struct drm_nouveau_gem_cpu_prep *req = data;
	struct drm_gem_object *gem;
	struct nouveau_bo *nvbo;
	int ret = -EINVAL;

	gem = drm_gem_object_lookup(dev, file_priv, req->handle);
	if (!gem)
		return ret;
	nvbo = nouveau_gem_object(gem);

	if (nvbo->cpu_filp != file_priv)
		goto out;
	nvbo->cpu_filp = NULL;

	ttm_bo_synccpu_write_release(&nvbo->bo);
	ret = 0;

out:
827
	drm_gem_object_unreference_unlocked(gem);
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
	return ret;
}

int
nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
	struct drm_nouveau_gem_info *req = data;
	struct drm_gem_object *gem;
	int ret;

	gem = drm_gem_object_lookup(dev, file_priv, req->handle);
	if (!gem)
		return -EINVAL;

	ret = nouveau_gem_info(gem, req);
844
	drm_gem_object_unreference_unlocked(gem);
845 846 847
	return ret;
}