nouveau_bo.c 27.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
/*
 * Copyright 2007 Dave Airlied
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */
/*
 * Authors: Dave Airlied <airlied@linux.ie>
 *	    Ben Skeggs   <darktama@iinet.net.au>
 *	    Jeremy Kolb  <jkolb@brandeis.edu>
 */

#include "drmP.h"

#include "nouveau_drm.h"
#include "nouveau_drv.h"
#include "nouveau_dma.h"
35 36
#include "nouveau_mm.h"
#include "nouveau_vm.h"
37

38
#include <linux/log2.h>
39
#include <linux/slab.h>
40

41 42 43 44
static void
nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
45
	struct drm_device *dev = dev_priv->dev;
46 47 48 49 50
	struct nouveau_bo *nvbo = nouveau_bo(bo);

	if (unlikely(nvbo->gem))
		DRM_ERROR("bo %p still attached to GEM object\n", bo);

51
	nv10_mem_put_tile_region(dev, nvbo->tile, NULL);
52
	nouveau_vm_put(&nvbo->vma);
53 54 55
	kfree(nvbo);
}

56
static void
57 58
nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, int *size,
		       int *page_shift)
59
{
60
	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
61

B
Ben Skeggs 已提交
62
	if (dev_priv->card_type < NV_50) {
63
		if (nvbo->tile_mode) {
64 65
			if (dev_priv->chipset >= 0x40) {
				*align = 65536;
66
				*size = roundup(*size, 64 * nvbo->tile_mode);
67 68 69

			} else if (dev_priv->chipset >= 0x30) {
				*align = 32768;
70
				*size = roundup(*size, 64 * nvbo->tile_mode);
71 72 73

			} else if (dev_priv->chipset >= 0x20) {
				*align = 16384;
74
				*size = roundup(*size, 64 * nvbo->tile_mode);
75 76 77

			} else if (dev_priv->chipset >= 0x10) {
				*align = 16384;
78
				*size = roundup(*size, 32 * nvbo->tile_mode);
79 80
			}
		}
81 82 83 84 85 86 87 88 89 90 91 92
	} else {
		if (likely(dev_priv->chan_vm)) {
			if (*size > 256 * 1024)
				*page_shift = dev_priv->chan_vm->lpg_shift;
			else
				*page_shift = dev_priv->chan_vm->spg_shift;
		} else {
			*page_shift = 12;
		}

		*size = roundup(*size, (1 << *page_shift));
		*align = max((1 << *page_shift), *align);
93 94
	}

95
	*size = roundup(*size, PAGE_SIZE);
96 97
}

98 99 100 101 102 103 104 105
int
nouveau_bo_new(struct drm_device *dev, struct nouveau_channel *chan,
	       int size, int align, uint32_t flags, uint32_t tile_mode,
	       uint32_t tile_flags, bool no_vm, bool mappable,
	       struct nouveau_bo **pnvbo)
{
	struct drm_nouveau_private *dev_priv = dev->dev_private;
	struct nouveau_bo *nvbo;
106
	int ret = 0, page_shift = 0;
107 108 109 110 111 112 113 114 115 116

	nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
	if (!nvbo)
		return -ENOMEM;
	INIT_LIST_HEAD(&nvbo->head);
	INIT_LIST_HEAD(&nvbo->entry);
	nvbo->mappable = mappable;
	nvbo->no_vm = no_vm;
	nvbo->tile_mode = tile_mode;
	nvbo->tile_flags = tile_flags;
117
	nvbo->bo.bdev = &dev_priv->ttm.bdev;
118

119
	nouveau_bo_fixup_align(nvbo, &align, &size, &page_shift);
120 121
	align >>= PAGE_SHIFT;

122
	if (!nvbo->no_vm && dev_priv->chan_vm) {
123
		ret = nouveau_vm_get(dev_priv->chan_vm, size, page_shift,
124 125 126 127 128 129 130
				     NV_MEM_ACCESS_RW, &nvbo->vma);
		if (ret) {
			kfree(nvbo);
			return ret;
		}
	}

131
	nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
132
	nouveau_bo_placement_set(nvbo, flags, 0);
133 134 135 136 137 138 139 140 141

	nvbo->channel = chan;
	ret = ttm_bo_init(&dev_priv->ttm.bdev, &nvbo->bo, size,
			  ttm_bo_type_device, &nvbo->placement, align, 0,
			  false, NULL, size, nouveau_bo_del_ttm);
	if (ret) {
		/* ttm will call nouveau_bo_del_ttm if it fails.. */
		return ret;
	}
142
	nvbo->channel = NULL;
143

144 145 146 147 148
	if (nvbo->vma.node) {
		if (nvbo->bo.mem.mem_type == TTM_PL_VRAM)
			nvbo->bo.offset = nvbo->vma.offset;
	}

149 150 151 152
	*pnvbo = nvbo;
	return 0;
}

153 154 155 156 157 158 159 160 161 162 163 164 165
static void
set_placement_list(uint32_t *pl, unsigned *n, uint32_t type, uint32_t flags)
{
	*n = 0;

	if (type & TTM_PL_FLAG_VRAM)
		pl[(*n)++] = TTM_PL_FLAG_VRAM | flags;
	if (type & TTM_PL_FLAG_TT)
		pl[(*n)++] = TTM_PL_FLAG_TT | flags;
	if (type & TTM_PL_FLAG_SYSTEM)
		pl[(*n)++] = TTM_PL_FLAG_SYSTEM | flags;
}

166 167 168 169
static void
set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
170
	int vram_pages = dev_priv->vram_size >> PAGE_SHIFT;
171 172

	if (dev_priv->card_type == NV_10 &&
173 174
	    nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
	    nvbo->bo.mem.num_pages < vram_pages / 2) {
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
		/*
		 * Make sure that the color and depth buffers are handled
		 * by independent memory controller units. Up to a 9x
		 * speed up when alpha-blending and depth-test are enabled
		 * at the same time.
		 */
		if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) {
			nvbo->placement.fpfn = vram_pages / 2;
			nvbo->placement.lpfn = ~0;
		} else {
			nvbo->placement.fpfn = 0;
			nvbo->placement.lpfn = vram_pages / 2;
		}
	}
}

191
void
192
nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
193
{
194 195 196 197 198 199 200 201 202 203 204
	struct ttm_placement *pl = &nvbo->placement;
	uint32_t flags = TTM_PL_MASK_CACHING |
		(nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);

	pl->placement = nvbo->placements;
	set_placement_list(nvbo->placements, &pl->num_placement,
			   type, flags);

	pl->busy_placement = nvbo->busy_placements;
	set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
			   type | busy, flags);
205 206

	set_placement_range(nvbo, type);
207 208 209 210 211 212 213
}

int
nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype)
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
	struct ttm_buffer_object *bo = &nvbo->bo;
214
	int ret;
215 216 217 218 219 220 221 222 223 224 225 226 227 228 229

	if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) {
		NV_ERROR(nouveau_bdev(bo->bdev)->dev,
			 "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo,
			 1 << bo->mem.mem_type, memtype);
		return -EINVAL;
	}

	if (nvbo->pin_refcnt++)
		return 0;

	ret = ttm_bo_reserve(bo, false, false, false, 0);
	if (ret)
		goto out;

230
	nouveau_bo_placement_set(nvbo, memtype, 0);
231

232
	ret = nouveau_bo_validate(nvbo, false, false, false);
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
	if (ret == 0) {
		switch (bo->mem.mem_type) {
		case TTM_PL_VRAM:
			dev_priv->fb_aper_free -= bo->mem.size;
			break;
		case TTM_PL_TT:
			dev_priv->gart_info.aper_free -= bo->mem.size;
			break;
		default:
			break;
		}
	}
	ttm_bo_unreserve(bo);
out:
	if (unlikely(ret))
		nvbo->pin_refcnt--;
	return ret;
}

int
nouveau_bo_unpin(struct nouveau_bo *nvbo)
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(nvbo->bo.bdev);
	struct ttm_buffer_object *bo = &nvbo->bo;
257
	int ret;
258 259 260 261 262 263 264 265

	if (--nvbo->pin_refcnt)
		return 0;

	ret = ttm_bo_reserve(bo, false, false, false, 0);
	if (ret)
		return ret;

266
	nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
267

268
	ret = nouveau_bo_validate(nvbo, false, false, false);
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
	if (ret == 0) {
		switch (bo->mem.mem_type) {
		case TTM_PL_VRAM:
			dev_priv->fb_aper_free += bo->mem.size;
			break;
		case TTM_PL_TT:
			dev_priv->gart_info.aper_free += bo->mem.size;
			break;
		default:
			break;
		}
	}

	ttm_bo_unreserve(bo);
	return ret;
}

int
nouveau_bo_map(struct nouveau_bo *nvbo)
{
	int ret;

	ret = ttm_bo_reserve(&nvbo->bo, false, false, false, 0);
	if (ret)
		return ret;

	ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);
	ttm_bo_unreserve(&nvbo->bo);
	return ret;
}

void
nouveau_bo_unmap(struct nouveau_bo *nvbo)
{
303 304
	if (nvbo)
		ttm_bo_kunmap(&nvbo->kmap);
305 306
}

307 308 309 310 311 312 313 314 315 316 317
int
nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
		    bool no_wait_reserve, bool no_wait_gpu)
{
	int ret;

	ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, interruptible,
			      no_wait_reserve, no_wait_gpu);
	if (ret)
		return ret;

318 319 320 321 322
	if (nvbo->vma.node) {
		if (nvbo->bo.mem.mem_type == TTM_PL_VRAM)
			nvbo->bo.offset = nvbo->vma.offset;
	}

323 324 325
	return 0;
}

326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380
u16
nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index)
{
	bool is_iomem;
	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
	mem = &mem[index];
	if (is_iomem)
		return ioread16_native((void __force __iomem *)mem);
	else
		return *mem;
}

void
nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
{
	bool is_iomem;
	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
	mem = &mem[index];
	if (is_iomem)
		iowrite16_native(val, (void __force __iomem *)mem);
	else
		*mem = val;
}

u32
nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
{
	bool is_iomem;
	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
	mem = &mem[index];
	if (is_iomem)
		return ioread32_native((void __force __iomem *)mem);
	else
		return *mem;
}

void
nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
{
	bool is_iomem;
	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
	mem = &mem[index];
	if (is_iomem)
		iowrite32_native(val, (void __force __iomem *)mem);
	else
		*mem = val;
}

static struct ttm_backend *
nouveau_bo_create_ttm_backend_entry(struct ttm_bo_device *bdev)
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
	struct drm_device *dev = dev_priv->dev;

	switch (dev_priv->gart_info.type) {
381
#if __OS_HAS_AGP
382 383
	case NOUVEAU_GART_AGP:
		return ttm_agp_backend_init(bdev, dev->agp->bridge);
384
#endif
385 386
	case NOUVEAU_GART_PDMA:
	case NOUVEAU_GART_HW:
387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417
		return nouveau_sgdma_init_ttm(dev);
	default:
		NV_ERROR(dev, "Unknown GART type %d\n",
			 dev_priv->gart_info.type);
		break;
	}

	return NULL;
}

static int
nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
{
	/* We'll do this from user space. */
	return 0;
}

static int
nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
			 struct ttm_mem_type_manager *man)
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
	struct drm_device *dev = dev_priv->dev;

	switch (type) {
	case TTM_PL_SYSTEM:
		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
		man->available_caching = TTM_PL_MASK_CACHING;
		man->default_caching = TTM_PL_FLAG_CACHED;
		break;
	case TTM_PL_VRAM:
418
		if (dev_priv->card_type >= NV_50) {
B
Ben Skeggs 已提交
419
			man->func = &nouveau_vram_manager;
420 421 422
			man->io_reserve_fastpath = false;
			man->use_io_reserve_lru = true;
		} else {
B
Ben Skeggs 已提交
423
			man->func = &ttm_bo_manager_func;
424
		}
425
		man->flags = TTM_MEMTYPE_FLAG_FIXED |
426
			     TTM_MEMTYPE_FLAG_MAPPABLE;
427 428 429 430 431
		man->available_caching = TTM_PL_FLAG_UNCACHED |
					 TTM_PL_FLAG_WC;
		man->default_caching = TTM_PL_FLAG_WC;
		break;
	case TTM_PL_TT:
432
		man->func = &ttm_bo_manager_func;
433 434
		switch (dev_priv->gart_info.type) {
		case NOUVEAU_GART_AGP:
435
			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
436 437 438
			man->available_caching = TTM_PL_FLAG_UNCACHED |
				TTM_PL_FLAG_WC;
			man->default_caching = TTM_PL_FLAG_WC;
439
			break;
440 441
		case NOUVEAU_GART_PDMA:
		case NOUVEAU_GART_HW:
442 443 444 445
			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
				     TTM_MEMTYPE_FLAG_CMA;
			man->available_caching = TTM_PL_MASK_CACHING;
			man->default_caching = TTM_PL_FLAG_CACHED;
446
			man->gpu_offset = dev_priv->gart_info.aper_base;
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
			break;
		default:
			NV_ERROR(dev, "Unknown GART type: %d\n",
				 dev_priv->gart_info.type);
			return -EINVAL;
		}
		break;
	default:
		NV_ERROR(dev, "Unsupported memory type %u\n", (unsigned)type);
		return -EINVAL;
	}
	return 0;
}

static void
nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
{
	struct nouveau_bo *nvbo = nouveau_bo(bo);

	switch (bo->mem.mem_type) {
467
	case TTM_PL_VRAM:
468 469
		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,
					 TTM_PL_FLAG_SYSTEM);
470
		break;
471
	default:
472
		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);
473 474
		break;
	}
475 476

	*pl = nvbo->placement;
477 478 479 480 481 482
}


/* GPU-assisted copy using NV_MEMORY_TO_MEMORY_FORMAT, can access
 * TTM_PL_{VRAM,TT} directly.
 */
483

484 485
static int
nouveau_bo_move_accel_cleanup(struct nouveau_channel *chan,
486 487
			      struct nouveau_bo *nvbo, bool evict,
			      bool no_wait_reserve, bool no_wait_gpu,
488 489 490 491 492 493 494 495 496
			      struct ttm_mem_reg *new_mem)
{
	struct nouveau_fence *fence = NULL;
	int ret;

	ret = nouveau_fence_new(chan, &fence, true);
	if (ret)
		return ret;

497
	ret = ttm_bo_move_accel_cleanup(&nvbo->bo, fence, NULL, evict,
498
					no_wait_reserve, no_wait_gpu, new_mem);
499
	nouveau_fence_unref(&fence);
500 501 502 503
	return ret;
}

static inline uint32_t
504 505
nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
		      struct nouveau_channel *chan, struct ttm_mem_reg *mem)
506
{
507 508 509
	struct nouveau_bo *nvbo = nouveau_bo(bo);

	if (nvbo->no_vm) {
510 511 512 513 514 515 516 517 518 519
		if (mem->mem_type == TTM_PL_TT)
			return NvDmaGART;
		return NvDmaVRAM;
	}

	if (mem->mem_type == TTM_PL_TT)
		return chan->gart_handle;
	return chan->vram_handle;
}

B
Ben Skeggs 已提交
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571
static int
nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
	struct nouveau_bo *nvbo = nouveau_bo(bo);
	u64 src_offset = old_mem->start << PAGE_SHIFT;
	u64 dst_offset = new_mem->start << PAGE_SHIFT;
	u32 page_count = new_mem->num_pages;
	int ret;

	if (!nvbo->no_vm) {
		if (old_mem->mem_type == TTM_PL_VRAM)
			src_offset  = nvbo->vma.offset;
		else
			src_offset += dev_priv->gart_info.aper_base;

		if (new_mem->mem_type == TTM_PL_VRAM)
			dst_offset  = nvbo->vma.offset;
		else
			dst_offset += dev_priv->gart_info.aper_base;
	}

	page_count = new_mem->num_pages;
	while (page_count) {
		int line_count = (page_count > 2047) ? 2047 : page_count;

		ret = RING_SPACE(chan, 12);
		if (ret)
			return ret;

		BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0238, 2);
		OUT_RING  (chan, upper_32_bits(dst_offset));
		OUT_RING  (chan, lower_32_bits(dst_offset));
		BEGIN_NVC0(chan, 2, NvSubM2MF, 0x030c, 6);
		OUT_RING  (chan, upper_32_bits(src_offset));
		OUT_RING  (chan, lower_32_bits(src_offset));
		OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* line_length */
		OUT_RING  (chan, line_count);
		BEGIN_NVC0(chan, 2, NvSubM2MF, 0x0300, 1);
		OUT_RING  (chan, 0x00100110);

		page_count -= line_count;
		src_offset += (PAGE_SIZE * line_count);
		dst_offset += (PAGE_SIZE * line_count);
	}

	return 0;
}

572
static int
573 574
nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
575 576
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
577 578 579
	struct nouveau_bo *nvbo = nouveau_bo(bo);
	u64 length = (new_mem->num_pages << PAGE_SHIFT);
	u64 src_offset, dst_offset;
580 581
	int ret;

582 583
	src_offset = old_mem->start << PAGE_SHIFT;
	dst_offset = new_mem->start << PAGE_SHIFT;
584 585
	if (!nvbo->no_vm) {
		if (old_mem->mem_type == TTM_PL_VRAM)
586
			src_offset  = nvbo->vma.offset;
587
		else
588
			src_offset += dev_priv->gart_info.aper_base;
589 590

		if (new_mem->mem_type == TTM_PL_VRAM)
591
			dst_offset  = nvbo->vma.offset;
592
		else
593
			dst_offset += dev_priv->gart_info.aper_base;
594 595 596 597 598 599
	}

	ret = RING_SPACE(chan, 3);
	if (ret)
		return ret;

600 601 602 603 604 605 606
	BEGIN_RING(chan, NvSubM2MF, 0x0184, 2);
	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));

	while (length) {
		u32 amount, stride, height;

607 608
		amount  = min(length, (u64)(4 * 1024 * 1024));
		stride  = 16 * 4;
609 610
		height  = amount / stride;

611 612
		if (new_mem->mem_type == TTM_PL_VRAM &&
		    nouveau_bo_tile_layout(nvbo)) {
613 614 615 616 617 618
			ret = RING_SPACE(chan, 8);
			if (ret)
				return ret;

			BEGIN_RING(chan, NvSubM2MF, 0x0200, 7);
			OUT_RING  (chan, 0);
619
			OUT_RING  (chan, 0);
620 621 622 623 624 625 626 627 628 629 630 631 632
			OUT_RING  (chan, stride);
			OUT_RING  (chan, height);
			OUT_RING  (chan, 1);
			OUT_RING  (chan, 0);
			OUT_RING  (chan, 0);
		} else {
			ret = RING_SPACE(chan, 2);
			if (ret)
				return ret;

			BEGIN_RING(chan, NvSubM2MF, 0x0200, 1);
			OUT_RING  (chan, 1);
		}
633 634
		if (old_mem->mem_type == TTM_PL_VRAM &&
		    nouveau_bo_tile_layout(nvbo)) {
635 636 637 638 639 640
			ret = RING_SPACE(chan, 8);
			if (ret)
				return ret;

			BEGIN_RING(chan, NvSubM2MF, 0x021c, 7);
			OUT_RING  (chan, 0);
641
			OUT_RING  (chan, 0);
642 643 644 645 646 647 648 649 650 651 652 653 654 655 656
			OUT_RING  (chan, stride);
			OUT_RING  (chan, height);
			OUT_RING  (chan, 1);
			OUT_RING  (chan, 0);
			OUT_RING  (chan, 0);
		} else {
			ret = RING_SPACE(chan, 2);
			if (ret)
				return ret;

			BEGIN_RING(chan, NvSubM2MF, 0x021c, 1);
			OUT_RING  (chan, 1);
		}

		ret = RING_SPACE(chan, 14);
657 658
		if (ret)
			return ret;
659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677

		BEGIN_RING(chan, NvSubM2MF, 0x0238, 2);
		OUT_RING  (chan, upper_32_bits(src_offset));
		OUT_RING  (chan, upper_32_bits(dst_offset));
		BEGIN_RING(chan, NvSubM2MF, 0x030c, 8);
		OUT_RING  (chan, lower_32_bits(src_offset));
		OUT_RING  (chan, lower_32_bits(dst_offset));
		OUT_RING  (chan, stride);
		OUT_RING  (chan, stride);
		OUT_RING  (chan, stride);
		OUT_RING  (chan, height);
		OUT_RING  (chan, 0x00000101);
		OUT_RING  (chan, 0x00000000);
		BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
		OUT_RING  (chan, 0);

		length -= amount;
		src_offset += amount;
		dst_offset += amount;
678 679
	}

680 681 682 683 684 685 686
	return 0;
}

static int
nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
{
687 688
	u32 src_offset = old_mem->start << PAGE_SHIFT;
	u32 dst_offset = new_mem->start << PAGE_SHIFT;
689 690 691 692 693 694 695 696 697 698 699
	u32 page_count = new_mem->num_pages;
	int ret;

	ret = RING_SPACE(chan, 3);
	if (ret)
		return ret;

	BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));

700 701 702 703 704 705 706
	page_count = new_mem->num_pages;
	while (page_count) {
		int line_count = (page_count > 2047) ? 2047 : page_count;

		ret = RING_SPACE(chan, 11);
		if (ret)
			return ret;
707

708 709
		BEGIN_RING(chan, NvSubM2MF,
				 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
710 711 712 713 714 715 716 717
		OUT_RING  (chan, src_offset);
		OUT_RING  (chan, dst_offset);
		OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* line_length */
		OUT_RING  (chan, line_count);
		OUT_RING  (chan, 0x00000101);
		OUT_RING  (chan, 0x00000000);
718
		BEGIN_RING(chan, NvSubM2MF, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
719
		OUT_RING  (chan, 0);
720 721 722 723 724 725

		page_count -= line_count;
		src_offset += (PAGE_SIZE * line_count);
		dst_offset += (PAGE_SIZE * line_count);
	}

726 727 728 729 730 731 732 733 734 735 736 737 738 739
	return 0;
}

static int
nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
		     bool no_wait_reserve, bool no_wait_gpu,
		     struct ttm_mem_reg *new_mem)
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
	struct nouveau_bo *nvbo = nouveau_bo(bo);
	struct nouveau_channel *chan;
	int ret;

	chan = nvbo->channel;
740
	if (!chan || nvbo->no_vm) {
741
		chan = dev_priv->channel;
742
		mutex_lock_nested(&chan->mutex, NOUVEAU_KCHANNEL_MUTEX);
743
	}
744 745 746 747

	if (dev_priv->card_type < NV_50)
		ret = nv04_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
	else
B
Ben Skeggs 已提交
748
	if (dev_priv->card_type < NV_C0)
749
		ret = nv50_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
B
Ben Skeggs 已提交
750 751
	else
		ret = nvc0_bo_move_m2mf(chan, bo, &bo->mem, new_mem);
752 753 754 755 756
	if (ret == 0) {
		ret = nouveau_bo_move_accel_cleanup(chan, nvbo, evict,
						    no_wait_reserve,
						    no_wait_gpu, new_mem);
	}
757

758 759 760
	if (chan == dev_priv->channel)
		mutex_unlock(&chan->mutex);
	return ret;
761 762 763 764
}

static int
nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
765 766
		      bool no_wait_reserve, bool no_wait_gpu,
		      struct ttm_mem_reg *new_mem)
767 768 769 770 771 772 773 774
{
	u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
	struct ttm_placement placement;
	struct ttm_mem_reg tmp_mem;
	int ret;

	placement.fpfn = placement.lpfn = 0;
	placement.num_placement = placement.num_busy_placement = 1;
775
	placement.placement = placement.busy_placement = &placement_memtype;
776 777 778

	tmp_mem = *new_mem;
	tmp_mem.mm_node = NULL;
779
	ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu);
780 781 782 783 784 785 786
	if (ret)
		return ret;

	ret = ttm_tt_bind(bo->ttm, &tmp_mem);
	if (ret)
		goto out;

787
	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, &tmp_mem);
788 789 790
	if (ret)
		goto out;

791
	ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, new_mem);
792
out:
793
	ttm_bo_mem_put(bo, &tmp_mem);
794 795 796 797 798
	return ret;
}

static int
nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
799 800
		      bool no_wait_reserve, bool no_wait_gpu,
		      struct ttm_mem_reg *new_mem)
801 802 803 804 805 806 807 808
{
	u32 placement_memtype = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING;
	struct ttm_placement placement;
	struct ttm_mem_reg tmp_mem;
	int ret;

	placement.fpfn = placement.lpfn = 0;
	placement.num_placement = placement.num_busy_placement = 1;
809
	placement.placement = placement.busy_placement = &placement_memtype;
810 811 812

	tmp_mem = *new_mem;
	tmp_mem.mm_node = NULL;
813
	ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_reserve, no_wait_gpu);
814 815 816
	if (ret)
		return ret;

817
	ret = ttm_bo_move_ttm(bo, true, no_wait_reserve, no_wait_gpu, &tmp_mem);
818 819 820
	if (ret)
		goto out;

821
	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_reserve, no_wait_gpu, new_mem);
822 823 824 825
	if (ret)
		goto out;

out:
826
	ttm_bo_mem_put(bo, &tmp_mem);
827 828 829 830
	return ret;
}

static int
831 832
nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
		   struct nouveau_tile_reg **new_tile)
833 834 835
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
	struct drm_device *dev = dev_priv->dev;
836 837
	struct nouveau_bo *nvbo = nouveau_bo(bo);
	uint64_t offset;
838

839 840 841 842 843 844
	if (nvbo->no_vm || new_mem->mem_type != TTM_PL_VRAM) {
		/* Nothing to do. */
		*new_tile = NULL;
		return 0;
	}

845
	offset = new_mem->start << PAGE_SHIFT;
846

847 848
	if (dev_priv->chan_vm) {
		nouveau_vm_map(&nvbo->vma, new_mem->mm_node);
849 850
	} else if (dev_priv->card_type >= NV_10) {
		*new_tile = nv10_mem_set_tiling(dev, offset, new_mem->size,
851 852
						nvbo->tile_mode,
						nvbo->tile_flags);
853 854
	}

855 856 857 858 859 860 861 862 863 864 865 866 867
	return 0;
}

static void
nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
		      struct nouveau_tile_reg *new_tile,
		      struct nouveau_tile_reg **old_tile)
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
	struct drm_device *dev = dev_priv->dev;

	if (dev_priv->card_type >= NV_10 &&
	    dev_priv->card_type < NV_50) {
868
		nv10_mem_put_tile_region(dev, *old_tile, bo->sync_obj);
869 870 871 872 873 874
		*old_tile = new_tile;
	}
}

static int
nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
875 876
		bool no_wait_reserve, bool no_wait_gpu,
		struct ttm_mem_reg *new_mem)
877 878 879 880 881 882 883 884 885 886 887 888
{
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
	struct nouveau_bo *nvbo = nouveau_bo(bo);
	struct ttm_mem_reg *old_mem = &bo->mem;
	struct nouveau_tile_reg *new_tile = NULL;
	int ret = 0;

	ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
	if (ret)
		return ret;

	/* Fake bo copy. */
889 890 891 892
	if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
		BUG_ON(bo->mem.mm_node != NULL);
		bo->mem = *new_mem;
		new_mem->mm_node = NULL;
893
		goto out;
894 895
	}

896
	/* Software copy if the card isn't up and running yet. */
B
Ben Skeggs 已提交
897
	if (!dev_priv->channel) {
898 899 900 901
		ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
		goto out;
	}

902 903
	/* Hardware assisted copy. */
	if (new_mem->mem_type == TTM_PL_SYSTEM)
904
		ret = nouveau_bo_move_flipd(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
905
	else if (old_mem->mem_type == TTM_PL_SYSTEM)
906
		ret = nouveau_bo_move_flips(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
907
	else
908
		ret = nouveau_bo_move_m2mf(bo, evict, intr, no_wait_reserve, no_wait_gpu, new_mem);
909

910 911 912 913
	if (!ret)
		goto out;

	/* Fallback to software copy. */
914
	ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, new_mem);
915 916 917 918 919 920 921 922

out:
	if (ret)
		nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
	else
		nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);

	return ret;
923 924 925 926 927 928 929 930
}

static int
nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
{
	return 0;
}

931 932 933 934 935 936
static int
nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
	struct drm_device *dev = dev_priv->dev;
937
	int ret;
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952

	mem->bus.addr = NULL;
	mem->bus.offset = 0;
	mem->bus.size = mem->num_pages << PAGE_SHIFT;
	mem->bus.base = 0;
	mem->bus.is_iomem = false;
	if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
		return -EINVAL;
	switch (mem->mem_type) {
	case TTM_PL_SYSTEM:
		/* System memory */
		return 0;
	case TTM_PL_TT:
#if __OS_HAS_AGP
		if (dev_priv->gart_info.type == NOUVEAU_GART_AGP) {
953
			mem->bus.offset = mem->start << PAGE_SHIFT;
954 955 956 957 958 959
			mem->bus.base = dev_priv->gart_info.aper_base;
			mem->bus.is_iomem = true;
		}
#endif
		break;
	case TTM_PL_VRAM:
960 961
	{
		struct nouveau_vram *vram = mem->mm_node;
962
		u8 page_shift;
963 964 965 966 967 968 969 970

		if (!dev_priv->bar1_vm) {
			mem->bus.offset = mem->start << PAGE_SHIFT;
			mem->bus.base = pci_resource_start(dev->pdev, 1);
			mem->bus.is_iomem = true;
			break;
		}

971 972 973 974 975
		if (dev_priv->card_type == NV_C0)
			page_shift = vram->page_shift;
		else
			page_shift = 12;

B
Ben Skeggs 已提交
976
		ret = nouveau_vm_get(dev_priv->bar1_vm, mem->bus.size,
977
				     page_shift, NV_MEM_ACCESS_RW,
B
Ben Skeggs 已提交
978
				     &vram->bar_vma);
979 980 981 982 983 984 985 986 987
		if (ret)
			return ret;

		nouveau_vm_map(&vram->bar_vma, vram);
		if (ret) {
			nouveau_vm_put(&vram->bar_vma);
			return ret;
		}

988 989 990
		mem->bus.offset = vram->bar_vma.offset;
		if (dev_priv->card_type == NV_50) /*XXX*/
			mem->bus.offset -= 0x0020000000ULL;
991
		mem->bus.base = pci_resource_start(dev->pdev, 1);
992
		mem->bus.is_iomem = true;
993
	}
994 995 996 997 998 999 1000 1001 1002 1003
		break;
	default:
		return -EINVAL;
	}
	return 0;
}

static void
nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bdev);
	struct nouveau_vram *vram = mem->mm_node;

	if (!dev_priv->bar1_vm || mem->mem_type != TTM_PL_VRAM)
		return;

	if (!vram->bar_vma.node)
		return;

	nouveau_vm_unmap(&vram->bar_vma);
	nouveau_vm_put(&vram->bar_vma);
1015 1016 1017 1018 1019
}

static int
nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
{
1020 1021 1022 1023 1024 1025 1026
	struct drm_nouveau_private *dev_priv = nouveau_bdev(bo->bdev);
	struct nouveau_bo *nvbo = nouveau_bo(bo);

	/* as long as the bo isn't in vram, and isn't tiled, we've got
	 * nothing to do here.
	 */
	if (bo->mem.mem_type != TTM_PL_VRAM) {
1027 1028
		if (dev_priv->card_type < NV_50 ||
		    !nouveau_bo_tile_layout(nvbo))
1029 1030 1031 1032
			return 0;
	}

	/* make sure bo is in mappable vram */
1033
	if (bo->mem.start + bo->mem.num_pages < dev_priv->fb_mappable_pages)
1034 1035 1036 1037 1038 1039
		return 0;


	nvbo->placement.fpfn = 0;
	nvbo->placement.lpfn = dev_priv->fb_mappable_pages;
	nouveau_bo_placement_set(nvbo, TTM_PL_VRAM, 0);
1040
	return nouveau_bo_validate(nvbo, false, true, false);
1041 1042
}

1043 1044 1045
void
nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence)
{
1046
	struct nouveau_fence *old_fence;
1047 1048

	if (likely(fence))
1049
		nouveau_fence_ref(fence);
1050

1051 1052 1053
	spin_lock(&nvbo->bo.bdev->fence_lock);
	old_fence = nvbo->bo.sync_obj;
	nvbo->bo.sync_obj = fence;
1054
	spin_unlock(&nvbo->bo.bdev->fence_lock);
1055 1056

	nouveau_fence_unref(&old_fence);
1057 1058
}

1059 1060 1061 1062 1063 1064 1065
struct ttm_bo_driver nouveau_bo_driver = {
	.create_ttm_backend_entry = nouveau_bo_create_ttm_backend_entry,
	.invalidate_caches = nouveau_bo_invalidate_caches,
	.init_mem_type = nouveau_bo_init_mem_type,
	.evict_flags = nouveau_bo_evict_flags,
	.move = nouveau_bo_move,
	.verify_access = nouveau_bo_verify_access,
1066 1067 1068 1069 1070
	.sync_obj_signaled = __nouveau_fence_signalled,
	.sync_obj_wait = __nouveau_fence_wait,
	.sync_obj_flush = __nouveau_fence_flush,
	.sync_obj_unref = __nouveau_fence_unref,
	.sync_obj_ref = __nouveau_fence_ref,
1071 1072 1073
	.fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
	.io_mem_reserve = &nouveau_ttm_io_mem_reserve,
	.io_mem_free = &nouveau_ttm_io_mem_free,
1074 1075
};