nouveau_bo.c 41.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 * Copyright 2007 Dave Airlied
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */
/*
 * Authors: Dave Airlied <airlied@linux.ie>
 *	    Ben Skeggs   <darktama@iinet.net.au>
 *	    Jeremy Kolb  <jkolb@brandeis.edu>
 */

30
#include <linux/dma-mapping.h>
31
#include <linux/swiotlb.h>
32

33
#include "nouveau_drv.h"
34
#include "nouveau_dma.h"
35
#include "nouveau_fence.h"
36

37 38 39
#include "nouveau_bo.h"
#include "nouveau_ttm.h"
#include "nouveau_gem.h"
40

41 42 43 44 45
/*
 * NV10-NV40 tiling helpers
 */

static void
46 47
nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
			   u32 addr, u32 size, u32 pitch, u32 flags)
48
{
49
	struct nouveau_drm *drm = nouveau_drm(dev);
50
	int i = reg - drm->tile.reg;
51
	struct nvkm_device *device = nvxx_device(&drm->client.device);
52
	struct nvkm_fb *fb = device->fb;
B
Ben Skeggs 已提交
53
	struct nvkm_fb_tile *tile = &fb->tile.region[i];
54

55
	nouveau_fence_unref(&reg->fence);
56 57

	if (tile->pitch)
58
		nvkm_fb_tile_fini(fb, i, tile);
59 60

	if (pitch)
61
		nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile);
62

63
	nvkm_fb_tile_prog(fb, i, tile);
64 65
}

66
static struct nouveau_drm_tile *
67 68
nv10_bo_get_tile_region(struct drm_device *dev, int i)
{
69
	struct nouveau_drm *drm = nouveau_drm(dev);
70
	struct nouveau_drm_tile *tile = &drm->tile.reg[i];
71

72
	spin_lock(&drm->tile.lock);
73 74 75 76 77 78 79

	if (!tile->used &&
	    (!tile->fence || nouveau_fence_done(tile->fence)))
		tile->used = true;
	else
		tile = NULL;

80
	spin_unlock(&drm->tile.lock);
81 82 83 84
	return tile;
}

static void
85
nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
86
			struct dma_fence *fence)
87
{
88
	struct nouveau_drm *drm = nouveau_drm(dev);
89 90

	if (tile) {
91
		spin_lock(&drm->tile.lock);
92
		tile->fence = (struct nouveau_fence *)dma_fence_get(fence);
93
		tile->used = false;
94
		spin_unlock(&drm->tile.lock);
95 96 97
	}
}

98 99 100
static struct nouveau_drm_tile *
nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
		   u32 size, u32 pitch, u32 flags)
101
{
102
	struct nouveau_drm *drm = nouveau_drm(dev);
103
	struct nvkm_fb *fb = nvxx_fb(&drm->client.device);
104
	struct nouveau_drm_tile *tile, *found = NULL;
105 106
	int i;

B
Ben Skeggs 已提交
107
	for (i = 0; i < fb->tile.regions; i++) {
108 109 110 111 112 113
		tile = nv10_bo_get_tile_region(dev, i);

		if (pitch && !found) {
			found = tile;
			continue;

B
Ben Skeggs 已提交
114
		} else if (tile && fb->tile.region[i].pitch) {
115 116 117 118 119 120 121 122 123 124 125 126 127
			/* Kill an unused tile region. */
			nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0);
		}

		nv10_bo_put_tile_region(dev, tile, NULL);
	}

	if (found)
		nv10_bo_update_tile_region(dev, found, addr, size,
					    pitch, flags);
	return found;
}

128 129 130
static void
nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
{
131 132
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
	struct drm_device *dev = drm->dev;
133 134
	struct nouveau_bo *nvbo = nouveau_bo(bo);

135
	if (unlikely(nvbo->gem.filp))
136
		DRM_ERROR("bo %p still attached to GEM object\n", bo);
137
	WARN_ON(nvbo->pin_refcnt > 0);
138
	nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
139 140 141
	kfree(nvbo);
}

142
static void
143
nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
144
		       int *align, int *size)
145
{
146
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
147
	struct nvif_device *device = &drm->client.device;
148

149
	if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
150
		if (nvbo->tile_mode) {
151
			if (device->info.chipset >= 0x40) {
152
				*align = 65536;
153
				*size = roundup(*size, 64 * nvbo->tile_mode);
154

155
			} else if (device->info.chipset >= 0x30) {
156
				*align = 32768;
157
				*size = roundup(*size, 64 * nvbo->tile_mode);
158

159
			} else if (device->info.chipset >= 0x20) {
160
				*align = 16384;
161
				*size = roundup(*size, 64 * nvbo->tile_mode);
162

163
			} else if (device->info.chipset >= 0x10) {
164
				*align = 16384;
165
				*size = roundup(*size, 32 * nvbo->tile_mode);
166 167
			}
		}
168
	} else {
169 170
		*size = roundup(*size, (1 << nvbo->page_shift));
		*align = max((1 <<  nvbo->page_shift), *align);
171 172
	}

173
	*size = roundup(*size, PAGE_SIZE);
174 175
}

176
int
177
nouveau_bo_new(struct nouveau_cli *cli, int size, int align,
178
	       uint32_t flags, uint32_t tile_mode, uint32_t tile_flags,
179
	       struct sg_table *sg, struct reservation_object *robj,
180
	       struct nouveau_bo **pnvbo)
181
{
182
	struct nouveau_drm *drm = nouveau_drm(cli->dev);
183
	struct nouveau_bo *nvbo;
184
	size_t acc_size;
185
	int ret;
D
Dave Airlie 已提交
186
	int type = ttm_bo_type_device;
187 188 189
	int lpg_shift = 12;
	int max_size;

190
	if (drm->client.vm)
191
		lpg_shift = drm->client.vm->mmu->lpg_shift;
192
	max_size = INT_MAX & ~((1 << lpg_shift) - 1);
193 194

	if (size <= 0 || size > max_size) {
195
		NV_WARN(drm, "skipped size %x\n", (u32)size);
196 197
		return -EINVAL;
	}
D
Dave Airlie 已提交
198 199 200

	if (sg)
		type = ttm_bo_type_sg;
201 202 203 204 205 206

	nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
	if (!nvbo)
		return -ENOMEM;
	INIT_LIST_HEAD(&nvbo->head);
	INIT_LIST_HEAD(&nvbo->entry);
207
	INIT_LIST_HEAD(&nvbo->vma_list);
208 209
	nvbo->tile_mode = tile_mode;
	nvbo->tile_flags = tile_flags;
210
	nvbo->bo.bdev = &drm->ttm.bdev;
211
	nvbo->cli = cli;
212

213
	if (!nvxx_device(&drm->client.device)->func->cpu_coherent)
214
		nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;
215

216
	nvbo->page_shift = 12;
217
	if (drm->client.vm) {
218
		if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
219
			nvbo->page_shift = drm->client.vm->mmu->lpg_shift;
220 221 222
	}

	nouveau_bo_fixup_align(nvbo, flags, &align, &size);
223 224
	nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
	nouveau_bo_placement_set(nvbo, flags, 0);
225

226
	acc_size = ttm_bo_dma_acc_size(&drm->ttm.bdev, size,
227 228
				       sizeof(struct nouveau_bo));

229
	ret = ttm_bo_init(&drm->ttm.bdev, &nvbo->bo, size,
D
Dave Airlie 已提交
230
			  type, &nvbo->placement,
231
			  align >> PAGE_SHIFT, false, NULL, acc_size, sg,
232
			  robj, nouveau_bo_del_ttm);
233 234 235 236 237 238 239 240 241
	if (ret) {
		/* ttm will call nouveau_bo_del_ttm if it fails.. */
		return ret;
	}

	*pnvbo = nvbo;
	return 0;
}

242
static void
243
set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t type, uint32_t flags)
244 245 246 247
{
	*n = 0;

	if (type & TTM_PL_FLAG_VRAM)
248
		pl[(*n)++].flags = TTM_PL_FLAG_VRAM | flags;
249
	if (type & TTM_PL_FLAG_TT)
250
		pl[(*n)++].flags = TTM_PL_FLAG_TT | flags;
251
	if (type & TTM_PL_FLAG_SYSTEM)
252
		pl[(*n)++].flags = TTM_PL_FLAG_SYSTEM | flags;
253 254
}

255 256 257
static void
set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
{
258
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
259
	u32 vram_pages = drm->client.device.info.ram_size >> PAGE_SHIFT;
260
	unsigned i, fpfn, lpfn;
261

262
	if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
263
	    nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
264
	    nvbo->bo.mem.num_pages < vram_pages / 4) {
265 266 267 268 269 270 271
		/*
		 * Make sure that the color and depth buffers are handled
		 * by independent memory controller units. Up to a 9x
		 * speed up when alpha-blending and depth-test are enabled
		 * at the same time.
		 */
		if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) {
272 273
			fpfn = vram_pages / 2;
			lpfn = ~0;
274
		} else {
275 276 277 278 279 280 281 282 283 284
			fpfn = 0;
			lpfn = vram_pages / 2;
		}
		for (i = 0; i < nvbo->placement.num_placement; ++i) {
			nvbo->placements[i].fpfn = fpfn;
			nvbo->placements[i].lpfn = lpfn;
		}
		for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
			nvbo->busy_placements[i].fpfn = fpfn;
			nvbo->busy_placements[i].lpfn = lpfn;
285 286 287 288
		}
	}
}

289
void
290
nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
291
{
292
	struct ttm_placement *pl = &nvbo->placement;
293 294 295
	uint32_t flags = (nvbo->force_coherent ? TTM_PL_FLAG_UNCACHED :
						 TTM_PL_MASK_CACHING) |
			 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
296 297 298 299 300 301 302 303

	pl->placement = nvbo->placements;
	set_placement_list(nvbo->placements, &pl->num_placement,
			   type, flags);

	pl->busy_placement = nvbo->busy_placements;
	set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
			   type | busy, flags);
304 305

	set_placement_range(nvbo, type);
306 307 308
}

int
309
nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype, bool contig)
310
{
311
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
312
	struct ttm_buffer_object *bo = &nvbo->bo;
313
	bool force = false, evict = false;
314
	int ret;
315

316
	ret = ttm_bo_reserve(bo, false, false, NULL);
317
	if (ret)
318
		return ret;
319

320
	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
321 322 323
	    memtype == TTM_PL_FLAG_VRAM && contig) {
		if (nvbo->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG) {
			if (bo->mem.mem_type == TTM_PL_VRAM) {
324
				struct nvkm_mem *mem = bo->mem.mm_node;
325
				if (!nvkm_mm_contiguous(mem->mem))
326 327 328 329 330
					evict = true;
			}
			nvbo->tile_flags &= ~NOUVEAU_GEM_TILE_NONCONTIG;
			force = true;
		}
331 332
	}

333 334 335 336 337 338 339 340
	if (nvbo->pin_refcnt) {
		if (!(memtype & (1 << bo->mem.mem_type)) || evict) {
			NV_ERROR(drm, "bo %p pinned elsewhere: "
				      "0x%08x vs 0x%08x\n", bo,
				 1 << bo->mem.mem_type, memtype);
			ret = -EBUSY;
		}
		nvbo->pin_refcnt++;
341
		goto out;
342 343 344 345 346 347 348 349
	}

	if (evict) {
		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT, 0);
		ret = nouveau_bo_validate(nvbo, false, false);
		if (ret)
			goto out;
	}
350

351
	nvbo->pin_refcnt++;
352
	nouveau_bo_placement_set(nvbo, memtype, 0);
353

354 355 356 357 358
	/* drop pin_refcnt temporarily, so we don't trip the assertion
	 * in nouveau_bo_move() that makes sure we're not trying to
	 * move a pinned buffer
	 */
	nvbo->pin_refcnt--;
359
	ret = nouveau_bo_validate(nvbo, false, false);
360 361
	if (ret)
		goto out;
362
	nvbo->pin_refcnt++;
363 364 365 366 367 368 369 370 371 372

	switch (bo->mem.mem_type) {
	case TTM_PL_VRAM:
		drm->gem.vram_available -= bo->mem.size;
		break;
	case TTM_PL_TT:
		drm->gem.gart_available -= bo->mem.size;
		break;
	default:
		break;
373
	}
374

375
out:
376 377
	if (force && ret)
		nvbo->tile_flags |= NOUVEAU_GEM_TILE_NONCONTIG;
378
	ttm_bo_unreserve(bo);
379 380 381 382 383 384
	return ret;
}

int
nouveau_bo_unpin(struct nouveau_bo *nvbo)
{
385
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
386
	struct ttm_buffer_object *bo = &nvbo->bo;
387
	int ret, ref;
388

389
	ret = ttm_bo_reserve(bo, false, false, NULL);
390 391 392
	if (ret)
		return ret;

393 394 395
	ref = --nvbo->pin_refcnt;
	WARN_ON_ONCE(ref < 0);
	if (ref)
396 397
		goto out;

398
	nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
399

400
	ret = nouveau_bo_validate(nvbo, false, false);
401 402 403
	if (ret == 0) {
		switch (bo->mem.mem_type) {
		case TTM_PL_VRAM:
404
			drm->gem.vram_available += bo->mem.size;
405 406
			break;
		case TTM_PL_TT:
407
			drm->gem.gart_available += bo->mem.size;
408 409 410 411 412 413
			break;
		default:
			break;
		}
	}

414
out:
415 416 417 418 419 420 421 422 423
	ttm_bo_unreserve(bo);
	return ret;
}

int
nouveau_bo_map(struct nouveau_bo *nvbo)
{
	int ret;

424
	ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
425 426 427
	if (ret)
		return ret;

428
	ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);
429

430 431 432 433 434 435 436
	ttm_bo_unreserve(&nvbo->bo);
	return ret;
}

void
nouveau_bo_unmap(struct nouveau_bo *nvbo)
{
437 438 439
	if (!nvbo)
		return;

440
	ttm_bo_kunmap(&nvbo->kmap);
441 442
}

443 444 445 446
void
nouveau_bo_sync_for_device(struct nouveau_bo *nvbo)
{
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
447
	struct nvkm_device *device = nvxx_device(&drm->client.device);
448 449 450 451 452 453 454 455 456 457 458
	struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
	int i;

	if (!ttm_dma)
		return;

	/* Don't waste time looping if the object is coherent */
	if (nvbo->force_coherent)
		return;

	for (i = 0; i < ttm_dma->ttm.num_pages; i++)
459 460
		dma_sync_single_for_device(device->dev, ttm_dma->dma_address[i],
					   PAGE_SIZE, DMA_TO_DEVICE);
461 462 463 464 465 466
}

void
nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo)
{
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
467
	struct nvkm_device *device = nvxx_device(&drm->client.device);
468 469 470 471 472 473 474 475 476 477 478
	struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
	int i;

	if (!ttm_dma)
		return;

	/* Don't waste time looping if the object is coherent */
	if (nvbo->force_coherent)
		return;

	for (i = 0; i < ttm_dma->ttm.num_pages; i++)
479 480
		dma_sync_single_for_cpu(device->dev, ttm_dma->dma_address[i],
					PAGE_SIZE, DMA_FROM_DEVICE);
481 482
}

483 484
int
nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
485
		    bool no_wait_gpu)
486 487 488
{
	int ret;

489 490
	ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement,
			      interruptible, no_wait_gpu);
491 492 493
	if (ret)
		return ret;

494 495
	nouveau_bo_sync_for_device(nvbo);

496 497 498
	return 0;
}

499 500 501 502 503
void
nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
{
	bool is_iomem;
	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
504

505
	mem += index;
506

507 508 509 510 511 512 513 514 515 516 517
	if (is_iomem)
		iowrite16_native(val, (void __force __iomem *)mem);
	else
		*mem = val;
}

u32
nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
{
	bool is_iomem;
	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
518

519
	mem += index;
520

521 522 523 524 525 526 527 528 529 530 531
	if (is_iomem)
		return ioread32_native((void __force __iomem *)mem);
	else
		return *mem;
}

void
nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
{
	bool is_iomem;
	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
532

533
	mem += index;
534

535 536 537 538 539 540
	if (is_iomem)
		iowrite32_native(val, (void __force __iomem *)mem);
	else
		*mem = val;
}

541
static struct ttm_tt *
542 543
nouveau_ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size,
		      uint32_t page_flags, struct page *dummy_read)
544
{
D
Daniel Vetter 已提交
545
#if IS_ENABLED(CONFIG_AGP)
546
	struct nouveau_drm *drm = nouveau_bdev(bdev);
547

548 549
	if (drm->agp.bridge) {
		return ttm_agp_tt_create(bdev, drm->agp.bridge, size,
550
					 page_flags, dummy_read);
551
	}
552
#endif
553

554
	return nouveau_sgdma_create_ttm(bdev, size, page_flags, dummy_read);
555 556 557 558 559 560 561 562 563 564 565 566 567
}

static int
nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
{
	/* We'll do this from user space. */
	return 0;
}

static int
nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
			 struct ttm_mem_type_manager *man)
{
568
	struct nouveau_drm *drm = nouveau_bdev(bdev);
569 570 571 572 573 574 575 576

	switch (type) {
	case TTM_PL_SYSTEM:
		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
		man->available_caching = TTM_PL_MASK_CACHING;
		man->default_caching = TTM_PL_FLAG_CACHED;
		break;
	case TTM_PL_VRAM:
577 578 579 580 581 582
		man->flags = TTM_MEMTYPE_FLAG_FIXED |
			     TTM_MEMTYPE_FLAG_MAPPABLE;
		man->available_caching = TTM_PL_FLAG_UNCACHED |
					 TTM_PL_FLAG_WC;
		man->default_caching = TTM_PL_FLAG_WC;

583
		if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
584
			/* Some BARs do not support being ioremapped WC */
585
			if (nvxx_bar(&drm->client.device)->iomap_uncached) {
586 587 588 589
				man->available_caching = TTM_PL_FLAG_UNCACHED;
				man->default_caching = TTM_PL_FLAG_UNCACHED;
			}

B
Ben Skeggs 已提交
590
			man->func = &nouveau_vram_manager;
591 592 593
			man->io_reserve_fastpath = false;
			man->use_io_reserve_lru = true;
		} else {
B
Ben Skeggs 已提交
594
			man->func = &ttm_bo_manager_func;
595
		}
596 597
		break;
	case TTM_PL_TT:
598
		if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
599
			man->func = &nouveau_gart_manager;
600
		else
601
		if (!drm->agp.bridge)
602
			man->func = &nv04_gart_manager;
603 604
		else
			man->func = &ttm_bo_manager_func;
605

606
		if (drm->agp.bridge) {
607
			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
608 609 610
			man->available_caching = TTM_PL_FLAG_UNCACHED |
				TTM_PL_FLAG_WC;
			man->default_caching = TTM_PL_FLAG_WC;
611
		} else {
612 613 614 615 616
			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
				     TTM_MEMTYPE_FLAG_CMA;
			man->available_caching = TTM_PL_MASK_CACHING;
			man->default_caching = TTM_PL_FLAG_CACHED;
		}
617

618 619 620 621 622 623 624 625 626 627 628 629 630
		break;
	default:
		return -EINVAL;
	}
	return 0;
}

static void
nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
{
	struct nouveau_bo *nvbo = nouveau_bo(bo);

	switch (bo->mem.mem_type) {
631
	case TTM_PL_VRAM:
632 633
		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,
					 TTM_PL_FLAG_SYSTEM);
634
		break;
635
	default:
636
		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);
637 638
		break;
	}
639 640

	*pl = nvbo->placement;
641 642 643
}


644 645 646 647 648 649
static int
nve0_bo_move_init(struct nouveau_channel *chan, u32 handle)
{
	int ret = RING_SPACE(chan, 2);
	if (ret == 0) {
		BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
650
		OUT_RING  (chan, handle & 0x0000ffff);
651 652 653 654 655
		FIRE_RING (chan);
	}
	return ret;
}

656 657
static int
nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
658
		  struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
659
{
660
	struct nvkm_mem *mem = old_reg->mm_node;
661 662
	int ret = RING_SPACE(chan, 10);
	if (ret == 0) {
663
		BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
664 665 666 667
		OUT_RING  (chan, upper_32_bits(mem->vma[0].offset));
		OUT_RING  (chan, lower_32_bits(mem->vma[0].offset));
		OUT_RING  (chan, upper_32_bits(mem->vma[1].offset));
		OUT_RING  (chan, lower_32_bits(mem->vma[1].offset));
668 669 670
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
671
		OUT_RING  (chan, new_reg->num_pages);
672
		BEGIN_IMC0(chan, NvSubCopy, 0x0300, 0x0386);
673 674 675 676
	}
	return ret;
}

677 678 679 680 681 682 683 684 685 686 687
static int
nvc0_bo_move_init(struct nouveau_channel *chan, u32 handle)
{
	int ret = RING_SPACE(chan, 2);
	if (ret == 0) {
		BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
		OUT_RING  (chan, handle);
	}
	return ret;
}

688 689
static int
nvc0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
690
		  struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
691
{
692 693 694 695
	struct nvkm_mem *mem = old_reg->mm_node;
	u64 src_offset = mem->vma[0].offset;
	u64 dst_offset = mem->vma[1].offset;
	u32 page_count = new_reg->num_pages;
696 697
	int ret;

698
	page_count = new_reg->num_pages;
699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
	while (page_count) {
		int line_count = (page_count > 8191) ? 8191 : page_count;

		ret = RING_SPACE(chan, 11);
		if (ret)
			return ret;

		BEGIN_NVC0(chan, NvSubCopy, 0x030c, 8);
		OUT_RING  (chan, upper_32_bits(src_offset));
		OUT_RING  (chan, lower_32_bits(src_offset));
		OUT_RING  (chan, upper_32_bits(dst_offset));
		OUT_RING  (chan, lower_32_bits(dst_offset));
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, line_count);
		BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
		OUT_RING  (chan, 0x00000110);

		page_count -= line_count;
		src_offset += (PAGE_SIZE * line_count);
		dst_offset += (PAGE_SIZE * line_count);
	}

	return 0;
}

B
Ben Skeggs 已提交
726 727
static int
nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
728
		  struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
B
Ben Skeggs 已提交
729
{
730 731 732 733
	struct nvkm_mem *mem = old_reg->mm_node;
	u64 src_offset = mem->vma[0].offset;
	u64 dst_offset = mem->vma[1].offset;
	u32 page_count = new_reg->num_pages;
B
Ben Skeggs 已提交
734 735
	int ret;

736
	page_count = new_reg->num_pages;
B
Ben Skeggs 已提交
737 738 739 740 741 742 743
	while (page_count) {
		int line_count = (page_count > 2047) ? 2047 : page_count;

		ret = RING_SPACE(chan, 12);
		if (ret)
			return ret;

744
		BEGIN_NVC0(chan, NvSubCopy, 0x0238, 2);
B
Ben Skeggs 已提交
745 746
		OUT_RING  (chan, upper_32_bits(dst_offset));
		OUT_RING  (chan, lower_32_bits(dst_offset));
747
		BEGIN_NVC0(chan, NvSubCopy, 0x030c, 6);
B
Ben Skeggs 已提交
748 749 750 751 752 753
		OUT_RING  (chan, upper_32_bits(src_offset));
		OUT_RING  (chan, lower_32_bits(src_offset));
		OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* line_length */
		OUT_RING  (chan, line_count);
754
		BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
B
Ben Skeggs 已提交
755 756 757 758 759 760 761 762 763 764
		OUT_RING  (chan, 0x00100110);

		page_count -= line_count;
		src_offset += (PAGE_SIZE * line_count);
		dst_offset += (PAGE_SIZE * line_count);
	}

	return 0;
}

765 766
static int
nva3_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
767
		  struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
768
{
769 770 771 772
	struct nvkm_mem *mem = old_reg->mm_node;
	u64 src_offset = mem->vma[0].offset;
	u64 dst_offset = mem->vma[1].offset;
	u32 page_count = new_reg->num_pages;
773 774
	int ret;

775
	page_count = new_reg->num_pages;
776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802
	while (page_count) {
		int line_count = (page_count > 8191) ? 8191 : page_count;

		ret = RING_SPACE(chan, 11);
		if (ret)
			return ret;

		BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
		OUT_RING  (chan, upper_32_bits(src_offset));
		OUT_RING  (chan, lower_32_bits(src_offset));
		OUT_RING  (chan, upper_32_bits(dst_offset));
		OUT_RING  (chan, lower_32_bits(dst_offset));
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, line_count);
		BEGIN_NV04(chan, NvSubCopy, 0x0300, 1);
		OUT_RING  (chan, 0x00000110);

		page_count -= line_count;
		src_offset += (PAGE_SIZE * line_count);
		dst_offset += (PAGE_SIZE * line_count);
	}

	return 0;
}

803 804
static int
nv98_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
805
		  struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
806
{
807
	struct nvkm_mem *mem = old_reg->mm_node;
808 809 810
	int ret = RING_SPACE(chan, 7);
	if (ret == 0) {
		BEGIN_NV04(chan, NvSubCopy, 0x0320, 6);
811 812 813 814
		OUT_RING  (chan, upper_32_bits(mem->vma[0].offset));
		OUT_RING  (chan, lower_32_bits(mem->vma[0].offset));
		OUT_RING  (chan, upper_32_bits(mem->vma[1].offset));
		OUT_RING  (chan, lower_32_bits(mem->vma[1].offset));
815
		OUT_RING  (chan, 0x00000000 /* COPY */);
816
		OUT_RING  (chan, new_reg->num_pages << PAGE_SHIFT);
817 818 819 820
	}
	return ret;
}

821 822
static int
nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
823
		  struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
824
{
825
	struct nvkm_mem *mem = old_reg->mm_node;
826 827 828
	int ret = RING_SPACE(chan, 7);
	if (ret == 0) {
		BEGIN_NV04(chan, NvSubCopy, 0x0304, 6);
829 830 831 832 833
		OUT_RING  (chan, new_reg->num_pages << PAGE_SHIFT);
		OUT_RING  (chan, upper_32_bits(mem->vma[0].offset));
		OUT_RING  (chan, lower_32_bits(mem->vma[0].offset));
		OUT_RING  (chan, upper_32_bits(mem->vma[1].offset));
		OUT_RING  (chan, lower_32_bits(mem->vma[1].offset));
834 835 836 837 838
		OUT_RING  (chan, 0x00000000 /* MODE_COPY, QUERY_NONE */);
	}
	return ret;
}

839 840 841
static int
nv50_bo_move_init(struct nouveau_channel *chan, u32 handle)
{
842
	int ret = RING_SPACE(chan, 6);
843
	if (ret == 0) {
844 845 846
		BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
		OUT_RING  (chan, handle);
		BEGIN_NV04(chan, NvSubCopy, 0x0180, 3);
847 848 849
		OUT_RING  (chan, chan->drm->ntfy.handle);
		OUT_RING  (chan, chan->vram.handle);
		OUT_RING  (chan, chan->vram.handle);
850 851 852 853 854
	}

	return ret;
}

855
static int
856
nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
857
		  struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
858
{
859 860 861 862 863 864
	struct nvkm_mem *mem = old_reg->mm_node;
	u64 length = (new_reg->num_pages << PAGE_SHIFT);
	u64 src_offset = mem->vma[0].offset;
	u64 dst_offset = mem->vma[1].offset;
	int src_tiled = !!mem->memtype;
	int dst_tiled = !!((struct nvkm_mem *)new_reg->mm_node)->memtype;
865 866
	int ret;

867 868 869
	while (length) {
		u32 amount, stride, height;

870 871 872 873
		ret = RING_SPACE(chan, 18 + 6 * (src_tiled + dst_tiled));
		if (ret)
			return ret;

874 875
		amount  = min(length, (u64)(4 * 1024 * 1024));
		stride  = 16 * 4;
876 877
		height  = amount / stride;

878
		if (src_tiled) {
879
			BEGIN_NV04(chan, NvSubCopy, 0x0200, 7);
880
			OUT_RING  (chan, 0);
881
			OUT_RING  (chan, 0);
882 883 884 885 886 887
			OUT_RING  (chan, stride);
			OUT_RING  (chan, height);
			OUT_RING  (chan, 1);
			OUT_RING  (chan, 0);
			OUT_RING  (chan, 0);
		} else {
888
			BEGIN_NV04(chan, NvSubCopy, 0x0200, 1);
889 890
			OUT_RING  (chan, 1);
		}
891
		if (dst_tiled) {
892
			BEGIN_NV04(chan, NvSubCopy, 0x021c, 7);
893
			OUT_RING  (chan, 0);
894
			OUT_RING  (chan, 0);
895 896 897 898 899 900
			OUT_RING  (chan, stride);
			OUT_RING  (chan, height);
			OUT_RING  (chan, 1);
			OUT_RING  (chan, 0);
			OUT_RING  (chan, 0);
		} else {
901
			BEGIN_NV04(chan, NvSubCopy, 0x021c, 1);
902 903 904
			OUT_RING  (chan, 1);
		}

905
		BEGIN_NV04(chan, NvSubCopy, 0x0238, 2);
906 907
		OUT_RING  (chan, upper_32_bits(src_offset));
		OUT_RING  (chan, upper_32_bits(dst_offset));
908
		BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
909 910 911 912 913 914 915 916
		OUT_RING  (chan, lower_32_bits(src_offset));
		OUT_RING  (chan, lower_32_bits(dst_offset));
		OUT_RING  (chan, stride);
		OUT_RING  (chan, stride);
		OUT_RING  (chan, stride);
		OUT_RING  (chan, height);
		OUT_RING  (chan, 0x00000101);
		OUT_RING  (chan, 0x00000000);
917
		BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
918 919 920 921 922
		OUT_RING  (chan, 0);

		length -= amount;
		src_offset += amount;
		dst_offset += amount;
923 924
	}

925 926 927
	return 0;
}

928 929 930
static int
nv04_bo_move_init(struct nouveau_channel *chan, u32 handle)
{
931
	int ret = RING_SPACE(chan, 4);
932
	if (ret == 0) {
933 934 935
		BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
		OUT_RING  (chan, handle);
		BEGIN_NV04(chan, NvSubCopy, 0x0180, 1);
936
		OUT_RING  (chan, chan->drm->ntfy.handle);
937 938 939 940 941
	}

	return ret;
}

942 943
static inline uint32_t
nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
944
		      struct nouveau_channel *chan, struct ttm_mem_reg *reg)
945
{
946
	if (reg->mem_type == TTM_PL_TT)
947
		return NvDmaTT;
948
	return chan->vram.handle;
949 950
}

951 952
static int
nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
953
		  struct ttm_mem_reg *old_reg, struct ttm_mem_reg *new_reg)
954
{
955 956 957
	u32 src_offset = old_reg->start << PAGE_SHIFT;
	u32 dst_offset = new_reg->start << PAGE_SHIFT;
	u32 page_count = new_reg->num_pages;
958 959 960 961 962 963
	int ret;

	ret = RING_SPACE(chan, 3);
	if (ret)
		return ret;

964
	BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
965 966
	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, old_reg));
	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, new_reg));
967

968
	page_count = new_reg->num_pages;
969 970 971 972 973 974
	while (page_count) {
		int line_count = (page_count > 2047) ? 2047 : page_count;

		ret = RING_SPACE(chan, 11);
		if (ret)
			return ret;
975

976
		BEGIN_NV04(chan, NvSubCopy,
977
				 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
978 979 980 981 982 983 984 985
		OUT_RING  (chan, src_offset);
		OUT_RING  (chan, dst_offset);
		OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* line_length */
		OUT_RING  (chan, line_count);
		OUT_RING  (chan, 0x00000101);
		OUT_RING  (chan, 0x00000000);
986
		BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
987
		OUT_RING  (chan, 0);
988 989 990 991 992 993

		page_count -= line_count;
		src_offset += (PAGE_SIZE * line_count);
		dst_offset += (PAGE_SIZE * line_count);
	}

994 995 996
	return 0;
}

997
static int
998
nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,
999
		     struct ttm_mem_reg *reg)
1000
{
1001 1002 1003
	struct nvkm_mem *old_mem = bo->mem.mm_node;
	struct nvkm_mem *new_mem = reg->mm_node;
	u64 size = (u64)reg->num_pages << PAGE_SHIFT;
1004 1005
	int ret;

1006 1007
	ret = nvkm_vm_get(drm->client.vm, size, old_mem->page_shift,
			  NV_MEM_ACCESS_RW, &old_mem->vma[0]);
1008 1009 1010
	if (ret)
		return ret;

1011 1012
	ret = nvkm_vm_get(drm->client.vm, size, new_mem->page_shift,
			  NV_MEM_ACCESS_RW, &old_mem->vma[1]);
1013
	if (ret) {
1014
		nvkm_vm_put(&old_mem->vma[0]);
1015 1016 1017
		return ret;
	}

1018 1019
	nvkm_vm_map(&old_mem->vma[0], old_mem);
	nvkm_vm_map(&old_mem->vma[1], new_mem);
1020 1021 1022
	return 0;
}

1023 1024
static int
nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
1025
		     bool no_wait_gpu, struct ttm_mem_reg *new_reg)
1026
{
1027
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1028
	struct nouveau_channel *chan = drm->ttm.chan;
1029
	struct nouveau_cli *cli = (void *)chan->user.client;
1030
	struct nouveau_fence *fence;
1031 1032
	int ret;

1033
	/* create temporary vmas for the transfer and attach them to the
1034
	 * old nvkm_mem node, these will get cleaned up after ttm has
1035
	 * destroyed the ttm_mem_reg
1036
	 */
1037
	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
1038
		ret = nouveau_bo_move_prep(drm, bo, new_reg);
1039
		if (ret)
1040
			return ret;
1041 1042
	}

1043
	mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING);
1044
	ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, intr);
1045
	if (ret == 0) {
1046
		ret = drm->ttm.move(chan, bo, &bo->mem, new_reg);
1047 1048 1049
		if (ret == 0) {
			ret = nouveau_fence_new(chan, false, &fence);
			if (ret == 0) {
1050 1051
				ret = ttm_bo_move_accel_cleanup(bo,
								&fence->base,
1052
								evict,
1053
								new_reg);
1054 1055 1056
				nouveau_fence_unref(&fence);
			}
		}
1057
	}
1058
	mutex_unlock(&cli->mutex);
1059
	return ret;
1060 1061
}

1062
void
1063
nouveau_bo_move_init(struct nouveau_drm *drm)
1064 1065 1066
{
	static const struct {
		const char *name;
1067
		int engine;
1068
		s32 oclass;
1069 1070 1071 1072 1073
		int (*exec)(struct nouveau_channel *,
			    struct ttm_buffer_object *,
			    struct ttm_mem_reg *, struct ttm_mem_reg *);
		int (*init)(struct nouveau_channel *, u32 handle);
	} _methods[] = {
1074 1075
		{  "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init },
		{  "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init },
1076 1077
		{  "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init },
		{  "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1078 1079
		{  "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init },
		{  "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1080
		{  "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
1081
		{  "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1082 1083 1084 1085 1086 1087 1088
		{ "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
		{ "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
		{  "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
		{ "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init },
		{  "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init },
		{  "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init },
		{  "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init },
1089
		{},
1090
		{ "CRYPT", 0, 0x88b4, nv98_bo_move_exec, nv50_bo_move_init },
1091 1092 1093 1094 1095
	}, *mthd = _methods;
	const char *name = "CPU";
	int ret;

	do {
1096
		struct nouveau_channel *chan;
1097

1098
		if (mthd->engine)
1099 1100 1101 1102 1103 1104
			chan = drm->cechan;
		else
			chan = drm->channel;
		if (chan == NULL)
			continue;

1105
		ret = nvif_object_init(&chan->user,
1106 1107 1108
				       mthd->oclass | (mthd->engine << 16),
				       mthd->oclass, NULL, 0,
				       &drm->ttm.copy);
1109
		if (ret == 0) {
1110
			ret = mthd->init(chan, drm->ttm.copy.handle);
1111
			if (ret) {
1112
				nvif_object_fini(&drm->ttm.copy);
1113
				continue;
1114
			}
1115 1116

			drm->ttm.move = mthd->exec;
1117
			drm->ttm.chan = chan;
1118 1119
			name = mthd->name;
			break;
1120 1121 1122
		}
	} while ((++mthd)->exec);

1123
	NV_INFO(drm, "MM: using %s for buffer copies\n", name);
1124 1125
}

1126 1127
static int
nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
1128
		      bool no_wait_gpu, struct ttm_mem_reg *new_reg)
1129
{
1130 1131 1132 1133 1134
	struct ttm_place placement_memtype = {
		.fpfn = 0,
		.lpfn = 0,
		.flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
	};
1135
	struct ttm_placement placement;
1136
	struct ttm_mem_reg tmp_reg;
1137 1138 1139
	int ret;

	placement.num_placement = placement.num_busy_placement = 1;
1140
	placement.placement = placement.busy_placement = &placement_memtype;
1141

1142 1143 1144
	tmp_reg = *new_reg;
	tmp_reg.mm_node = NULL;
	ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, intr, no_wait_gpu);
1145 1146 1147
	if (ret)
		return ret;

1148
	ret = ttm_tt_bind(bo->ttm, &tmp_reg);
1149 1150 1151
	if (ret)
		goto out;

1152
	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, &tmp_reg);
1153 1154 1155
	if (ret)
		goto out;

1156
	ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, new_reg);
1157
out:
1158
	ttm_bo_mem_put(bo, &tmp_reg);
1159 1160 1161 1162 1163
	return ret;
}

static int
nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
1164
		      bool no_wait_gpu, struct ttm_mem_reg *new_reg)
1165
{
1166 1167 1168 1169 1170
	struct ttm_place placement_memtype = {
		.fpfn = 0,
		.lpfn = 0,
		.flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
	};
1171
	struct ttm_placement placement;
1172
	struct ttm_mem_reg tmp_reg;
1173 1174 1175
	int ret;

	placement.num_placement = placement.num_busy_placement = 1;
1176
	placement.placement = placement.busy_placement = &placement_memtype;
1177

1178 1179 1180
	tmp_reg = *new_reg;
	tmp_reg.mm_node = NULL;
	ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, intr, no_wait_gpu);
1181 1182 1183
	if (ret)
		return ret;

1184
	ret = ttm_bo_move_ttm(bo, intr, no_wait_gpu, &tmp_reg);
1185 1186 1187
	if (ret)
		goto out;

1188
	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, new_reg);
1189 1190 1191 1192
	if (ret)
		goto out;

out:
1193
	ttm_bo_mem_put(bo, &tmp_reg);
1194 1195 1196
	return ret;
}

1197
static void
1198
nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict,
1199
		     struct ttm_mem_reg *new_reg)
1200 1201
{
	struct nouveau_bo *nvbo = nouveau_bo(bo);
1202
	struct nvkm_vma *vma;
1203

1204 1205 1206 1207
	/* ttm can now (stupidly) pass the driver bos it didn't create... */
	if (bo->destroy != nouveau_bo_del_ttm)
		return;

1208
	list_for_each_entry(vma, &nvbo->vma_list, head) {
1209 1210
		if (new_reg && new_reg->mem_type != TTM_PL_SYSTEM &&
			      (new_reg->mem_type == TTM_PL_VRAM ||
1211
			       nvbo->page_shift != vma->vm->mmu->lpg_shift)) {
1212
			nvkm_vm_map(vma, new_reg->mm_node);
1213
		} else {
1214
			WARN_ON(ttm_bo_wait(bo, false, false));
1215
			nvkm_vm_unmap(vma);
1216
		}
1217 1218 1219
	}
}

1220
static int
1221
nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_reg,
1222
		   struct nouveau_drm_tile **new_tile)
1223
{
1224 1225
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
	struct drm_device *dev = drm->dev;
1226
	struct nouveau_bo *nvbo = nouveau_bo(bo);
1227
	u64 offset = new_reg->start << PAGE_SHIFT;
1228

1229
	*new_tile = NULL;
1230
	if (new_reg->mem_type != TTM_PL_VRAM)
1231 1232
		return 0;

1233
	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
1234
		*new_tile = nv10_bo_set_tiling(dev, offset, new_reg->size,
1235 1236
						nvbo->tile_mode,
						nvbo->tile_flags);
1237 1238
	}

1239 1240 1241 1242 1243
	return 0;
}

static void
nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
1244 1245
		      struct nouveau_drm_tile *new_tile,
		      struct nouveau_drm_tile **old_tile)
1246
{
1247 1248
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
	struct drm_device *dev = drm->dev;
1249
	struct dma_fence *fence = reservation_object_get_excl(bo->resv);
1250

1251
	nv10_bo_put_tile_region(dev, *old_tile, fence);
1252
	*old_tile = new_tile;
1253 1254 1255 1256
}

static int
nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
1257
		bool no_wait_gpu, struct ttm_mem_reg *new_reg)
1258
{
1259
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1260
	struct nouveau_bo *nvbo = nouveau_bo(bo);
1261
	struct ttm_mem_reg *old_reg = &bo->mem;
1262
	struct nouveau_drm_tile *new_tile = NULL;
1263 1264
	int ret = 0;

1265 1266 1267 1268
	ret = ttm_bo_wait(bo, intr, no_wait_gpu);
	if (ret)
		return ret;

1269 1270 1271
	if (nvbo->pin_refcnt)
		NV_WARN(drm, "Moving pinned object %p!\n", nvbo);

1272
	if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1273
		ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile);
1274 1275 1276
		if (ret)
			return ret;
	}
1277 1278

	/* Fake bo copy. */
1279
	if (old_reg->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
1280
		BUG_ON(bo->mem.mm_node != NULL);
1281 1282
		bo->mem = *new_reg;
		new_reg->mm_node = NULL;
1283
		goto out;
1284 1285
	}

1286
	/* Hardware assisted copy. */
1287
	if (drm->ttm.move) {
1288
		if (new_reg->mem_type == TTM_PL_SYSTEM)
1289
			ret = nouveau_bo_move_flipd(bo, evict, intr,
1290 1291
						    no_wait_gpu, new_reg);
		else if (old_reg->mem_type == TTM_PL_SYSTEM)
1292
			ret = nouveau_bo_move_flips(bo, evict, intr,
1293
						    no_wait_gpu, new_reg);
1294 1295
		else
			ret = nouveau_bo_move_m2mf(bo, evict, intr,
1296
						   no_wait_gpu, new_reg);
1297 1298 1299
		if (!ret)
			goto out;
	}
1300 1301

	/* Fallback to software copy. */
1302
	ret = ttm_bo_wait(bo, intr, no_wait_gpu);
1303
	if (ret == 0)
1304
		ret = ttm_bo_move_memcpy(bo, intr, no_wait_gpu, new_reg);
1305 1306

out:
1307
	if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1308 1309 1310 1311 1312
		if (ret)
			nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
		else
			nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
	}
1313 1314

	return ret;
1315 1316 1317 1318 1319
}

static int
nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
{
1320 1321
	struct nouveau_bo *nvbo = nouveau_bo(bo);

D
David Herrmann 已提交
1322 1323
	return drm_vma_node_verify_access(&nvbo->gem.vma_node,
					  filp->private_data);
1324 1325
}

1326
static int
1327
nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg)
1328
{
1329
	struct ttm_mem_type_manager *man = &bdev->man[reg->mem_type];
1330
	struct nouveau_drm *drm = nouveau_bdev(bdev);
1331
	struct nvkm_device *device = nvxx_device(&drm->client.device);
1332
	struct nvkm_mem *mem = reg->mm_node;
1333
	int ret;
1334

1335 1336 1337 1338 1339
	reg->bus.addr = NULL;
	reg->bus.offset = 0;
	reg->bus.size = reg->num_pages << PAGE_SHIFT;
	reg->bus.base = 0;
	reg->bus.is_iomem = false;
1340 1341
	if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
		return -EINVAL;
1342
	switch (reg->mem_type) {
1343 1344 1345 1346
	case TTM_PL_SYSTEM:
		/* System memory */
		return 0;
	case TTM_PL_TT:
D
Daniel Vetter 已提交
1347
#if IS_ENABLED(CONFIG_AGP)
1348
		if (drm->agp.bridge) {
1349 1350 1351
			reg->bus.offset = reg->start << PAGE_SHIFT;
			reg->bus.base = drm->agp.base;
			reg->bus.is_iomem = !drm->agp.cma;
1352 1353
		}
#endif
1354
		if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || !mem->memtype)
1355 1356 1357
			/* untiled */
			break;
		/* fallthrough, tiled memory */
1358
	case TTM_PL_VRAM:
1359 1360 1361
		reg->bus.offset = reg->start << PAGE_SHIFT;
		reg->bus.base = device->func->resource_addr(device, 1);
		reg->bus.is_iomem = true;
1362 1363
		if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
			struct nvkm_bar *bar = nvxx_bar(&drm->client.device);
1364
			int page_shift = 12;
1365
			if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_FERMI)
1366
				page_shift = mem->page_shift;
1367

1368 1369
			ret = nvkm_bar_umap(bar, mem->size << 12, page_shift,
					    &mem->bar_vma);
1370 1371
			if (ret)
				return ret;
1372

1373 1374
			nvkm_vm_map(&mem->bar_vma, mem);
			reg->bus.offset = mem->bar_vma.offset;
1375
		}
1376 1377 1378 1379 1380 1381 1382 1383
		break;
	default:
		return -EINVAL;
	}
	return 0;
}

static void
1384
nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *reg)
1385
{
1386
	struct nvkm_mem *mem = reg->mm_node;
1387

1388
	if (!mem->bar_vma.node)
1389 1390
		return;

1391 1392
	nvkm_vm_unmap(&mem->bar_vma);
	nvkm_vm_put(&mem->bar_vma);
1393 1394 1395 1396 1397
}

static int
nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
{
1398
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1399
	struct nouveau_bo *nvbo = nouveau_bo(bo);
1400
	struct nvkm_device *device = nvxx_device(&drm->client.device);
1401
	u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT;
1402
	int i, ret;
1403 1404 1405 1406 1407

	/* as long as the bo isn't in vram, and isn't tiled, we've got
	 * nothing to do here.
	 */
	if (bo->mem.mem_type != TTM_PL_VRAM) {
1408
		if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA ||
1409
		    !nouveau_bo_tile_layout(nvbo))
1410
			return 0;
1411 1412 1413 1414 1415 1416 1417 1418 1419

		if (bo->mem.mem_type == TTM_PL_SYSTEM) {
			nouveau_bo_placement_set(nvbo, TTM_PL_TT, 0);

			ret = nouveau_bo_validate(nvbo, false, false);
			if (ret)
				return ret;
		}
		return 0;
1420 1421 1422
	}

	/* make sure bo is in mappable vram */
1423
	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA ||
1424
	    bo->mem.start + bo->mem.num_pages < mappable)
1425 1426
		return 0;

1427 1428 1429 1430 1431 1432 1433 1434 1435
	for (i = 0; i < nvbo->placement.num_placement; ++i) {
		nvbo->placements[i].fpfn = 0;
		nvbo->placements[i].lpfn = mappable;
	}

	for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
		nvbo->busy_placements[i].fpfn = 0;
		nvbo->busy_placements[i].lpfn = mappable;
	}
1436

1437
	nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0);
1438
	return nouveau_bo_validate(nvbo, false, false);
1439 1440
}

1441 1442 1443
static int
nouveau_ttm_tt_populate(struct ttm_tt *ttm)
{
1444
	struct ttm_dma_tt *ttm_dma = (void *)ttm;
1445
	struct nouveau_drm *drm;
1446
	struct nvkm_device *device;
1447
	struct drm_device *dev;
1448
	struct device *pdev;
1449 1450
	unsigned i;
	int r;
D
Dave Airlie 已提交
1451
	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1452 1453 1454 1455

	if (ttm->state != tt_unpopulated)
		return 0;

D
Dave Airlie 已提交
1456 1457 1458 1459 1460 1461 1462 1463
	if (slave && ttm->sg) {
		/* make userspace faulting work */
		drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
						 ttm_dma->dma_address, ttm->num_pages);
		ttm->state = tt_unbound;
		return 0;
	}

1464
	drm = nouveau_bdev(ttm->bdev);
1465
	device = nvxx_device(&drm->client.device);
1466
	dev = drm->dev;
1467
	pdev = device->dev;
1468

D
Daniel Vetter 已提交
1469
#if IS_ENABLED(CONFIG_AGP)
1470
	if (drm->agp.bridge) {
J
Jerome Glisse 已提交
1471 1472 1473 1474
		return ttm_agp_tt_populate(ttm);
	}
#endif

1475
#if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
1476
	if (swiotlb_nr_tbl()) {
1477
		return ttm_dma_populate((void *)ttm, dev->dev);
1478 1479 1480 1481 1482 1483 1484 1485 1486
	}
#endif

	r = ttm_pool_populate(ttm);
	if (r) {
		return r;
	}

	for (i = 0; i < ttm->num_pages; i++) {
1487 1488 1489 1490 1491 1492
		dma_addr_t addr;

		addr = dma_map_page(pdev, ttm->pages[i], 0, PAGE_SIZE,
				    DMA_BIDIRECTIONAL);

		if (dma_mapping_error(pdev, addr)) {
1493
			while (i--) {
1494 1495
				dma_unmap_page(pdev, ttm_dma->dma_address[i],
					       PAGE_SIZE, DMA_BIDIRECTIONAL);
1496
				ttm_dma->dma_address[i] = 0;
1497 1498 1499 1500
			}
			ttm_pool_unpopulate(ttm);
			return -EFAULT;
		}
1501 1502

		ttm_dma->dma_address[i] = addr;
1503 1504 1505 1506 1507 1508 1509
	}
	return 0;
}

static void
nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
{
1510
	struct ttm_dma_tt *ttm_dma = (void *)ttm;
1511
	struct nouveau_drm *drm;
1512
	struct nvkm_device *device;
1513
	struct drm_device *dev;
1514
	struct device *pdev;
1515
	unsigned i;
D
Dave Airlie 已提交
1516 1517 1518 1519
	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);

	if (slave)
		return;
1520

1521
	drm = nouveau_bdev(ttm->bdev);
1522
	device = nvxx_device(&drm->client.device);
1523
	dev = drm->dev;
1524
	pdev = device->dev;
1525

D
Daniel Vetter 已提交
1526
#if IS_ENABLED(CONFIG_AGP)
1527
	if (drm->agp.bridge) {
J
Jerome Glisse 已提交
1528 1529 1530 1531 1532
		ttm_agp_tt_unpopulate(ttm);
		return;
	}
#endif

1533
#if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
1534
	if (swiotlb_nr_tbl()) {
1535
		ttm_dma_unpopulate((void *)ttm, dev->dev);
1536 1537 1538 1539 1540
		return;
	}
#endif

	for (i = 0; i < ttm->num_pages; i++) {
1541
		if (ttm_dma->dma_address[i]) {
1542 1543
			dma_unmap_page(pdev, ttm_dma->dma_address[i], PAGE_SIZE,
				       DMA_BIDIRECTIONAL);
1544 1545 1546 1547 1548 1549
		}
	}

	ttm_pool_unpopulate(ttm);
}

1550
void
1551
nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive)
1552
{
1553
	struct reservation_object *resv = nvbo->bo.resv;
1554

1555 1556 1557 1558
	if (exclusive)
		reservation_object_add_excl_fence(resv, &fence->base);
	else if (fence)
		reservation_object_add_shared_fence(resv, &fence->base);
1559 1560
}

1561
struct ttm_bo_driver nouveau_bo_driver = {
1562
	.ttm_tt_create = &nouveau_ttm_tt_create,
1563 1564
	.ttm_tt_populate = &nouveau_ttm_tt_populate,
	.ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
1565 1566
	.invalidate_caches = nouveau_bo_invalidate_caches,
	.init_mem_type = nouveau_bo_init_mem_type,
1567
	.eviction_valuable = ttm_bo_eviction_valuable,
1568
	.evict_flags = nouveau_bo_evict_flags,
1569
	.move_notify = nouveau_bo_move_ntfy,
1570 1571
	.move = nouveau_bo_move,
	.verify_access = nouveau_bo_verify_access,
1572 1573 1574
	.fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
	.io_mem_reserve = &nouveau_ttm_io_mem_reserve,
	.io_mem_free = &nouveau_ttm_io_mem_free,
1575 1576
};

1577 1578
struct nvkm_vma *
nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nvkm_vm *vm)
1579
{
1580
	struct nvkm_vma *vma;
1581 1582 1583 1584 1585 1586 1587 1588 1589
	list_for_each_entry(vma, &nvbo->vma_list, head) {
		if (vma->vm == vm)
			return vma;
	}

	return NULL;
}

int
1590 1591
nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nvkm_vm *vm,
		   struct nvkm_vma *vma)
1592 1593 1594 1595
{
	const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
	int ret;

1596
	ret = nvkm_vm_get(vm, size, nvbo->page_shift,
1597 1598 1599 1600
			     NV_MEM_ACCESS_RW, vma);
	if (ret)
		return ret;

1601 1602
	if ( nvbo->bo.mem.mem_type != TTM_PL_SYSTEM &&
	    (nvbo->bo.mem.mem_type == TTM_PL_VRAM ||
1603
	     nvbo->page_shift != vma->vm->mmu->lpg_shift))
1604
		nvkm_vm_map(vma, nvbo->bo.mem.mm_node);
1605 1606

	list_add_tail(&vma->head, &nvbo->vma_list);
1607
	vma->refcount = 1;
1608 1609 1610 1611
	return 0;
}

void
1612
nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nvkm_vma *vma)
1613 1614
{
	if (vma->node) {
1615
		if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM)
1616 1617
			nvkm_vm_unmap(vma);
		nvkm_vm_put(vma);
1618 1619 1620
		list_del(&vma->head);
	}
}