nouveau_bo.c 41.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
/*
 * Copyright 2007 Dave Airlied
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 */
/*
 * Authors: Dave Airlied <airlied@linux.ie>
 *	    Ben Skeggs   <darktama@iinet.net.au>
 *	    Jeremy Kolb  <jkolb@brandeis.edu>
 */

30
#include <linux/dma-mapping.h>
31
#include <linux/swiotlb.h>
32

33
#include "nouveau_drm.h"
34
#include "nouveau_dma.h"
35
#include "nouveau_fence.h"
36

37 38 39
#include "nouveau_bo.h"
#include "nouveau_ttm.h"
#include "nouveau_gem.h"
40

41 42 43 44 45
/*
 * NV10-NV40 tiling helpers
 */

static void
46 47
nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
			   u32 addr, u32 size, u32 pitch, u32 flags)
48
{
49
	struct nouveau_drm *drm = nouveau_drm(dev);
50
	int i = reg - drm->tile.reg;
51
	struct nouveau_fb *pfb = nvkm_fb(&drm->device);
52 53
	struct nouveau_fb_tile *tile = &pfb->tile.region[i];
	struct nouveau_engine *engine;
54

55
	nouveau_fence_unref(&reg->fence);
56 57

	if (tile->pitch)
58
		pfb->tile.fini(pfb, i, tile);
59 60

	if (pitch)
61
		pfb->tile.init(pfb, i, addr, size, pitch, flags, tile);
62

63
	pfb->tile.prog(pfb, i, tile);
64

65 66 67 68
	if ((engine = nouveau_engine(pfb, NVDEV_ENGINE_GR)))
		engine->tile_prog(engine, i);
	if ((engine = nouveau_engine(pfb, NVDEV_ENGINE_MPEG)))
		engine->tile_prog(engine, i);
69 70
}

71
static struct nouveau_drm_tile *
72 73
nv10_bo_get_tile_region(struct drm_device *dev, int i)
{
74
	struct nouveau_drm *drm = nouveau_drm(dev);
75
	struct nouveau_drm_tile *tile = &drm->tile.reg[i];
76

77
	spin_lock(&drm->tile.lock);
78 79 80 81 82 83 84

	if (!tile->used &&
	    (!tile->fence || nouveau_fence_done(tile->fence)))
		tile->used = true;
	else
		tile = NULL;

85
	spin_unlock(&drm->tile.lock);
86 87 88 89
	return tile;
}

static void
90
nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
91
			struct fence *fence)
92
{
93
	struct nouveau_drm *drm = nouveau_drm(dev);
94 95

	if (tile) {
96
		spin_lock(&drm->tile.lock);
97
		tile->fence = (struct nouveau_fence *)fence_get(fence);
98
		tile->used = false;
99
		spin_unlock(&drm->tile.lock);
100 101 102
	}
}

103 104 105
static struct nouveau_drm_tile *
nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
		   u32 size, u32 pitch, u32 flags)
106
{
107
	struct nouveau_drm *drm = nouveau_drm(dev);
108
	struct nouveau_fb *pfb = nvkm_fb(&drm->device);
109
	struct nouveau_drm_tile *tile, *found = NULL;
110 111
	int i;

112
	for (i = 0; i < pfb->tile.regions; i++) {
113 114 115 116 117 118
		tile = nv10_bo_get_tile_region(dev, i);

		if (pitch && !found) {
			found = tile;
			continue;

119
		} else if (tile && pfb->tile.region[i].pitch) {
120 121 122 123 124 125 126 127 128 129 130 131 132
			/* Kill an unused tile region. */
			nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0);
		}

		nv10_bo_put_tile_region(dev, tile, NULL);
	}

	if (found)
		nv10_bo_update_tile_region(dev, found, addr, size,
					    pitch, flags);
	return found;
}

133 134 135
static void
nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
{
136 137
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
	struct drm_device *dev = drm->dev;
138 139
	struct nouveau_bo *nvbo = nouveau_bo(bo);

140
	if (unlikely(nvbo->gem.filp))
141
		DRM_ERROR("bo %p still attached to GEM object\n", bo);
142
	WARN_ON(nvbo->pin_refcnt > 0);
143
	nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
144 145 146
	kfree(nvbo);
}

147
static void
148
nouveau_bo_fixup_align(struct nouveau_bo *nvbo, u32 flags,
149
		       int *align, int *size)
150
{
151
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
152
	struct nvif_device *device = &drm->device;
153

154
	if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
155
		if (nvbo->tile_mode) {
156
			if (device->info.chipset >= 0x40) {
157
				*align = 65536;
158
				*size = roundup(*size, 64 * nvbo->tile_mode);
159

160
			} else if (device->info.chipset >= 0x30) {
161
				*align = 32768;
162
				*size = roundup(*size, 64 * nvbo->tile_mode);
163

164
			} else if (device->info.chipset >= 0x20) {
165
				*align = 16384;
166
				*size = roundup(*size, 64 * nvbo->tile_mode);
167

168
			} else if (device->info.chipset >= 0x10) {
169
				*align = 16384;
170
				*size = roundup(*size, 32 * nvbo->tile_mode);
171 172
			}
		}
173
	} else {
174 175
		*size = roundup(*size, (1 << nvbo->page_shift));
		*align = max((1 <<  nvbo->page_shift), *align);
176 177
	}

178
	*size = roundup(*size, PAGE_SIZE);
179 180
}

181
int
182 183
nouveau_bo_new(struct drm_device *dev, int size, int align,
	       uint32_t flags, uint32_t tile_mode, uint32_t tile_flags,
184
	       struct sg_table *sg, struct reservation_object *robj,
185
	       struct nouveau_bo **pnvbo)
186
{
187
	struct nouveau_drm *drm = nouveau_drm(dev);
188
	struct nouveau_bo *nvbo;
189
	size_t acc_size;
190
	int ret;
D
Dave Airlie 已提交
191
	int type = ttm_bo_type_device;
192 193 194
	int lpg_shift = 12;
	int max_size;

195 196
	if (drm->client.vm)
		lpg_shift = drm->client.vm->vmm->lpg_shift;
197
	max_size = INT_MAX & ~((1 << lpg_shift) - 1);
198 199

	if (size <= 0 || size > max_size) {
200
		NV_WARN(drm, "skipped size %x\n", (u32)size);
201 202
		return -EINVAL;
	}
D
Dave Airlie 已提交
203 204 205

	if (sg)
		type = ttm_bo_type_sg;
206 207 208 209 210 211

	nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
	if (!nvbo)
		return -ENOMEM;
	INIT_LIST_HEAD(&nvbo->head);
	INIT_LIST_HEAD(&nvbo->entry);
212
	INIT_LIST_HEAD(&nvbo->vma_list);
213 214
	nvbo->tile_mode = tile_mode;
	nvbo->tile_flags = tile_flags;
215
	nvbo->bo.bdev = &drm->ttm.bdev;
216

217 218 219
	if (!nv_device_is_cpu_coherent(nvkm_device(&drm->device)))
		nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;

220
	nvbo->page_shift = 12;
221
	if (drm->client.vm) {
222
		if (!(flags & TTM_PL_FLAG_TT) && size > 256 * 1024)
223
			nvbo->page_shift = drm->client.vm->vmm->lpg_shift;
224 225 226
	}

	nouveau_bo_fixup_align(nvbo, flags, &align, &size);
227 228
	nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
	nouveau_bo_placement_set(nvbo, flags, 0);
229

230
	acc_size = ttm_bo_dma_acc_size(&drm->ttm.bdev, size,
231 232
				       sizeof(struct nouveau_bo));

233
	ret = ttm_bo_init(&drm->ttm.bdev, &nvbo->bo, size,
D
Dave Airlie 已提交
234
			  type, &nvbo->placement,
235
			  align >> PAGE_SHIFT, false, NULL, acc_size, sg,
236
			  robj, nouveau_bo_del_ttm);
237 238 239 240 241 242 243 244 245
	if (ret) {
		/* ttm will call nouveau_bo_del_ttm if it fails.. */
		return ret;
	}

	*pnvbo = nvbo;
	return 0;
}

246
static void
247
set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t type, uint32_t flags)
248 249 250 251
{
	*n = 0;

	if (type & TTM_PL_FLAG_VRAM)
252
		pl[(*n)++].flags = TTM_PL_FLAG_VRAM | flags;
253
	if (type & TTM_PL_FLAG_TT)
254
		pl[(*n)++].flags = TTM_PL_FLAG_TT | flags;
255
	if (type & TTM_PL_FLAG_SYSTEM)
256
		pl[(*n)++].flags = TTM_PL_FLAG_SYSTEM | flags;
257 258
}

259 260 261
static void
set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
{
262
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
263
	u32 vram_pages = drm->device.info.ram_size >> PAGE_SHIFT;
264
	unsigned i, fpfn, lpfn;
265

266
	if (drm->device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
267
	    nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
268
	    nvbo->bo.mem.num_pages < vram_pages / 4) {
269 270 271 272 273 274 275
		/*
		 * Make sure that the color and depth buffers are handled
		 * by independent memory controller units. Up to a 9x
		 * speed up when alpha-blending and depth-test are enabled
		 * at the same time.
		 */
		if (nvbo->tile_flags & NOUVEAU_GEM_TILE_ZETA) {
276 277
			fpfn = vram_pages / 2;
			lpfn = ~0;
278
		} else {
279 280 281 282 283 284 285 286 287 288
			fpfn = 0;
			lpfn = vram_pages / 2;
		}
		for (i = 0; i < nvbo->placement.num_placement; ++i) {
			nvbo->placements[i].fpfn = fpfn;
			nvbo->placements[i].lpfn = lpfn;
		}
		for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
			nvbo->busy_placements[i].fpfn = fpfn;
			nvbo->busy_placements[i].lpfn = lpfn;
289 290 291 292
		}
	}
}

293
void
294
nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t type, uint32_t busy)
295
{
296
	struct ttm_placement *pl = &nvbo->placement;
297 298 299
	uint32_t flags = (nvbo->force_coherent ? TTM_PL_FLAG_UNCACHED :
						 TTM_PL_MASK_CACHING) |
			 (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
300 301 302 303 304 305 306 307

	pl->placement = nvbo->placements;
	set_placement_list(nvbo->placements, &pl->num_placement,
			   type, flags);

	pl->busy_placement = nvbo->busy_placements;
	set_placement_list(nvbo->busy_placements, &pl->num_busy_placement,
			   type | busy, flags);
308 309

	set_placement_range(nvbo, type);
310 311 312 313 314
}

int
nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t memtype)
{
315
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
316
	struct ttm_buffer_object *bo = &nvbo->bo;
317
	int ret;
318

319
	ret = ttm_bo_reserve(bo, false, false, false, NULL);
320 321 322
	if (ret)
		goto out;

323
	if (nvbo->pin_refcnt && !(memtype & (1 << bo->mem.mem_type))) {
324
		NV_ERROR(drm, "bo %p pinned elsewhere: 0x%08x vs 0x%08x\n", bo,
325
			 1 << bo->mem.mem_type, memtype);
326 327
		ret = -EINVAL;
		goto out;
328 329
	}

330 331
	if (nvbo->pin_refcnt)
		goto ref_inc;
332

333
	nouveau_bo_placement_set(nvbo, memtype, 0);
334

335
	ret = nouveau_bo_validate(nvbo, false, false);
336 337 338
	if (ret == 0) {
		switch (bo->mem.mem_type) {
		case TTM_PL_VRAM:
339
			drm->gem.vram_available -= bo->mem.size;
340 341
			break;
		case TTM_PL_TT:
342
			drm->gem.gart_available -= bo->mem.size;
343 344 345 346 347
			break;
		default:
			break;
		}
	}
348 349 350 351

ref_inc:
	nvbo->pin_refcnt++;

352
out:
353
	ttm_bo_unreserve(bo);
354 355 356 357 358 359
	return ret;
}

int
nouveau_bo_unpin(struct nouveau_bo *nvbo)
{
360
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
361
	struct ttm_buffer_object *bo = &nvbo->bo;
362
	int ret, ref;
363

364
	ret = ttm_bo_reserve(bo, false, false, false, NULL);
365 366 367
	if (ret)
		return ret;

368 369 370
	ref = --nvbo->pin_refcnt;
	WARN_ON_ONCE(ref < 0);
	if (ref)
371 372
		goto out;

373
	nouveau_bo_placement_set(nvbo, bo->mem.placement, 0);
374

375
	ret = nouveau_bo_validate(nvbo, false, false);
376 377 378
	if (ret == 0) {
		switch (bo->mem.mem_type) {
		case TTM_PL_VRAM:
379
			drm->gem.vram_available += bo->mem.size;
380 381
			break;
		case TTM_PL_TT:
382
			drm->gem.gart_available += bo->mem.size;
383 384 385 386 387 388
			break;
		default:
			break;
		}
	}

389
out:
390 391 392 393 394 395 396 397 398
	ttm_bo_unreserve(bo);
	return ret;
}

int
nouveau_bo_map(struct nouveau_bo *nvbo)
{
	int ret;

399
	ret = ttm_bo_reserve(&nvbo->bo, false, false, false, NULL);
400 401 402
	if (ret)
		return ret;

403 404 405 406 407 408 409 410
	/*
	 * TTM buffers allocated using the DMA API already have a mapping, let's
	 * use it instead.
	 */
	if (!nvbo->force_coherent)
		ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
				  &nvbo->kmap);

411 412 413 414 415 416 417
	ttm_bo_unreserve(&nvbo->bo);
	return ret;
}

void
nouveau_bo_unmap(struct nouveau_bo *nvbo)
{
418 419 420 421 422 423 424 425
	if (!nvbo)
		return;

	/*
	 * TTM buffers allocated using the DMA API already had a coherent
	 * mapping which we used, no need to unmap.
	 */
	if (!nvbo->force_coherent)
426
		ttm_bo_kunmap(&nvbo->kmap);
427 428
}

429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468
void
nouveau_bo_sync_for_device(struct nouveau_bo *nvbo)
{
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
	struct nouveau_device *device = nvkm_device(&drm->device);
	struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
	int i;

	if (!ttm_dma)
		return;

	/* Don't waste time looping if the object is coherent */
	if (nvbo->force_coherent)
		return;

	for (i = 0; i < ttm_dma->ttm.num_pages; i++)
		dma_sync_single_for_device(nv_device_base(device),
			ttm_dma->dma_address[i], PAGE_SIZE, DMA_TO_DEVICE);
}

void
nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo)
{
	struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
	struct nouveau_device *device = nvkm_device(&drm->device);
	struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
	int i;

	if (!ttm_dma)
		return;

	/* Don't waste time looping if the object is coherent */
	if (nvbo->force_coherent)
		return;

	for (i = 0; i < ttm_dma->ttm.num_pages; i++)
		dma_sync_single_for_cpu(nv_device_base(device),
			ttm_dma->dma_address[i], PAGE_SIZE, DMA_FROM_DEVICE);
}

469 470
int
nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
471
		    bool no_wait_gpu)
472 473 474
{
	int ret;

475 476
	ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement,
			      interruptible, no_wait_gpu);
477 478 479
	if (ret)
		return ret;

480 481
	nouveau_bo_sync_for_device(nvbo);

482 483 484
	return 0;
}

485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
static inline void *
_nouveau_bo_mem_index(struct nouveau_bo *nvbo, unsigned index, void *mem, u8 sz)
{
	struct ttm_dma_tt *dma_tt;
	u8 *m = mem;

	index *= sz;

	if (m) {
		/* kmap'd address, return the corresponding offset */
		m += index;
	} else {
		/* DMA-API mapping, lookup the right address */
		dma_tt = (struct ttm_dma_tt *)nvbo->bo.ttm;
		m = dma_tt->cpu_address[index / PAGE_SIZE];
		m += index % PAGE_SIZE;
	}

	return m;
}
#define nouveau_bo_mem_index(o, i, m) _nouveau_bo_mem_index(o, i, m, sizeof(*m))

507 508 509 510 511
u16
nouveau_bo_rd16(struct nouveau_bo *nvbo, unsigned index)
{
	bool is_iomem;
	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
512 513 514

	mem = nouveau_bo_mem_index(nvbo, index, mem);

515 516 517 518 519 520 521 522 523 524 525
	if (is_iomem)
		return ioread16_native((void __force __iomem *)mem);
	else
		return *mem;
}

void
nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
{
	bool is_iomem;
	u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
526 527 528

	mem = nouveau_bo_mem_index(nvbo, index, mem);

529 530 531 532 533 534 535 536 537 538 539
	if (is_iomem)
		iowrite16_native(val, (void __force __iomem *)mem);
	else
		*mem = val;
}

u32
nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
{
	bool is_iomem;
	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
540 541 542

	mem = nouveau_bo_mem_index(nvbo, index, mem);

543 544 545 546 547 548 549 550 551 552 553
	if (is_iomem)
		return ioread32_native((void __force __iomem *)mem);
	else
		return *mem;
}

void
nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
{
	bool is_iomem;
	u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
554 555 556

	mem = nouveau_bo_mem_index(nvbo, index, mem);

557 558 559 560 561 562
	if (is_iomem)
		iowrite32_native(val, (void __force __iomem *)mem);
	else
		*mem = val;
}

563
static struct ttm_tt *
564 565
nouveau_ttm_tt_create(struct ttm_bo_device *bdev, unsigned long size,
		      uint32_t page_flags, struct page *dummy_read)
566
{
567
#if __OS_HAS_AGP
568 569
	struct nouveau_drm *drm = nouveau_bdev(bdev);
	struct drm_device *dev = drm->dev;
570

571 572 573
	if (drm->agp.stat == ENABLED) {
		return ttm_agp_tt_create(bdev, dev->agp->bridge, size,
					 page_flags, dummy_read);
574
	}
575
#endif
576

577
	return nouveau_sgdma_create_ttm(bdev, size, page_flags, dummy_read);
578 579 580 581 582 583 584 585 586 587 588 589 590
}

static int
nouveau_bo_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
{
	/* We'll do this from user space. */
	return 0;
}

static int
nouveau_bo_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
			 struct ttm_mem_type_manager *man)
{
591
	struct nouveau_drm *drm = nouveau_bdev(bdev);
592 593 594 595 596 597 598 599

	switch (type) {
	case TTM_PL_SYSTEM:
		man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
		man->available_caching = TTM_PL_MASK_CACHING;
		man->default_caching = TTM_PL_FLAG_CACHED;
		break;
	case TTM_PL_VRAM:
600 601 602 603 604 605
		man->flags = TTM_MEMTYPE_FLAG_FIXED |
			     TTM_MEMTYPE_FLAG_MAPPABLE;
		man->available_caching = TTM_PL_FLAG_UNCACHED |
					 TTM_PL_FLAG_WC;
		man->default_caching = TTM_PL_FLAG_WC;

606
		if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
607
			/* Some BARs do not support being ioremapped WC */
608
			if (nvkm_bar(&drm->device)->iomap_uncached) {
609 610 611 612
				man->available_caching = TTM_PL_FLAG_UNCACHED;
				man->default_caching = TTM_PL_FLAG_UNCACHED;
			}

B
Ben Skeggs 已提交
613
			man->func = &nouveau_vram_manager;
614 615 616
			man->io_reserve_fastpath = false;
			man->use_io_reserve_lru = true;
		} else {
B
Ben Skeggs 已提交
617
			man->func = &ttm_bo_manager_func;
618
		}
619 620
		break;
	case TTM_PL_TT:
621
		if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
622
			man->func = &nouveau_gart_manager;
623
		else
624
		if (drm->agp.stat != ENABLED)
625
			man->func = &nv04_gart_manager;
626 627
		else
			man->func = &ttm_bo_manager_func;
628 629

		if (drm->agp.stat == ENABLED) {
630
			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
631 632 633
			man->available_caching = TTM_PL_FLAG_UNCACHED |
				TTM_PL_FLAG_WC;
			man->default_caching = TTM_PL_FLAG_WC;
634
		} else {
635 636 637 638 639
			man->flags = TTM_MEMTYPE_FLAG_MAPPABLE |
				     TTM_MEMTYPE_FLAG_CMA;
			man->available_caching = TTM_PL_MASK_CACHING;
			man->default_caching = TTM_PL_FLAG_CACHED;
		}
640

641 642 643 644 645 646 647 648 649 650 651 652 653
		break;
	default:
		return -EINVAL;
	}
	return 0;
}

static void
nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
{
	struct nouveau_bo *nvbo = nouveau_bo(bo);

	switch (bo->mem.mem_type) {
654
	case TTM_PL_VRAM:
655 656
		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_TT,
					 TTM_PL_FLAG_SYSTEM);
657
		break;
658
	default:
659
		nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_SYSTEM, 0);
660 661
		break;
	}
662 663

	*pl = nvbo->placement;
664 665 666
}


667 668 669 670 671 672
static int
nve0_bo_move_init(struct nouveau_channel *chan, u32 handle)
{
	int ret = RING_SPACE(chan, 2);
	if (ret == 0) {
		BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
673
		OUT_RING  (chan, handle & 0x0000ffff);
674 675 676 677 678
		FIRE_RING (chan);
	}
	return ret;
}

679 680 681 682 683 684 685
static int
nve0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
{
	struct nouveau_mem *node = old_mem->mm_node;
	int ret = RING_SPACE(chan, 10);
	if (ret == 0) {
686
		BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
687 688 689 690 691 692 693 694
		OUT_RING  (chan, upper_32_bits(node->vma[0].offset));
		OUT_RING  (chan, lower_32_bits(node->vma[0].offset));
		OUT_RING  (chan, upper_32_bits(node->vma[1].offset));
		OUT_RING  (chan, lower_32_bits(node->vma[1].offset));
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, new_mem->num_pages);
695
		BEGIN_IMC0(chan, NvSubCopy, 0x0300, 0x0386);
696 697 698 699
	}
	return ret;
}

700 701 702 703 704 705 706 707 708 709 710
static int
nvc0_bo_move_init(struct nouveau_channel *chan, u32 handle)
{
	int ret = RING_SPACE(chan, 2);
	if (ret == 0) {
		BEGIN_NVC0(chan, NvSubCopy, 0x0000, 1);
		OUT_RING  (chan, handle);
	}
	return ret;
}

711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748
static int
nvc0_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
{
	struct nouveau_mem *node = old_mem->mm_node;
	u64 src_offset = node->vma[0].offset;
	u64 dst_offset = node->vma[1].offset;
	u32 page_count = new_mem->num_pages;
	int ret;

	page_count = new_mem->num_pages;
	while (page_count) {
		int line_count = (page_count > 8191) ? 8191 : page_count;

		ret = RING_SPACE(chan, 11);
		if (ret)
			return ret;

		BEGIN_NVC0(chan, NvSubCopy, 0x030c, 8);
		OUT_RING  (chan, upper_32_bits(src_offset));
		OUT_RING  (chan, lower_32_bits(src_offset));
		OUT_RING  (chan, upper_32_bits(dst_offset));
		OUT_RING  (chan, lower_32_bits(dst_offset));
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, line_count);
		BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
		OUT_RING  (chan, 0x00000110);

		page_count -= line_count;
		src_offset += (PAGE_SIZE * line_count);
		dst_offset += (PAGE_SIZE * line_count);
	}

	return 0;
}

B
Ben Skeggs 已提交
749 750 751 752
static int
nvc0_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
{
753 754 755
	struct nouveau_mem *node = old_mem->mm_node;
	u64 src_offset = node->vma[0].offset;
	u64 dst_offset = node->vma[1].offset;
B
Ben Skeggs 已提交
756 757 758 759 760 761 762 763 764 765 766
	u32 page_count = new_mem->num_pages;
	int ret;

	page_count = new_mem->num_pages;
	while (page_count) {
		int line_count = (page_count > 2047) ? 2047 : page_count;

		ret = RING_SPACE(chan, 12);
		if (ret)
			return ret;

767
		BEGIN_NVC0(chan, NvSubCopy, 0x0238, 2);
B
Ben Skeggs 已提交
768 769
		OUT_RING  (chan, upper_32_bits(dst_offset));
		OUT_RING  (chan, lower_32_bits(dst_offset));
770
		BEGIN_NVC0(chan, NvSubCopy, 0x030c, 6);
B
Ben Skeggs 已提交
771 772 773 774 775 776
		OUT_RING  (chan, upper_32_bits(src_offset));
		OUT_RING  (chan, lower_32_bits(src_offset));
		OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* line_length */
		OUT_RING  (chan, line_count);
777
		BEGIN_NVC0(chan, NvSubCopy, 0x0300, 1);
B
Ben Skeggs 已提交
778 779 780 781 782 783 784 785 786 787
		OUT_RING  (chan, 0x00100110);

		page_count -= line_count;
		src_offset += (PAGE_SIZE * line_count);
		dst_offset += (PAGE_SIZE * line_count);
	}

	return 0;
}

788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825
static int
nva3_bo_move_copy(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
{
	struct nouveau_mem *node = old_mem->mm_node;
	u64 src_offset = node->vma[0].offset;
	u64 dst_offset = node->vma[1].offset;
	u32 page_count = new_mem->num_pages;
	int ret;

	page_count = new_mem->num_pages;
	while (page_count) {
		int line_count = (page_count > 8191) ? 8191 : page_count;

		ret = RING_SPACE(chan, 11);
		if (ret)
			return ret;

		BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
		OUT_RING  (chan, upper_32_bits(src_offset));
		OUT_RING  (chan, lower_32_bits(src_offset));
		OUT_RING  (chan, upper_32_bits(dst_offset));
		OUT_RING  (chan, lower_32_bits(dst_offset));
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, PAGE_SIZE);
		OUT_RING  (chan, line_count);
		BEGIN_NV04(chan, NvSubCopy, 0x0300, 1);
		OUT_RING  (chan, 0x00000110);

		page_count -= line_count;
		src_offset += (PAGE_SIZE * line_count);
		dst_offset += (PAGE_SIZE * line_count);
	}

	return 0;
}

826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
static int
nv98_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
{
	struct nouveau_mem *node = old_mem->mm_node;
	int ret = RING_SPACE(chan, 7);
	if (ret == 0) {
		BEGIN_NV04(chan, NvSubCopy, 0x0320, 6);
		OUT_RING  (chan, upper_32_bits(node->vma[0].offset));
		OUT_RING  (chan, lower_32_bits(node->vma[0].offset));
		OUT_RING  (chan, upper_32_bits(node->vma[1].offset));
		OUT_RING  (chan, lower_32_bits(node->vma[1].offset));
		OUT_RING  (chan, 0x00000000 /* COPY */);
		OUT_RING  (chan, new_mem->num_pages << PAGE_SHIFT);
	}
	return ret;
}

844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
static int
nv84_bo_move_exec(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
{
	struct nouveau_mem *node = old_mem->mm_node;
	int ret = RING_SPACE(chan, 7);
	if (ret == 0) {
		BEGIN_NV04(chan, NvSubCopy, 0x0304, 6);
		OUT_RING  (chan, new_mem->num_pages << PAGE_SHIFT);
		OUT_RING  (chan, upper_32_bits(node->vma[0].offset));
		OUT_RING  (chan, lower_32_bits(node->vma[0].offset));
		OUT_RING  (chan, upper_32_bits(node->vma[1].offset));
		OUT_RING  (chan, lower_32_bits(node->vma[1].offset));
		OUT_RING  (chan, 0x00000000 /* MODE_COPY, QUERY_NONE */);
	}
	return ret;
}

862 863 864
static int
nv50_bo_move_init(struct nouveau_channel *chan, u32 handle)
{
865
	int ret = RING_SPACE(chan, 6);
866
	if (ret == 0) {
867 868 869
		BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
		OUT_RING  (chan, handle);
		BEGIN_NV04(chan, NvSubCopy, 0x0180, 3);
870 871 872
		OUT_RING  (chan, chan->drm->ntfy.handle);
		OUT_RING  (chan, chan->vram.handle);
		OUT_RING  (chan, chan->vram.handle);
873 874 875 876 877
	}

	return ret;
}

878
static int
879 880
nv50_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
881
{
882
	struct nouveau_mem *node = old_mem->mm_node;
883
	u64 length = (new_mem->num_pages << PAGE_SHIFT);
884 885
	u64 src_offset = node->vma[0].offset;
	u64 dst_offset = node->vma[1].offset;
886 887
	int src_tiled = !!node->memtype;
	int dst_tiled = !!((struct nouveau_mem *)new_mem->mm_node)->memtype;
888 889
	int ret;

890 891 892
	while (length) {
		u32 amount, stride, height;

893 894 895 896
		ret = RING_SPACE(chan, 18 + 6 * (src_tiled + dst_tiled));
		if (ret)
			return ret;

897 898
		amount  = min(length, (u64)(4 * 1024 * 1024));
		stride  = 16 * 4;
899 900
		height  = amount / stride;

901
		if (src_tiled) {
902
			BEGIN_NV04(chan, NvSubCopy, 0x0200, 7);
903
			OUT_RING  (chan, 0);
904
			OUT_RING  (chan, 0);
905 906 907 908 909 910
			OUT_RING  (chan, stride);
			OUT_RING  (chan, height);
			OUT_RING  (chan, 1);
			OUT_RING  (chan, 0);
			OUT_RING  (chan, 0);
		} else {
911
			BEGIN_NV04(chan, NvSubCopy, 0x0200, 1);
912 913
			OUT_RING  (chan, 1);
		}
914
		if (dst_tiled) {
915
			BEGIN_NV04(chan, NvSubCopy, 0x021c, 7);
916
			OUT_RING  (chan, 0);
917
			OUT_RING  (chan, 0);
918 919 920 921 922 923
			OUT_RING  (chan, stride);
			OUT_RING  (chan, height);
			OUT_RING  (chan, 1);
			OUT_RING  (chan, 0);
			OUT_RING  (chan, 0);
		} else {
924
			BEGIN_NV04(chan, NvSubCopy, 0x021c, 1);
925 926 927
			OUT_RING  (chan, 1);
		}

928
		BEGIN_NV04(chan, NvSubCopy, 0x0238, 2);
929 930
		OUT_RING  (chan, upper_32_bits(src_offset));
		OUT_RING  (chan, upper_32_bits(dst_offset));
931
		BEGIN_NV04(chan, NvSubCopy, 0x030c, 8);
932 933 934 935 936 937 938 939
		OUT_RING  (chan, lower_32_bits(src_offset));
		OUT_RING  (chan, lower_32_bits(dst_offset));
		OUT_RING  (chan, stride);
		OUT_RING  (chan, stride);
		OUT_RING  (chan, stride);
		OUT_RING  (chan, height);
		OUT_RING  (chan, 0x00000101);
		OUT_RING  (chan, 0x00000000);
940
		BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
941 942 943 944 945
		OUT_RING  (chan, 0);

		length -= amount;
		src_offset += amount;
		dst_offset += amount;
946 947
	}

948 949 950
	return 0;
}

951 952 953
static int
nv04_bo_move_init(struct nouveau_channel *chan, u32 handle)
{
954
	int ret = RING_SPACE(chan, 4);
955
	if (ret == 0) {
956 957 958
		BEGIN_NV04(chan, NvSubCopy, 0x0000, 1);
		OUT_RING  (chan, handle);
		BEGIN_NV04(chan, NvSubCopy, 0x0180, 1);
959
		OUT_RING  (chan, chan->drm->ntfy.handle);
960 961 962 963 964
	}

	return ret;
}

965 966 967 968 969
static inline uint32_t
nouveau_bo_mem_ctxdma(struct ttm_buffer_object *bo,
		      struct nouveau_channel *chan, struct ttm_mem_reg *mem)
{
	if (mem->mem_type == TTM_PL_TT)
970
		return NvDmaTT;
971
	return chan->vram.handle;
972 973
}

974 975 976 977
static int
nv04_bo_move_m2mf(struct nouveau_channel *chan, struct ttm_buffer_object *bo,
		  struct ttm_mem_reg *old_mem, struct ttm_mem_reg *new_mem)
{
978 979
	u32 src_offset = old_mem->start << PAGE_SHIFT;
	u32 dst_offset = new_mem->start << PAGE_SHIFT;
980 981 982 983 984 985 986
	u32 page_count = new_mem->num_pages;
	int ret;

	ret = RING_SPACE(chan, 3);
	if (ret)
		return ret;

987
	BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_DMA_SOURCE, 2);
988 989 990
	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, old_mem));
	OUT_RING  (chan, nouveau_bo_mem_ctxdma(bo, chan, new_mem));

991 992 993 994 995 996 997
	page_count = new_mem->num_pages;
	while (page_count) {
		int line_count = (page_count > 2047) ? 2047 : page_count;

		ret = RING_SPACE(chan, 11);
		if (ret)
			return ret;
998

999
		BEGIN_NV04(chan, NvSubCopy,
1000
				 NV_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 8);
1001 1002 1003 1004 1005 1006 1007 1008
		OUT_RING  (chan, src_offset);
		OUT_RING  (chan, dst_offset);
		OUT_RING  (chan, PAGE_SIZE); /* src_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* dst_pitch */
		OUT_RING  (chan, PAGE_SIZE); /* line_length */
		OUT_RING  (chan, line_count);
		OUT_RING  (chan, 0x00000101);
		OUT_RING  (chan, 0x00000000);
1009
		BEGIN_NV04(chan, NvSubCopy, NV_MEMORY_TO_MEMORY_FORMAT_NOP, 1);
1010
		OUT_RING  (chan, 0);
1011 1012 1013 1014 1015 1016

		page_count -= line_count;
		src_offset += (PAGE_SIZE * line_count);
		dst_offset += (PAGE_SIZE * line_count);
	}

1017 1018 1019
	return 0;
}

1020
static int
1021 1022
nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,
		     struct ttm_mem_reg *mem)
1023
{
1024 1025 1026
	struct nouveau_mem *old_node = bo->mem.mm_node;
	struct nouveau_mem *new_node = mem->mm_node;
	u64 size = (u64)mem->num_pages << PAGE_SHIFT;
1027 1028
	int ret;

1029
	ret = nouveau_vm_get(drm->client.vm, size, old_node->page_shift,
1030
			     NV_MEM_ACCESS_RW, &old_node->vma[0]);
1031 1032 1033
	if (ret)
		return ret;

1034
	ret = nouveau_vm_get(drm->client.vm, size, new_node->page_shift,
1035 1036 1037 1038 1039 1040 1041 1042
			     NV_MEM_ACCESS_RW, &old_node->vma[1]);
	if (ret) {
		nouveau_vm_put(&old_node->vma[0]);
		return ret;
	}

	nouveau_vm_map(&old_node->vma[0], old_node);
	nouveau_vm_map(&old_node->vma[1], new_node);
1043 1044 1045
	return 0;
}

1046 1047
static int
nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
1048
		     bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1049
{
1050
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1051
	struct nouveau_channel *chan = drm->ttm.chan;
1052
	struct nouveau_cli *cli = (void *)nvif_client(&chan->device->base);
1053
	struct nouveau_fence *fence;
1054 1055
	int ret;

1056 1057 1058
	/* create temporary vmas for the transfer and attach them to the
	 * old nouveau_mem node, these will get cleaned up after ttm has
	 * destroyed the ttm_mem_reg
1059
	 */
1060
	if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
1061
		ret = nouveau_bo_move_prep(drm, bo, new_mem);
1062
		if (ret)
1063
			return ret;
1064 1065
	}

1066
	mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING);
1067
	ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, intr);
1068
	if (ret == 0) {
1069 1070 1071 1072
		ret = drm->ttm.move(chan, bo, &bo->mem, new_mem);
		if (ret == 0) {
			ret = nouveau_fence_new(chan, false, &fence);
			if (ret == 0) {
1073 1074
				ret = ttm_bo_move_accel_cleanup(bo,
								&fence->base,
1075 1076 1077 1078 1079 1080
								evict,
								no_wait_gpu,
								new_mem);
				nouveau_fence_unref(&fence);
			}
		}
1081
	}
1082
	mutex_unlock(&cli->mutex);
1083
	return ret;
1084 1085
}

1086
void
1087
nouveau_bo_move_init(struct nouveau_drm *drm)
1088 1089 1090
{
	static const struct {
		const char *name;
1091
		int engine;
1092 1093 1094 1095 1096 1097
		u32 oclass;
		int (*exec)(struct nouveau_channel *,
			    struct ttm_buffer_object *,
			    struct ttm_mem_reg *, struct ttm_mem_reg *);
		int (*init)(struct nouveau_channel *, u32 handle);
	} _methods[] = {
1098
		{  "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
1099
		{  "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
1100 1101 1102 1103 1104 1105 1106
		{ "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
		{ "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
		{  "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
		{ "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init },
		{  "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init },
		{  "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init },
		{  "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init },
1107
		{},
1108
		{ "CRYPT", 0, 0x88b4, nv98_bo_move_exec, nv50_bo_move_init },
1109 1110 1111 1112 1113
	}, *mthd = _methods;
	const char *name = "CPU";
	int ret;

	do {
1114
		struct nouveau_channel *chan;
1115

1116
		if (mthd->engine)
1117 1118 1119 1120 1121 1122
			chan = drm->cechan;
		else
			chan = drm->channel;
		if (chan == NULL)
			continue;

1123 1124 1125 1126
		ret = nvif_object_init(chan->object, NULL,
				       mthd->oclass | (mthd->engine << 16),
				       mthd->oclass, NULL, 0,
				       &drm->ttm.copy);
1127
		if (ret == 0) {
1128
			ret = mthd->init(chan, drm->ttm.copy.handle);
1129
			if (ret) {
1130
				nvif_object_fini(&drm->ttm.copy);
1131
				continue;
1132
			}
1133 1134

			drm->ttm.move = mthd->exec;
1135
			drm->ttm.chan = chan;
1136 1137
			name = mthd->name;
			break;
1138 1139 1140
		}
	} while ((++mthd)->exec);

1141
	NV_INFO(drm, "MM: using %s for buffer copies\n", name);
1142 1143
}

1144 1145
static int
nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
1146
		      bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1147
{
1148 1149 1150 1151 1152
	struct ttm_place placement_memtype = {
		.fpfn = 0,
		.lpfn = 0,
		.flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
	};
1153 1154 1155 1156 1157
	struct ttm_placement placement;
	struct ttm_mem_reg tmp_mem;
	int ret;

	placement.num_placement = placement.num_busy_placement = 1;
1158
	placement.placement = placement.busy_placement = &placement_memtype;
1159 1160 1161

	tmp_mem = *new_mem;
	tmp_mem.mm_node = NULL;
1162
	ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_gpu);
1163 1164 1165 1166 1167 1168 1169
	if (ret)
		return ret;

	ret = ttm_tt_bind(bo->ttm, &tmp_mem);
	if (ret)
		goto out;

1170
	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, &tmp_mem);
1171 1172 1173
	if (ret)
		goto out;

1174
	ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, new_mem);
1175
out:
1176
	ttm_bo_mem_put(bo, &tmp_mem);
1177 1178 1179 1180 1181
	return ret;
}

static int
nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
1182
		      bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1183
{
1184 1185 1186 1187 1188
	struct ttm_place placement_memtype = {
		.fpfn = 0,
		.lpfn = 0,
		.flags = TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING
	};
1189 1190 1191 1192 1193
	struct ttm_placement placement;
	struct ttm_mem_reg tmp_mem;
	int ret;

	placement.num_placement = placement.num_busy_placement = 1;
1194
	placement.placement = placement.busy_placement = &placement_memtype;
1195 1196 1197

	tmp_mem = *new_mem;
	tmp_mem.mm_node = NULL;
1198
	ret = ttm_bo_mem_space(bo, &placement, &tmp_mem, intr, no_wait_gpu);
1199 1200 1201
	if (ret)
		return ret;

1202
	ret = ttm_bo_move_ttm(bo, true, no_wait_gpu, &tmp_mem);
1203 1204 1205
	if (ret)
		goto out;

1206
	ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, new_mem);
1207 1208 1209 1210
	if (ret)
		goto out;

out:
1211
	ttm_bo_mem_put(bo, &tmp_mem);
1212 1213 1214
	return ret;
}

1215 1216 1217 1218
static void
nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem)
{
	struct nouveau_bo *nvbo = nouveau_bo(bo);
1219 1220
	struct nouveau_vma *vma;

1221 1222 1223 1224
	/* ttm can now (stupidly) pass the driver bos it didn't create... */
	if (bo->destroy != nouveau_bo_del_ttm)
		return;

1225
	list_for_each_entry(vma, &nvbo->vma_list, head) {
1226 1227 1228
		if (new_mem && new_mem->mem_type != TTM_PL_SYSTEM &&
			      (new_mem->mem_type == TTM_PL_VRAM ||
			       nvbo->page_shift != vma->vm->vmm->lpg_shift)) {
1229 1230 1231 1232
			nouveau_vm_map(vma, new_mem->mm_node);
		} else {
			nouveau_vm_unmap(vma);
		}
1233 1234 1235
	}
}

1236
static int
1237
nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_mem_reg *new_mem,
1238
		   struct nouveau_drm_tile **new_tile)
1239
{
1240 1241
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
	struct drm_device *dev = drm->dev;
1242
	struct nouveau_bo *nvbo = nouveau_bo(bo);
1243
	u64 offset = new_mem->start << PAGE_SHIFT;
1244

1245 1246
	*new_tile = NULL;
	if (new_mem->mem_type != TTM_PL_VRAM)
1247 1248
		return 0;

1249
	if (drm->device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
1250
		*new_tile = nv10_bo_set_tiling(dev, offset, new_mem->size,
1251 1252
						nvbo->tile_mode,
						nvbo->tile_flags);
1253 1254
	}

1255 1256 1257 1258 1259
	return 0;
}

static void
nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
1260 1261
		      struct nouveau_drm_tile *new_tile,
		      struct nouveau_drm_tile **old_tile)
1262
{
1263 1264
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
	struct drm_device *dev = drm->dev;
1265
	struct fence *fence = reservation_object_get_excl(bo->resv);
1266

1267
	nv10_bo_put_tile_region(dev, *old_tile, fence);
1268
	*old_tile = new_tile;
1269 1270 1271 1272
}

static int
nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
1273
		bool no_wait_gpu, struct ttm_mem_reg *new_mem)
1274
{
1275
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1276 1277
	struct nouveau_bo *nvbo = nouveau_bo(bo);
	struct ttm_mem_reg *old_mem = &bo->mem;
1278
	struct nouveau_drm_tile *new_tile = NULL;
1279 1280
	int ret = 0;

1281 1282 1283
	if (nvbo->pin_refcnt)
		NV_WARN(drm, "Moving pinned object %p!\n", nvbo);

1284
	if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1285 1286 1287 1288
		ret = nouveau_bo_vm_bind(bo, new_mem, &new_tile);
		if (ret)
			return ret;
	}
1289 1290

	/* Fake bo copy. */
1291 1292 1293 1294
	if (old_mem->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
		BUG_ON(bo->mem.mm_node != NULL);
		bo->mem = *new_mem;
		new_mem->mm_node = NULL;
1295
		goto out;
1296 1297
	}

1298
	/* Hardware assisted copy. */
1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
	if (drm->ttm.move) {
		if (new_mem->mem_type == TTM_PL_SYSTEM)
			ret = nouveau_bo_move_flipd(bo, evict, intr,
						    no_wait_gpu, new_mem);
		else if (old_mem->mem_type == TTM_PL_SYSTEM)
			ret = nouveau_bo_move_flips(bo, evict, intr,
						    no_wait_gpu, new_mem);
		else
			ret = nouveau_bo_move_m2mf(bo, evict, intr,
						   no_wait_gpu, new_mem);
		if (!ret)
			goto out;
	}
1312 1313

	/* Fallback to software copy. */
1314 1315 1316
	ret = ttm_bo_wait(bo, true, intr, no_wait_gpu);
	if (ret == 0)
		ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
1317 1318

out:
1319
	if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
1320 1321 1322 1323 1324
		if (ret)
			nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
		else
			nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
	}
1325 1326

	return ret;
1327 1328 1329 1330 1331
}

static int
nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
{
1332 1333
	struct nouveau_bo *nvbo = nouveau_bo(bo);

1334
	return drm_vma_node_verify_access(&nvbo->gem.vma_node, filp);
1335 1336
}

1337 1338 1339 1340
static int
nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1341
	struct nouveau_drm *drm = nouveau_bdev(bdev);
1342
	struct nouveau_mem *node = mem->mm_node;
1343
	int ret;
1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357

	mem->bus.addr = NULL;
	mem->bus.offset = 0;
	mem->bus.size = mem->num_pages << PAGE_SHIFT;
	mem->bus.base = 0;
	mem->bus.is_iomem = false;
	if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
		return -EINVAL;
	switch (mem->mem_type) {
	case TTM_PL_SYSTEM:
		/* System memory */
		return 0;
	case TTM_PL_TT:
#if __OS_HAS_AGP
1358
		if (drm->agp.stat == ENABLED) {
1359
			mem->bus.offset = mem->start << PAGE_SHIFT;
1360
			mem->bus.base = drm->agp.base;
1361
			mem->bus.is_iomem = !drm->dev->agp->cant_use_aperture;
1362 1363
		}
#endif
1364
		if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA || !node->memtype)
1365 1366 1367
			/* untiled */
			break;
		/* fallthrough, tiled memory */
1368
	case TTM_PL_VRAM:
1369
		mem->bus.offset = mem->start << PAGE_SHIFT;
1370
		mem->bus.base = nv_device_resource_start(nvkm_device(&drm->device), 1);
1371
		mem->bus.is_iomem = true;
1372 1373
		if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
			struct nouveau_bar *bar = nvkm_bar(&drm->device);
1374

1375
			ret = bar->umap(bar, node, NV_MEM_ACCESS_RW,
1376 1377 1378
					&node->bar_vma);
			if (ret)
				return ret;
1379

1380
			mem->bus.offset = node->bar_vma.offset;
1381
		}
1382 1383 1384 1385 1386 1387 1388 1389 1390 1391
		break;
	default:
		return -EINVAL;
	}
	return 0;
}

static void
nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
1392
	struct nouveau_drm *drm = nouveau_bdev(bdev);
1393
	struct nouveau_bar *bar = nvkm_bar(&drm->device);
1394
	struct nouveau_mem *node = mem->mm_node;
1395

1396
	if (!node->bar_vma.node)
1397 1398
		return;

1399
	bar->unmap(bar, &node->bar_vma);
1400 1401 1402 1403 1404
}

static int
nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
{
1405
	struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
1406
	struct nouveau_bo *nvbo = nouveau_bo(bo);
1407 1408
	struct nvif_device *device = &drm->device;
	u32 mappable = nv_device_resource_len(nvkm_device(device), 1) >> PAGE_SHIFT;
1409
	int i, ret;
1410 1411 1412 1413 1414

	/* as long as the bo isn't in vram, and isn't tiled, we've got
	 * nothing to do here.
	 */
	if (bo->mem.mem_type != TTM_PL_VRAM) {
1415
		if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA ||
1416
		    !nouveau_bo_tile_layout(nvbo))
1417
			return 0;
1418 1419 1420 1421 1422 1423 1424 1425 1426

		if (bo->mem.mem_type == TTM_PL_SYSTEM) {
			nouveau_bo_placement_set(nvbo, TTM_PL_TT, 0);

			ret = nouveau_bo_validate(nvbo, false, false);
			if (ret)
				return ret;
		}
		return 0;
1427 1428 1429
	}

	/* make sure bo is in mappable vram */
1430
	if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA ||
1431
	    bo->mem.start + bo->mem.num_pages < mappable)
1432 1433
		return 0;

1434 1435 1436 1437 1438 1439 1440 1441 1442
	for (i = 0; i < nvbo->placement.num_placement; ++i) {
		nvbo->placements[i].fpfn = 0;
		nvbo->placements[i].lpfn = mappable;
	}

	for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
		nvbo->busy_placements[i].fpfn = 0;
		nvbo->busy_placements[i].lpfn = mappable;
	}
1443

1444
	nouveau_bo_placement_set(nvbo, TTM_PL_FLAG_VRAM, 0);
1445
	return nouveau_bo_validate(nvbo, false, false);
1446 1447
}

1448 1449 1450
static int
nouveau_ttm_tt_populate(struct ttm_tt *ttm)
{
1451
	struct ttm_dma_tt *ttm_dma = (void *)ttm;
1452
	struct nouveau_drm *drm;
A
Alexandre Courbot 已提交
1453
	struct nouveau_device *device;
1454
	struct drm_device *dev;
1455
	struct device *pdev;
1456 1457
	unsigned i;
	int r;
D
Dave Airlie 已提交
1458
	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
1459 1460 1461 1462

	if (ttm->state != tt_unpopulated)
		return 0;

D
Dave Airlie 已提交
1463 1464 1465 1466 1467 1468 1469 1470
	if (slave && ttm->sg) {
		/* make userspace faulting work */
		drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
						 ttm_dma->dma_address, ttm->num_pages);
		ttm->state = tt_unbound;
		return 0;
	}

1471
	drm = nouveau_bdev(ttm->bdev);
1472
	device = nvkm_device(&drm->device);
1473
	dev = drm->dev;
1474
	pdev = nv_device_base(device);
1475

1476 1477 1478 1479 1480 1481 1482 1483
	/*
	 * Objects matching this condition have been marked as force_coherent,
	 * so use the DMA API for them.
	 */
	if (!nv_device_is_cpu_coherent(device) &&
	    ttm->caching_state == tt_uncached)
		return ttm_dma_populate(ttm_dma, dev->dev);

J
Jerome Glisse 已提交
1484
#if __OS_HAS_AGP
1485
	if (drm->agp.stat == ENABLED) {
J
Jerome Glisse 已提交
1486 1487 1488 1489
		return ttm_agp_tt_populate(ttm);
	}
#endif

1490 1491
#ifdef CONFIG_SWIOTLB
	if (swiotlb_nr_tbl()) {
1492
		return ttm_dma_populate((void *)ttm, dev->dev);
1493 1494 1495 1496 1497 1498 1499 1500 1501
	}
#endif

	r = ttm_pool_populate(ttm);
	if (r) {
		return r;
	}

	for (i = 0; i < ttm->num_pages; i++) {
1502 1503 1504 1505 1506 1507
		dma_addr_t addr;

		addr = dma_map_page(pdev, ttm->pages[i], 0, PAGE_SIZE,
				    DMA_BIDIRECTIONAL);

		if (dma_mapping_error(pdev, addr)) {
1508
			while (--i) {
1509 1510
				dma_unmap_page(pdev, ttm_dma->dma_address[i],
					       PAGE_SIZE, DMA_BIDIRECTIONAL);
1511
				ttm_dma->dma_address[i] = 0;
1512 1513 1514 1515
			}
			ttm_pool_unpopulate(ttm);
			return -EFAULT;
		}
1516 1517

		ttm_dma->dma_address[i] = addr;
1518 1519 1520 1521 1522 1523 1524
	}
	return 0;
}

static void
nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
{
1525
	struct ttm_dma_tt *ttm_dma = (void *)ttm;
1526
	struct nouveau_drm *drm;
A
Alexandre Courbot 已提交
1527
	struct nouveau_device *device;
1528
	struct drm_device *dev;
1529
	struct device *pdev;
1530
	unsigned i;
D
Dave Airlie 已提交
1531 1532 1533 1534
	bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);

	if (slave)
		return;
1535

1536
	drm = nouveau_bdev(ttm->bdev);
1537
	device = nvkm_device(&drm->device);
1538
	dev = drm->dev;
1539
	pdev = nv_device_base(device);
1540

1541 1542 1543 1544 1545 1546 1547 1548
	/*
	 * Objects matching this condition have been marked as force_coherent,
	 * so use the DMA API for them.
	 */
	if (!nv_device_is_cpu_coherent(device) &&
	    ttm->caching_state == tt_uncached)
		ttm_dma_unpopulate(ttm_dma, dev->dev);

J
Jerome Glisse 已提交
1549
#if __OS_HAS_AGP
1550
	if (drm->agp.stat == ENABLED) {
J
Jerome Glisse 已提交
1551 1552 1553 1554 1555
		ttm_agp_tt_unpopulate(ttm);
		return;
	}
#endif

1556 1557
#ifdef CONFIG_SWIOTLB
	if (swiotlb_nr_tbl()) {
1558
		ttm_dma_unpopulate((void *)ttm, dev->dev);
1559 1560 1561 1562 1563
		return;
	}
#endif

	for (i = 0; i < ttm->num_pages; i++) {
1564
		if (ttm_dma->dma_address[i]) {
1565 1566
			dma_unmap_page(pdev, ttm_dma->dma_address[i], PAGE_SIZE,
				       DMA_BIDIRECTIONAL);
1567 1568 1569 1570 1571 1572
		}
	}

	ttm_pool_unpopulate(ttm);
}

1573
void
1574
nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive)
1575
{
1576
	struct reservation_object *resv = nvbo->bo.resv;
1577

1578 1579 1580 1581
	if (exclusive)
		reservation_object_add_excl_fence(resv, &fence->base);
	else if (fence)
		reservation_object_add_shared_fence(resv, &fence->base);
1582 1583
}

1584
struct ttm_bo_driver nouveau_bo_driver = {
1585
	.ttm_tt_create = &nouveau_ttm_tt_create,
1586 1587
	.ttm_tt_populate = &nouveau_ttm_tt_populate,
	.ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
1588 1589 1590
	.invalidate_caches = nouveau_bo_invalidate_caches,
	.init_mem_type = nouveau_bo_init_mem_type,
	.evict_flags = nouveau_bo_evict_flags,
1591
	.move_notify = nouveau_bo_move_ntfy,
1592 1593
	.move = nouveau_bo_move,
	.verify_access = nouveau_bo_verify_access,
1594 1595 1596
	.fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
	.io_mem_reserve = &nouveau_ttm_io_mem_reserve,
	.io_mem_free = &nouveau_ttm_io_mem_free,
1597 1598
};

1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622
struct nouveau_vma *
nouveau_bo_vma_find(struct nouveau_bo *nvbo, struct nouveau_vm *vm)
{
	struct nouveau_vma *vma;
	list_for_each_entry(vma, &nvbo->vma_list, head) {
		if (vma->vm == vm)
			return vma;
	}

	return NULL;
}

int
nouveau_bo_vma_add(struct nouveau_bo *nvbo, struct nouveau_vm *vm,
		   struct nouveau_vma *vma)
{
	const u32 size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
	int ret;

	ret = nouveau_vm_get(vm, size, nvbo->page_shift,
			     NV_MEM_ACCESS_RW, vma);
	if (ret)
		return ret;

1623 1624 1625
	if ( nvbo->bo.mem.mem_type != TTM_PL_SYSTEM &&
	    (nvbo->bo.mem.mem_type == TTM_PL_VRAM ||
	     nvbo->page_shift != vma->vm->vmm->lpg_shift))
1626 1627 1628
		nouveau_vm_map(vma, nvbo->bo.mem.mm_node);

	list_add_tail(&vma->head, &nvbo->vma_list);
1629
	vma->refcount = 1;
1630 1631 1632 1633 1634 1635 1636
	return 0;
}

void
nouveau_bo_vma_del(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
{
	if (vma->node) {
1637
		if (nvbo->bo.mem.mem_type != TTM_PL_SYSTEM)
1638 1639 1640 1641 1642
			nouveau_vm_unmap(vma);
		nouveau_vm_put(vma);
		list_del(&vma->head);
	}
}