ttm_bo.c 43.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
/**************************************************************************
 *
 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **************************************************************************/
/*
 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
 */

J
Joe Perches 已提交
31 32
#define pr_fmt(fmt) "[TTM] " fmt

33 34 35
#include <drm/ttm/ttm_module.h>
#include <drm/ttm/ttm_bo_driver.h>
#include <drm/ttm/ttm_placement.h>
36 37 38 39 40 41
#include <linux/jiffies.h>
#include <linux/slab.h>
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/file.h>
#include <linux/module.h>
A
Arun Sharma 已提交
42
#include <linux/atomic.h>
43
#include <linux/reservation.h>
44 45

static int ttm_bo_swapout(struct ttm_mem_shrink *shrink);
46 47 48 49 50 51 52
static void ttm_bo_global_kobj_release(struct kobject *kobj);

static struct attribute ttm_bo_count = {
	.name = "bo_count",
	.mode = S_IRUGO
};

53 54
static inline int ttm_mem_type_from_place(const struct ttm_place *place,
					  uint32_t *mem_type)
55
{
56
	int pos;
57

58 59 60 61 62 63
	pos = ffs(place->flags & TTM_PL_MASK_MEM);
	if (unlikely(!pos))
		return -EINVAL;

	*mem_type = pos - 1;
	return 0;
64 65
}

66
static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
67
{
68
	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
69
	struct drm_printer p = drm_debug_printer(TTM_PFX);
70

J
Joe Perches 已提交
71 72 73
	pr_err("    has_type: %d\n", man->has_type);
	pr_err("    use_type: %d\n", man->use_type);
	pr_err("    flags: 0x%08X\n", man->flags);
74
	pr_err("    gpu_offset: 0x%08llX\n", man->gpu_offset);
J
Joe Perches 已提交
75 76 77
	pr_err("    size: %llu\n", man->size);
	pr_err("    available_caching: 0x%08X\n", man->available_caching);
	pr_err("    default_caching: 0x%08X\n", man->default_caching);
78
	if (mem_type != TTM_PL_SYSTEM)
79
		(*man->func->debug)(man, &p);
80 81 82 83 84 85 86
}

static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
					struct ttm_placement *placement)
{
	int i, ret, mem_type;

J
Joe Perches 已提交
87 88 89
	pr_err("No space for %p (%lu pages, %luK, %luM)\n",
	       bo, bo->mem.num_pages, bo->mem.size >> 10,
	       bo->mem.size >> 20);
90
	for (i = 0; i < placement->num_placement; i++) {
91
		ret = ttm_mem_type_from_place(&placement->placement[i],
92 93 94
						&mem_type);
		if (ret)
			return;
J
Joe Perches 已提交
95
		pr_err("  placement[%d]=0x%08X (%d)\n",
96
		       i, placement->placement[i].flags, mem_type);
97
		ttm_mem_type_debug(bo->bdev, mem_type);
98 99 100
	}
}

101 102 103 104 105 106 107
static ssize_t ttm_bo_global_show(struct kobject *kobj,
				  struct attribute *attr,
				  char *buffer)
{
	struct ttm_bo_global *glob =
		container_of(kobj, struct ttm_bo_global, kobj);

108 109
	return snprintf(buffer, PAGE_SIZE, "%d\n",
				atomic_read(&glob->bo_count));
110 111 112 113 114 115 116
}

static struct attribute *ttm_bo_global_attrs[] = {
	&ttm_bo_count,
	NULL
};

117
static const struct sysfs_ops ttm_bo_global_ops = {
118 119 120 121 122 123 124 125 126
	.show = &ttm_bo_global_show
};

static struct kobj_type ttm_bo_glob_kobj_type  = {
	.release = &ttm_bo_global_kobj_release,
	.sysfs_ops = &ttm_bo_global_ops,
	.default_attrs = ttm_bo_global_attrs
};

127 128 129 130 131 132 133 134 135 136 137

static inline uint32_t ttm_bo_type_flags(unsigned type)
{
	return 1 << (type);
}

static void ttm_bo_release_list(struct kref *list_kref)
{
	struct ttm_buffer_object *bo =
	    container_of(list_kref, struct ttm_buffer_object, list_kref);
	struct ttm_bo_device *bdev = bo->bdev;
138
	size_t acc_size = bo->acc_size;
139

140 141
	BUG_ON(kref_read(&bo->list_kref));
	BUG_ON(kref_read(&bo->kref));
142 143 144 145
	BUG_ON(atomic_read(&bo->cpu_writers));
	BUG_ON(bo->mem.mm_node != NULL);
	BUG_ON(!list_empty(&bo->lru));
	BUG_ON(!list_empty(&bo->ddestroy));
146
	ttm_tt_destroy(bo->ttm);
147
	atomic_dec(&bo->glob->bo_count);
148
	dma_fence_put(bo->moving);
149
	reservation_object_fini(&bo->ttm_resv);
150
	mutex_destroy(&bo->wu_mutex);
151 152 153 154 155
	if (bo->destroy)
		bo->destroy(bo);
	else {
		kfree(bo);
	}
156
	ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
157 158
}

159
void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
160 161
{
	struct ttm_bo_device *bdev = bo->bdev;
162
	struct ttm_mem_type_manager *man;
163

164
	reservation_object_assert_held(bo->resv);
165 166 167 168 169

	if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {

		BUG_ON(!list_empty(&bo->lru));

170 171
		man = &bdev->man[bo->mem.mem_type];
		list_add_tail(&bo->lru, &man->lru[bo->priority]);
172 173
		kref_get(&bo->list_kref);

174
		if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) {
175 176
			list_add_tail(&bo->swap,
				      &bo->glob->swap_lru[bo->priority]);
177 178 179 180
			kref_get(&bo->list_kref);
		}
	}
}
181
EXPORT_SYMBOL(ttm_bo_add_to_lru);
182

183 184 185 186 187 188
static void ttm_bo_ref_bug(struct kref *list_kref)
{
	BUG();
}

void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
189 190 191
{
	if (!list_empty(&bo->swap)) {
		list_del_init(&bo->swap);
192
		kref_put(&bo->list_kref, ttm_bo_ref_bug);
193 194 195
	}
	if (!list_empty(&bo->lru)) {
		list_del_init(&bo->lru);
196
		kref_put(&bo->list_kref, ttm_bo_ref_bug);
197 198
	}

199 200 201 202
	/*
	 * TODO: Add a driver hook to delete from
	 * driver-specific LRU's here.
	 */
203 204
}

205
void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo)
206
{
207
	spin_lock(&bo->glob->lru_lock);
208
	ttm_bo_del_from_lru(bo);
209
	spin_unlock(&bo->glob->lru_lock);
210
}
211
EXPORT_SYMBOL(ttm_bo_del_sub_from_lru);
212

213 214
void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo)
{
215
	reservation_object_assert_held(bo->resv);
216

217
	ttm_bo_del_from_lru(bo);
218
	ttm_bo_add_to_lru(bo);
219 220 221
}
EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);

222 223 224 225 226 227
/*
 * Call bo->mutex locked.
 */
static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
{
	struct ttm_bo_device *bdev = bo->bdev;
228
	struct ttm_bo_global *glob = bo->glob;
229 230 231
	int ret = 0;
	uint32_t page_flags = 0;

232
	reservation_object_assert_held(bo->resv);
233 234
	bo->ttm = NULL;

D
Dave Airlie 已提交
235 236 237
	if (bdev->need_dma32)
		page_flags |= TTM_PAGE_FLAG_DMA32;

238 239 240 241 242
	switch (bo->type) {
	case ttm_bo_type_device:
		if (zero_alloc)
			page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC;
	case ttm_bo_type_kernel:
243 244
		bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
						      page_flags, glob->dummy_read_page);
245 246 247
		if (unlikely(bo->ttm == NULL))
			ret = -ENOMEM;
		break;
248 249 250 251 252 253 254 255 256 257
	case ttm_bo_type_sg:
		bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
						      page_flags | TTM_PAGE_FLAG_SG,
						      glob->dummy_read_page);
		if (unlikely(bo->ttm == NULL)) {
			ret = -ENOMEM;
			break;
		}
		bo->ttm->sg = bo->sg;
		break;
258
	default:
J
Joe Perches 已提交
259
		pr_err("Illegal buffer object type\n");
260 261 262 263 264 265 266 267
		ret = -EINVAL;
		break;
	}

	return ret;
}

static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
268 269
				  struct ttm_mem_reg *mem, bool evict,
				  struct ttm_operation_ctx *ctx)
270 271 272 273 274 275 276 277 278
{
	struct ttm_bo_device *bdev = bo->bdev;
	bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
	bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
	struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
	struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
	int ret = 0;

	if (old_is_pci || new_is_pci ||
279 280 281 282 283 284 285
	    ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) {
		ret = ttm_mem_io_lock(old_man, true);
		if (unlikely(ret != 0))
			goto out_err;
		ttm_bo_unmap_virtual_locked(bo);
		ttm_mem_io_unlock(old_man);
	}
286 287 288 289 290

	/*
	 * Create and bind a ttm if required.
	 */

291 292
	if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
		if (bo->ttm == NULL) {
293 294
			bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED);
			ret = ttm_bo_add_ttm(bo, zero);
295 296 297
			if (ret)
				goto out_err;
		}
298 299 300

		ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
		if (ret)
301
			goto out_err;
302 303 304 305 306 307 308 309

		if (mem->mem_type != TTM_PL_SYSTEM) {
			ret = ttm_tt_bind(bo->ttm, mem);
			if (ret)
				goto out_err;
		}

		if (bo->mem.mem_type == TTM_PL_SYSTEM) {
310
			if (bdev->driver->move_notify)
311
				bdev->driver->move_notify(bo, evict, mem);
312
			bo->mem = *mem;
313 314 315 316 317
			mem->mm_node = NULL;
			goto moved;
		}
	}

318
	if (bdev->driver->move_notify)
319
		bdev->driver->move_notify(bo, evict, mem);
320

321 322
	if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
	    !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
323
		ret = ttm_bo_move_ttm(bo, ctx, mem);
324
	else if (bdev->driver->move)
325
		ret = bdev->driver->move(bo, evict, ctx, mem);
326
	else
327
		ret = ttm_bo_move_memcpy(bo, ctx, mem);
328

329 330 331 332 333
	if (ret) {
		if (bdev->driver->move_notify) {
			struct ttm_mem_reg tmp_mem = *mem;
			*mem = bo->mem;
			bo->mem = tmp_mem;
334
			bdev->driver->move_notify(bo, false, mem);
335
			bo->mem = *mem;
336
			*mem = tmp_mem;
337
		}
338

339 340
		goto out_err;
	}
341

342 343
moved:
	if (bo->evicted) {
344 345 346 347 348
		if (bdev->driver->invalidate_caches) {
			ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
			if (ret)
				pr_err("Can not flush read caches\n");
		}
349 350 351
		bo->evicted = false;
	}

C
Christian König 已提交
352
	if (bo->mem.mm_node)
353
		bo->offset = (bo->mem.start << PAGE_SHIFT) +
354
		    bdev->man[bo->mem.mem_type].gpu_offset;
C
Christian König 已提交
355
	else
356
		bo->offset = 0;
357

358
	ctx->bytes_moved += bo->num_pages << PAGE_SHIFT;
359 360 361 362
	return 0;

out_err:
	new_man = &bdev->man[bo->mem.mem_type];
363
	if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) {
364 365 366 367 368 369 370
		ttm_tt_destroy(bo->ttm);
		bo->ttm = NULL;
	}

	return ret;
}

371
/**
372
 * Call bo::reserved.
373
 * Will release GPU memory type usage on destruction.
374 375 376
 * This is the place to put in driver specific hooks to release
 * driver private resources.
 * Will release the bo::reserved lock.
377 378 379 380
 */

static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
{
381
	if (bo->bdev->driver->move_notify)
382
		bo->bdev->driver->move_notify(bo, false, NULL);
383

384 385
	ttm_tt_destroy(bo->ttm);
	bo->ttm = NULL;
386
	ttm_bo_mem_put(bo, &bo->mem);
387 388
}

389 390 391 392 393 394 395
static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo)
{
	int r;

	if (bo->resv == &bo->ttm_resv)
		return 0;

396
	BUG_ON(!reservation_object_trylock(&bo->ttm_resv));
397 398

	r = reservation_object_copy_fences(&bo->ttm_resv, bo->resv);
399
	if (r)
400 401 402 403 404
		reservation_object_unlock(&bo->ttm_resv);

	return r;
}

405 406 407
static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo)
{
	struct reservation_object_list *fobj;
408
	struct dma_fence *fence;
409 410
	int i;

411 412
	fobj = reservation_object_get_list(&bo->ttm_resv);
	fence = reservation_object_get_excl(&bo->ttm_resv);
413
	if (fence && !fence->ops->signaled)
414
		dma_fence_enable_sw_signaling(fence);
415 416 417 418 419 420

	for (i = 0; fobj && i < fobj->shared_count; ++i) {
		fence = rcu_dereference_protected(fobj->shared[i],
					reservation_object_held(bo->resv));

		if (!fence->ops->signaled)
421
			dma_fence_enable_sw_signaling(fence);
422 423 424
	}
}

425
static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
426 427
{
	struct ttm_bo_device *bdev = bo->bdev;
428
	struct ttm_bo_global *glob = bo->glob;
429 430
	int ret;

431 432 433 434 435 436 437 438 439 440 441
	ret = ttm_bo_individualize_resv(bo);
	if (ret) {
		/* Last resort, if we fail to allocate memory for the
		 * fences block for the BO to become idle
		 */
		reservation_object_wait_timeout_rcu(bo->resv, true, false,
						    30 * HZ);
		spin_lock(&glob->lru_lock);
		goto error;
	}

442
	spin_lock(&glob->lru_lock);
443
	ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY;
M
Maarten Lankhorst 已提交
444
	if (!ret) {
445
		if (reservation_object_test_signaled_rcu(&bo->ttm_resv, true)) {
446
			ttm_bo_del_from_lru(bo);
M
Maarten Lankhorst 已提交
447
			spin_unlock(&glob->lru_lock);
448
			if (bo->resv != &bo->ttm_resv)
449
				reservation_object_unlock(&bo->ttm_resv);
450

M
Maarten Lankhorst 已提交
451
			ttm_bo_cleanup_memtype_use(bo);
452
			reservation_object_unlock(bo->resv);
M
Maarten Lankhorst 已提交
453
			return;
454 455 456
		}

		ttm_bo_flush_all_fences(bo);
457 458 459 460 461 462 463 464 465 466 467

		/*
		 * Make NO_EVICT bos immediately available to
		 * shrinkers, now that they are queued for
		 * destruction.
		 */
		if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
			bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
			ttm_bo_add_to_lru(bo);
		}

468
		reservation_object_unlock(bo->resv);
469
	}
470 471
	if (bo->resv != &bo->ttm_resv)
		reservation_object_unlock(&bo->ttm_resv);
472

473
error:
474 475 476 477 478 479 480 481 482
	kref_get(&bo->list_kref);
	list_add_tail(&bo->ddestroy, &bdev->ddestroy);
	spin_unlock(&glob->lru_lock);

	schedule_delayed_work(&bdev->wq,
			      ((HZ / 100) < 1) ? 1 : HZ / 100);
}

/**
483
 * function ttm_bo_cleanup_refs
484 485 486
 * If bo idle, remove from delayed- and lru lists, and unref.
 * If not idle, do nothing.
 *
487
 * Must be called with lru_lock and reservation held, this function
488
 * will drop the lru lock and optionally the reservation lock before returning.
489
 *
490 491
 * @interruptible         Any sleeps should occur interruptibly.
 * @no_wait_gpu           Never wait for gpu. Return -EBUSY instead.
492
 * @unlock_resv           Unlock the reservation lock as well.
493 494
 */

495 496 497
static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
			       bool interruptible, bool no_wait_gpu,
			       bool unlock_resv)
498 499
{
	struct ttm_bo_global *glob = bo->glob;
500
	struct reservation_object *resv;
501
	int ret;
502

503 504 505 506 507 508 509 510 511
	if (unlikely(list_empty(&bo->ddestroy)))
		resv = bo->resv;
	else
		resv = &bo->ttm_resv;

	if (reservation_object_test_signaled_rcu(resv, true))
		ret = 0;
	else
		ret = -EBUSY;
512

513
	if (ret && !no_wait_gpu) {
M
Maarten Lankhorst 已提交
514
		long lret;
515

516 517
		if (unlock_resv)
			reservation_object_unlock(bo->resv);
M
Maarten Lankhorst 已提交
518 519
		spin_unlock(&glob->lru_lock);

520
		lret = reservation_object_wait_timeout_rcu(resv, true,
M
Maarten Lankhorst 已提交
521 522 523 524 525 526 527
							   interruptible,
							   30 * HZ);

		if (lret < 0)
			return lret;
		else if (lret == 0)
			return -EBUSY;
528

529
		spin_lock(&glob->lru_lock);
530 531 532 533 534 535 536 537 538
		if (unlock_resv && !reservation_object_trylock(bo->resv)) {
			/*
			 * We raced, and lost, someone else holds the reservation now,
			 * and is probably busy in ttm_bo_cleanup_memtype_use.
			 *
			 * Even if it's not the case, because we finished waiting any
			 * delayed destruction would succeed, so just return success
			 * here.
			 */
539 540 541
			spin_unlock(&glob->lru_lock);
			return 0;
		}
542
		ret = 0;
543
	}
544

545
	if (ret || unlikely(list_empty(&bo->ddestroy))) {
546 547
		if (unlock_resv)
			reservation_object_unlock(bo->resv);
548
		spin_unlock(&glob->lru_lock);
549
		return ret;
550 551
	}

552
	ttm_bo_del_from_lru(bo);
553
	list_del_init(&bo->ddestroy);
554
	kref_put(&bo->list_kref, ttm_bo_ref_bug);
555 556 557

	spin_unlock(&glob->lru_lock);
	ttm_bo_cleanup_memtype_use(bo);
558 559 560

	if (unlock_resv)
		reservation_object_unlock(bo->resv);
561 562

	return 0;
563 564 565 566 567 568
}

/**
 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 * encountered buffers.
 */
569
static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
570
{
571
	struct ttm_bo_global *glob = bdev->glob;
572 573
	struct list_head removed;
	bool empty;
574

575
	INIT_LIST_HEAD(&removed);
576

577 578 579
	spin_lock(&glob->lru_lock);
	while (!list_empty(&bdev->ddestroy)) {
		struct ttm_buffer_object *bo;
580

581 582 583 584
		bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object,
				      ddestroy);
		kref_get(&bo->list_kref);
		list_move_tail(&bo->ddestroy, &removed);
585

586 587 588
		if (remove_all || bo->resv != &bo->ttm_resv) {
			spin_unlock(&glob->lru_lock);
			reservation_object_lock(bo->resv, NULL);
589

590 591 592 593 594 595
			spin_lock(&glob->lru_lock);
			ttm_bo_cleanup_refs(bo, false, !remove_all, true);

		} else if (reservation_object_trylock(bo->resv)) {
			ttm_bo_cleanup_refs(bo, false, !remove_all, true);
		}
596

597
		kref_put(&bo->list_kref, ttm_bo_release_list);
598
		spin_lock(&glob->lru_lock);
599
	}
600 601
	list_splice_tail(&removed, &bdev->ddestroy);
	empty = list_empty(&bdev->ddestroy);
602
	spin_unlock(&glob->lru_lock);
603 604

	return empty;
605 606 607 608 609 610 611
}

static void ttm_bo_delayed_workqueue(struct work_struct *work)
{
	struct ttm_bo_device *bdev =
	    container_of(work, struct ttm_bo_device, wq.work);

612
	if (!ttm_bo_delayed_delete(bdev, false)) {
613 614 615 616 617 618 619 620 621 622
		schedule_delayed_work(&bdev->wq,
				      ((HZ / 100) < 1) ? 1 : HZ / 100);
	}
}

static void ttm_bo_release(struct kref *kref)
{
	struct ttm_buffer_object *bo =
	    container_of(kref, struct ttm_buffer_object, kref);
	struct ttm_bo_device *bdev = bo->bdev;
623
	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];
624

625
	drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node);
626 627 628
	ttm_mem_io_lock(man, false);
	ttm_mem_io_free_vm(bo);
	ttm_mem_io_unlock(man);
629
	ttm_bo_cleanup_refs_or_queue(bo);
630 631 632 633 634 635 636 637 638 639 640 641
	kref_put(&bo->list_kref, ttm_bo_release_list);
}

void ttm_bo_unref(struct ttm_buffer_object **p_bo)
{
	struct ttm_buffer_object *bo = *p_bo;

	*p_bo = NULL;
	kref_put(&bo->kref, ttm_bo_release);
}
EXPORT_SYMBOL(ttm_bo_unref);

642 643 644 645 646 647 648 649 650 651 652 653 654 655
int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
{
	return cancel_delayed_work_sync(&bdev->wq);
}
EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);

void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
{
	if (resched)
		schedule_delayed_work(&bdev->wq,
				      ((HZ / 100) < 1) ? 1 : HZ / 100);
}
EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);

656 657
static int ttm_bo_evict(struct ttm_buffer_object *bo,
			struct ttm_operation_ctx *ctx)
658 659 660
{
	struct ttm_bo_device *bdev = bo->bdev;
	struct ttm_mem_reg evict_mem;
661 662
	struct ttm_placement placement;
	int ret = 0;
663

664
	reservation_object_assert_held(bo->resv);
665 666 667

	evict_mem = bo->mem;
	evict_mem.mm_node = NULL;
668 669
	evict_mem.bus.io_reserved_vm = false;
	evict_mem.bus.io_reserved_count = 0;
670

671 672
	placement.num_placement = 0;
	placement.num_busy_placement = 0;
673
	bdev->driver->evict_flags(bo, &placement);
674
	ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx);
675
	if (ret) {
676
		if (ret != -ERESTARTSYS) {
J
Joe Perches 已提交
677 678
			pr_err("Failed to find memory space for buffer 0x%p eviction\n",
			       bo);
679 680
			ttm_bo_mem_space_debug(bo, &placement);
		}
681 682 683
		goto out;
	}

684
	ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx);
685
	if (unlikely(ret)) {
686
		if (ret != -ERESTARTSYS)
J
Joe Perches 已提交
687
			pr_err("Buffer eviction failed\n");
688
		ttm_bo_mem_put(bo, &evict_mem);
689 690
		goto out;
	}
691 692 693 694 695
	bo->evicted = true;
out:
	return ret;
}

696 697 698 699 700 701 702 703 704 705 706 707 708 709
bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
			      const struct ttm_place *place)
{
	/* Don't evict this BO if it's outside of the
	 * requested placement range
	 */
	if (place->fpfn >= (bo->mem.start + bo->mem.size) ||
	    (place->lpfn && place->lpfn <= bo->mem.start))
		return false;

	return true;
}
EXPORT_SYMBOL(ttm_bo_eviction_valuable);

710
static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
711 712
			       uint32_t mem_type,
			       const struct ttm_place *place,
713
			       struct ttm_operation_ctx *ctx)
714 715 716
{
	struct ttm_bo_global *glob = bdev->glob;
	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
717 718
	struct ttm_buffer_object *bo = NULL;
	bool locked = false;
719
	unsigned i;
720
	int ret;
721

722
	spin_lock(&glob->lru_lock);
723 724
	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
		list_for_each_entry(bo, &man->lru[i], lru) {
R
Roger He 已提交
725 726 727
			if (bo->resv == ctx->resv) {
				if (!ctx->allow_reserved_eviction &&
				    list_empty(&bo->ddestroy))
728 729 730 731 732 733
					continue;
			} else {
				locked = reservation_object_trylock(bo->resv);
				if (!locked)
					continue;
			}
734

735 736
			if (place && !bdev->driver->eviction_valuable(bo,
								      place)) {
737 738
				if (locked)
					reservation_object_unlock(bo->resv);
739
				locked = false;
740 741 742
				continue;
			}
			break;
743
		}
744

745 746
		/* If the inner loop terminated early, we have our candidate */
		if (&bo->lru != &man->lru[i])
747
			break;
748 749

		bo = NULL;
750 751
	}

752
	if (!bo) {
753
		spin_unlock(&glob->lru_lock);
754
		return -EBUSY;
755 756
	}

757
	kref_get(&bo->list_kref);
758

759
	if (!list_empty(&bo->ddestroy)) {
760 761
		ret = ttm_bo_cleanup_refs(bo, ctx->interruptible,
					  ctx->no_wait_gpu, locked);
762
		kref_put(&bo->list_kref, ttm_bo_release_list);
763
		return ret;
764 765
	}

766
	ttm_bo_del_from_lru(bo);
767
	spin_unlock(&glob->lru_lock);
768

769
	ret = ttm_bo_evict(bo, ctx);
770
	if (locked) {
771
		ttm_bo_unreserve(bo);
772 773
	} else {
		spin_lock(&glob->lru_lock);
774
		ttm_bo_add_to_lru(bo);
775 776
		spin_unlock(&glob->lru_lock);
	}
777

778
	kref_put(&bo->list_kref, ttm_bo_release_list);
779 780 781
	return ret;
}

782 783
void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
{
784
	struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type];
785

786 787
	if (mem->mm_node)
		(*man->func->put_node)(man, mem);
788 789 790
}
EXPORT_SYMBOL(ttm_bo_mem_put);

791 792 793 794 795 796 797
/**
 * Add the last move fence to the BO and reserve a new shared slot.
 */
static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
				 struct ttm_mem_type_manager *man,
				 struct ttm_mem_reg *mem)
{
798
	struct dma_fence *fence;
799 800 801
	int ret;

	spin_lock(&man->move_lock);
802
	fence = dma_fence_get(man->move);
803 804 805 806 807 808 809 810 811
	spin_unlock(&man->move_lock);

	if (fence) {
		reservation_object_add_shared_fence(bo->resv, fence);

		ret = reservation_object_reserve_shared(bo->resv);
		if (unlikely(ret))
			return ret;

812
		dma_fence_put(bo->moving);
813 814 815 816 817 818
		bo->moving = fence;
	}

	return 0;
}

819 820 821 822
/**
 * Repeatedly evict memory from the LRU for @mem_type until we create enough
 * space, or we've evicted everything and there isn't enough space.
 */
823 824
static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
					uint32_t mem_type,
825
					const struct ttm_place *place,
826
					struct ttm_mem_reg *mem,
827
					struct ttm_operation_ctx *ctx)
828
{
829
	struct ttm_bo_device *bdev = bo->bdev;
830 831 832 833
	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
	int ret;

	do {
834
		ret = (*man->func->get_node)(man, bo, place, mem);
835 836
		if (unlikely(ret != 0))
			return ret;
837
		if (mem->mm_node)
838
			break;
R
Roger He 已提交
839
		ret = ttm_mem_evict_first(bdev, mem_type, place, ctx);
840 841 842 843
		if (unlikely(ret != 0))
			return ret;
	} while (1);
	mem->mem_type = mem_type;
844
	return ttm_bo_add_move_fence(bo, man, mem);
845 846
}

847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871
static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
				      uint32_t cur_placement,
				      uint32_t proposed_placement)
{
	uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
	uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;

	/**
	 * Keep current caching if possible.
	 */

	if ((cur_placement & caching) != 0)
		result |= (cur_placement & caching);
	else if ((man->default_caching & caching) != 0)
		result |= man->default_caching;
	else if ((TTM_PL_FLAG_CACHED & caching) != 0)
		result |= TTM_PL_FLAG_CACHED;
	else if ((TTM_PL_FLAG_WC & caching) != 0)
		result |= TTM_PL_FLAG_WC;
	else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
		result |= TTM_PL_FLAG_UNCACHED;

	return result;
}

872 873
static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
				 uint32_t mem_type,
874
				 const struct ttm_place *place,
875
				 uint32_t *masked_placement)
876 877 878
{
	uint32_t cur_flags = ttm_bo_type_flags(mem_type);

879
	if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0)
880 881
		return false;

882
	if ((place->flags & man->available_caching) == 0)
883 884
		return false;

885
	cur_flags |= (place->flags & man->available_caching);
886 887

	*masked_placement = cur_flags;
888 889 890 891 892 893 894 895 896 897 898 899
	return true;
}

/**
 * Creates space for memory region @mem according to its type.
 *
 * This function first searches for free space in compatible memory types in
 * the priority order defined by the driver.  If free space isn't found, then
 * ttm_bo_mem_force_space is attempted in priority order to evict and find
 * space.
 */
int ttm_bo_mem_space(struct ttm_buffer_object *bo,
900 901
			struct ttm_placement *placement,
			struct ttm_mem_reg *mem,
902
			struct ttm_operation_ctx *ctx)
903 904 905 906 907 908 909
{
	struct ttm_bo_device *bdev = bo->bdev;
	struct ttm_mem_type_manager *man;
	uint32_t mem_type = TTM_PL_SYSTEM;
	uint32_t cur_flags = 0;
	bool type_found = false;
	bool type_ok = false;
910
	bool has_erestartsys = false;
911
	int i, ret;
912

913 914 915 916
	ret = reservation_object_reserve_shared(bo->resv);
	if (unlikely(ret))
		return ret;

917
	mem->mm_node = NULL;
918
	for (i = 0; i < placement->num_placement; ++i) {
919 920 921
		const struct ttm_place *place = &placement->placement[i];

		ret = ttm_mem_type_from_place(place, &mem_type);
922 923
		if (ret)
			return ret;
924
		man = &bdev->man[mem_type];
925 926
		if (!man->has_type || !man->use_type)
			continue;
927

928
		type_ok = ttm_bo_mt_compatible(man, mem_type, place,
929
						&cur_flags);
930 931 932 933

		if (!type_ok)
			continue;

934
		type_found = true;
935 936
		cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
						  cur_flags);
937 938 939 940
		/*
		 * Use the access and other non-mapping-related flag bits from
		 * the memory placement flags to the current flags
		 */
941
		ttm_flag_masked(&cur_flags, place->flags,
942
				~TTM_PL_MASK_MEMTYPE);
943

944 945 946
		if (mem_type == TTM_PL_SYSTEM)
			break;

947 948 949
		ret = (*man->func->get_node)(man, bo, place, mem);
		if (unlikely(ret))
			return ret;
950 951 952 953 954 955 956

		if (mem->mm_node) {
			ret = ttm_bo_add_move_fence(bo, man, mem);
			if (unlikely(ret)) {
				(*man->func->put_node)(man, mem);
				return ret;
			}
957
			break;
958
		}
959 960
	}

961
	if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
962 963 964 965 966
		mem->mem_type = mem_type;
		mem->placement = cur_flags;
		return 0;
	}

967
	for (i = 0; i < placement->num_busy_placement; ++i) {
968 969 970
		const struct ttm_place *place = &placement->busy_placement[i];

		ret = ttm_mem_type_from_place(place, &mem_type);
971 972
		if (ret)
			return ret;
973
		man = &bdev->man[mem_type];
974
		if (!man->has_type || !man->use_type)
975
			continue;
976
		if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags))
977 978
			continue;

979
		type_found = true;
980 981
		cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
						  cur_flags);
982 983 984 985
		/*
		 * Use the access and other non-mapping-related flag bits from
		 * the memory placement flags to the current flags
		 */
986
		ttm_flag_masked(&cur_flags, place->flags,
987
				~TTM_PL_MASK_MEMTYPE);
988

989 990 991 992 993 994 995
		if (mem_type == TTM_PL_SYSTEM) {
			mem->mem_type = mem_type;
			mem->placement = cur_flags;
			mem->mm_node = NULL;
			return 0;
		}

996
		ret = ttm_bo_mem_force_space(bo, mem_type, place, mem, ctx);
997 998 999 1000
		if (ret == 0 && mem->mm_node) {
			mem->placement = cur_flags;
			return 0;
		}
1001 1002
		if (ret == -ERESTARTSYS)
			has_erestartsys = true;
1003
	}
1004 1005

	if (!type_found) {
1006
		pr_err(TTM_PFX "No compatible memory type found\n");
1007 1008 1009 1010
		return -EINVAL;
	}

	return (has_erestartsys) ? -ERESTARTSYS : -ENOMEM;
1011 1012 1013
}
EXPORT_SYMBOL(ttm_bo_mem_space);

1014
static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
1015 1016
			      struct ttm_placement *placement,
			      struct ttm_operation_ctx *ctx)
1017 1018 1019 1020
{
	int ret = 0;
	struct ttm_mem_reg mem;

1021
	reservation_object_assert_held(bo->resv);
1022 1023 1024 1025

	mem.num_pages = bo->num_pages;
	mem.size = mem.num_pages << PAGE_SHIFT;
	mem.page_alignment = bo->mem.page_alignment;
1026 1027
	mem.bus.io_reserved_vm = false;
	mem.bus.io_reserved_count = 0;
1028 1029 1030
	/*
	 * Determine where to move the buffer.
	 */
1031
	ret = ttm_bo_mem_space(bo, placement, &mem, ctx);
1032 1033
	if (ret)
		goto out_unlock;
1034
	ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx);
1035
out_unlock:
1036 1037
	if (ret && mem.mm_node)
		ttm_bo_mem_put(bo, &mem);
1038 1039 1040
	return ret;
}

1041 1042 1043 1044
static bool ttm_bo_places_compat(const struct ttm_place *places,
				 unsigned num_placement,
				 struct ttm_mem_reg *mem,
				 uint32_t *new_flags)
1045
{
1046
	unsigned i;
1047

1048 1049
	for (i = 0; i < num_placement; i++) {
		const struct ttm_place *heap = &places[i];
1050

1051
		if (mem->mm_node && (mem->start < heap->fpfn ||
1052
		     (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn)))
1053 1054 1055
			continue;

		*new_flags = heap->flags;
1056
		if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
1057 1058 1059
		    (*new_flags & mem->placement & TTM_PL_MASK_MEM) &&
		    (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) ||
		     (mem->placement & TTM_PL_FLAG_CONTIGUOUS)))
1060
			return true;
1061
	}
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
	return false;
}

bool ttm_bo_mem_compat(struct ttm_placement *placement,
		       struct ttm_mem_reg *mem,
		       uint32_t *new_flags)
{
	if (ttm_bo_places_compat(placement->placement, placement->num_placement,
				 mem, new_flags))
		return true;

	if ((placement->busy_placement != placement->placement ||
	     placement->num_busy_placement > placement->num_placement) &&
	    ttm_bo_places_compat(placement->busy_placement,
				 placement->num_busy_placement,
				 mem, new_flags))
		return true;
1079 1080

	return false;
1081
}
1082
EXPORT_SYMBOL(ttm_bo_mem_compat);
1083

1084
int ttm_bo_validate(struct ttm_buffer_object *bo,
1085 1086
		    struct ttm_placement *placement,
		    struct ttm_operation_ctx *ctx)
1087 1088
{
	int ret;
1089
	uint32_t new_flags;
1090

1091
	reservation_object_assert_held(bo->resv);
1092 1093 1094
	/*
	 * Check whether we need to move buffer.
	 */
1095
	if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
1096
		ret = ttm_bo_move_buffer(bo, placement, ctx);
1097
		if (ret)
1098
			return ret;
1099 1100 1101 1102 1103
	} else {
		/*
		 * Use the access and other non-mapping-related flag bits from
		 * the compatible memory placement flags to the active flags
		 */
1104
		ttm_flag_masked(&bo->mem.placement, new_flags,
1105
				~TTM_PL_MASK_MEMTYPE);
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
	}
	/*
	 * We might need to add a TTM.
	 */
	if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
		ret = ttm_bo_add_ttm(bo, true);
		if (ret)
			return ret;
	}
	return 0;
}
1117
EXPORT_SYMBOL(ttm_bo_validate);
1118

1119 1120 1121 1122 1123 1124
int ttm_bo_init_reserved(struct ttm_bo_device *bdev,
			 struct ttm_buffer_object *bo,
			 unsigned long size,
			 enum ttm_bo_type type,
			 struct ttm_placement *placement,
			 uint32_t page_alignment,
1125
			 struct ttm_operation_ctx *ctx,
1126 1127 1128 1129 1130
			 struct file *persistent_swap_storage,
			 size_t acc_size,
			 struct sg_table *sg,
			 struct reservation_object *resv,
			 void (*destroy) (struct ttm_buffer_object *))
1131
{
1132
	int ret = 0;
1133
	unsigned long num_pages;
1134
	struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
1135
	bool locked;
1136 1137 1138

	ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false);
	if (ret) {
J
Joe Perches 已提交
1139
		pr_err("Out of kernel memory\n");
1140 1141 1142 1143 1144 1145
		if (destroy)
			(*destroy)(bo);
		else
			kfree(bo);
		return -ENOMEM;
	}
1146 1147 1148

	num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
	if (num_pages == 0) {
J
Joe Perches 已提交
1149
		pr_err("Illegal buffer object size\n");
1150 1151 1152 1153
		if (destroy)
			(*destroy)(bo);
		else
			kfree(bo);
1154
		ttm_mem_global_free(mem_glob, acc_size);
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
		return -EINVAL;
	}
	bo->destroy = destroy;

	kref_init(&bo->kref);
	kref_init(&bo->list_kref);
	atomic_set(&bo->cpu_writers, 0);
	INIT_LIST_HEAD(&bo->lru);
	INIT_LIST_HEAD(&bo->ddestroy);
	INIT_LIST_HEAD(&bo->swap);
1165
	INIT_LIST_HEAD(&bo->io_reserve_lru);
1166
	mutex_init(&bo->wu_mutex);
1167
	bo->bdev = bdev;
1168
	bo->glob = bdev->glob;
1169 1170
	bo->type = type;
	bo->num_pages = num_pages;
1171
	bo->mem.size = num_pages << PAGE_SHIFT;
1172 1173 1174 1175
	bo->mem.mem_type = TTM_PL_SYSTEM;
	bo->mem.num_pages = bo->num_pages;
	bo->mem.mm_node = NULL;
	bo->mem.page_alignment = page_alignment;
1176 1177
	bo->mem.bus.io_reserved_vm = false;
	bo->mem.bus.io_reserved_count = 0;
1178
	bo->moving = NULL;
1179
	bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
J
Jan Engelhardt 已提交
1180
	bo->persistent_swap_storage = persistent_swap_storage;
1181
	bo->acc_size = acc_size;
1182
	bo->sg = sg;
1183 1184
	if (resv) {
		bo->resv = resv;
1185
		reservation_object_assert_held(bo->resv);
1186 1187 1188
	} else {
		bo->resv = &bo->ttm_resv;
	}
1189
	reservation_object_init(&bo->ttm_resv);
1190
	atomic_inc(&bo->glob->bo_count);
1191
	drm_vma_node_reset(&bo->vma_node);
1192
	bo->priority = 0;
1193 1194 1195 1196 1197

	/*
	 * For ttm_bo_type_device buffers, allocate
	 * address space from the device.
	 */
1198 1199
	if (bo->type == ttm_bo_type_device ||
	    bo->type == ttm_bo_type_sg)
1200 1201
		ret = drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node,
					 bo->mem.num_pages);
1202

1203 1204 1205 1206
	/* passed reservation objects should already be locked,
	 * since otherwise lockdep will be angered in radeon.
	 */
	if (!resv) {
1207
		locked = reservation_object_trylock(bo->resv);
1208 1209
		WARN_ON(!locked);
	}
1210

1211 1212
	if (likely(!ret))
		ret = ttm_bo_validate(bo, placement, ctx);
1213

1214
	if (unlikely(ret)) {
1215 1216 1217
		if (!resv)
			ttm_bo_unreserve(bo);

1218 1219 1220 1221 1222
		ttm_bo_unref(&bo);
		return ret;
	}

	if (resv && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
1223 1224 1225 1226 1227
		spin_lock(&bo->glob->lru_lock);
		ttm_bo_add_to_lru(bo);
		spin_unlock(&bo->glob->lru_lock);
	}

1228 1229
	return ret;
}
1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244
EXPORT_SYMBOL(ttm_bo_init_reserved);

int ttm_bo_init(struct ttm_bo_device *bdev,
		struct ttm_buffer_object *bo,
		unsigned long size,
		enum ttm_bo_type type,
		struct ttm_placement *placement,
		uint32_t page_alignment,
		bool interruptible,
		struct file *persistent_swap_storage,
		size_t acc_size,
		struct sg_table *sg,
		struct reservation_object *resv,
		void (*destroy) (struct ttm_buffer_object *))
{
1245
	struct ttm_operation_ctx ctx = { interruptible, false };
1246 1247 1248
	int ret;

	ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
1249
				   page_alignment, &ctx,
1250 1251 1252 1253 1254 1255 1256 1257 1258 1259
				   persistent_swap_storage, acc_size,
				   sg, resv, destroy);
	if (ret)
		return ret;

	if (!resv)
		ttm_bo_unreserve(bo);

	return 0;
}
1260
EXPORT_SYMBOL(ttm_bo_init);
1261

1262 1263 1264
size_t ttm_bo_acc_size(struct ttm_bo_device *bdev,
		       unsigned long bo_size,
		       unsigned struct_size)
1265
{
1266 1267
	unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
	size_t size = 0;
1268

1269
	size += ttm_round_pot(struct_size);
F
Felix Kuehling 已提交
1270
	size += ttm_round_pot(npages * sizeof(void *));
1271 1272
	size += ttm_round_pot(sizeof(struct ttm_tt));
	return size;
1273
}
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
EXPORT_SYMBOL(ttm_bo_acc_size);

size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev,
			   unsigned long bo_size,
			   unsigned struct_size)
{
	unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT;
	size_t size = 0;

	size += ttm_round_pot(struct_size);
F
Felix Kuehling 已提交
1284
	size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t)));
1285 1286 1287 1288
	size += ttm_round_pot(sizeof(struct ttm_dma_tt));
	return size;
}
EXPORT_SYMBOL(ttm_bo_dma_acc_size);
1289

1290 1291 1292 1293 1294 1295
int ttm_bo_create(struct ttm_bo_device *bdev,
			unsigned long size,
			enum ttm_bo_type type,
			struct ttm_placement *placement,
			uint32_t page_alignment,
			bool interruptible,
J
Jan Engelhardt 已提交
1296
			struct file *persistent_swap_storage,
1297
			struct ttm_buffer_object **p_bo)
1298 1299
{
	struct ttm_buffer_object *bo;
1300
	size_t acc_size;
1301
	int ret;
1302 1303

	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1304
	if (unlikely(bo == NULL))
1305 1306
		return -ENOMEM;

1307
	acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object));
1308
	ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1309
			  interruptible, persistent_swap_storage, acc_size,
1310
			  NULL, NULL, NULL);
1311 1312 1313 1314 1315
	if (likely(ret == 0))
		*p_bo = bo;

	return ret;
}
T
Thomas Hellstrom 已提交
1316
EXPORT_SYMBOL(ttm_bo_create);
1317 1318

static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1319
				   unsigned mem_type)
1320
{
1321
	struct ttm_operation_ctx ctx = { false, false };
1322
	struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1323
	struct ttm_bo_global *glob = bdev->glob;
1324
	struct dma_fence *fence;
1325
	int ret;
1326
	unsigned i;
1327 1328 1329 1330 1331

	/*
	 * Can't use standard list traversal since we're unlocking.
	 */

1332
	spin_lock(&glob->lru_lock);
1333 1334 1335
	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
		while (!list_empty(&man->lru[i])) {
			spin_unlock(&glob->lru_lock);
R
Roger He 已提交
1336
			ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx);
1337
			if (ret)
1338
				return ret;
1339
			spin_lock(&glob->lru_lock);
1340
		}
1341
	}
1342
	spin_unlock(&glob->lru_lock);
1343 1344

	spin_lock(&man->move_lock);
1345
	fence = dma_fence_get(man->move);
1346 1347 1348
	spin_unlock(&man->move_lock);

	if (fence) {
1349 1350
		ret = dma_fence_wait(fence, false);
		dma_fence_put(fence);
1351 1352
		if (ret)
			return ret;
1353 1354
	}

1355 1356 1357 1358 1359
	return 0;
}

int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
{
R
Roel Kluin 已提交
1360
	struct ttm_mem_type_manager *man;
1361 1362 1363
	int ret = -EINVAL;

	if (mem_type >= TTM_NUM_MEM_TYPES) {
J
Joe Perches 已提交
1364
		pr_err("Illegal memory type %d\n", mem_type);
1365 1366
		return ret;
	}
R
Roel Kluin 已提交
1367
	man = &bdev->man[mem_type];
1368 1369

	if (!man->has_type) {
J
Joe Perches 已提交
1370 1371
		pr_err("Trying to take down uninitialized memory manager type %u\n",
		       mem_type);
1372 1373 1374 1375 1376 1377 1378 1379
		return ret;
	}

	man->use_type = false;
	man->has_type = false;

	ret = 0;
	if (mem_type > 0) {
1380 1381 1382 1383 1384
		ret = ttm_bo_force_list_clean(bdev, mem_type);
		if (ret) {
			pr_err("Cleanup eviction failed\n");
			return ret;
		}
1385

1386
		ret = (*man->func->takedown)(man);
1387 1388
	}

1389 1390 1391
	dma_fence_put(man->move);
	man->move = NULL;

1392 1393 1394 1395 1396 1397 1398 1399 1400
	return ret;
}
EXPORT_SYMBOL(ttm_bo_clean_mm);

int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
{
	struct ttm_mem_type_manager *man = &bdev->man[mem_type];

	if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
J
Joe Perches 已提交
1401
		pr_err("Illegal memory manager memory type %u\n", mem_type);
1402 1403 1404 1405
		return -EINVAL;
	}

	if (!man->has_type) {
J
Joe Perches 已提交
1406
		pr_err("Memory type %u has not been initialized\n", mem_type);
1407 1408 1409
		return 0;
	}

1410
	return ttm_bo_force_list_clean(bdev, mem_type);
1411 1412 1413 1414
}
EXPORT_SYMBOL(ttm_bo_evict_mm);

int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1415
			unsigned long p_size)
1416
{
H
Huang Rui 已提交
1417
	int ret;
1418
	struct ttm_mem_type_manager *man;
1419
	unsigned i;
1420

1421
	BUG_ON(type >= TTM_NUM_MEM_TYPES);
1422
	man = &bdev->man[type];
1423
	BUG_ON(man->has_type);
1424 1425 1426
	man->io_reserve_fastpath = true;
	man->use_io_reserve_lru = false;
	mutex_init(&man->io_reserve_mutex);
1427
	spin_lock_init(&man->move_lock);
1428
	INIT_LIST_HEAD(&man->io_reserve_lru);
1429 1430 1431 1432

	ret = bdev->driver->init_mem_type(bdev, type, man);
	if (ret)
		return ret;
1433
	man->bdev = bdev;
1434 1435

	if (type != TTM_PL_SYSTEM) {
1436
		ret = (*man->func->init)(man, p_size);
1437 1438 1439 1440 1441 1442 1443
		if (ret)
			return ret;
	}
	man->has_type = true;
	man->use_type = true;
	man->size = p_size;

1444 1445
	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
		INIT_LIST_HEAD(&man->lru[i]);
1446
	man->move = NULL;
1447 1448 1449 1450 1451

	return 0;
}
EXPORT_SYMBOL(ttm_bo_init_mm);

1452 1453 1454 1455 1456 1457 1458 1459 1460 1461
static void ttm_bo_global_kobj_release(struct kobject *kobj)
{
	struct ttm_bo_global *glob =
		container_of(kobj, struct ttm_bo_global, kobj);

	ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink);
	__free_page(glob->dummy_read_page);
	kfree(glob);
}

1462
void ttm_bo_global_release(struct drm_global_reference *ref)
1463 1464 1465 1466 1467 1468 1469 1470
{
	struct ttm_bo_global *glob = ref->object;

	kobject_del(&glob->kobj);
	kobject_put(&glob->kobj);
}
EXPORT_SYMBOL(ttm_bo_global_release);

1471
int ttm_bo_global_init(struct drm_global_reference *ref)
1472 1473 1474 1475 1476
{
	struct ttm_bo_global_ref *bo_ref =
		container_of(ref, struct ttm_bo_global_ref, ref);
	struct ttm_bo_global *glob = ref->object;
	int ret;
1477
	unsigned i;
1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488

	mutex_init(&glob->device_list_mutex);
	spin_lock_init(&glob->lru_lock);
	glob->mem_glob = bo_ref->mem_glob;
	glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);

	if (unlikely(glob->dummy_read_page == NULL)) {
		ret = -ENOMEM;
		goto out_no_drp;
	}

1489 1490
	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
		INIT_LIST_HEAD(&glob->swap_lru[i]);
1491 1492 1493 1494 1495
	INIT_LIST_HEAD(&glob->device_list);

	ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout);
	ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink);
	if (unlikely(ret != 0)) {
J
Joe Perches 已提交
1496
		pr_err("Could not register buffer object swapout\n");
1497 1498 1499 1500 1501
		goto out_no_shrink;
	}

	atomic_set(&glob->bo_count, 0);

1502 1503
	ret = kobject_init_and_add(
		&glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
	if (unlikely(ret != 0))
		kobject_put(&glob->kobj);
	return ret;
out_no_shrink:
	__free_page(glob->dummy_read_page);
out_no_drp:
	kfree(glob);
	return ret;
}
EXPORT_SYMBOL(ttm_bo_global_init);


1516 1517 1518 1519 1520
int ttm_bo_device_release(struct ttm_bo_device *bdev)
{
	int ret = 0;
	unsigned i = TTM_NUM_MEM_TYPES;
	struct ttm_mem_type_manager *man;
1521
	struct ttm_bo_global *glob = bdev->glob;
1522 1523 1524 1525 1526 1527 1528

	while (i--) {
		man = &bdev->man[i];
		if (man->has_type) {
			man->use_type = false;
			if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
				ret = -EBUSY;
J
Joe Perches 已提交
1529 1530
				pr_err("DRM memory manager type %d is not clean\n",
				       i);
1531 1532 1533 1534 1535
			}
			man->has_type = false;
		}
	}

1536 1537 1538 1539
	mutex_lock(&glob->device_list_mutex);
	list_del(&bdev->device_list);
	mutex_unlock(&glob->device_list_mutex);

1540
	cancel_delayed_work_sync(&bdev->wq);
1541

1542
	if (ttm_bo_delayed_delete(bdev, true))
1543
		pr_debug("Delayed destroy list was clean\n");
1544

1545
	spin_lock(&glob->lru_lock);
1546 1547
	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
		if (list_empty(&bdev->man[0].lru[0]))
1548
			pr_debug("Swap list %d was clean\n", i);
1549
	spin_unlock(&glob->lru_lock);
1550

1551
	drm_vma_offset_manager_destroy(&bdev->vma_manager);
1552 1553 1554 1555 1556 1557

	return ret;
}
EXPORT_SYMBOL(ttm_bo_device_release);

int ttm_bo_device_init(struct ttm_bo_device *bdev,
1558 1559
		       struct ttm_bo_global *glob,
		       struct ttm_bo_driver *driver,
1560
		       struct address_space *mapping,
D
Dave Airlie 已提交
1561
		       uint64_t file_page_offset,
D
Dave Airlie 已提交
1562
		       bool need_dma32)
1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573
{
	int ret = -EINVAL;

	bdev->driver = driver;

	memset(bdev->man, 0, sizeof(bdev->man));

	/*
	 * Initialize the system memory buffer type.
	 * Other types need to be driver / IOCTL initialized.
	 */
1574
	ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1575
	if (unlikely(ret != 0))
1576
		goto out_no_sys;
1577

1578 1579
	drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset,
				    0x10000000);
1580 1581
	INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
	INIT_LIST_HEAD(&bdev->ddestroy);
1582
	bdev->dev_mapping = mapping;
1583
	bdev->glob = glob;
D
Dave Airlie 已提交
1584
	bdev->need_dma32 = need_dma32;
1585 1586 1587
	mutex_lock(&glob->device_list_mutex);
	list_add_tail(&bdev->device_list, &glob->device_list);
	mutex_unlock(&glob->device_list_mutex);
1588 1589

	return 0;
1590
out_no_sys:
1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
	return ret;
}
EXPORT_SYMBOL(ttm_bo_device_init);

/*
 * buffer object vm functions.
 */

bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
{
	struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];

	if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
		if (mem->mem_type == TTM_PL_SYSTEM)
			return false;

		if (man->flags & TTM_MEMTYPE_FLAG_CMA)
			return false;

		if (mem->placement & TTM_PL_FLAG_CACHED)
			return false;
	}
	return true;
}

1616
void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo)
1617 1618 1619
{
	struct ttm_bo_device *bdev = bo->bdev;

1620
	drm_vma_node_unmap(&bo->vma_node, bdev->dev_mapping);
1621
	ttm_mem_io_free_vm(bo);
1622
}
1623 1624 1625 1626 1627 1628 1629 1630 1631

void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
{
	struct ttm_bo_device *bdev = bo->bdev;
	struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type];

	ttm_mem_io_lock(man, false);
	ttm_bo_unmap_virtual_locked(bo);
	ttm_mem_io_unlock(man);
1632
}
1633 1634


1635
EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1636 1637

int ttm_bo_wait(struct ttm_buffer_object *bo,
1638
		bool interruptible, bool no_wait)
1639
{
C
Christian König 已提交
1640 1641 1642 1643 1644 1645 1646 1647
	long timeout = 15 * HZ;

	if (no_wait) {
		if (reservation_object_test_signaled_rcu(bo->resv, true))
			return 0;
		else
			return -EBUSY;
	}
1648

C
Christian König 已提交
1649 1650
	timeout = reservation_object_wait_timeout_rcu(bo->resv, true,
						      interruptible, timeout);
1651 1652 1653 1654 1655 1656
	if (timeout < 0)
		return timeout;

	if (timeout == 0)
		return -EBUSY;

C
Christian König 已提交
1657
	reservation_object_add_excl_fence(bo->resv, NULL);
1658
	return 0;
1659 1660 1661 1662 1663 1664 1665 1666
}
EXPORT_SYMBOL(ttm_bo_wait);

int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
{
	int ret = 0;

	/*
1667
	 * Using ttm_bo_reserve makes sure the lru lists are updated.
1668 1669
	 */

1670
	ret = ttm_bo_reserve(bo, true, no_wait, NULL);
1671 1672
	if (unlikely(ret != 0))
		return ret;
1673
	ret = ttm_bo_wait(bo, true, no_wait);
1674 1675 1676 1677 1678
	if (likely(ret == 0))
		atomic_inc(&bo->cpu_writers);
	ttm_bo_unreserve(bo);
	return ret;
}
1679
EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
1680 1681 1682

void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
{
1683
	atomic_dec(&bo->cpu_writers);
1684
}
1685
EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
1686 1687 1688 1689 1690 1691 1692 1693

/**
 * A buffer object shrink method that tries to swap out the first
 * buffer object on the bo_global::swap_lru list.
 */

static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
{
1694 1695
	struct ttm_bo_global *glob =
	    container_of(shrink, struct ttm_bo_global, shrink);
1696 1697
	struct ttm_buffer_object *bo;
	int ret = -EBUSY;
1698
	unsigned i;
1699

1700
	spin_lock(&glob->lru_lock);
1701 1702
	for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
		list_for_each_entry(bo, &glob->swap_lru[i], swap) {
1703
			ret = reservation_object_trylock(bo->resv) ? 0 : -EBUSY;
1704 1705 1706
			if (!ret)
				break;
		}
1707 1708 1709
		if (!ret)
			break;
	}
1710

1711 1712 1713 1714
	if (ret) {
		spin_unlock(&glob->lru_lock);
		return ret;
	}
1715

1716
	kref_get(&bo->list_kref);
1717

1718
	if (!list_empty(&bo->ddestroy)) {
1719
		ret = ttm_bo_cleanup_refs(bo, false, false, true);
1720 1721
		kref_put(&bo->list_kref, ttm_bo_release_list);
		return ret;
1722 1723
	}

1724
	ttm_bo_del_from_lru(bo);
1725
	spin_unlock(&glob->lru_lock);
1726 1727

	/**
1728
	 * Move to system cached
1729 1730
	 */

1731 1732
	if (bo->mem.mem_type != TTM_PL_SYSTEM ||
	    bo->ttm->caching_state != tt_cached) {
1733
		struct ttm_operation_ctx ctx = { false, false };
1734 1735 1736 1737 1738 1739 1740
		struct ttm_mem_reg evict_mem;

		evict_mem = bo->mem;
		evict_mem.mm_node = NULL;
		evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
		evict_mem.mem_type = TTM_PL_SYSTEM;

1741
		ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx);
1742 1743 1744 1745
		if (unlikely(ret != 0))
			goto out;
	}

1746 1747 1748 1749 1750 1751 1752 1753
	/**
	 * Make sure BO is idle.
	 */

	ret = ttm_bo_wait(bo, false, false);
	if (unlikely(ret != 0))
		goto out;

1754 1755 1756 1757 1758 1759 1760
	ttm_bo_unmap_virtual(bo);

	/**
	 * Swap out. Buffer will be swapped in again as soon as
	 * anyone tries to access a ttm page.
	 */

1761 1762 1763
	if (bo->bdev->driver->swap_notify)
		bo->bdev->driver->swap_notify(bo);

J
Jan Engelhardt 已提交
1764
	ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage);
1765 1766 1767 1768 1769 1770 1771 1772
out:

	/**
	 *
	 * Unreserve without putting on LRU to avoid swapping out an
	 * already swapped buffer.
	 */

1773
	reservation_object_unlock(bo->resv);
1774 1775 1776 1777 1778 1779
	kref_put(&bo->list_kref, ttm_bo_release_list);
	return ret;
}

void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
{
1780
	while (ttm_bo_swapout(&bdev->glob->shrink) == 0)
1781 1782
		;
}
1783
EXPORT_SYMBOL(ttm_bo_swapout_all);
1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806

/**
 * ttm_bo_wait_unreserved - interruptible wait for a buffer object to become
 * unreserved
 *
 * @bo: Pointer to buffer
 */
int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo)
{
	int ret;

	/*
	 * In the absense of a wait_unlocked API,
	 * Use the bo::wu_mutex to avoid triggering livelocks due to
	 * concurrent use of this function. Note that this use of
	 * bo::wu_mutex can go away if we change locking order to
	 * mmap_sem -> bo::reserve.
	 */
	ret = mutex_lock_interruptible(&bo->wu_mutex);
	if (unlikely(ret != 0))
		return -ERESTARTSYS;
	if (!ww_mutex_is_locked(&bo->resv->lock))
		goto out_unlock;
1807 1808 1809
	ret = reservation_object_lock_interruptible(bo->resv, NULL);
	if (ret == -EINTR)
		ret = -ERESTARTSYS;
1810 1811
	if (unlikely(ret != 0))
		goto out_unlock;
1812
	reservation_object_unlock(bo->resv);
1813 1814 1815 1816 1817

out_unlock:
	mutex_unlock(&bo->wu_mutex);
	return ret;
}