ttm_bo_vm.c 11.6 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 OR MIT */
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/**************************************************************************
 *
 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sub license, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice (including the
 * next paragraph) shall be included in all copies or substantial portions
 * of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 **************************************************************************/
/*
 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
 */

J
Joe Perches 已提交
32 33
#define pr_fmt(fmt) "[TTM] " fmt

34 35 36
#include <drm/ttm/ttm_module.h>
#include <drm/ttm/ttm_bo_driver.h>
#include <drm/ttm/ttm_placement.h>
37
#include <drm/drm_vma_manager.h>
38
#include <linux/mm.h>
39
#include <linux/pfn_t.h>
40 41 42
#include <linux/rbtree.h>
#include <linux/module.h>
#include <linux/uaccess.h>
43
#include <linux/mem_encrypt.h>
44 45 46

#define TTM_BO_VM_NUM_PREFAULT 16

47
static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
T
Thomas Hellstrom 已提交
48 49
				struct vm_fault *vmf)
{
50 51
	vm_fault_t ret = 0;
	int err = 0;
T
Thomas Hellstrom 已提交
52

53
	if (likely(!bo->moving))
T
Thomas Hellstrom 已提交
54 55 56 57 58
		goto out_unlock;

	/*
	 * Quick non-stalling check for idle.
	 */
59
	if (dma_fence_is_signaled(bo->moving))
60
		goto out_clear;
T
Thomas Hellstrom 已提交
61 62 63 64 65 66 67 68 69 70

	/*
	 * If possible, avoid waiting for GPU with mmap_sem
	 * held.
	 */
	if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) {
		ret = VM_FAULT_RETRY;
		if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
			goto out_unlock;

71
		ttm_bo_get(bo);
72
		up_read(&vmf->vma->vm_mm->mmap_sem);
73
		(void) dma_fence_wait(bo->moving, true);
74
		ttm_bo_unreserve(bo);
75
		ttm_bo_put(bo);
T
Thomas Hellstrom 已提交
76 77 78 79 80 81
		goto out_unlock;
	}

	/*
	 * Ordinary wait.
	 */
82 83 84
	err = dma_fence_wait(bo->moving, true);
	if (unlikely(err != 0)) {
		ret = (err != -ERESTARTSYS) ? VM_FAULT_SIGBUS :
T
Thomas Hellstrom 已提交
85
			VM_FAULT_NOPAGE;
86 87 88 89
		goto out_unlock;
	}

out_clear:
90
	dma_fence_put(bo->moving);
91
	bo->moving = NULL;
T
Thomas Hellstrom 已提交
92 93 94 95 96

out_unlock:
	return ret;
}

97 98 99 100 101 102 103 104
static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo,
				       unsigned long page_offset)
{
	struct ttm_bo_device *bdev = bo->bdev;

	if (bdev->driver->io_mem_pfn)
		return bdev->driver->io_mem_pfn(bo, page_offset);

105 106
	return ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT)
		+ page_offset;
107 108
}

109
static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf)
110
{
111
	struct vm_area_struct *vma = vmf->vma;
112 113 114 115 116 117 118 119
	struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
	    vma->vm_private_data;
	struct ttm_bo_device *bdev = bo->bdev;
	unsigned long page_offset;
	unsigned long page_last;
	unsigned long pfn;
	struct ttm_tt *ttm = NULL;
	struct page *page;
120
	int err;
121
	int i;
122
	vm_fault_t ret = VM_FAULT_NOPAGE;
123
	unsigned long address = vmf->address;
124 125
	struct ttm_mem_type_manager *man =
		&bdev->man[bo->mem.mem_type];
126
	struct vm_area_struct cvma;
127 128 129 130

	/*
	 * Work around locking order reversal in fault / nopfn
	 * between mmap_sem and bo_reserve: Perform a trylock operation
131 132
	 * for reserve, and if it fails, retry the fault after waiting
	 * for the buffer to become unreserved.
133
	 */
134 135 136
	err = ttm_bo_reserve(bo, true, true, NULL);
	if (unlikely(err != 0)) {
		if (err != -EBUSY)
137 138 139 140
			return VM_FAULT_NOPAGE;

		if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) {
			if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
141
				ttm_bo_get(bo);
142
				up_read(&vmf->vma->vm_mm->mmap_sem);
143
				(void) ttm_bo_wait_unreserved(bo);
144
				ttm_bo_put(bo);
145 146 147 148 149 150 151 152 153 154
			}

			return VM_FAULT_RETRY;
		}

		/*
		 * If we'd want to change locking order to
		 * mmap_sem -> bo::reserve, we'd use a blocking reserve here
		 * instead of retrying the fault...
		 */
155 156 157
		return VM_FAULT_NOPAGE;
	}

158 159 160 161 162
	/*
	 * Refuse to fault imported pages. This should be handled
	 * (if at all) by redirecting mmap to the exporter.
	 */
	if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) {
163
		ret = VM_FAULT_SIGBUS;
164 165 166
		goto out_unlock;
	}

167
	if (bdev->driver->fault_reserve_notify) {
168 169
		err = bdev->driver->fault_reserve_notify(bo);
		switch (err) {
170 171 172 173
		case 0:
			break;
		case -EBUSY:
		case -ERESTARTSYS:
174
			ret = VM_FAULT_NOPAGE;
175 176
			goto out_unlock;
		default:
177
			ret = VM_FAULT_SIGBUS;
178 179 180
			goto out_unlock;
		}
	}
181

182 183 184 185
	/*
	 * Wait for buffer data in transit, due to a pipelined
	 * move.
	 */
186
	ret = ttm_bo_vm_fault_idle(bo, vmf);
T
Thomas Hellstrom 已提交
187
	if (unlikely(ret != 0)) {
188
		if (ret == VM_FAULT_RETRY &&
189 190
		    !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
			/* The BO has already been unreserved. */
191
			return ret;
192 193
		}

T
Thomas Hellstrom 已提交
194 195
		goto out_unlock;
	}
196

197 198
	err = ttm_mem_io_lock(man, true);
	if (unlikely(err != 0)) {
199
		ret = VM_FAULT_NOPAGE;
200 201
		goto out_unlock;
	}
202 203
	err = ttm_mem_io_reserve_vm(bo);
	if (unlikely(err != 0)) {
204
		ret = VM_FAULT_SIGBUS;
205 206
		goto out_io_unlock;
	}
207 208

	page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) +
209 210 211
		vma->vm_pgoff - drm_vma_node_start(&bo->vma_node);
	page_last = vma_pages(vma) + vma->vm_pgoff -
		drm_vma_node_start(&bo->vma_node);
212 213

	if (unlikely(page_offset >= bo->num_pages)) {
214
		ret = VM_FAULT_SIGBUS;
215
		goto out_io_unlock;
216 217 218
	}

	/*
219 220 221
	 * Make a local vma copy to modify the page_prot member
	 * and vm_flags if necessary. The vma parameter is protected
	 * by mmap_sem in write mode.
222
	 */
223 224 225
	cvma = *vma;
	cvma.vm_page_prot = vm_get_page_prot(cvma.vm_flags);

226
	if (bo->mem.bus.is_iomem) {
227 228
		cvma.vm_page_prot = ttm_io_prot(bo->mem.placement,
						cvma.vm_page_prot);
229
	} else {
230 231
		struct ttm_operation_ctx ctx = {
			.interruptible = false,
232 233 234
			.no_wait_gpu = false,
			.flags = TTM_OPT_FLAG_FORCE_ALLOC

235 236
		};

237
		ttm = bo->ttm;
238 239
		cvma.vm_page_prot = ttm_io_prot(bo->mem.placement,
						cvma.vm_page_prot);
240 241

		/* Allocate all page at once, most common usage */
242
		if (ttm_tt_populate(ttm, &ctx)) {
243
			ret = VM_FAULT_OOM;
244 245
			goto out_io_unlock;
		}
246 247 248 249 250 251 252
	}

	/*
	 * Speculatively prefault a number of pages. Only error on
	 * first page.
	 */
	for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) {
253 254 255
		if (bo->mem.bus.is_iomem) {
			/* Iomem should not be marked encrypted */
			cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot);
256
			pfn = ttm_bo_io_mem_pfn(bo, page_offset);
257
		} else {
258
			page = ttm->pages[page_offset];
259
			if (unlikely(!page && i == 0)) {
260
				ret = VM_FAULT_OOM;
261
				goto out_io_unlock;
262 263 264
			} else if (unlikely(!page)) {
				break;
			}
265 266
			page->index = drm_vma_node_start(&bo->vma_node) +
				page_offset;
267 268 269
			pfn = page_to_pfn(page);
		}

270
		if (vma->vm_flags & VM_MIXEDMAP)
271
			ret = vmf_insert_mixed(&cvma, address,
272
					__pfn_to_pfn_t(pfn, PFN_DEV));
273
		else
274
			ret = vmf_insert_pfn(&cvma, address, pfn);
275

276 277 278 279 280 281 282
		/* Never error on prefaulted PTEs */
		if (unlikely((ret & VM_FAULT_ERROR))) {
			if (i == 0)
				goto out_io_unlock;
			else
				break;
		}
283 284 285 286 287

		address += PAGE_SIZE;
		if (unlikely(++page_offset >= page_last))
			break;
	}
288
	ret = VM_FAULT_NOPAGE;
289 290
out_io_unlock:
	ttm_mem_io_unlock(man);
291 292
out_unlock:
	ttm_bo_unreserve(bo);
293
	return ret;
294 295 296 297 298 299 300
}

static void ttm_bo_vm_open(struct vm_area_struct *vma)
{
	struct ttm_buffer_object *bo =
	    (struct ttm_buffer_object *)vma->vm_private_data;

301 302
	WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping);

303
	ttm_bo_get(bo);
304 305 306 307
}

static void ttm_bo_vm_close(struct vm_area_struct *vma)
{
308
	struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data;
309

310
	ttm_bo_put(bo);
311 312 313
	vma->vm_private_data = NULL;
}

314 315
static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo,
				 unsigned long offset,
316
				 uint8_t *buf, int len, int write)
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
{
	unsigned long page = offset >> PAGE_SHIFT;
	unsigned long bytes_left = len;
	int ret;

	/* Copy a page at a time, that way no extra virtual address
	 * mapping is needed
	 */
	offset -= page << PAGE_SHIFT;
	do {
		unsigned long bytes = min(bytes_left, PAGE_SIZE - offset);
		struct ttm_bo_kmap_obj map;
		void *ptr;
		bool is_iomem;

		ret = ttm_bo_kmap(bo, page, 1, &map);
		if (ret)
			return ret;

		ptr = (uint8_t *)ttm_kmap_obj_virtual(&map, &is_iomem) + offset;
		WARN_ON_ONCE(is_iomem);
		if (write)
			memcpy(ptr, buf, bytes);
		else
			memcpy(buf, ptr, bytes);
		ttm_bo_kunmap(&map);

		page++;
345
		buf += bytes;
346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390
		bytes_left -= bytes;
		offset = 0;
	} while (bytes_left);

	return len;
}

static int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr,
			    void *buf, int len, int write)
{
	unsigned long offset = (addr) - vma->vm_start;
	struct ttm_buffer_object *bo = vma->vm_private_data;
	int ret;

	if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->num_pages)
		return -EIO;

	ret = ttm_bo_reserve(bo, true, false, NULL);
	if (ret)
		return ret;

	switch (bo->mem.mem_type) {
	case TTM_PL_SYSTEM:
		if (unlikely(bo->ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
			ret = ttm_tt_swapin(bo->ttm);
			if (unlikely(ret != 0))
				return ret;
		}
		/* fall through */
	case TTM_PL_TT:
		ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write);
		break;
	default:
		if (bo->bdev->driver->access_memory)
			ret = bo->bdev->driver->access_memory(
				bo, offset, buf, len, write);
		else
			ret = -EIO;
	}

	ttm_bo_unreserve(bo);

	return ret;
}

391
static const struct vm_operations_struct ttm_bo_vm_ops = {
392 393
	.fault = ttm_bo_vm_fault,
	.open = ttm_bo_vm_open,
394 395
	.close = ttm_bo_vm_close,
	.access = ttm_bo_vm_access
396 397
};

398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev,
						  unsigned long offset,
						  unsigned long pages)
{
	struct drm_vma_offset_node *node;
	struct ttm_buffer_object *bo = NULL;

	drm_vma_offset_lock_lookup(&bdev->vma_manager);

	node = drm_vma_offset_lookup_locked(&bdev->vma_manager, offset, pages);
	if (likely(node)) {
		bo = container_of(node, struct ttm_buffer_object, vma_node);
		if (!kref_get_unless_zero(&bo->kref))
			bo = NULL;
	}

	drm_vma_offset_unlock_lookup(&bdev->vma_manager);

	if (!bo)
		pr_err("Could not find buffer object to map\n");

	return bo;
}

422 423 424 425 426 427 428
int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma,
		struct ttm_bo_device *bdev)
{
	struct ttm_bo_driver *driver;
	struct ttm_buffer_object *bo;
	int ret;

429 430
	bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma));
	if (unlikely(!bo))
431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449
		return -EINVAL;

	driver = bo->bdev->driver;
	if (unlikely(!driver->verify_access)) {
		ret = -EPERM;
		goto out_unref;
	}
	ret = driver->verify_access(bo, filp);
	if (unlikely(ret != 0))
		goto out_unref;

	vma->vm_ops = &ttm_bo_vm_ops;

	/*
	 * Note: We're transferring the bo reference to
	 * vma->vm_private_data here.
	 */

	vma->vm_private_data = bo;
450 451

	/*
452 453 454 455 456
	 * We'd like to use VM_PFNMAP on shared mappings, where
	 * (vma->vm_flags & VM_SHARED) != 0, for performance reasons,
	 * but for some reason VM_PFNMAP + x86 PAT + write-combine is very
	 * bad for performance. Until that has been sorted out, use
	 * VM_MIXEDMAP on all mappings. See freedesktop.org bug #75719
457
	 */
458
	vma->vm_flags |= VM_MIXEDMAP;
459
	vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP;
460 461
	return 0;
out_unref:
462
	ttm_bo_put(bo);
463 464 465 466 467 468 469 470 471
	return ret;
}
EXPORT_SYMBOL(ttm_bo_mmap);

int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
{
	if (vma->vm_pgoff != 0)
		return -EACCES;

472 473
	ttm_bo_get(bo);

474
	vma->vm_ops = &ttm_bo_vm_ops;
475
	vma->vm_private_data = bo;
476
	vma->vm_flags |= VM_MIXEDMAP;
477
	vma->vm_flags |= VM_IO | VM_DONTEXPAND;
478 479 480
	return 0;
}
EXPORT_SYMBOL(ttm_fbdev_mmap);