i915_gem_gtt.c 106.1 KB
Newer Older
1 2
/*
 * Copyright © 2010 Daniel Vetter
3
 * Copyright © 2011-2014 Intel Corporation
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

26 27 28
#include <linux/slab.h> /* fault-inject.h is not standalone! */

#include <linux/fault-inject.h>
29
#include <linux/log2.h>
30
#include <linux/random.h>
31
#include <linux/seq_file.h>
32
#include <linux/stop_machine.h>
33

L
Laura Abbott 已提交
34 35
#include <asm/set_memory.h>

36
#include <drm/i915_drm.h>
37

38 39
#include "display/intel_frontbuffer.h"

40
#include "i915_drv.h"
41
#include "i915_scatterlist.h"
42
#include "i915_trace.h"
43
#include "i915_vgpu.h"
44 45
#include "intel_drv.h"

46
#define I915_GFP_ALLOW_FAIL (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN)
47

48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
/**
 * DOC: Global GTT views
 *
 * Background and previous state
 *
 * Historically objects could exists (be bound) in global GTT space only as
 * singular instances with a view representing all of the object's backing pages
 * in a linear fashion. This view will be called a normal view.
 *
 * To support multiple views of the same object, where the number of mapped
 * pages is not equal to the backing store, or where the layout of the pages
 * is not linear, concept of a GGTT view was added.
 *
 * One example of an alternative view is a stereo display driven by a single
 * image. In this case we would have a framebuffer looking like this
 * (2x2 pages):
 *
 *    12
 *    34
 *
 * Above would represent a normal GGTT view as normally mapped for GPU or CPU
 * rendering. In contrast, fed to the display engine would be an alternative
 * view which could look something like this:
 *
 *   1212
 *   3434
 *
 * In this example both the size and layout of pages in the alternative view is
 * different from the normal view.
 *
 * Implementation and usage
 *
 * GGTT views are implemented using VMAs and are distinguished via enum
 * i915_ggtt_view_type and struct i915_ggtt_view.
 *
 * A new flavour of core GEM functions which work with GGTT bound objects were
84 85 86
 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
 * renaming  in large amounts of code. They take the struct i915_ggtt_view
 * parameter encapsulating all metadata required to implement a view.
87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
 *
 * As a helper for callers which are only interested in the normal view,
 * globally const i915_ggtt_view_normal singleton instance exists. All old core
 * GEM API functions, the ones not taking the view parameter, are operating on,
 * or with the normal GGTT view.
 *
 * Code wanting to add or use a new GGTT view needs to:
 *
 * 1. Add a new enum with a suitable name.
 * 2. Extend the metadata in the i915_ggtt_view structure if required.
 * 3. Add support to i915_get_vma_pages().
 *
 * New views are required to build a scatter-gather table from within the
 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
 * exists for the lifetime of an VMA.
 *
 * Core API is designed to have copy semantics which means that passed in
 * struct i915_ggtt_view does not need to be persistent (left around after
 * calling the core API functions).
 *
 */

109 110 111
static int
i915_get_ggtt_vma_pages(struct i915_vma *vma);

112
static void gen6_ggtt_invalidate(struct drm_i915_private *i915)
113
{
114 115
	struct intel_uncore *uncore = &i915->uncore;

116 117
	/*
	 * Note that as an uncached mmio write, this will flush the
118 119
	 * WCB of the writes into the GGTT before it triggers the invalidate.
	 */
120
	intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
121 122
}

123
static void guc_ggtt_invalidate(struct drm_i915_private *i915)
124
{
125 126 127 128
	struct intel_uncore *uncore = &i915->uncore;

	gen6_ggtt_invalidate(i915);
	intel_uncore_write_fw(uncore, GEN8_GTCR, GEN8_GTCR_INVALIDATE);
129 130
}

131
static void gmch_ggtt_invalidate(struct drm_i915_private *i915)
132 133 134 135 136 137 138 139 140
{
	intel_gtt_chipset_flush();
}

static inline void i915_ggtt_invalidate(struct drm_i915_private *i915)
{
	i915->ggtt.invalidate(i915);
}

141 142 143
static int ppgtt_bind_vma(struct i915_vma *vma,
			  enum i915_cache_level cache_level,
			  u32 unused)
144
{
145
	u32 pte_flags;
146 147 148 149 150 151 152 153
	int err;

	if (!(vma->flags & I915_VMA_LOCAL_BIND)) {
		err = vma->vm->allocate_va_range(vma->vm,
						 vma->node.start, vma->size);
		if (err)
			return err;
	}
154

155
	/* Applicable to VLV, and gen8+ */
156
	pte_flags = 0;
157
	if (i915_gem_object_is_readonly(vma->obj))
158 159
		pte_flags |= PTE_READ_ONLY;

160
	vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
161 162

	return 0;
163 164 165 166
}

static void ppgtt_unbind_vma(struct i915_vma *vma)
{
167
	vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
168
}
169

170 171 172 173 174 175
static int ppgtt_set_pages(struct i915_vma *vma)
{
	GEM_BUG_ON(vma->pages);

	vma->pages = vma->obj->mm.pages;

176 177
	vma->page_sizes = vma->obj->mm.page_sizes;

178 179 180 181 182 183 184 185 186 187 188 189
	return 0;
}

static void clear_pages(struct i915_vma *vma)
{
	GEM_BUG_ON(!vma->pages);

	if (vma->pages != vma->obj->mm.pages) {
		sg_free_table(vma->pages);
		kfree(vma->pages);
	}
	vma->pages = NULL;
190 191

	memset(&vma->page_sizes, 0, sizeof(vma->page_sizes));
192 193
}

194 195 196
static u64 gen8_pte_encode(dma_addr_t addr,
			   enum i915_cache_level level,
			   u32 flags)
B
Ben Widawsky 已提交
197
{
198 199 200 201
	gen8_pte_t pte = addr | _PAGE_PRESENT | _PAGE_RW;

	if (unlikely(flags & PTE_READ_ONLY))
		pte &= ~_PAGE_RW;
202 203 204

	switch (level) {
	case I915_CACHE_NONE:
205
		pte |= PPAT_UNCACHED;
206 207
		break;
	case I915_CACHE_WT:
208
		pte |= PPAT_DISPLAY_ELLC;
209 210
		break;
	default:
211
		pte |= PPAT_CACHED;
212 213 214
		break;
	}

B
Ben Widawsky 已提交
215 216 217
	return pte;
}

218 219
static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
				  const enum i915_cache_level level)
B
Ben Widawsky 已提交
220
{
221
	gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
B
Ben Widawsky 已提交
222 223
	pde |= addr;
	if (level != I915_CACHE_NONE)
224
		pde |= PPAT_CACHED_PDE;
B
Ben Widawsky 已提交
225
	else
226
		pde |= PPAT_UNCACHED;
B
Ben Widawsky 已提交
227 228 229
	return pde;
}

230 231 232
#define gen8_pdpe_encode gen8_pde_encode
#define gen8_pml4e_encode gen8_pde_encode

233 234 235
static u64 snb_pte_encode(dma_addr_t addr,
			  enum i915_cache_level level,
			  u32 flags)
236
{
237
	gen6_pte_t pte = GEN6_PTE_VALID;
238
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
239 240

	switch (level) {
241 242 243 244 245 246 247 248
	case I915_CACHE_L3_LLC:
	case I915_CACHE_LLC:
		pte |= GEN6_PTE_CACHE_LLC;
		break;
	case I915_CACHE_NONE:
		pte |= GEN6_PTE_UNCACHED;
		break;
	default:
249
		MISSING_CASE(level);
250 251 252 253 254
	}

	return pte;
}

255 256 257
static u64 ivb_pte_encode(dma_addr_t addr,
			  enum i915_cache_level level,
			  u32 flags)
258
{
259
	gen6_pte_t pte = GEN6_PTE_VALID;
260 261 262 263 264
	pte |= GEN6_PTE_ADDR_ENCODE(addr);

	switch (level) {
	case I915_CACHE_L3_LLC:
		pte |= GEN7_PTE_CACHE_L3_LLC;
265 266 267 268 269
		break;
	case I915_CACHE_LLC:
		pte |= GEN6_PTE_CACHE_LLC;
		break;
	case I915_CACHE_NONE:
270
		pte |= GEN6_PTE_UNCACHED;
271 272
		break;
	default:
273
		MISSING_CASE(level);
274 275
	}

276 277 278
	return pte;
}

279 280 281
static u64 byt_pte_encode(dma_addr_t addr,
			  enum i915_cache_level level,
			  u32 flags)
282
{
283
	gen6_pte_t pte = GEN6_PTE_VALID;
284 285
	pte |= GEN6_PTE_ADDR_ENCODE(addr);

286 287
	if (!(flags & PTE_READ_ONLY))
		pte |= BYT_PTE_WRITEABLE;
288 289 290 291 292 293 294

	if (level != I915_CACHE_NONE)
		pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;

	return pte;
}

295 296 297
static u64 hsw_pte_encode(dma_addr_t addr,
			  enum i915_cache_level level,
			  u32 flags)
298
{
299
	gen6_pte_t pte = GEN6_PTE_VALID;
300
	pte |= HSW_PTE_ADDR_ENCODE(addr);
301 302

	if (level != I915_CACHE_NONE)
303
		pte |= HSW_WB_LLC_AGE3;
304 305 306 307

	return pte;
}

308 309 310
static u64 iris_pte_encode(dma_addr_t addr,
			   enum i915_cache_level level,
			   u32 flags)
311
{
312
	gen6_pte_t pte = GEN6_PTE_VALID;
313 314
	pte |= HSW_PTE_ADDR_ENCODE(addr);

315 316 317 318
	switch (level) {
	case I915_CACHE_NONE:
		break;
	case I915_CACHE_WT:
319
		pte |= HSW_WT_ELLC_LLC_AGE3;
320 321
		break;
	default:
322
		pte |= HSW_WB_ELLC_LLC_AGE3;
323 324
		break;
	}
325 326 327 328

	return pte;
}

329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
static void stash_init(struct pagestash *stash)
{
	pagevec_init(&stash->pvec);
	spin_lock_init(&stash->lock);
}

static struct page *stash_pop_page(struct pagestash *stash)
{
	struct page *page = NULL;

	spin_lock(&stash->lock);
	if (likely(stash->pvec.nr))
		page = stash->pvec.pages[--stash->pvec.nr];
	spin_unlock(&stash->lock);

	return page;
}

static void stash_push_pagevec(struct pagestash *stash, struct pagevec *pvec)
{
349
	unsigned int nr;
350 351 352

	spin_lock_nested(&stash->lock, SINGLE_DEPTH_NESTING);

353
	nr = min_t(typeof(nr), pvec->nr, pagevec_space(&stash->pvec));
354 355 356 357 358 359 360 361 362 363
	memcpy(stash->pvec.pages + stash->pvec.nr,
	       pvec->pages + pvec->nr - nr,
	       sizeof(pvec->pages[0]) * nr);
	stash->pvec.nr += nr;

	spin_unlock(&stash->lock);

	pvec->nr -= nr;
}

364
static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
365
{
366 367
	struct pagevec stack;
	struct page *page;
368

369 370
	if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
		i915_gem_shrink_all(vm->i915);
371

372 373 374
	page = stash_pop_page(&vm->free_pages);
	if (page)
		return page;
375 376 377 378 379

	if (!vm->pt_kmap_wc)
		return alloc_page(gfp);

	/* Look in our global stash of WC pages... */
380 381 382
	page = stash_pop_page(&vm->i915->mm.wc_stash);
	if (page)
		return page;
383

384
	/*
385
	 * Otherwise batch allocate pages to amortize cost of set_pages_wc.
386 387 388 389 390 391
	 *
	 * We have to be careful as page allocation may trigger the shrinker
	 * (via direct reclaim) which will fill up the WC stash underneath us.
	 * So we add our WB pages into a temporary pvec on the stack and merge
	 * them into the WC stash after all the allocations are complete.
	 */
392
	pagevec_init(&stack);
393 394
	do {
		struct page *page;
395

396 397 398 399
		page = alloc_page(gfp);
		if (unlikely(!page))
			break;

400 401
		stack.pages[stack.nr++] = page;
	} while (pagevec_space(&stack));
402

403 404
	if (stack.nr && !set_pages_array_wc(stack.pages, stack.nr)) {
		page = stack.pages[--stack.nr];
405

406
		/* Merge spare WC pages to the global stash */
407 408
		if (stack.nr)
			stash_push_pagevec(&vm->i915->mm.wc_stash, &stack);
409

410 411 412
		/* Push any surplus WC pages onto the local VM stash */
		if (stack.nr)
			stash_push_pagevec(&vm->free_pages, &stack);
413
	}
414

415 416 417 418 419 420 421
	/* Return unwanted leftovers */
	if (unlikely(stack.nr)) {
		WARN_ON_ONCE(set_pages_array_wb(stack.pages, stack.nr));
		__pagevec_release(&stack);
	}

	return page;
422 423
}

424 425
static void vm_free_pages_release(struct i915_address_space *vm,
				  bool immediate)
426
{
427 428
	struct pagevec *pvec = &vm->free_pages.pvec;
	struct pagevec stack;
429

430
	lockdep_assert_held(&vm->free_pages.lock);
431
	GEM_BUG_ON(!pagevec_count(pvec));
432

433
	if (vm->pt_kmap_wc) {
434 435
		/*
		 * When we use WC, first fill up the global stash and then
436 437
		 * only if full immediately free the overflow.
		 */
438
		stash_push_pagevec(&vm->i915->mm.wc_stash, pvec);
439

440 441 442 443 444 445 446 447
		/*
		 * As we have made some room in the VM's free_pages,
		 * we can wait for it to fill again. Unless we are
		 * inside i915_address_space_fini() and must
		 * immediately release the pages!
		 */
		if (pvec->nr <= (immediate ? 0 : PAGEVEC_SIZE - 1))
			return;
448

449 450 451 452 453 454 455 456 457 458
		/*
		 * We have to drop the lock to allow ourselves to sleep,
		 * so take a copy of the pvec and clear the stash for
		 * others to use it as we sleep.
		 */
		stack = *pvec;
		pagevec_reinit(pvec);
		spin_unlock(&vm->free_pages.lock);

		pvec = &stack;
459
		set_pages_array_wb(pvec->pages, pvec->nr);
460 461

		spin_lock(&vm->free_pages.lock);
462 463 464
	}

	__pagevec_release(pvec);
465 466 467 468
}

static void vm_free_page(struct i915_address_space *vm, struct page *page)
{
469 470 471 472 473 474 475 476
	/*
	 * On !llc, we need to change the pages back to WB. We only do so
	 * in bulk, so we rarely need to change the page attributes here,
	 * but doing so requires a stop_machine() from deep inside arch/x86/mm.
	 * To make detection of the possible sleep more likely, use an
	 * unconditional might_sleep() for everybody.
	 */
	might_sleep();
477
	spin_lock(&vm->free_pages.lock);
478
	while (!pagevec_space(&vm->free_pages.pvec))
479
		vm_free_pages_release(vm, false);
480 481
	GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec) >= PAGEVEC_SIZE);
	pagevec_add(&vm->free_pages.pvec, page);
482 483 484
	spin_unlock(&vm->free_pages.lock);
}

485
static void i915_address_space_init(struct i915_address_space *vm, int subclass)
486
{
487 488
	kref_init(&vm->ref);

489 490 491 492 493 494
	/*
	 * The vm->mutex must be reclaim safe (for use in the shrinker).
	 * Do a dummy acquire now under fs_reclaim so that any allocation
	 * attempt holding the lock is immediately reported by lockdep.
	 */
	mutex_init(&vm->mutex);
495
	lockdep_set_subclass(&vm->mutex, subclass);
496
	i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex);
497

498 499 500 501 502 503 504
	GEM_BUG_ON(!vm->total);
	drm_mm_init(&vm->mm, 0, vm->total);
	vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;

	stash_init(&vm->free_pages);

	INIT_LIST_HEAD(&vm->unbound_list);
505
	INIT_LIST_HEAD(&vm->bound_list);
506 507 508 509 510 511 512 513 514 515 516
}

static void i915_address_space_fini(struct i915_address_space *vm)
{
	spin_lock(&vm->free_pages.lock);
	if (pagevec_count(&vm->free_pages.pvec))
		vm_free_pages_release(vm, true);
	GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec));
	spin_unlock(&vm->free_pages.lock);

	drm_mm_takedown(&vm->mm);
517 518

	mutex_destroy(&vm->mutex);
519
}
520

521 522 523 524
static int __setup_page_dma(struct i915_address_space *vm,
			    struct i915_page_dma *p,
			    gfp_t gfp)
{
525
	p->page = vm_alloc_page(vm, gfp | I915_GFP_ALLOW_FAIL);
526 527
	if (unlikely(!p->page))
		return -ENOMEM;
528

529 530 531
	p->daddr = dma_map_page_attrs(vm->dma,
				      p->page, 0, PAGE_SIZE,
				      PCI_DMA_BIDIRECTIONAL,
532
				      DMA_ATTR_SKIP_CPU_SYNC |
533
				      DMA_ATTR_NO_WARN);
534 535 536
	if (unlikely(dma_mapping_error(vm->dma, p->daddr))) {
		vm_free_page(vm, p->page);
		return -ENOMEM;
537
	}
538 539

	return 0;
540 541
}

542
static int setup_page_dma(struct i915_address_space *vm,
543
			  struct i915_page_dma *p)
544
{
545
	return __setup_page_dma(vm, p, __GFP_HIGHMEM);
546 547
}

548
static void cleanup_page_dma(struct i915_address_space *vm,
549
			     struct i915_page_dma *p)
550
{
551 552
	dma_unmap_page(vm->dma, p->daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
	vm_free_page(vm, p->page);
553 554
}

555
#define kmap_atomic_px(px) kmap_atomic(px_base(px)->page)
556

557 558
#define setup_px(vm, px) setup_page_dma((vm), px_base(px))
#define cleanup_px(vm, px) cleanup_page_dma((vm), px_base(px))
559 560
#define fill_px(vm, px, v) fill_page_dma((vm), px_base(px), (v))
#define fill32_px(vm, px, v) fill_page_dma_32((vm), px_base(px), (v))
561

562 563 564
static void fill_page_dma(struct i915_address_space *vm,
			  struct i915_page_dma *p,
			  const u64 val)
565
{
566
	u64 * const vaddr = kmap_atomic(p->page);
567

568
	memset64(vaddr, val, PAGE_SIZE / sizeof(val));
569

570
	kunmap_atomic(vaddr);
571 572
}

573 574 575
static void fill_page_dma_32(struct i915_address_space *vm,
			     struct i915_page_dma *p,
			     const u32 v)
576
{
577
	fill_page_dma(vm, p, (u64)v << 32 | v);
578 579
}

580
static int
581
setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
582
{
583
	unsigned long size;
584

585 586 587 588 589 590 591
	/*
	 * In order to utilize 64K pages for an object with a size < 2M, we will
	 * need to support a 64K scratch page, given that every 16th entry for a
	 * page-table operating in 64K mode must point to a properly aligned 64K
	 * region, including any PTEs which happen to point to scratch.
	 *
	 * This is only relevant for the 48b PPGTT where we support
592 593 594
	 * huge-gtt-pages, see also i915_vma_insert(). However, as we share the
	 * scratch (read-only) between all vm, we create one 64k scratch page
	 * for all.
595
	 */
596
	size = I915_GTT_PAGE_SIZE_4K;
597
	if (i915_vm_is_4lvl(vm) &&
598
	    HAS_PAGE_SIZES(vm->i915, I915_GTT_PAGE_SIZE_64K)) {
599 600
		size = I915_GTT_PAGE_SIZE_64K;
		gfp |= __GFP_NOWARN;
601
	}
602 603 604 605 606 607
	gfp |= __GFP_ZERO | __GFP_RETRY_MAYFAIL;

	do {
		int order = get_order(size);
		struct page *page;
		dma_addr_t addr;
608

609
		page = alloc_pages(gfp, order);
610
		if (unlikely(!page))
611
			goto skip;
612

613 614 615
		addr = dma_map_page_attrs(vm->dma,
					  page, 0, size,
					  PCI_DMA_BIDIRECTIONAL,
616
					  DMA_ATTR_SKIP_CPU_SYNC |
617
					  DMA_ATTR_NO_WARN);
618 619
		if (unlikely(dma_mapping_error(vm->dma, addr)))
			goto free_page;
620

621 622
		if (unlikely(!IS_ALIGNED(addr, size)))
			goto unmap_page;
623

624 625
		vm->scratch_page.page = page;
		vm->scratch_page.daddr = addr;
626
		vm->scratch_order = order;
627 628 629 630 631 632 633 634 635 636 637 638 639
		return 0;

unmap_page:
		dma_unmap_page(vm->dma, addr, size, PCI_DMA_BIDIRECTIONAL);
free_page:
		__free_pages(page, order);
skip:
		if (size == I915_GTT_PAGE_SIZE_4K)
			return -ENOMEM;

		size = I915_GTT_PAGE_SIZE_4K;
		gfp &= ~__GFP_NOWARN;
	} while (1);
640 641
}

642
static void cleanup_scratch_page(struct i915_address_space *vm)
643
{
644
	struct i915_page_dma *p = &vm->scratch_page;
645
	int order = vm->scratch_order;
646

647
	dma_unmap_page(vm->dma, p->daddr, BIT(order) << PAGE_SHIFT,
648
		       PCI_DMA_BIDIRECTIONAL);
649
	__free_pages(p->page, order);
650 651
}

652
static struct i915_page_table *alloc_pt(struct i915_address_space *vm)
653
{
654
	struct i915_page_table *pt;
655

656
	pt = kmalloc(sizeof(*pt), I915_GFP_ALLOW_FAIL);
657
	if (unlikely(!pt))
658 659
		return ERR_PTR(-ENOMEM);

660 661 662 663
	if (unlikely(setup_px(vm, pt))) {
		kfree(pt);
		return ERR_PTR(-ENOMEM);
	}
664

665 666
	atomic_set(&pt->used, 0);

667 668 669
	return pt;
}

670
static void free_pt(struct i915_address_space *vm, struct i915_page_table *pt)
671
{
672
	cleanup_px(vm, pt);
673 674 675 676 677 678
	kfree(pt);
}

static void gen8_initialize_pt(struct i915_address_space *vm,
			       struct i915_page_table *pt)
{
679
	fill_px(vm, pt, vm->scratch_pte);
680 681
}

682
static void gen6_initialize_pt(struct i915_address_space *vm,
683 684
			       struct i915_page_table *pt)
{
685
	fill32_px(vm, pt, vm->scratch_pte);
686 687
}

688
static struct i915_page_directory *__alloc_pd(void)
689
{
690
	struct i915_page_directory *pd;
691

692
	pd = kmalloc(sizeof(*pd), I915_GFP_ALLOW_FAIL);
693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711

	if (unlikely(!pd))
		return NULL;

	memset(&pd->base, 0, sizeof(pd->base));
	atomic_set(&pd->used, 0);
	spin_lock_init(&pd->lock);

	/* for safety */
	pd->entry[0] = NULL;

	return pd;
}

static struct i915_page_directory *alloc_pd(struct i915_address_space *vm)
{
	struct i915_page_directory *pd;

	pd = __alloc_pd();
712
	if (unlikely(!pd))
713 714
		return ERR_PTR(-ENOMEM);

715 716 717 718
	if (unlikely(setup_px(vm, pd))) {
		kfree(pd);
		return ERR_PTR(-ENOMEM);
	}
719

720 721 722
	return pd;
}

723 724 725 726 727
static inline bool pd_has_phys_page(const struct i915_page_directory * const pd)
{
	return pd->base.page;
}

728
static void free_pd(struct i915_address_space *vm,
729
		    struct i915_page_directory *pd)
730
{
731 732 733
	if (likely(pd_has_phys_page(pd)))
		cleanup_px(vm, pd);

734
	kfree(pd);
735 736
}

737 738 739
static void init_pd_with_page(struct i915_address_space *vm,
			      struct i915_page_directory * const pd,
			      struct i915_page_table *pt)
740
{
741 742
	fill_px(vm, pd, gen8_pde_encode(px_dma(pt), I915_CACHE_LLC));
	memset_p(pd->entry, pt, 512);
743 744
}

M
Mika Kuoppala 已提交
745 746 747
static void init_pd(struct i915_address_space *vm,
		    struct i915_page_directory * const pd,
		    struct i915_page_directory * const to)
748
{
749 750
	GEM_DEBUG_BUG_ON(!pd_has_phys_page(pd));

M
Mika Kuoppala 已提交
751 752
	fill_px(vm, pd, gen8_pdpe_encode(px_dma(to), I915_CACHE_LLC));
	memset_p(pd->entry, to, 512);
753 754
}

755 756
/*
 * PDE TLBs are a pain to invalidate on GEN8+. When we modify
757 758 759 760
 * the page table structures, we mark them dirty so that
 * context switching/execlist queuing code takes extra steps
 * to ensure that tlbs are flushed.
 */
761
static void mark_tlbs_dirty(struct i915_ppgtt *ppgtt)
762
{
763
	ppgtt->pd_dirty_engines = ALL_ENGINES;
764 765
}

766 767 768
/* Removes entries from a single page table, releasing it if it's empty.
 * Caller can use the return value to update higher-level entries.
 */
769
static bool gen8_ppgtt_clear_pt(const struct i915_address_space *vm,
770
				struct i915_page_table *pt,
771
				u64 start, u64 length)
772
{
773
	unsigned int num_entries = gen8_pte_count(start, length);
774
	gen8_pte_t *vaddr;
775

776
	vaddr = kmap_atomic_px(pt);
777
	memset64(vaddr + gen8_pte_index(start), vm->scratch_pte, num_entries);
778
	kunmap_atomic(vaddr);
779

780 781
	GEM_BUG_ON(num_entries > atomic_read(&pt->used));
	return !atomic_sub_return(num_entries, &pt->used);
782
}
783

784 785 786 787 788 789 790 791 792 793 794 795
static void gen8_ppgtt_set_pde(struct i915_address_space *vm,
			       struct i915_page_directory *pd,
			       struct i915_page_table *pt,
			       unsigned int pde)
{
	gen8_pde_t *vaddr;

	vaddr = kmap_atomic_px(pd);
	vaddr[pde] = gen8_pde_encode(px_dma(pt), I915_CACHE_LLC);
	kunmap_atomic(vaddr);
}

796
static bool gen8_ppgtt_clear_pd(struct i915_address_space *vm,
797
				struct i915_page_directory *pd,
798
				u64 start, u64 length)
799 800
{
	struct i915_page_table *pt;
801
	u32 pde;
802 803

	gen8_for_each_pde(pt, pd, start, length, pde) {
804 805
		bool free = false;

806 807
		GEM_BUG_ON(pt == vm->scratch_pt);

808 809
		if (!gen8_ppgtt_clear_pt(vm, pt, start, length))
			continue;
810

811
		spin_lock(&pd->lock);
812
		if (!atomic_read(&pt->used)) {
813
			gen8_ppgtt_set_pde(vm, pd, vm->scratch_pt, pde);
814
			pd->entry[pde] = vm->scratch_pt;
815

816 817
			GEM_BUG_ON(!atomic_read(&pd->used));
			atomic_dec(&pd->used);
818 819 820 821 822
			free = true;
		}
		spin_unlock(&pd->lock);
		if (free)
			free_pt(vm, pt);
823 824
	}

825
	return !atomic_read(&pd->used);
826
}
827

828
static void gen8_ppgtt_set_pdpe(struct i915_page_directory *pdp,
829 830 831 832 833
				struct i915_page_directory *pd,
				unsigned int pdpe)
{
	gen8_ppgtt_pdpe_t *vaddr;

834
	if (!pd_has_phys_page(pdp))
835 836 837 838 839
		return;

	vaddr = kmap_atomic_px(pdp);
	vaddr[pdpe] = gen8_pdpe_encode(px_dma(pd), I915_CACHE_LLC);
	kunmap_atomic(vaddr);
840
}
841

842 843 844 845
/* Removes entries from a single page dir pointer, releasing it if it's empty.
 * Caller can use the return value to update higher-level entries
 */
static bool gen8_ppgtt_clear_pdp(struct i915_address_space *vm,
846
				 struct i915_page_directory * const pdp,
847
				 u64 start, u64 length)
848 849
{
	struct i915_page_directory *pd;
850
	unsigned int pdpe;
851

852
	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
853 854
		bool free = false;

855 856
		GEM_BUG_ON(pd == vm->scratch_pd);

857 858
		if (!gen8_ppgtt_clear_pd(vm, pd, start, length))
			continue;
859

860
		spin_lock(&pdp->lock);
861
		if (!atomic_read(&pd->used)) {
862
			gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe);
863
			pdp->entry[pdpe] = vm->scratch_pd;
864

865 866
			GEM_BUG_ON(!atomic_read(&pdp->used));
			atomic_dec(&pdp->used);
867 868 869 870 871
			free = true;
		}
		spin_unlock(&pdp->lock);
		if (free)
			free_pd(vm, pd);
872
	}
873

874
	return !atomic_read(&pdp->used);
875
}
876

877 878 879
static void gen8_ppgtt_clear_3lvl(struct i915_address_space *vm,
				  u64 start, u64 length)
{
880
	gen8_ppgtt_clear_pdp(vm, i915_vm_to_ppgtt(vm)->pd, start, length);
881 882
}

883 884
static void gen8_ppgtt_set_pml4e(struct i915_page_directory *pml4,
				 struct i915_page_directory *pdp,
885 886 887 888 889 890 891 892 893
				 unsigned int pml4e)
{
	gen8_ppgtt_pml4e_t *vaddr;

	vaddr = kmap_atomic_px(pml4);
	vaddr[pml4e] = gen8_pml4e_encode(px_dma(pdp), I915_CACHE_LLC);
	kunmap_atomic(vaddr);
}

894 895 896 897
/* Removes entries from a single pml4.
 * This is the top-level structure in 4-level page tables used on gen8+.
 * Empty entries are always scratch pml4e.
 */
898 899
static void gen8_ppgtt_clear_4lvl(struct i915_address_space *vm,
				  u64 start, u64 length)
900
{
901
	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
902 903
	struct i915_page_directory * const pml4 = ppgtt->pd;
	struct i915_page_directory *pdp;
904
	unsigned int pml4e;
905

906
	GEM_BUG_ON(!i915_vm_is_4lvl(vm));
907

908
	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
909
		bool free = false;
910 911
		GEM_BUG_ON(pdp == vm->scratch_pdp);

912 913
		if (!gen8_ppgtt_clear_pdp(vm, pdp, start, length))
			continue;
914

915
		spin_lock(&pml4->lock);
916
		if (!atomic_read(&pdp->used)) {
917
			gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
918
			pml4->entry[pml4e] = vm->scratch_pdp;
919 920 921 922
			free = true;
		}
		spin_unlock(&pml4->lock);
		if (free)
923
			free_pd(vm, pdp);
924 925 926
	}
}

927
static inline struct sgt_dma {
928 929
	struct scatterlist *sg;
	dma_addr_t dma, max;
930 931 932 933 934
} sgt_dma(struct i915_vma *vma) {
	struct scatterlist *sg = vma->pages->sgl;
	dma_addr_t addr = sg_dma_address(sg);
	return (struct sgt_dma) { sg, addr, addr + sg->length };
}
935

936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952
struct gen8_insert_pte {
	u16 pml4e;
	u16 pdpe;
	u16 pde;
	u16 pte;
};

static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
{
	return (struct gen8_insert_pte) {
		 gen8_pml4e_index(start),
		 gen8_pdpe_index(start),
		 gen8_pde_index(start),
		 gen8_pte_index(start),
	};
}

953
static __always_inline bool
954
gen8_ppgtt_insert_pte_entries(struct i915_ppgtt *ppgtt,
955
			      struct i915_page_directory *pdp,
956
			      struct sgt_dma *iter,
957
			      struct gen8_insert_pte *idx,
958 959
			      enum i915_cache_level cache_level,
			      u32 flags)
960
{
961
	struct i915_page_directory *pd;
962
	const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);
963 964
	gen8_pte_t *vaddr;
	bool ret;
965

966
	GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
967 968
	pd = i915_pd_entry(pdp, idx->pdpe);
	vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
969
	do {
970 971
		vaddr[idx->pte] = pte_encode | iter->dma;

972
		iter->dma += I915_GTT_PAGE_SIZE;
973 974 975 976 977 978
		if (iter->dma >= iter->max) {
			iter->sg = __sg_next(iter->sg);
			if (!iter->sg) {
				ret = false;
				break;
			}
979

980 981
			iter->dma = sg_dma_address(iter->sg);
			iter->max = iter->dma + iter->sg->length;
B
Ben Widawsky 已提交
982
		}
983

984 985 986 987 988 989
		if (++idx->pte == GEN8_PTES) {
			idx->pte = 0;

			if (++idx->pde == I915_PDES) {
				idx->pde = 0;

990
				/* Limited by sg length for 3lvl */
991 992
				if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {
					idx->pdpe = 0;
993
					ret = true;
994
					break;
995 996
				}

997
				GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->vm));
998
				pd = pdp->entry[idx->pdpe];
999
			}
1000

1001
			kunmap_atomic(vaddr);
1002
			vaddr = kmap_atomic_px(i915_pt_entry(pd, idx->pde));
1003
		}
1004
	} while (1);
1005
	kunmap_atomic(vaddr);
1006

1007
	return ret;
1008 1009
}

1010
static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
1011
				   struct i915_vma *vma,
1012
				   enum i915_cache_level cache_level,
1013
				   u32 flags)
1014
{
1015
	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1016
	struct sgt_dma iter = sgt_dma(vma);
1017
	struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);
1018

1019
	gen8_ppgtt_insert_pte_entries(ppgtt, ppgtt->pd, &iter, &idx,
1020
				      cache_level, flags);
1021 1022

	vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1023
}
1024

1025
static void gen8_ppgtt_insert_huge_entries(struct i915_vma *vma,
1026
					   struct i915_page_directory *pml4,
1027
					   struct sgt_dma *iter,
1028 1029
					   enum i915_cache_level cache_level,
					   u32 flags)
1030
{
1031
	const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level, flags);
1032 1033 1034 1035 1036
	u64 start = vma->node.start;
	dma_addr_t rem = iter->sg->length;

	do {
		struct gen8_insert_pte idx = gen8_insert_pte(start);
1037 1038 1039
		struct i915_page_directory *pdp =
			i915_pdp_entry(pml4, idx.pml4e);
		struct i915_page_directory *pd = i915_pd_entry(pdp, idx.pdpe);
1040
		unsigned int page_size;
1041
		bool maybe_64K = false;
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
		gen8_pte_t encode = pte_encode;
		gen8_pte_t *vaddr;
		u16 index, max;

		if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_2M &&
		    IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_2M) &&
		    rem >= I915_GTT_PAGE_SIZE_2M && !idx.pte) {
			index = idx.pde;
			max = I915_PDES;
			page_size = I915_GTT_PAGE_SIZE_2M;

			encode |= GEN8_PDE_PS_2M;

			vaddr = kmap_atomic_px(pd);
		} else {
1057
			struct i915_page_table *pt = i915_pt_entry(pd, idx.pde);
1058 1059 1060 1061 1062

			index = idx.pte;
			max = GEN8_PTES;
			page_size = I915_GTT_PAGE_SIZE;

1063 1064 1065 1066
			if (!index &&
			    vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K &&
			    IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
			    (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
1067
			     rem >= (max - index) * I915_GTT_PAGE_SIZE))
1068 1069
				maybe_64K = true;

1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
			vaddr = kmap_atomic_px(pt);
		}

		do {
			GEM_BUG_ON(iter->sg->length < page_size);
			vaddr[index++] = encode | iter->dma;

			start += page_size;
			iter->dma += page_size;
			rem -= page_size;
			if (iter->dma >= iter->max) {
				iter->sg = __sg_next(iter->sg);
				if (!iter->sg)
					break;

				rem = iter->sg->length;
				iter->dma = sg_dma_address(iter->sg);
				iter->max = iter->dma + rem;

1089 1090 1091
				if (maybe_64K && index < max &&
				    !(IS_ALIGNED(iter->dma, I915_GTT_PAGE_SIZE_64K) &&
				      (IS_ALIGNED(rem, I915_GTT_PAGE_SIZE_64K) ||
1092
				       rem >= (max - index) * I915_GTT_PAGE_SIZE)))
1093 1094
					maybe_64K = false;

1095 1096 1097 1098 1099 1100
				if (unlikely(!IS_ALIGNED(iter->dma, page_size)))
					break;
			}
		} while (rem >= page_size && index < max);

		kunmap_atomic(vaddr);
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116

		/*
		 * Is it safe to mark the 2M block as 64K? -- Either we have
		 * filled whole page-table with 64K entries, or filled part of
		 * it and have reached the end of the sg table and we have
		 * enough padding.
		 */
		if (maybe_64K &&
		    (index == max ||
		     (i915_vm_has_scratch_64K(vma->vm) &&
		      !iter->sg && IS_ALIGNED(vma->node.start +
					      vma->node.size,
					      I915_GTT_PAGE_SIZE_2M)))) {
			vaddr = kmap_atomic_px(pd);
			vaddr[idx.pde] |= GEN8_PDE_IPS_64K;
			kunmap_atomic(vaddr);
1117
			page_size = I915_GTT_PAGE_SIZE_64K;
M
Matthew Auld 已提交
1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130

			/*
			 * We write all 4K page entries, even when using 64K
			 * pages. In order to verify that the HW isn't cheating
			 * by using the 4K PTE instead of the 64K PTE, we want
			 * to remove all the surplus entries. If the HW skipped
			 * the 64K PTE, it will read/write into the scratch page
			 * instead - which we detect as missing results during
			 * selftests.
			 */
			if (I915_SELFTEST_ONLY(vma->vm->scrub_64K)) {
				u16 i;

1131
				encode = vma->vm->scratch_pte;
1132 1133
				vaddr = kmap_atomic_px(i915_pt_entry(pd,
								     idx.pde));
M
Matthew Auld 已提交
1134 1135 1136 1137 1138 1139

				for (i = 1; i < index; i += 16)
					memset64(vaddr + i, encode, 15);

				kunmap_atomic(vaddr);
			}
1140
		}
1141 1142

		vma->page_sizes.gtt |= page_size;
1143 1144 1145
	} while (iter->sg);
}

1146
static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
1147
				   struct i915_vma *vma,
1148
				   enum i915_cache_level cache_level,
1149
				   u32 flags)
1150
{
1151
	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1152
	struct sgt_dma iter = sgt_dma(vma);
1153
	struct i915_page_directory * const pml4 = ppgtt->pd;
1154

1155
	if (vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
1156
		gen8_ppgtt_insert_huge_entries(vma, pml4, &iter, cache_level,
1157
					       flags);
1158 1159 1160
	} else {
		struct gen8_insert_pte idx = gen8_insert_pte(vma->node.start);

1161 1162
		while (gen8_ppgtt_insert_pte_entries(ppgtt,
						     i915_pdp_entry(pml4, idx.pml4e++),
1163 1164
						     &iter, &idx, cache_level,
						     flags))
1165
			GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4);
1166 1167

		vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
1168
	}
1169 1170
}

1171
static void gen8_free_page_tables(struct i915_address_space *vm,
1172
				  struct i915_page_directory *pd)
1173 1174 1175
{
	int i;

1176
	for (i = 0; i < I915_PDES; i++) {
1177 1178
		if (pd->entry[i] != vm->scratch_pt)
			free_pt(vm, pd->entry[i]);
1179
	}
B
Ben Widawsky 已提交
1180 1181
}

1182 1183
static int gen8_init_scratch(struct i915_address_space *vm)
{
1184
	int ret;
1185

1186 1187 1188 1189 1190 1191
	/*
	 * If everybody agrees to not to write into the scratch page,
	 * we can reuse it for all vm, keeping contexts and processes separate.
	 */
	if (vm->has_read_only &&
	    vm->i915->kernel_context &&
1192 1193
	    vm->i915->kernel_context->vm) {
		struct i915_address_space *clone = vm->i915->kernel_context->vm;
1194 1195 1196

		GEM_BUG_ON(!clone->has_read_only);

1197
		vm->scratch_order = clone->scratch_order;
1198 1199 1200 1201 1202 1203 1204
		vm->scratch_pte = clone->scratch_pte;
		vm->scratch_pt  = clone->scratch_pt;
		vm->scratch_pd  = clone->scratch_pd;
		vm->scratch_pdp = clone->scratch_pdp;
		return 0;
	}

1205
	ret = setup_scratch_page(vm, __GFP_HIGHMEM);
1206 1207
	if (ret)
		return ret;
1208

1209 1210 1211
	vm->scratch_pte =
		gen8_pte_encode(vm->scratch_page.daddr,
				I915_CACHE_LLC,
1212
				vm->has_read_only);
1213

1214
	vm->scratch_pt = alloc_pt(vm);
1215
	if (IS_ERR(vm->scratch_pt)) {
1216 1217
		ret = PTR_ERR(vm->scratch_pt);
		goto free_scratch_page;
1218 1219
	}

1220
	vm->scratch_pd = alloc_pd(vm);
1221
	if (IS_ERR(vm->scratch_pd)) {
1222 1223
		ret = PTR_ERR(vm->scratch_pd);
		goto free_pt;
1224 1225
	}

1226
	if (i915_vm_is_4lvl(vm)) {
1227
		vm->scratch_pdp = alloc_pd(vm);
1228
		if (IS_ERR(vm->scratch_pdp)) {
1229 1230
			ret = PTR_ERR(vm->scratch_pdp);
			goto free_pd;
1231 1232 1233
		}
	}

1234
	gen8_initialize_pt(vm, vm->scratch_pt);
1235
	init_pd_with_page(vm, vm->scratch_pd, vm->scratch_pt);
1236
	if (i915_vm_is_4lvl(vm))
M
Mika Kuoppala 已提交
1237
		init_pd(vm, vm->scratch_pdp, vm->scratch_pd);
1238 1239

	return 0;
1240 1241

free_pd:
1242
	free_pd(vm, vm->scratch_pd);
1243
free_pt:
1244
	free_pt(vm, vm->scratch_pt);
1245
free_scratch_page:
1246
	cleanup_scratch_page(vm);
1247 1248

	return ret;
1249 1250
}

1251
static int gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create)
1252
{
1253
	struct i915_address_space *vm = &ppgtt->vm;
1254
	struct drm_i915_private *dev_priv = vm->i915;
1255 1256 1257
	enum vgt_g2v_type msg;
	int i;

1258
	if (i915_vm_is_4lvl(vm)) {
1259
		const u64 daddr = px_dma(ppgtt->pd);
1260

1261 1262
		I915_WRITE(vgtif_reg(pdp[0].lo), lower_32_bits(daddr));
		I915_WRITE(vgtif_reg(pdp[0].hi), upper_32_bits(daddr));
1263 1264 1265 1266

		msg = (create ? VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE :
				VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY);
	} else {
1267
		for (i = 0; i < GEN8_3LVL_PDPES; i++) {
1268
			const u64 daddr = i915_page_dir_dma_addr(ppgtt, i);
1269

1270 1271
			I915_WRITE(vgtif_reg(pdp[i].lo), lower_32_bits(daddr));
			I915_WRITE(vgtif_reg(pdp[i].hi), upper_32_bits(daddr));
1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282
		}

		msg = (create ? VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE :
				VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY);
	}

	I915_WRITE(vgtif_reg(g2v_notify), msg);

	return 0;
}

1283 1284
static void gen8_free_scratch(struct i915_address_space *vm)
{
1285 1286 1287
	if (!vm->scratch_page.daddr)
		return;

1288
	if (i915_vm_is_4lvl(vm))
1289
		free_pd(vm, vm->scratch_pdp);
1290 1291 1292
	free_pd(vm, vm->scratch_pd);
	free_pt(vm, vm->scratch_pt);
	cleanup_scratch_page(vm);
1293 1294
}

1295
static void gen8_ppgtt_cleanup_3lvl(struct i915_address_space *vm,
1296
				    struct i915_page_directory *pdp)
1297
{
1298
	const unsigned int pdpes = i915_pdpes_per_pdp(vm);
1299 1300
	int i;

1301
	for (i = 0; i < pdpes; i++) {
1302
		if (pdp->entry[i] == vm->scratch_pd)
1303 1304
			continue;

1305 1306
		gen8_free_page_tables(vm, pdp->entry[i]);
		free_pd(vm, pdp->entry[i]);
1307
	}
1308

1309
	free_pd(vm, pdp);
1310 1311
}

1312
static void gen8_ppgtt_cleanup_4lvl(struct i915_ppgtt *ppgtt)
1313
{
1314
	struct i915_page_directory * const pml4 = ppgtt->pd;
1315 1316
	int i;

1317
	for (i = 0; i < GEN8_PML4ES_PER_PML4; i++) {
1318 1319 1320
		struct i915_page_directory *pdp = i915_pdp_entry(pml4, i);

		if (pdp == ppgtt->vm.scratch_pdp)
1321 1322
			continue;

1323
		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, pdp);
1324 1325
	}

1326
	free_pd(&ppgtt->vm, pml4);
1327 1328 1329 1330
}

static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
{
1331
	struct drm_i915_private *i915 = vm->i915;
1332
	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1333

1334
	if (intel_vgpu_active(i915))
1335 1336
		gen8_ppgtt_notify_vgt(ppgtt, false);

1337
	if (i915_vm_is_4lvl(vm))
1338
		gen8_ppgtt_cleanup_4lvl(ppgtt);
1339
	else
1340
		gen8_ppgtt_cleanup_3lvl(&ppgtt->vm, ppgtt->pd);
1341

1342
	gen8_free_scratch(vm);
1343 1344
}

1345 1346 1347
static int gen8_ppgtt_alloc_pd(struct i915_address_space *vm,
			       struct i915_page_directory *pd,
			       u64 start, u64 length)
1348
{
1349
	struct i915_page_table *pt, *alloc = NULL;
1350
	u64 from = start;
1351
	unsigned int pde;
1352
	int ret = 0;
1353

1354
	spin_lock(&pd->lock);
1355
	gen8_for_each_pde(pt, pd, start, length, pde) {
1356
		const int count = gen8_pte_count(start, length);
1357

1358
		if (pt == vm->scratch_pt) {
1359
			spin_unlock(&pd->lock);
1360

1361 1362 1363 1364 1365
			pt = fetch_and_zero(&alloc);
			if (!pt)
				pt = alloc_pt(vm);
			if (IS_ERR(pt)) {
				ret = PTR_ERR(pt);
1366
				goto unwind;
1367
			}
1368

1369
			if (count < GEN8_PTES || intel_vgpu_active(vm->i915))
1370
				gen8_initialize_pt(vm, pt);
1371

1372 1373
			spin_lock(&pd->lock);
			if (pd->entry[pde] == vm->scratch_pt) {
1374
				gen8_ppgtt_set_pde(vm, pd, pt, pde);
1375
				pd->entry[pde] = pt;
1376
				atomic_inc(&pd->used);
1377
			} else {
1378 1379
				alloc = pt;
				pt = pd->entry[pde];
1380
			}
1381
		}
1382

1383
		atomic_add(count, &pt->used);
1384
	}
1385
	spin_unlock(&pd->lock);
1386
	goto out;
1387

1388 1389
unwind:
	gen8_ppgtt_clear_pd(vm, pd, from, start - from);
1390 1391 1392 1393
out:
	if (alloc)
		free_pt(vm, alloc);
	return ret;
1394 1395
}

1396
static int gen8_ppgtt_alloc_pdp(struct i915_address_space *vm,
1397
				struct i915_page_directory *pdp,
1398
				u64 start, u64 length)
1399
{
1400
	struct i915_page_directory *pd, *alloc = NULL;
1401 1402
	u64 from = start;
	unsigned int pdpe;
1403
	int ret = 0;
1404

1405
	spin_lock(&pdp->lock);
1406
	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
1407
		if (pd == vm->scratch_pd) {
1408
			spin_unlock(&pdp->lock);
1409

1410 1411 1412 1413 1414
			pd = fetch_and_zero(&alloc);
			if (!pd)
				pd = alloc_pd(vm);
			if (IS_ERR(pd)) {
				ret = PTR_ERR(pd);
1415
				goto unwind;
1416
			}
1417

1418
			init_pd_with_page(vm, pd, vm->scratch_pt);
1419

1420 1421
			spin_lock(&pdp->lock);
			if (pdp->entry[pdpe] == vm->scratch_pd) {
1422
				gen8_ppgtt_set_pdpe(pdp, pd, pdpe);
1423
				pdp->entry[pdpe] = pd;
1424
				atomic_inc(&pdp->used);
1425
			} else {
1426 1427
				alloc = pd;
				pd = pdp->entry[pdpe];
1428
			}
1429
		}
1430
		atomic_inc(&pd->used);
1431
		spin_unlock(&pdp->lock);
1432 1433

		ret = gen8_ppgtt_alloc_pd(vm, pd, start, length);
1434 1435
		if (unlikely(ret))
			goto unwind_pd;
1436 1437

		spin_lock(&pdp->lock);
1438
		atomic_dec(&pd->used);
1439
	}
1440
	spin_unlock(&pdp->lock);
1441
	goto out;
1442

1443
unwind_pd:
1444
	spin_lock(&pdp->lock);
1445
	if (atomic_dec_and_test(&pd->used)) {
1446
		gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe);
1447 1448
		GEM_BUG_ON(!atomic_read(&pdp->used));
		atomic_dec(&pdp->used);
1449 1450
		free_pd(vm, pd);
	}
1451
	spin_unlock(&pdp->lock);
1452 1453
unwind:
	gen8_ppgtt_clear_pdp(vm, pdp, from, start - from);
1454 1455 1456 1457
out:
	if (alloc)
		free_pd(vm, alloc);
	return ret;
1458 1459
}

1460 1461
static int gen8_ppgtt_alloc_3lvl(struct i915_address_space *vm,
				 u64 start, u64 length)
1462
{
1463
	return gen8_ppgtt_alloc_pdp(vm,
1464
				    i915_vm_to_ppgtt(vm)->pd, start, length);
1465
}
1466

1467 1468 1469
static int gen8_ppgtt_alloc_4lvl(struct i915_address_space *vm,
				 u64 start, u64 length)
{
1470
	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1471
	struct i915_page_directory * const pml4 = ppgtt->pd;
1472
	struct i915_page_directory *pdp, *alloc = NULL;
1473
	u64 from = start;
1474
	int ret = 0;
1475
	u32 pml4e;
1476

1477
	spin_lock(&pml4->lock);
1478
	gen8_for_each_pml4e(pdp, pml4, start, length, pml4e) {
1479 1480 1481
		if (pdp == vm->scratch_pdp) {
			spin_unlock(&pml4->lock);

1482 1483 1484 1485 1486
			pdp = fetch_and_zero(&alloc);
			if (!pdp)
				pdp = alloc_pd(vm);
			if (IS_ERR(pdp)) {
				ret = PTR_ERR(pdp);
1487
				goto unwind;
1488
			}
1489

M
Mika Kuoppala 已提交
1490
			init_pd(vm, pdp, vm->scratch_pd);
1491

1492 1493
			spin_lock(&pml4->lock);
			if (pml4->entry[pml4e] == vm->scratch_pdp) {
1494
				gen8_ppgtt_set_pml4e(pml4, pdp, pml4e);
1495
				pml4->entry[pml4e] = pdp;
1496
			} else {
1497 1498
				alloc = pdp;
				pdp = pml4->entry[pml4e];
1499
			}
1500
		}
1501
		atomic_inc(&pdp->used);
1502
		spin_unlock(&pml4->lock);
1503

1504
		ret = gen8_ppgtt_alloc_pdp(vm, pdp, start, length);
1505 1506
		if (unlikely(ret))
			goto unwind_pdp;
1507 1508

		spin_lock(&pml4->lock);
1509
		atomic_dec(&pdp->used);
1510
	}
1511
	spin_unlock(&pml4->lock);
1512
	goto out;
1513

1514
unwind_pdp:
1515
	spin_lock(&pml4->lock);
1516
	if (atomic_dec_and_test(&pdp->used)) {
1517
		gen8_ppgtt_set_pml4e(pml4, vm->scratch_pdp, pml4e);
1518
		free_pd(vm, pdp);
1519
	}
1520
	spin_unlock(&pml4->lock);
1521 1522
unwind:
	gen8_ppgtt_clear_4lvl(vm, from, start - from);
1523 1524 1525 1526
out:
	if (alloc)
		free_pd(vm, alloc);
	return ret;
1527 1528
}

1529
static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
1530
{
1531
	struct i915_address_space *vm = &ppgtt->vm;
1532
	struct i915_page_directory *pdp = ppgtt->pd;
1533
	struct i915_page_directory *pd;
1534
	u64 start = 0, length = ppgtt->vm.total;
1535 1536
	u64 from = start;
	unsigned int pdpe;
1537

1538 1539 1540 1541
	gen8_for_each_pdpe(pd, pdp, start, length, pdpe) {
		pd = alloc_pd(vm);
		if (IS_ERR(pd))
			goto unwind;
1542

1543
		init_pd_with_page(vm, pd, vm->scratch_pt);
1544
		gen8_ppgtt_set_pdpe(pdp, pd, pdpe);
1545 1546

		atomic_inc(&pdp->used);
1547
	}
1548

1549 1550
	atomic_inc(&pdp->used); /* never remove */

1551
	return 0;
1552

1553 1554 1555
unwind:
	start -= from;
	gen8_for_each_pdpe(pd, pdp, from, start, pdpe) {
1556
		gen8_ppgtt_set_pdpe(pdp, vm->scratch_pd, pdpe);
1557 1558
		free_pd(vm, pd);
	}
1559
	atomic_set(&pdp->used, 0);
1560
	return -ENOMEM;
1561 1562
}

1563
static void ppgtt_init(struct drm_i915_private *i915,
1564
		       struct i915_ppgtt *ppgtt)
1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577
{
	ppgtt->vm.i915 = i915;
	ppgtt->vm.dma = &i915->drm.pdev->dev;
	ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size);

	i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT);

	ppgtt->vm.vma_ops.bind_vma    = ppgtt_bind_vma;
	ppgtt->vm.vma_ops.unbind_vma  = ppgtt_unbind_vma;
	ppgtt->vm.vma_ops.set_pages   = ppgtt_set_pages;
	ppgtt->vm.vma_ops.clear_pages = clear_pages;
}

1578
/*
1579 1580 1581 1582
 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
 * with a net effect resembling a 2-level page table in normal x86 terms. Each
 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
 * space.
B
Ben Widawsky 已提交
1583
 *
1584
 */
1585
static struct i915_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
B
Ben Widawsky 已提交
1586
{
1587
	struct i915_ppgtt *ppgtt;
1588 1589 1590 1591 1592 1593
	int err;

	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
	if (!ppgtt)
		return ERR_PTR(-ENOMEM);

1594
	ppgtt_init(i915, ppgtt);
1595

1596 1597 1598 1599 1600 1601 1602
	/*
	 * From bdw, there is hw support for read-only pages in the PPGTT.
	 *
	 * Gen11 has HSDES#:1807136187 unresolved. Disable ro support
	 * for now.
	 */
	ppgtt->vm.has_read_only = INTEL_GEN(i915) != 11;
1603

1604 1605 1606
	/* There are only few exceptions for gen >=6. chv and bxt.
	 * And we are not sure about the latter so play safe for now.
	 */
1607
	if (IS_CHERRYVIEW(i915) || IS_BROXTON(i915))
1608
		ppgtt->vm.pt_kmap_wc = true;
1609

1610 1611 1612
	err = gen8_init_scratch(&ppgtt->vm);
	if (err)
		goto err_free;
1613

1614 1615 1616 1617
	ppgtt->pd = __alloc_pd();
	if (!ppgtt->pd) {
		err = -ENOMEM;
		goto err_free_scratch;
1618
	}
1619

1620
	if (i915_vm_is_4lvl(&ppgtt->vm)) {
1621 1622 1623 1624
		err = setup_px(&ppgtt->vm, ppgtt->pd);
		if (err)
			goto err_free_pdp;

M
Mika Kuoppala 已提交
1625
		init_pd(&ppgtt->vm, ppgtt->pd, ppgtt->vm.scratch_pdp);
1626

1627 1628 1629
		ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_4lvl;
		ppgtt->vm.insert_entries = gen8_ppgtt_insert_4lvl;
		ppgtt->vm.clear_range = gen8_ppgtt_clear_4lvl;
1630
	} else {
M
Mika Kuoppala 已提交
1631 1632 1633 1634 1635 1636
		/*
		 * We don't need to setup dma for top level pdp, only
		 * for entries. So point entries to scratch.
		 */
		memset_p(ppgtt->pd->entry, ppgtt->vm.scratch_pd,
			 GEN8_3LVL_PDPES);
1637

1638 1639
		if (intel_vgpu_active(i915)) {
			err = gen8_preallocate_top_level_pdp(ppgtt);
1640
			if (err)
1641
				goto err_free_pdp;
1642
		}
1643

1644 1645 1646
		ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc_3lvl;
		ppgtt->vm.insert_entries = gen8_ppgtt_insert_3lvl;
		ppgtt->vm.clear_range = gen8_ppgtt_clear_3lvl;
1647
	}
1648

1649
	if (intel_vgpu_active(i915))
1650 1651
		gen8_ppgtt_notify_vgt(ppgtt, true);

1652
	ppgtt->vm.cleanup = gen8_ppgtt_cleanup;
1653

1654
	return ppgtt;
1655

1656 1657 1658
err_free_pdp:
	free_pd(&ppgtt->vm, ppgtt->pd);
err_free_scratch:
1659
	gen8_free_scratch(&ppgtt->vm);
1660 1661 1662
err_free:
	kfree(ppgtt);
	return ERR_PTR(err);
1663 1664
}

1665
/* Write pde (index) from the page directory @pd to the page table @pt */
1666
static inline void gen6_write_pde(const struct gen6_ppgtt *ppgtt,
C
Chris Wilson 已提交
1667 1668
				  const unsigned int pde,
				  const struct i915_page_table *pt)
B
Ben Widawsky 已提交
1669
{
1670
	/* Caller needs to make sure the write completes if necessary */
1671 1672
	iowrite32(GEN6_PDE_ADDR_ENCODE(px_dma(pt)) | GEN6_PDE_VALID,
		  ppgtt->pd_addr + pde);
1673
}
B
Ben Widawsky 已提交
1674

1675
static void gen7_ppgtt_enable(struct drm_i915_private *dev_priv)
B
Ben Widawsky 已提交
1676
{
1677
	struct intel_engine_cs *engine;
1678
	u32 ecochk, ecobits;
1679
	enum intel_engine_id id;
B
Ben Widawsky 已提交
1680

1681 1682
	ecobits = I915_READ(GAC_ECO_BITS);
	I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1683

1684
	ecochk = I915_READ(GAM_ECOCHK);
1685
	if (IS_HASWELL(dev_priv)) {
1686 1687 1688 1689 1690 1691
		ecochk |= ECOCHK_PPGTT_WB_HSW;
	} else {
		ecochk |= ECOCHK_PPGTT_LLC_IVB;
		ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
	}
	I915_WRITE(GAM_ECOCHK, ecochk);
1692

1693
	for_each_engine(engine, dev_priv, id) {
B
Ben Widawsky 已提交
1694
		/* GFX_MODE is per-ring on gen7+ */
1695 1696 1697
		ENGINE_WRITE(engine,
			     RING_MODE_GEN7,
			     _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
B
Ben Widawsky 已提交
1698
	}
1699
}
B
Ben Widawsky 已提交
1700

1701
static void gen6_ppgtt_enable(struct drm_i915_private *dev_priv)
1702
{
1703
	u32 ecochk, gab_ctl, ecobits;
1704

1705 1706 1707
	ecobits = I915_READ(GAC_ECO_BITS);
	I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
		   ECOBITS_PPGTT_CACHE64B);
B
Ben Widawsky 已提交
1708

1709 1710 1711 1712 1713 1714
	gab_ctl = I915_READ(GAB_CTL);
	I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);

	ecochk = I915_READ(GAM_ECOCHK);
	I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);

1715 1716
	if (HAS_PPGTT(dev_priv)) /* may be disabled for VT-d */
		I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
B
Ben Widawsky 已提交
1717 1718
}

1719
/* PPGTT support for Sandybdrige/Gen6 and later */
1720
static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1721
				   u64 start, u64 length)
1722
{
1723 1724 1725
	struct gen6_ppgtt * const ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
	const unsigned int first_entry = start / I915_GTT_PAGE_SIZE;
	const gen6_pte_t scratch_pte = vm->scratch_pte;
1726 1727
	unsigned int pde = first_entry / GEN6_PTES;
	unsigned int pte = first_entry % GEN6_PTES;
1728
	unsigned int num_entries = length / I915_GTT_PAGE_SIZE;
1729

1730
	while (num_entries) {
1731 1732
		struct i915_page_table * const pt =
			i915_pt_entry(ppgtt->base.pd, pde++);
1733
		const unsigned int count = min(num_entries, GEN6_PTES - pte);
1734
		gen6_pte_t *vaddr;
1735

1736 1737 1738 1739
		GEM_BUG_ON(pt == vm->scratch_pt);

		num_entries -= count;

1740 1741
		GEM_BUG_ON(count > atomic_read(&pt->used));
		if (!atomic_sub_return(count, &pt->used))
1742
			ppgtt->scan_for_unused_pt = true;
1743

1744 1745
		/*
		 * Note that the hw doesn't support removing PDE on the fly
1746 1747 1748 1749
		 * (they are cached inside the context with no means to
		 * invalidate the cache), so we can only reset the PTE
		 * entries back to scratch.
		 */
1750

1751
		vaddr = kmap_atomic_px(pt);
1752
		memset32(vaddr + pte, scratch_pte, count);
1753
		kunmap_atomic(vaddr);
1754

1755
		pte = 0;
1756
	}
1757 1758
}

1759
static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
1760
				      struct i915_vma *vma,
1761 1762
				      enum i915_cache_level cache_level,
				      u32 flags)
D
Daniel Vetter 已提交
1763
{
1764
	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
1765
	struct i915_page_directory * const pd = ppgtt->pd;
1766
	unsigned first_entry = vma->node.start / I915_GTT_PAGE_SIZE;
1767 1768
	unsigned act_pt = first_entry / GEN6_PTES;
	unsigned act_pte = first_entry % GEN6_PTES;
1769
	const u32 pte_encode = vm->pte_encode(0, cache_level, flags);
1770
	struct sgt_dma iter = sgt_dma(vma);
1771 1772
	gen6_pte_t *vaddr;

1773
	GEM_BUG_ON(i915_pt_entry(pd, act_pt) == vm->scratch_pt);
1774

1775
	vaddr = kmap_atomic_px(i915_pt_entry(pd, act_pt));
1776 1777
	do {
		vaddr[act_pte] = pte_encode | GEN6_PTE_ADDR_ENCODE(iter.dma);
1778

1779
		iter.dma += I915_GTT_PAGE_SIZE;
1780 1781 1782 1783
		if (iter.dma == iter.max) {
			iter.sg = __sg_next(iter.sg);
			if (!iter.sg)
				break;
1784

1785 1786 1787
			iter.dma = sg_dma_address(iter.sg);
			iter.max = iter.dma + iter.sg->length;
		}
1788

1789
		if (++act_pte == GEN6_PTES) {
1790
			kunmap_atomic(vaddr);
1791
			vaddr = kmap_atomic_px(i915_pt_entry(pd, ++act_pt));
1792
			act_pte = 0;
D
Daniel Vetter 已提交
1793
		}
1794
	} while (1);
1795
	kunmap_atomic(vaddr);
1796 1797

	vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;
D
Daniel Vetter 已提交
1798 1799
}

1800
static int gen6_alloc_va_range(struct i915_address_space *vm,
1801
			       u64 start, u64 length)
1802
{
1803
	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
1804
	struct i915_page_directory * const pd = ppgtt->base.pd;
1805
	struct i915_page_table *pt, *alloc = NULL;
1806
	intel_wakeref_t wakeref;
1807 1808 1809
	u64 from = start;
	unsigned int pde;
	bool flush = false;
1810
	int ret = 0;
1811

1812
	wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm);
1813

1814 1815
	spin_lock(&pd->lock);
	gen6_for_each_pde(pt, pd, start, length, pde) {
1816 1817
		const unsigned int count = gen6_pte_count(start, length);

1818
		if (pt == vm->scratch_pt) {
1819
			spin_unlock(&pd->lock);
1820

1821 1822 1823 1824 1825
			pt = fetch_and_zero(&alloc);
			if (!pt)
				pt = alloc_pt(vm);
			if (IS_ERR(pt)) {
				ret = PTR_ERR(pt);
1826
				goto unwind_out;
1827
			}
1828

1829
			gen6_initialize_pt(vm, pt);
1830

1831 1832 1833
			spin_lock(&pd->lock);
			if (pd->entry[pde] == vm->scratch_pt) {
				pd->entry[pde] = pt;
1834 1835 1836 1837 1838 1839
				if (i915_vma_is_bound(ppgtt->vma,
						      I915_VMA_GLOBAL_BIND)) {
					gen6_write_pde(ppgtt, pde, pt);
					flush = true;
				}
			} else {
1840 1841
				alloc = pt;
				pt = pd->entry[pde];
1842
			}
1843
		}
1844

1845
		atomic_add(count, &pt->used);
1846
	}
1847
	spin_unlock(&pd->lock);
1848

1849
	if (flush) {
1850
		mark_tlbs_dirty(&ppgtt->base);
1851
		gen6_ggtt_invalidate(vm->i915);
1852 1853
	}

1854
	goto out;
1855 1856

unwind_out:
1857
	gen6_ppgtt_clear_range(vm, from, start - from);
1858 1859 1860 1861 1862
out:
	if (alloc)
		free_pt(vm, alloc);
	intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref);
	return ret;
1863 1864
}

1865
static int gen6_ppgtt_init_scratch(struct gen6_ppgtt *ppgtt)
1866
{
1867
	struct i915_address_space * const vm = &ppgtt->base.vm;
1868
	struct i915_page_directory * const pd = ppgtt->base.pd;
1869 1870
	struct i915_page_table *unused;
	u32 pde;
1871
	int ret;
1872

1873
	ret = setup_scratch_page(vm, __GFP_HIGHMEM);
1874 1875
	if (ret)
		return ret;
1876

1877 1878 1879
	vm->scratch_pte = vm->pte_encode(vm->scratch_page.daddr,
					 I915_CACHE_NONE,
					 PTE_READ_ONLY);
1880

1881
	vm->scratch_pt = alloc_pt(vm);
1882
	if (IS_ERR(vm->scratch_pt)) {
1883
		cleanup_scratch_page(vm);
1884 1885 1886
		return PTR_ERR(vm->scratch_pt);
	}

1887
	gen6_initialize_pt(vm, vm->scratch_pt);
1888 1889 1890

	gen6_for_all_pdes(unused, pd, pde)
		pd->entry[pde] = vm->scratch_pt;
1891 1892 1893 1894

	return 0;
}

1895
static void gen6_ppgtt_free_scratch(struct i915_address_space *vm)
1896
{
1897 1898
	free_pt(vm, vm->scratch_pt);
	cleanup_scratch_page(vm);
1899 1900
}

1901
static void gen6_ppgtt_free_pd(struct gen6_ppgtt *ppgtt)
1902
{
1903
	struct i915_page_directory * const pd = ppgtt->base.pd;
1904
	struct i915_page_table *pt;
1905
	u32 pde;
1906

1907
	gen6_for_all_pdes(pt, pd, pde)
1908 1909 1910 1911
		if (pt != ppgtt->base.vm.scratch_pt)
			free_pt(&ppgtt->base.vm, pt);
}

1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930
struct gen6_ppgtt_cleanup_work {
	struct work_struct base;
	struct i915_vma *vma;
};

static void gen6_ppgtt_cleanup_work(struct work_struct *wrk)
{
	struct gen6_ppgtt_cleanup_work *work =
		container_of(wrk, typeof(*work), base);
	/* Side note, vma->vm is the GGTT not the ppgtt we just destroyed! */
	struct drm_i915_private *i915 = work->vma->vm->i915;

	mutex_lock(&i915->drm.struct_mutex);
	i915_vma_destroy(work->vma);
	mutex_unlock(&i915->drm.struct_mutex);

	kfree(work);
}

1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957
static int nop_set_pages(struct i915_vma *vma)
{
	return -ENODEV;
}

static void nop_clear_pages(struct i915_vma *vma)
{
}

static int nop_bind(struct i915_vma *vma,
		    enum i915_cache_level cache_level,
		    u32 unused)
{
	return -ENODEV;
}

static void nop_unbind(struct i915_vma *vma)
{
}

static const struct i915_vma_ops nop_vma_ops = {
	.set_pages = nop_set_pages,
	.clear_pages = nop_clear_pages,
	.bind_vma = nop_bind,
	.unbind_vma = nop_unbind,
};

1958 1959
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
{
1960
	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(i915_vm_to_ppgtt(vm));
1961
	struct gen6_ppgtt_cleanup_work *work = ppgtt->work;
1962

1963 1964 1965
	/* FIXME remove the struct_mutex to bring the locking under control */
	INIT_WORK(&work->base, gen6_ppgtt_cleanup_work);
	work->vma = ppgtt->vma;
1966
	work->vma->ops = &nop_vma_ops;
1967
	schedule_work(&work->base);
1968 1969 1970

	gen6_ppgtt_free_pd(ppgtt);
	gen6_ppgtt_free_scratch(vm);
1971
	kfree(ppgtt->base.pd);
1972 1973
}

1974
static int pd_vma_set_pages(struct i915_vma *vma)
1975
{
1976 1977 1978
	vma->pages = ERR_PTR(-ENODEV);
	return 0;
}
1979

1980 1981 1982
static void pd_vma_clear_pages(struct i915_vma *vma)
{
	GEM_BUG_ON(!vma->pages);
1983

1984 1985 1986 1987 1988 1989 1990 1991
	vma->pages = NULL;
}

static int pd_vma_bind(struct i915_vma *vma,
		       enum i915_cache_level cache_level,
		       u32 unused)
{
	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vma->vm);
1992
	struct gen6_ppgtt *ppgtt = vma->private;
1993
	u32 ggtt_offset = i915_ggtt_offset(vma) / I915_GTT_PAGE_SIZE;
1994 1995
	struct i915_page_table *pt;
	unsigned int pde;
1996

1997
	ppgtt->base.pd->base.ggtt_offset = ggtt_offset * sizeof(gen6_pte_t);
1998
	ppgtt->pd_addr = (gen6_pte_t __iomem *)ggtt->gsm + ggtt_offset;
1999

2000
	gen6_for_all_pdes(pt, ppgtt->base.pd, pde)
2001
		gen6_write_pde(ppgtt, pde, pt);
2002

2003 2004
	mark_tlbs_dirty(&ppgtt->base);
	gen6_ggtt_invalidate(ppgtt->base.vm.i915);
2005

2006
	return 0;
2007
}
2008

2009
static void pd_vma_unbind(struct i915_vma *vma)
2010
{
2011
	struct gen6_ppgtt *ppgtt = vma->private;
2012
	struct i915_page_directory * const pd = ppgtt->base.pd;
2013 2014 2015 2016 2017 2018 2019 2020
	struct i915_page_table * const scratch_pt = ppgtt->base.vm.scratch_pt;
	struct i915_page_table *pt;
	unsigned int pde;

	if (!ppgtt->scan_for_unused_pt)
		return;

	/* Free all no longer used page tables */
2021 2022
	gen6_for_all_pdes(pt, ppgtt->base.pd, pde) {
		if (atomic_read(&pt->used) || pt == scratch_pt)
2023 2024 2025
			continue;

		free_pt(&ppgtt->base.vm, pt);
2026
		pd->entry[pde] = scratch_pt;
2027 2028 2029
	}

	ppgtt->scan_for_unused_pt = false;
2030 2031 2032 2033 2034 2035 2036 2037 2038
}

static const struct i915_vma_ops pd_vma_ops = {
	.set_pages = pd_vma_set_pages,
	.clear_pages = pd_vma_clear_pages,
	.bind_vma = pd_vma_bind,
	.unbind_vma = pd_vma_unbind,
};

2039
static struct i915_vma *pd_vma_create(struct gen6_ppgtt *ppgtt, int size)
2040 2041 2042 2043 2044 2045 2046 2047
{
	struct drm_i915_private *i915 = ppgtt->base.vm.i915;
	struct i915_ggtt *ggtt = &i915->ggtt;
	struct i915_vma *vma;

	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
	GEM_BUG_ON(size > ggtt->vm.total);

2048
	vma = i915_vma_alloc();
2049 2050 2051
	if (!vma)
		return ERR_PTR(-ENOMEM);

2052
	i915_active_init(i915, &vma->active, NULL);
2053
	INIT_ACTIVE_REQUEST(&vma->last_fence);
2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064

	vma->vm = &ggtt->vm;
	vma->ops = &pd_vma_ops;
	vma->private = ppgtt;

	vma->size = size;
	vma->fence_size = size;
	vma->flags = I915_VMA_GGTT;
	vma->ggtt_view.type = I915_GGTT_VIEW_ROTATED; /* prevent fencing */

	INIT_LIST_HEAD(&vma->obj_link);
2065
	INIT_LIST_HEAD(&vma->closed_link);
2066 2067

	mutex_lock(&vma->vm->mutex);
2068
	list_add(&vma->vm_link, &vma->vm->unbound_list);
2069
	mutex_unlock(&vma->vm->mutex);
2070 2071 2072

	return vma;
}
2073

2074
int gen6_ppgtt_pin(struct i915_ppgtt *base)
2075
{
2076
	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
2077
	int err;
2078

2079 2080
	GEM_BUG_ON(ppgtt->base.vm.closed);

2081 2082 2083 2084 2085 2086 2087 2088 2089
	/*
	 * Workaround the limited maximum vma->pin_count and the aliasing_ppgtt
	 * which will be pinned into every active context.
	 * (When vma->pin_count becomes atomic, I expect we will naturally
	 * need a larger, unpacked, type and kill this redundancy.)
	 */
	if (ppgtt->pin_count++)
		return 0;

2090 2091 2092 2093 2094
	/*
	 * PPGTT PDEs reside in the GGTT and consists of 512 entries. The
	 * allocator works in address space sizes, so it's multiplied by page
	 * size. We allocate at the top of the GTT to avoid fragmentation.
	 */
2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105
	err = i915_vma_pin(ppgtt->vma,
			   0, GEN6_PD_ALIGN,
			   PIN_GLOBAL | PIN_HIGH);
	if (err)
		goto unpin;

	return 0;

unpin:
	ppgtt->pin_count = 0;
	return err;
2106 2107
}

2108
void gen6_ppgtt_unpin(struct i915_ppgtt *base)
2109
{
2110
	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
2111 2112 2113 2114 2115 2116 2117 2118

	GEM_BUG_ON(!ppgtt->pin_count);
	if (--ppgtt->pin_count)
		return;

	i915_vma_unpin(ppgtt->vma);
}

2119
void gen6_ppgtt_unpin_all(struct i915_ppgtt *base)
2120
{
2121
	struct gen6_ppgtt *ppgtt = to_gen6_ppgtt(base);
2122 2123 2124 2125 2126 2127 2128 2129

	if (!ppgtt->pin_count)
		return;

	ppgtt->pin_count = 0;
	i915_vma_unpin(ppgtt->vma);
}

2130
static struct i915_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
2131
{
2132
	struct i915_ggtt * const ggtt = &i915->ggtt;
2133
	struct gen6_ppgtt *ppgtt;
2134 2135 2136 2137 2138 2139
	int err;

	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
	if (!ppgtt)
		return ERR_PTR(-ENOMEM);

2140
	ppgtt_init(i915, &ppgtt->base);
2141

2142
	ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
2143 2144 2145
	ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
	ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
	ppgtt->base.vm.cleanup = gen6_ppgtt_cleanup;
2146

2147 2148
	ppgtt->base.vm.pte_encode = ggtt->vm.pte_encode;

2149
	ppgtt->work = kmalloc(sizeof(*ppgtt->work), GFP_KERNEL);
2150 2151
	if (!ppgtt->work) {
		err = -ENOMEM;
2152
		goto err_free;
2153
	}
2154

2155 2156 2157 2158 2159 2160
	ppgtt->base.pd = __alloc_pd();
	if (!ppgtt->base.pd) {
		err = -ENOMEM;
		goto err_work;
	}

2161
	err = gen6_ppgtt_init_scratch(ppgtt);
2162
	if (err)
2163
		goto err_pd;
2164

2165 2166 2167
	ppgtt->vma = pd_vma_create(ppgtt, GEN6_PD_SIZE);
	if (IS_ERR(ppgtt->vma)) {
		err = PTR_ERR(ppgtt->vma);
2168
		goto err_scratch;
2169
	}
2170

2171
	return &ppgtt->base;
2172

2173 2174
err_scratch:
	gen6_ppgtt_free_scratch(&ppgtt->base.vm);
2175 2176
err_pd:
	kfree(ppgtt->base.pd);
2177 2178
err_work:
	kfree(ppgtt->work);
2179 2180 2181
err_free:
	kfree(ppgtt);
	return ERR_PTR(err);
2182
}
2183

2184
static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
2185 2186 2187 2188 2189
{
	/* This function is for gtt related workarounds. This function is
	 * called on driver load and after a GPU reset, so you can place
	 * workarounds here even if they get overwritten by GPU reset.
	 */
2190
	/* WaIncreaseDefaultTLBEntries:chv,bdw,skl,bxt,kbl,glk,cfl,cnl,icl */
2191
	if (IS_BROADWELL(dev_priv))
2192
		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_BDW);
2193
	else if (IS_CHERRYVIEW(dev_priv))
2194
		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN8_L3_LRA_1_GPGPU_DEFAULT_VALUE_CHV);
2195
	else if (IS_GEN9_LP(dev_priv))
2196
		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
2197 2198
	else if (INTEL_GEN(dev_priv) >= 9)
		I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_SKL);
2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215

	/*
	 * To support 64K PTEs we need to first enable the use of the
	 * Intermediate-Page-Size(IPS) bit of the PDE field via some magical
	 * mmio, otherwise the page-walker will simply ignore the IPS bit. This
	 * shouldn't be needed after GEN10.
	 *
	 * 64K pages were first introduced from BDW+, although technically they
	 * only *work* from gen9+. For pre-BDW we instead have the option for
	 * 32K pages, but we don't currently have any support for it in our
	 * driver.
	 */
	if (HAS_PAGE_SIZES(dev_priv, I915_GTT_PAGE_SIZE_64K) &&
	    INTEL_GEN(dev_priv) <= 10)
		I915_WRITE(GEN8_GAMW_ECO_DEV_RW_IA,
			   I915_READ(GEN8_GAMW_ECO_DEV_RW_IA) |
			   GAMW_ECO_ENABLE_64K_IPS_FIELD);
2216 2217
}

2218
int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv)
2219
{
2220
	gtt_write_workarounds(dev_priv);
2221

2222
	if (IS_GEN(dev_priv, 6))
2223
		gen6_ppgtt_enable(dev_priv);
2224
	else if (IS_GEN(dev_priv, 7))
2225
		gen7_ppgtt_enable(dev_priv);
2226

2227 2228
	return 0;
}
2229

2230 2231
static struct i915_ppgtt *
__ppgtt_create(struct drm_i915_private *i915)
2232 2233 2234 2235 2236 2237 2238
{
	if (INTEL_GEN(i915) < 8)
		return gen6_ppgtt_create(i915);
	else
		return gen8_ppgtt_create(i915);
}

2239
struct i915_ppgtt *
2240
i915_ppgtt_create(struct drm_i915_private *i915)
2241
{
2242
	struct i915_ppgtt *ppgtt;
2243

2244
	ppgtt = __ppgtt_create(i915);
2245 2246
	if (IS_ERR(ppgtt))
		return ppgtt;
2247

2248
	trace_i915_ppgtt_create(&ppgtt->vm);
2249

2250 2251 2252
	return ppgtt;
}

2253
static void ppgtt_destroy_vma(struct i915_address_space *vm)
2254 2255
{
	struct list_head *phases[] = {
2256
		&vm->bound_list,
2257 2258 2259 2260 2261 2262 2263 2264 2265
		&vm->unbound_list,
		NULL,
	}, **phase;

	vm->closed = true;
	for (phase = phases; *phase; phase++) {
		struct i915_vma *vma, *vn;

		list_for_each_entry_safe(vma, vn, *phase, vm_link)
2266
			i915_vma_destroy(vma);
2267 2268 2269
	}
}

2270
void i915_vm_release(struct kref *kref)
2271
{
2272 2273
	struct i915_address_space *vm =
		container_of(kref, struct i915_address_space, ref);
2274

2275 2276
	GEM_BUG_ON(i915_is_ggtt(vm));
	trace_i915_ppgtt_release(vm);
2277

2278
	ppgtt_destroy_vma(vm);
2279

2280 2281
	GEM_BUG_ON(!list_empty(&vm->bound_list));
	GEM_BUG_ON(!list_empty(&vm->unbound_list));
2282

2283 2284 2285 2286
	vm->cleanup(vm);
	i915_address_space_fini(vm);

	kfree(vm);
2287
}
2288

2289 2290 2291
/* Certain Gen5 chipsets require require idling the GPU before
 * unmapping anything from the GTT when VT-d is enabled.
 */
2292
static bool needs_idle_maps(struct drm_i915_private *dev_priv)
2293 2294 2295 2296
{
	/* Query intel_iommu to see if we need the workaround. Presumably that
	 * was loaded first.
	 */
2297
	return IS_GEN(dev_priv, 5) && IS_MOBILE(dev_priv) && intel_vtd_active();
2298 2299
}

2300
void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
2301
{
2302
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
2303 2304 2305 2306

	/* Don't bother messing with faults pre GEN6 as we have little
	 * documentation supporting that it's a good idea.
	 */
2307
	if (INTEL_GEN(dev_priv) < 6)
2308 2309
		return;

2310
	i915_check_and_clear_faults(dev_priv);
2311

2312
	ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total);
2313

2314
	i915_ggtt_invalidate(dev_priv);
2315 2316
}

2317 2318
int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
			       struct sg_table *pages)
2319
{
2320
	do {
2321 2322 2323 2324
		if (dma_map_sg_attrs(&obj->base.dev->pdev->dev,
				     pages->sgl, pages->nents,
				     PCI_DMA_BIDIRECTIONAL,
				     DMA_ATTR_NO_WARN))
2325 2326
			return 0;

2327 2328
		/*
		 * If the DMA remap fails, one cause can be that we have
2329 2330 2331 2332 2333 2334 2335
		 * too many objects pinned in a small remapping table,
		 * such as swiotlb. Incrementally purge all other objects and
		 * try again - if there are no more pages to remove from
		 * the DMA remapper, i915_gem_shrink will return 0.
		 */
		GEM_BUG_ON(obj->mm.pages == pages);
	} while (i915_gem_shrink(to_i915(obj->base.dev),
2336
				 obj->base.size >> PAGE_SHIFT, NULL,
2337
				 I915_SHRINK_BOUND |
2338
				 I915_SHRINK_UNBOUND));
2339

2340
	return -ENOSPC;
2341 2342
}

2343
static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
B
Ben Widawsky 已提交
2344 2345 2346 2347
{
	writeq(pte, addr);
}

2348 2349
static void gen8_ggtt_insert_page(struct i915_address_space *vm,
				  dma_addr_t addr,
2350
				  u64 offset,
2351 2352 2353
				  enum i915_cache_level level,
				  u32 unused)
{
2354
	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2355
	gen8_pte_t __iomem *pte =
2356
		(gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
2357

2358
	gen8_set_pte(pte, gen8_pte_encode(addr, level, 0));
2359

2360
	ggtt->invalidate(vm->i915);
2361 2362
}

B
Ben Widawsky 已提交
2363
static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
2364
				     struct i915_vma *vma,
2365
				     enum i915_cache_level level,
2366
				     u32 flags)
B
Ben Widawsky 已提交
2367
{
2368
	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2369 2370
	struct sgt_iter sgt_iter;
	gen8_pte_t __iomem *gtt_entries;
2371
	const gen8_pte_t pte_encode = gen8_pte_encode(0, level, 0);
2372
	dma_addr_t addr;
2373

2374 2375 2376 2377
	/*
	 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
	 * not to allow the user to override access to a read only page.
	 */
2378

2379
	gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm;
2380
	gtt_entries += vma->node.start / I915_GTT_PAGE_SIZE;
2381
	for_each_sgt_dma(addr, sgt_iter, vma->pages)
2382
		gen8_set_pte(gtt_entries++, pte_encode | addr);
2383

2384 2385 2386
	/*
	 * We want to flush the TLBs only after we're certain all the PTE
	 * updates have finished.
B
Ben Widawsky 已提交
2387
	 */
2388
	ggtt->invalidate(vm->i915);
B
Ben Widawsky 已提交
2389 2390
}

2391 2392
static void gen6_ggtt_insert_page(struct i915_address_space *vm,
				  dma_addr_t addr,
2393
				  u64 offset,
2394 2395 2396
				  enum i915_cache_level level,
				  u32 flags)
{
2397
	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2398
	gen6_pte_t __iomem *pte =
2399
		(gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE;
2400

2401
	iowrite32(vm->pte_encode(addr, level, flags), pte);
2402

2403
	ggtt->invalidate(vm->i915);
2404 2405
}

2406 2407 2408 2409 2410 2411
/*
 * Binds an object into the global gtt with the specified cache level. The object
 * will be accessible to the GPU via commands whose operands reference offsets
 * within the global GTT as well as accessible by the GPU through the GMADR
 * mapped BAR (dev_priv->mm.gtt->gtt).
 */
2412
static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
2413
				     struct i915_vma *vma,
2414 2415
				     enum i915_cache_level level,
				     u32 flags)
2416
{
2417
	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2418
	gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm;
2419
	unsigned int i = vma->node.start / I915_GTT_PAGE_SIZE;
2420
	struct sgt_iter iter;
2421
	dma_addr_t addr;
2422
	for_each_sgt_dma(addr, iter, vma->pages)
2423
		iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]);
2424

2425 2426 2427
	/*
	 * We want to flush the TLBs only after we're certain all the PTE
	 * updates have finished.
2428
	 */
2429
	ggtt->invalidate(vm->i915);
2430 2431
}

2432
static void nop_clear_range(struct i915_address_space *vm,
2433
			    u64 start, u64 length)
2434 2435 2436
{
}

B
Ben Widawsky 已提交
2437
static void gen8_ggtt_clear_range(struct i915_address_space *vm,
2438
				  u64 start, u64 length)
B
Ben Widawsky 已提交
2439
{
2440
	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2441 2442
	unsigned first_entry = start / I915_GTT_PAGE_SIZE;
	unsigned num_entries = length / I915_GTT_PAGE_SIZE;
2443
	const gen8_pte_t scratch_pte = vm->scratch_pte;
2444
	gen8_pte_t __iomem *gtt_base =
2445 2446
		(gen8_pte_t __iomem *)ggtt->gsm + first_entry;
	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
B
Ben Widawsky 已提交
2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457
	int i;

	if (WARN(num_entries > max_entries,
		 "First entry = %d; Num entries = %d (max=%d)\n",
		 first_entry, num_entries, max_entries))
		num_entries = max_entries;

	for (i = 0; i < num_entries; i++)
		gen8_set_pte(&gtt_base[i], scratch_pte);
}

2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501
static void bxt_vtd_ggtt_wa(struct i915_address_space *vm)
{
	struct drm_i915_private *dev_priv = vm->i915;

	/*
	 * Make sure the internal GAM fifo has been cleared of all GTT
	 * writes before exiting stop_machine(). This guarantees that
	 * any aperture accesses waiting to start in another process
	 * cannot back up behind the GTT writes causing a hang.
	 * The register can be any arbitrary GAM register.
	 */
	POSTING_READ(GFX_FLSH_CNTL_GEN6);
}

struct insert_page {
	struct i915_address_space *vm;
	dma_addr_t addr;
	u64 offset;
	enum i915_cache_level level;
};

static int bxt_vtd_ggtt_insert_page__cb(void *_arg)
{
	struct insert_page *arg = _arg;

	gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0);
	bxt_vtd_ggtt_wa(arg->vm);

	return 0;
}

static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm,
					  dma_addr_t addr,
					  u64 offset,
					  enum i915_cache_level level,
					  u32 unused)
{
	struct insert_page arg = { vm, addr, offset, level };

	stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL);
}

struct insert_entries {
	struct i915_address_space *vm;
2502
	struct i915_vma *vma;
2503
	enum i915_cache_level level;
2504
	u32 flags;
2505 2506 2507 2508 2509 2510
};

static int bxt_vtd_ggtt_insert_entries__cb(void *_arg)
{
	struct insert_entries *arg = _arg;

2511
	gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, arg->flags);
2512 2513 2514 2515 2516 2517
	bxt_vtd_ggtt_wa(arg->vm);

	return 0;
}

static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm,
2518
					     struct i915_vma *vma,
2519
					     enum i915_cache_level level,
2520
					     u32 flags)
2521
{
2522
	struct insert_entries arg = { vm, vma, level, flags };
2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551

	stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL);
}

struct clear_range {
	struct i915_address_space *vm;
	u64 start;
	u64 length;
};

static int bxt_vtd_ggtt_clear_range__cb(void *_arg)
{
	struct clear_range *arg = _arg;

	gen8_ggtt_clear_range(arg->vm, arg->start, arg->length);
	bxt_vtd_ggtt_wa(arg->vm);

	return 0;
}

static void bxt_vtd_ggtt_clear_range__BKL(struct i915_address_space *vm,
					  u64 start,
					  u64 length)
{
	struct clear_range arg = { vm, start, length };

	stop_machine(bxt_vtd_ggtt_clear_range__cb, &arg, NULL);
}

2552
static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2553
				  u64 start, u64 length)
2554
{
2555
	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
2556 2557
	unsigned first_entry = start / I915_GTT_PAGE_SIZE;
	unsigned num_entries = length / I915_GTT_PAGE_SIZE;
2558
	gen6_pte_t scratch_pte, __iomem *gtt_base =
2559 2560
		(gen6_pte_t __iomem *)ggtt->gsm + first_entry;
	const int max_entries = ggtt_total_entries(ggtt) - first_entry;
2561 2562 2563 2564 2565 2566 2567
	int i;

	if (WARN(num_entries > max_entries,
		 "First entry = %d; Num entries = %d (max=%d)\n",
		 first_entry, num_entries, max_entries))
		num_entries = max_entries;

2568
	scratch_pte = vm->scratch_pte;
2569

2570 2571 2572 2573
	for (i = 0; i < num_entries; i++)
		iowrite32(scratch_pte, &gtt_base[i]);
}

2574 2575
static void i915_ggtt_insert_page(struct i915_address_space *vm,
				  dma_addr_t addr,
2576
				  u64 offset,
2577 2578 2579 2580 2581 2582 2583 2584 2585
				  enum i915_cache_level cache_level,
				  u32 unused)
{
	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;

	intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
}

2586
static void i915_ggtt_insert_entries(struct i915_address_space *vm,
2587
				     struct i915_vma *vma,
2588 2589
				     enum i915_cache_level cache_level,
				     u32 unused)
2590 2591 2592 2593
{
	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;

2594 2595
	intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT,
				    flags);
2596 2597
}

2598
static void i915_ggtt_clear_range(struct i915_address_space *vm,
2599
				  u64 start, u64 length)
2600
{
2601
	intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT);
2602 2603
}

2604 2605 2606
static int ggtt_bind_vma(struct i915_vma *vma,
			 enum i915_cache_level cache_level,
			 u32 flags)
2607
{
2608
	struct drm_i915_private *i915 = vma->vm->i915;
2609
	struct drm_i915_gem_object *obj = vma->obj;
2610
	intel_wakeref_t wakeref;
2611
	u32 pte_flags;
2612

2613
	/* Applicable to VLV (gen8+ do not support RO in the GGTT) */
2614
	pte_flags = 0;
2615
	if (i915_gem_object_is_readonly(obj))
2616 2617
		pte_flags |= PTE_READ_ONLY;

2618
	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
2619
		vma->vm->insert_entries(vma->vm, vma, cache_level, pte_flags);
2620

2621 2622
	vma->page_sizes.gtt = I915_GTT_PAGE_SIZE;

2623 2624 2625 2626 2627
	/*
	 * Without aliasing PPGTT there's no difference between
	 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
	 * upgrade to both bound if we bind either to avoid double-binding.
	 */
2628
	vma->flags |= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
2629 2630 2631 2632

	return 0;
}

2633 2634 2635
static void ggtt_unbind_vma(struct i915_vma *vma)
{
	struct drm_i915_private *i915 = vma->vm->i915;
2636
	intel_wakeref_t wakeref;
2637

2638
	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
2639
		vma->vm->clear_range(vma->vm, vma->node.start, vma->size);
2640 2641
}

2642 2643 2644
static int aliasing_gtt_bind_vma(struct i915_vma *vma,
				 enum i915_cache_level cache_level,
				 u32 flags)
2645
{
2646
	struct drm_i915_private *i915 = vma->vm->i915;
2647
	u32 pte_flags;
2648
	int ret;
2649

2650
	/* Currently applicable only to VLV */
2651
	pte_flags = 0;
2652
	if (i915_gem_object_is_readonly(vma->obj))
2653
		pte_flags |= PTE_READ_ONLY;
2654

2655
	if (flags & I915_VMA_LOCAL_BIND) {
2656
		struct i915_ppgtt *appgtt = i915->mm.aliasing_ppgtt;
2657

2658
		if (!(vma->flags & I915_VMA_LOCAL_BIND)) {
2659 2660 2661
			ret = appgtt->vm.allocate_va_range(&appgtt->vm,
							   vma->node.start,
							   vma->size);
2662
			if (ret)
2663
				return ret;
2664 2665
		}

2666 2667
		appgtt->vm.insert_entries(&appgtt->vm, vma, cache_level,
					  pte_flags);
2668 2669
	}

2670
	if (flags & I915_VMA_GLOBAL_BIND) {
2671 2672
		intel_wakeref_t wakeref;

2673
		with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2674 2675 2676
			vma->vm->insert_entries(vma->vm, vma,
						cache_level, pte_flags);
		}
2677
	}
2678

2679
	return 0;
2680 2681
}

2682
static void aliasing_gtt_unbind_vma(struct i915_vma *vma)
2683
{
2684
	struct drm_i915_private *i915 = vma->vm->i915;
2685

2686
	if (vma->flags & I915_VMA_GLOBAL_BIND) {
2687
		struct i915_address_space *vm = vma->vm;
2688 2689
		intel_wakeref_t wakeref;

2690
		with_intel_runtime_pm(&i915->runtime_pm, wakeref)
2691
			vm->clear_range(vm, vma->node.start, vma->size);
2692
	}
2693

2694
	if (vma->flags & I915_VMA_LOCAL_BIND) {
2695
		struct i915_address_space *vm = &i915->mm.aliasing_ppgtt->vm;
2696 2697 2698

		vm->clear_range(vm, vma->node.start, vma->size);
	}
2699 2700
}

2701 2702
void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
			       struct sg_table *pages)
2703
{
D
David Weinehall 已提交
2704 2705
	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
	struct device *kdev = &dev_priv->drm.pdev->dev;
2706
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
B
Ben Widawsky 已提交
2707

2708
	if (unlikely(ggtt->do_idle_maps)) {
2709
		if (i915_gem_wait_for_idle(dev_priv, 0, MAX_SCHEDULE_TIMEOUT)) {
2710 2711 2712 2713 2714
			DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
			/* Wait a bit, in hopes it avoids the hang */
			udelay(10);
		}
	}
B
Ben Widawsky 已提交
2715

2716
	dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
2717
}
2718

2719 2720 2721 2722 2723 2724 2725 2726 2727 2728
static int ggtt_set_pages(struct i915_vma *vma)
{
	int ret;

	GEM_BUG_ON(vma->pages);

	ret = i915_get_ggtt_vma_pages(vma);
	if (ret)
		return ret;

2729 2730
	vma->page_sizes = vma->obj->mm.page_sizes;

2731 2732 2733
	return 0;
}

C
Chris Wilson 已提交
2734
static void i915_gtt_color_adjust(const struct drm_mm_node *node,
2735
				  unsigned long color,
2736 2737
				  u64 *start,
				  u64 *end)
2738
{
2739
	if (node->allocated && node->color != color)
2740
		*start += I915_GTT_PAGE_SIZE;
2741

2742 2743 2744 2745 2746
	/* Also leave a space between the unallocated reserved node after the
	 * GTT and any objects within the GTT, i.e. we use the color adjustment
	 * to insert a guard page to prevent prefetches crossing over the
	 * GTT boundary.
	 */
2747
	node = list_next_entry(node, node_list);
2748
	if (node->color != color)
2749
		*end -= I915_GTT_PAGE_SIZE;
2750
}
B
Ben Widawsky 已提交
2751

2752
static int init_aliasing_ppgtt(struct drm_i915_private *i915)
2753 2754
{
	struct i915_ggtt *ggtt = &i915->ggtt;
2755
	struct i915_ppgtt *ppgtt;
2756 2757
	int err;

2758
	ppgtt = i915_ppgtt_create(i915);
2759 2760
	if (IS_ERR(ppgtt))
		return PTR_ERR(ppgtt);
2761

2762
	if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) {
2763 2764 2765 2766
		err = -ENODEV;
		goto err_ppgtt;
	}

2767 2768 2769 2770 2771 2772 2773 2774 2775
	/*
	 * Note we only pre-allocate as far as the end of the global
	 * GTT. On 48b / 4-level page-tables, the difference is very,
	 * very significant! We have to preallocate as GVT/vgpu does
	 * not like the page directory disappearing.
	 */
	err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, ggtt->vm.total);
	if (err)
		goto err_ppgtt;
2776 2777

	i915->mm.aliasing_ppgtt = ppgtt;
2778

2779 2780
	GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != ggtt_bind_vma);
	ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma;
2781

2782 2783
	GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != ggtt_unbind_vma);
	ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma;
2784

2785 2786 2787
	return 0;

err_ppgtt:
2788
	i915_vm_put(&ppgtt->vm);
2789 2790 2791
	return err;
}

2792
static void fini_aliasing_ppgtt(struct drm_i915_private *i915)
2793 2794
{
	struct i915_ggtt *ggtt = &i915->ggtt;
2795
	struct i915_ppgtt *ppgtt;
2796 2797 2798 2799 2800

	ppgtt = fetch_and_zero(&i915->mm.aliasing_ppgtt);
	if (!ppgtt)
		return;

2801
	i915_vm_put(&ppgtt->vm);
2802

2803 2804
	ggtt->vm.vma_ops.bind_vma   = ggtt_bind_vma;
	ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma;
2805 2806
}

2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832
static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt)
{
	u64 size;
	int ret;

	if (!USES_GUC(ggtt->vm.i915))
		return 0;

	GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP);
	size = ggtt->vm.total - GUC_GGTT_TOP;

	ret = i915_gem_gtt_reserve(&ggtt->vm, &ggtt->uc_fw, size,
				   GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE,
				   PIN_NOEVICT);
	if (ret)
		DRM_DEBUG_DRIVER("Failed to reserve top of GGTT for GuC\n");

	return ret;
}

static void ggtt_release_guc_top(struct i915_ggtt *ggtt)
{
	if (drm_mm_node_allocated(&ggtt->uc_fw))
		drm_mm_remove_node(&ggtt->uc_fw);
}

2833
int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
2834
{
2835 2836 2837 2838 2839 2840 2841 2842 2843
	/* Let GEM Manage all of the aperture.
	 *
	 * However, leave one page at the end still bound to the scratch page.
	 * There are a number of places where the hardware apparently prefetches
	 * past the end of the object, and we've seen multiple hangs with the
	 * GPU head pointer stuck in a batchbuffer bound at the last page of the
	 * aperture.  One page should be enough to keep any prefetching inside
	 * of the aperture.
	 */
2844
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
2845
	unsigned long hole_start, hole_end;
2846
	struct drm_mm_node *entry;
2847
	int ret;
2848

2849 2850 2851 2852 2853 2854 2855
	/*
	 * GuC requires all resources that we're sharing with it to be placed in
	 * non-WOPCM memory. If GuC is not present or not in use we still need a
	 * small bias as ring wraparound at offset 0 sometimes hangs. No idea
	 * why.
	 */
	ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE,
2856
			       intel_wopcm_guc_size(&dev_priv->wopcm));
2857

2858 2859 2860
	ret = intel_vgt_balloon(dev_priv);
	if (ret)
		return ret;
2861

2862
	/* Reserve a mappable slot for our lockless error capture */
2863
	ret = drm_mm_insert_node_in_range(&ggtt->vm.mm, &ggtt->error_capture,
2864 2865 2866
					  PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
					  0, ggtt->mappable_end,
					  DRM_MM_INSERT_LOW);
2867 2868 2869
	if (ret)
		return ret;

2870 2871 2872 2873 2874 2875 2876 2877
	/*
	 * The upper portion of the GuC address space has a sizeable hole
	 * (several MB) that is inaccessible by GuC. Reserve this range within
	 * GGTT as it can comfortably hold GuC/HuC firmware images.
	 */
	ret = ggtt_reserve_guc_top(ggtt);
	if (ret)
		goto err_reserve;
2878

2879
	/* Clear any non-preallocated blocks */
2880
	drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) {
2881 2882
		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
			      hole_start, hole_end);
2883 2884
		ggtt->vm.clear_range(&ggtt->vm, hole_start,
				     hole_end - hole_start);
2885 2886 2887
	}

	/* And finally clear the reserved guard page */
2888
	ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE);
2889

2890
	if (INTEL_PPGTT(dev_priv) == INTEL_PPGTT_ALIASING) {
2891
		ret = init_aliasing_ppgtt(dev_priv);
2892
		if (ret)
2893
			goto err_appgtt;
2894 2895
	}

2896
	return 0;
2897

2898
err_appgtt:
2899
	ggtt_release_guc_top(ggtt);
2900
err_reserve:
2901 2902
	drm_mm_remove_node(&ggtt->error_capture);
	return ret;
2903 2904
}

2905 2906
/**
 * i915_ggtt_cleanup_hw - Clean up GGTT hardware initialization
2907
 * @dev_priv: i915 device
2908
 */
2909
void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
2910
{
2911
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
2912
	struct i915_vma *vma, *vn;
2913
	struct pagevec *pvec;
2914

2915
	ggtt->vm.closed = true;
2916 2917

	mutex_lock(&dev_priv->drm.struct_mutex);
2918
	fini_aliasing_ppgtt(dev_priv);
2919

2920
	list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link)
2921
		WARN_ON(i915_vma_unbind(vma));
2922

2923 2924 2925
	if (drm_mm_node_allocated(&ggtt->error_capture))
		drm_mm_remove_node(&ggtt->error_capture);

2926
	ggtt_release_guc_top(ggtt);
2927

2928
	if (drm_mm_initialized(&ggtt->vm.mm)) {
2929
		intel_vgt_deballoon(dev_priv);
2930
		i915_address_space_fini(&ggtt->vm);
2931 2932
	}

2933
	ggtt->vm.cleanup(&ggtt->vm);
2934

2935
	pvec = &dev_priv->mm.wc_stash.pvec;
2936 2937 2938 2939 2940
	if (pvec->nr) {
		set_pages_array_wb(pvec->pages, pvec->nr);
		__pagevec_release(pvec);
	}

2941
	mutex_unlock(&dev_priv->drm.struct_mutex);
2942 2943

	arch_phys_wc_del(ggtt->mtrr);
2944
	io_mapping_fini(&ggtt->iomap);
2945

2946
	i915_gem_cleanup_stolen(dev_priv);
2947
}
2948

2949
static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2950 2951 2952 2953 2954 2955
{
	snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
	snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
	return snb_gmch_ctl << 20;
}

2956
static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2957 2958 2959 2960 2961
{
	bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
	bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
	if (bdw_gmch_ctl)
		bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2962 2963

#ifdef CONFIG_X86_32
2964
	/* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */
2965 2966 2967 2968
	if (bdw_gmch_ctl > 4)
		bdw_gmch_ctl = 4;
#endif

2969 2970 2971
	return bdw_gmch_ctl << 20;
}

2972
static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2973 2974 2975 2976 2977 2978 2979 2980 2981 2982
{
	gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
	gmch_ctrl &= SNB_GMCH_GGMS_MASK;

	if (gmch_ctrl)
		return 1 << (20 + gmch_ctrl);

	return 0;
}

2983
static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
B
Ben Widawsky 已提交
2984
{
2985
	struct drm_i915_private *dev_priv = ggtt->vm.i915;
2986
	struct pci_dev *pdev = dev_priv->drm.pdev;
2987
	phys_addr_t phys_addr;
2988
	int ret;
B
Ben Widawsky 已提交
2989 2990

	/* For Modern GENs the PTEs and register space are split in the BAR */
2991
	phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2;
B
Ben Widawsky 已提交
2992

I
Imre Deak 已提交
2993
	/*
2994 2995 2996
	 * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range
	 * will be dropped. For WC mappings in general we have 64 byte burst
	 * writes when the WC buffer is flushed, so we can't use it, but have to
I
Imre Deak 已提交
2997 2998 2999
	 * resort to an uncached mapping. The WC issue is easily caught by the
	 * readback check when writing GTT PTE entries.
	 */
3000
	if (IS_GEN9_LP(dev_priv) || INTEL_GEN(dev_priv) >= 10)
3001
		ggtt->gsm = ioremap_nocache(phys_addr, size);
I
Imre Deak 已提交
3002
	else
3003
		ggtt->gsm = ioremap_wc(phys_addr, size);
3004
	if (!ggtt->gsm) {
3005
		DRM_ERROR("Failed to map the ggtt page table\n");
B
Ben Widawsky 已提交
3006 3007 3008
		return -ENOMEM;
	}

3009
	ret = setup_scratch_page(&ggtt->vm, GFP_DMA32);
3010
	if (ret) {
B
Ben Widawsky 已提交
3011 3012
		DRM_ERROR("Scratch setup failed\n");
		/* iounmap will also get called at remove, but meh */
3013
		iounmap(ggtt->gsm);
3014
		return ret;
B
Ben Widawsky 已提交
3015 3016
	}

3017 3018 3019 3020
	ggtt->vm.scratch_pte =
		ggtt->vm.pte_encode(ggtt->vm.scratch_page.daddr,
				    I915_CACHE_NONE, 0);

3021
	return 0;
B
Ben Widawsky 已提交
3022 3023
}

3024 3025
static struct intel_ppat_entry *
__alloc_ppat_entry(struct intel_ppat *ppat, unsigned int index, u8 value)
R
Rodrigo Vivi 已提交
3026
{
3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069
	struct intel_ppat_entry *entry = &ppat->entries[index];

	GEM_BUG_ON(index >= ppat->max_entries);
	GEM_BUG_ON(test_bit(index, ppat->used));

	entry->ppat = ppat;
	entry->value = value;
	kref_init(&entry->ref);
	set_bit(index, ppat->used);
	set_bit(index, ppat->dirty);

	return entry;
}

static void __free_ppat_entry(struct intel_ppat_entry *entry)
{
	struct intel_ppat *ppat = entry->ppat;
	unsigned int index = entry - ppat->entries;

	GEM_BUG_ON(index >= ppat->max_entries);
	GEM_BUG_ON(!test_bit(index, ppat->used));

	entry->value = ppat->clear_value;
	clear_bit(index, ppat->used);
	set_bit(index, ppat->dirty);
}

/**
 * intel_ppat_get - get a usable PPAT entry
 * @i915: i915 device instance
 * @value: the PPAT value required by the caller
 *
 * The function tries to search if there is an existing PPAT entry which
 * matches with the required value. If perfectly matched, the existing PPAT
 * entry will be used. If only partially matched, it will try to check if
 * there is any available PPAT index. If yes, it will allocate a new PPAT
 * index for the required entry and update the HW. If not, the partially
 * matched entry will be used.
 */
const struct intel_ppat_entry *
intel_ppat_get(struct drm_i915_private *i915, u8 value)
{
	struct intel_ppat *ppat = &i915->ppat;
3070
	struct intel_ppat_entry *entry = NULL;
3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092
	unsigned int scanned, best_score;
	int i;

	GEM_BUG_ON(!ppat->max_entries);

	scanned = best_score = 0;
	for_each_set_bit(i, ppat->used, ppat->max_entries) {
		unsigned int score;

		score = ppat->match(ppat->entries[i].value, value);
		if (score > best_score) {
			entry = &ppat->entries[i];
			if (score == INTEL_PPAT_PERFECT_MATCH) {
				kref_get(&entry->ref);
				return entry;
			}
			best_score = score;
		}
		scanned++;
	}

	if (scanned == ppat->max_entries) {
3093
		if (!entry)
3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169
			return ERR_PTR(-ENOSPC);

		kref_get(&entry->ref);
		return entry;
	}

	i = find_first_zero_bit(ppat->used, ppat->max_entries);
	entry = __alloc_ppat_entry(ppat, i, value);
	ppat->update_hw(i915);
	return entry;
}

static void release_ppat(struct kref *kref)
{
	struct intel_ppat_entry *entry =
		container_of(kref, struct intel_ppat_entry, ref);
	struct drm_i915_private *i915 = entry->ppat->i915;

	__free_ppat_entry(entry);
	entry->ppat->update_hw(i915);
}

/**
 * intel_ppat_put - put back the PPAT entry got from intel_ppat_get()
 * @entry: an intel PPAT entry
 *
 * Put back the PPAT entry got from intel_ppat_get(). If the PPAT index of the
 * entry is dynamically allocated, its reference count will be decreased. Once
 * the reference count becomes into zero, the PPAT index becomes free again.
 */
void intel_ppat_put(const struct intel_ppat_entry *entry)
{
	struct intel_ppat *ppat = entry->ppat;
	unsigned int index = entry - ppat->entries;

	GEM_BUG_ON(!ppat->max_entries);

	kref_put(&ppat->entries[index].ref, release_ppat);
}

static void cnl_private_pat_update_hw(struct drm_i915_private *dev_priv)
{
	struct intel_ppat *ppat = &dev_priv->ppat;
	int i;

	for_each_set_bit(i, ppat->dirty, ppat->max_entries) {
		I915_WRITE(GEN10_PAT_INDEX(i), ppat->entries[i].value);
		clear_bit(i, ppat->dirty);
	}
}

static void bdw_private_pat_update_hw(struct drm_i915_private *dev_priv)
{
	struct intel_ppat *ppat = &dev_priv->ppat;
	u64 pat = 0;
	int i;

	for (i = 0; i < ppat->max_entries; i++)
		pat |= GEN8_PPAT(i, ppat->entries[i].value);

	bitmap_clear(ppat->dirty, 0, ppat->max_entries);

	I915_WRITE(GEN8_PRIVATE_PAT_LO, lower_32_bits(pat));
	I915_WRITE(GEN8_PRIVATE_PAT_HI, upper_32_bits(pat));
}

static unsigned int bdw_private_pat_match(u8 src, u8 dst)
{
	unsigned int score = 0;
	enum {
		AGE_MATCH = BIT(0),
		TC_MATCH = BIT(1),
		CA_MATCH = BIT(2),
	};

	/* Cache attribute has to be matched. */
3170
	if (GEN8_PPAT_GET_CA(src) != GEN8_PPAT_GET_CA(dst))
3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207
		return 0;

	score |= CA_MATCH;

	if (GEN8_PPAT_GET_TC(src) == GEN8_PPAT_GET_TC(dst))
		score |= TC_MATCH;

	if (GEN8_PPAT_GET_AGE(src) == GEN8_PPAT_GET_AGE(dst))
		score |= AGE_MATCH;

	if (score == (AGE_MATCH | TC_MATCH | CA_MATCH))
		return INTEL_PPAT_PERFECT_MATCH;

	return score;
}

static unsigned int chv_private_pat_match(u8 src, u8 dst)
{
	return (CHV_PPAT_GET_SNOOP(src) == CHV_PPAT_GET_SNOOP(dst)) ?
		INTEL_PPAT_PERFECT_MATCH : 0;
}

static void cnl_setup_private_ppat(struct intel_ppat *ppat)
{
	ppat->max_entries = 8;
	ppat->update_hw = cnl_private_pat_update_hw;
	ppat->match = bdw_private_pat_match;
	ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);

	__alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC);
	__alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
	__alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);
	__alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC);
	__alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
	__alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
	__alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
	__alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
R
Rodrigo Vivi 已提交
3208 3209
}

B
Ben Widawsky 已提交
3210 3211 3212
/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
 * bits. When using advanced contexts each context stores its own PAT, but
 * writing this data shouldn't be harmful even in those cases. */
3213
static void bdw_setup_private_ppat(struct intel_ppat *ppat)
B
Ben Widawsky 已提交
3214
{
3215 3216 3217 3218
	ppat->max_entries = 8;
	ppat->update_hw = bdw_private_pat_update_hw;
	ppat->match = bdw_private_pat_match;
	ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);
B
Ben Widawsky 已提交
3219

3220
	if (!HAS_PPGTT(ppat->i915)) {
3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233
		/* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
		 * so RTL will always use the value corresponding to
		 * pat_sel = 000".
		 * So let's disable cache for GGTT to avoid screen corruptions.
		 * MOCS still can be used though.
		 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
		 * before this patch, i.e. the same uncached + snooping access
		 * like on gen6/7 seems to be in effect.
		 * - So this just fixes blitter/render access. Again it looks
		 * like it's not just uncached access, but uncached + snooping.
		 * So we can still hold onto all our assumptions wrt cpu
		 * clflushing on LLC machines.
		 */
3234 3235 3236
		__alloc_ppat_entry(ppat, 0, GEN8_PPAT_UC);
		return;
	}
3237

3238 3239 3240 3241 3242 3243 3244 3245
	__alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC);      /* for normal objects, no eLLC */
	__alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);  /* for something pointing to ptes? */
	__alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);  /* for scanout with eLLC */
	__alloc_ppat_entry(ppat, 3, GEN8_PPAT_UC);                      /* Uncached objects, mostly for scanout */
	__alloc_ppat_entry(ppat, 4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0));
	__alloc_ppat_entry(ppat, 5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1));
	__alloc_ppat_entry(ppat, 6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2));
	__alloc_ppat_entry(ppat, 7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
B
Ben Widawsky 已提交
3246 3247
}

3248
static void chv_setup_private_ppat(struct intel_ppat *ppat)
3249
{
3250 3251 3252 3253
	ppat->max_entries = 8;
	ppat->update_hw = bdw_private_pat_update_hw;
	ppat->match = chv_private_pat_match;
	ppat->clear_value = CHV_PPAT_SNOOP;
3254 3255 3256 3257 3258 3259 3260

	/*
	 * Map WB on BDW to snooped on CHV.
	 *
	 * Only the snoop bit has meaning for CHV, the rest is
	 * ignored.
	 *
3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271
	 * The hardware will never snoop for certain types of accesses:
	 * - CPU GTT (GMADR->GGTT->no snoop->memory)
	 * - PPGTT page tables
	 * - some other special cycles
	 *
	 * As with BDW, we also need to consider the following for GT accesses:
	 * "For GGTT, there is NO pat_sel[2:0] from the entry,
	 * so RTL will always use the value corresponding to
	 * pat_sel = 000".
	 * Which means we must set the snoop bit in PAT entry 0
	 * in order to keep the global status page working.
3272 3273
	 */

3274 3275 3276 3277 3278 3279 3280 3281
	__alloc_ppat_entry(ppat, 0, CHV_PPAT_SNOOP);
	__alloc_ppat_entry(ppat, 1, 0);
	__alloc_ppat_entry(ppat, 2, 0);
	__alloc_ppat_entry(ppat, 3, 0);
	__alloc_ppat_entry(ppat, 4, CHV_PPAT_SNOOP);
	__alloc_ppat_entry(ppat, 5, CHV_PPAT_SNOOP);
	__alloc_ppat_entry(ppat, 6, CHV_PPAT_SNOOP);
	__alloc_ppat_entry(ppat, 7, CHV_PPAT_SNOOP);
3282 3283
}

3284 3285 3286 3287 3288
static void gen6_gmch_remove(struct i915_address_space *vm)
{
	struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);

	iounmap(ggtt->gsm);
3289
	cleanup_scratch_page(vm);
3290 3291
}

3292 3293
static void setup_private_pat(struct drm_i915_private *dev_priv)
{
3294 3295 3296 3297 3298
	struct intel_ppat *ppat = &dev_priv->ppat;
	int i;

	ppat->i915 = dev_priv;

3299
	if (INTEL_GEN(dev_priv) >= 10)
3300
		cnl_setup_private_ppat(ppat);
3301
	else if (IS_CHERRYVIEW(dev_priv) || IS_GEN9_LP(dev_priv))
3302
		chv_setup_private_ppat(ppat);
3303
	else
3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314
		bdw_setup_private_ppat(ppat);

	GEM_BUG_ON(ppat->max_entries > INTEL_MAX_PPAT_ENTRIES);

	for_each_clear_bit(i, ppat->used, ppat->max_entries) {
		ppat->entries[i].value = ppat->clear_value;
		ppat->entries[i].ppat = ppat;
		set_bit(i, ppat->dirty);
	}

	ppat->update_hw(dev_priv);
3315 3316
}

3317
static int gen8_gmch_probe(struct i915_ggtt *ggtt)
B
Ben Widawsky 已提交
3318
{
3319
	struct drm_i915_private *dev_priv = ggtt->vm.i915;
3320
	struct pci_dev *pdev = dev_priv->drm.pdev;
3321
	unsigned int size;
B
Ben Widawsky 已提交
3322
	u16 snb_gmch_ctl;
3323
	int err;
B
Ben Widawsky 已提交
3324 3325

	/* TODO: We're not aware of mappable constraints on gen8 yet */
3326 3327 3328 3329
	ggtt->gmadr =
		(struct resource) DEFINE_RES_MEM(pci_resource_start(pdev, 2),
						 pci_resource_len(pdev, 2));
	ggtt->mappable_end = resource_size(&ggtt->gmadr);
B
Ben Widawsky 已提交
3330

3331 3332 3333 3334 3335
	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(39));
	if (!err)
		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39));
	if (err)
		DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
B
Ben Widawsky 已提交
3336

3337
	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3338
	if (IS_CHERRYVIEW(dev_priv))
3339
		size = chv_get_total_gtt_size(snb_gmch_ctl);
3340
	else
3341
		size = gen8_get_total_gtt_size(snb_gmch_ctl);
B
Ben Widawsky 已提交
3342

3343
	ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
3344 3345 3346
	ggtt->vm.cleanup = gen6_gmch_remove;
	ggtt->vm.insert_page = gen8_ggtt_insert_page;
	ggtt->vm.clear_range = nop_clear_range;
3347
	if (intel_scanout_needs_vtd_wa(dev_priv))
3348
		ggtt->vm.clear_range = gen8_ggtt_clear_range;
3349

3350
	ggtt->vm.insert_entries = gen8_ggtt_insert_entries;
3351

3352
	/* Serialize GTT updates with aperture access on BXT if VT-d is on. */
3353 3354
	if (intel_ggtt_update_needs_vtd_wa(dev_priv) ||
	    IS_CHERRYVIEW(dev_priv) /* fails with concurrent use/update */) {
3355 3356 3357 3358
		ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL;
		ggtt->vm.insert_page    = bxt_vtd_ggtt_insert_page__BKL;
		if (ggtt->vm.clear_range != nop_clear_range)
			ggtt->vm.clear_range = bxt_vtd_ggtt_clear_range__BKL;
3359 3360 3361 3362 3363

		/* Prevent recursively calling stop_machine() and deadlocks. */
		dev_info(dev_priv->drm.dev,
			 "Disabling error capture for VT-d workaround\n");
		i915_disable_error_state(dev_priv, -ENODEV);
3364 3365
	}

3366 3367
	ggtt->invalidate = gen6_ggtt_invalidate;

3368 3369 3370 3371 3372
	ggtt->vm.vma_ops.bind_vma    = ggtt_bind_vma;
	ggtt->vm.vma_ops.unbind_vma  = ggtt_unbind_vma;
	ggtt->vm.vma_ops.set_pages   = ggtt_set_pages;
	ggtt->vm.vma_ops.clear_pages = clear_pages;

3373 3374
	ggtt->vm.pte_encode = gen8_pte_encode;

3375 3376
	setup_private_pat(dev_priv);

3377
	return ggtt_probe_common(ggtt, size);
B
Ben Widawsky 已提交
3378 3379
}

3380
static int gen6_gmch_probe(struct i915_ggtt *ggtt)
3381
{
3382
	struct drm_i915_private *dev_priv = ggtt->vm.i915;
3383
	struct pci_dev *pdev = dev_priv->drm.pdev;
3384
	unsigned int size;
3385
	u16 snb_gmch_ctl;
3386
	int err;
3387

3388 3389 3390 3391
	ggtt->gmadr =
		(struct resource) DEFINE_RES_MEM(pci_resource_start(pdev, 2),
						 pci_resource_len(pdev, 2));
	ggtt->mappable_end = resource_size(&ggtt->gmadr);
3392

3393 3394
	/* 64/512MB is the current min/max we actually know of, but this is just
	 * a coarse sanity check.
3395
	 */
3396
	if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) {
3397
		DRM_ERROR("Unknown GMADR size (%pa)\n", &ggtt->mappable_end);
3398
		return -ENXIO;
3399 3400
	}

3401 3402 3403 3404 3405
	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(40));
	if (!err)
		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40));
	if (err)
		DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err);
3406
	pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
3407

3408
	size = gen6_get_total_gtt_size(snb_gmch_ctl);
3409
	ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE;
3410

3411 3412 3413
	ggtt->vm.clear_range = nop_clear_range;
	if (!HAS_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv))
		ggtt->vm.clear_range = gen6_ggtt_clear_range;
3414 3415 3416
	ggtt->vm.insert_page = gen6_ggtt_insert_page;
	ggtt->vm.insert_entries = gen6_ggtt_insert_entries;
	ggtt->vm.cleanup = gen6_gmch_remove;
3417

3418 3419
	ggtt->invalidate = gen6_ggtt_invalidate;

3420
	if (HAS_EDRAM(dev_priv))
3421
		ggtt->vm.pte_encode = iris_pte_encode;
3422
	else if (IS_HASWELL(dev_priv))
3423
		ggtt->vm.pte_encode = hsw_pte_encode;
3424
	else if (IS_VALLEYVIEW(dev_priv))
3425
		ggtt->vm.pte_encode = byt_pte_encode;
3426
	else if (INTEL_GEN(dev_priv) >= 7)
3427
		ggtt->vm.pte_encode = ivb_pte_encode;
3428
	else
3429
		ggtt->vm.pte_encode = snb_pte_encode;
3430

3431 3432 3433 3434 3435
	ggtt->vm.vma_ops.bind_vma    = ggtt_bind_vma;
	ggtt->vm.vma_ops.unbind_vma  = ggtt_unbind_vma;
	ggtt->vm.vma_ops.set_pages   = ggtt_set_pages;
	ggtt->vm.vma_ops.clear_pages = clear_pages;

3436
	return ggtt_probe_common(ggtt, size);
3437 3438
}

3439
static void i915_gmch_remove(struct i915_address_space *vm)
3440
{
3441
	intel_gmch_remove();
3442
}
3443

3444
static int i915_gmch_probe(struct i915_ggtt *ggtt)
3445
{
3446
	struct drm_i915_private *dev_priv = ggtt->vm.i915;
3447
	phys_addr_t gmadr_base;
3448 3449
	int ret;

3450
	ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
3451 3452 3453 3454 3455
	if (!ret) {
		DRM_ERROR("failed to set up gmch\n");
		return -EIO;
	}

3456
	intel_gtt_get(&ggtt->vm.total, &gmadr_base, &ggtt->mappable_end);
3457

3458 3459 3460 3461
	ggtt->gmadr =
		(struct resource) DEFINE_RES_MEM(gmadr_base,
						 ggtt->mappable_end);

3462
	ggtt->do_idle_maps = needs_idle_maps(dev_priv);
3463 3464 3465 3466
	ggtt->vm.insert_page = i915_ggtt_insert_page;
	ggtt->vm.insert_entries = i915_ggtt_insert_entries;
	ggtt->vm.clear_range = i915_ggtt_clear_range;
	ggtt->vm.cleanup = i915_gmch_remove;
3467

3468 3469
	ggtt->invalidate = gmch_ggtt_invalidate;

3470 3471 3472 3473 3474
	ggtt->vm.vma_ops.bind_vma    = ggtt_bind_vma;
	ggtt->vm.vma_ops.unbind_vma  = ggtt_unbind_vma;
	ggtt->vm.vma_ops.set_pages   = ggtt_set_pages;
	ggtt->vm.vma_ops.clear_pages = clear_pages;

3475
	if (unlikely(ggtt->do_idle_maps))
3476 3477
		DRM_INFO("applying Ironlake quirks for intel_iommu\n");

3478 3479 3480
	return 0;
}

3481
/**
3482
 * i915_ggtt_probe_hw - Probe GGTT hardware location
3483
 * @dev_priv: i915 device
3484
 */
3485
int i915_ggtt_probe_hw(struct drm_i915_private *dev_priv)
3486
{
3487
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
3488 3489
	int ret;

3490 3491
	ggtt->vm.i915 = dev_priv;
	ggtt->vm.dma = &dev_priv->drm.pdev->dev;
3492

3493 3494 3495 3496 3497 3498
	if (INTEL_GEN(dev_priv) <= 5)
		ret = i915_gmch_probe(ggtt);
	else if (INTEL_GEN(dev_priv) < 8)
		ret = gen6_gmch_probe(ggtt);
	else
		ret = gen8_gmch_probe(ggtt);
3499
	if (ret)
3500 3501
		return ret;

3502
	if ((ggtt->vm.total - 1) >> 32) {
3503
		DRM_ERROR("We never expected a Global GTT with more than 32bits"
3504
			  " of address space! Found %lldM!\n",
3505 3506 3507 3508
			  ggtt->vm.total >> 20);
		ggtt->vm.total = 1ULL << 32;
		ggtt->mappable_end =
			min_t(u64, ggtt->mappable_end, ggtt->vm.total);
3509 3510
	}

3511
	if (ggtt->mappable_end > ggtt->vm.total) {
3512
		DRM_ERROR("mappable aperture extends past end of GGTT,"
3513
			  " aperture=%pa, total=%llx\n",
3514 3515
			  &ggtt->mappable_end, ggtt->vm.total);
		ggtt->mappable_end = ggtt->vm.total;
3516 3517
	}

3518
	/* GMADR is the PCI mmio aperture into the global GTT. */
3519
	DRM_DEBUG_DRIVER("GGTT size = %lluM\n", ggtt->vm.total >> 20);
3520
	DRM_DEBUG_DRIVER("GMADR size = %lluM\n", (u64)ggtt->mappable_end >> 20);
3521
	DRM_DEBUG_DRIVER("DSM size = %lluM\n",
3522
			 (u64)resource_size(&intel_graphics_stolen_res) >> 20);
3523
	if (intel_vtd_active())
3524
		DRM_INFO("VT-d active for gfx access\n");
3525 3526

	return 0;
3527 3528 3529 3530
}

/**
 * i915_ggtt_init_hw - Initialize GGTT hardware
3531
 * @dev_priv: i915 device
3532
 */
3533
int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
3534 3535 3536 3537
{
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
	int ret;

3538 3539
	stash_init(&dev_priv->mm.wc_stash);

3540 3541 3542 3543
	/* Note that we use page colouring to enforce a guard page at the
	 * end of the address space. This is required as the CS may prefetch
	 * beyond the end of the batch buffer, across the page boundary,
	 * and beyond the end of the GTT if we do not provide a guard.
3544
	 */
C
Chris Wilson 已提交
3545
	mutex_lock(&dev_priv->drm.struct_mutex);
3546
	i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT);
3547

3548 3549
	ggtt->vm.is_ggtt = true;

3550 3551 3552
	/* Only VLV supports read-only GGTT mappings */
	ggtt->vm.has_read_only = IS_VALLEYVIEW(dev_priv);

3553
	if (!HAS_LLC(dev_priv) && !HAS_PPGTT(dev_priv))
3554
		ggtt->vm.mm.color_adjust = i915_gtt_color_adjust;
C
Chris Wilson 已提交
3555
	mutex_unlock(&dev_priv->drm.struct_mutex);
3556

3557 3558
	if (!io_mapping_init_wc(&dev_priv->ggtt.iomap,
				dev_priv->ggtt.gmadr.start,
3559
				dev_priv->ggtt.mappable_end)) {
3560 3561 3562 3563
		ret = -EIO;
		goto out_gtt_cleanup;
	}

3564
	ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, ggtt->mappable_end);
3565

3566 3567
	i915_ggtt_init_fences(ggtt);

3568 3569 3570 3571
	/*
	 * Initialise stolen early so that we may reserve preallocated
	 * objects for the BIOS to KMS transition.
	 */
3572
	ret = i915_gem_init_stolen(dev_priv);
3573 3574 3575 3576
	if (ret)
		goto out_gtt_cleanup;

	return 0;
3577 3578

out_gtt_cleanup:
3579
	ggtt->vm.cleanup(&ggtt->vm);
3580
	return ret;
3581
}
3582

3583
int i915_ggtt_enable_hw(struct drm_i915_private *dev_priv)
3584
{
3585
	if (INTEL_GEN(dev_priv) < 6 && !intel_enable_gtt())
3586 3587 3588 3589 3590
		return -EIO;

	return 0;
}

3591 3592
void i915_ggtt_enable_guc(struct drm_i915_private *i915)
{
3593 3594
	GEM_BUG_ON(i915->ggtt.invalidate != gen6_ggtt_invalidate);

3595
	i915->ggtt.invalidate = guc_ggtt_invalidate;
3596 3597

	i915_ggtt_invalidate(i915);
3598 3599 3600 3601
}

void i915_ggtt_disable_guc(struct drm_i915_private *i915)
{
3602 3603 3604 3605
	/* XXX Temporary pardon for error unload */
	if (i915->ggtt.invalidate == gen6_ggtt_invalidate)
		return;

3606 3607 3608 3609
	/* We should only be called after i915_ggtt_enable_guc() */
	GEM_BUG_ON(i915->ggtt.invalidate != guc_ggtt_invalidate);

	i915->ggtt.invalidate = gen6_ggtt_invalidate;
3610 3611

	i915_ggtt_invalidate(i915);
3612 3613
}

3614
void i915_gem_restore_gtt_mappings(struct drm_i915_private *dev_priv)
3615
{
3616
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
3617
	struct i915_vma *vma, *vn;
3618

3619
	i915_check_and_clear_faults(dev_priv);
3620

3621 3622
	mutex_lock(&ggtt->vm.mutex);

3623
	/* First fill our portion of the GTT with scratch pages */
3624 3625
	ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total);
	ggtt->vm.closed = true; /* skip rewriting PTE on VMA unbind */
3626 3627

	/* clflush objects bound into the GGTT and rebind them. */
3628
	list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) {
3629
		struct drm_i915_gem_object *obj = vma->obj;
3630

3631 3632
		if (!(vma->flags & I915_VMA_GLOBAL_BIND))
			continue;
3633

3634 3635
		mutex_unlock(&ggtt->vm.mutex);

3636
		if (!i915_vma_unbind(vma))
3637
			goto lock;
3638

3639 3640 3641
		WARN_ON(i915_vma_bind(vma,
				      obj ? obj->cache_level : 0,
				      PIN_UPDATE));
3642 3643
		if (obj) {
			i915_gem_object_lock(obj);
3644
			WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
3645 3646
			i915_gem_object_unlock(obj);
		}
3647 3648 3649

lock:
		mutex_lock(&ggtt->vm.mutex);
3650
	}
3651

3652
	ggtt->vm.closed = false;
3653
	i915_ggtt_invalidate(dev_priv);
3654

3655 3656
	mutex_unlock(&ggtt->vm.mutex);

3657
	if (INTEL_GEN(dev_priv) >= 8) {
3658
		struct intel_ppat *ppat = &dev_priv->ppat;
3659

3660 3661
		bitmap_set(ppat->dirty, 0, ppat->max_entries);
		dev_priv->ppat.update_hw(dev_priv);
3662 3663 3664 3665
		return;
	}
}

3666
static struct scatterlist *
3667
rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset,
3668
	     unsigned int width, unsigned int height,
3669
	     unsigned int stride,
3670
	     struct sg_table *st, struct scatterlist *sg)
3671 3672 3673 3674 3675
{
	unsigned int column, row;
	unsigned int src_idx;

	for (column = 0; column < width; column++) {
3676
		src_idx = stride * (height - 1) + column + offset;
3677 3678 3679 3680 3681 3682
		for (row = 0; row < height; row++) {
			st->nents++;
			/* We don't need the pages, but need to initialize
			 * the entries so the sg list can be happily traversed.
			 * The only thing we need are DMA addresses.
			 */
3683
			sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0);
3684 3685
			sg_dma_address(sg) =
				i915_gem_object_get_dma_address(obj, src_idx);
3686
			sg_dma_len(sg) = I915_GTT_PAGE_SIZE;
3687
			sg = sg_next(sg);
3688
			src_idx -= stride;
3689 3690
		}
	}
3691 3692

	return sg;
3693 3694
}

3695 3696 3697
static noinline struct sg_table *
intel_rotate_pages(struct intel_rotation_info *rot_info,
		   struct drm_i915_gem_object *obj)
3698
{
3699
	unsigned int size = intel_rotation_info_size(rot_info);
3700
	struct sg_table *st;
3701
	struct scatterlist *sg;
3702
	int ret = -ENOMEM;
3703
	int i;
3704 3705 3706 3707 3708 3709

	/* Allocate target SG list. */
	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (!st)
		goto err_st_alloc;

3710
	ret = sg_alloc_table(st, size, GFP_KERNEL);
3711 3712 3713
	if (ret)
		goto err_sg_alloc;

3714 3715 3716
	st->nents = 0;
	sg = st->sgl;

3717
	for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) {
3718
		sg = rotate_pages(obj, rot_info->plane[i].offset,
3719 3720
				  rot_info->plane[i].width, rot_info->plane[i].height,
				  rot_info->plane[i].stride, st, sg);
3721 3722
	}

3723 3724 3725 3726 3727 3728
	return st;

err_sg_alloc:
	kfree(st);
err_st_alloc:

3729 3730
	DRM_DEBUG_DRIVER("Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n",
			 obj->base.size, rot_info->plane[0].width, rot_info->plane[0].height, size);
3731

3732 3733
	return ERR_PTR(ret);
}
3734

3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817
static struct scatterlist *
remap_pages(struct drm_i915_gem_object *obj, unsigned int offset,
	    unsigned int width, unsigned int height,
	    unsigned int stride,
	    struct sg_table *st, struct scatterlist *sg)
{
	unsigned int row;

	for (row = 0; row < height; row++) {
		unsigned int left = width * I915_GTT_PAGE_SIZE;

		while (left) {
			dma_addr_t addr;
			unsigned int length;

			/* We don't need the pages, but need to initialize
			 * the entries so the sg list can be happily traversed.
			 * The only thing we need are DMA addresses.
			 */

			addr = i915_gem_object_get_dma_address_len(obj, offset, &length);

			length = min(left, length);

			st->nents++;

			sg_set_page(sg, NULL, length, 0);
			sg_dma_address(sg) = addr;
			sg_dma_len(sg) = length;
			sg = sg_next(sg);

			offset += length / I915_GTT_PAGE_SIZE;
			left -= length;
		}

		offset += stride - width;
	}

	return sg;
}

static noinline struct sg_table *
intel_remap_pages(struct intel_remapped_info *rem_info,
		  struct drm_i915_gem_object *obj)
{
	unsigned int size = intel_remapped_info_size(rem_info);
	struct sg_table *st;
	struct scatterlist *sg;
	int ret = -ENOMEM;
	int i;

	/* Allocate target SG list. */
	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (!st)
		goto err_st_alloc;

	ret = sg_alloc_table(st, size, GFP_KERNEL);
	if (ret)
		goto err_sg_alloc;

	st->nents = 0;
	sg = st->sgl;

	for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) {
		sg = remap_pages(obj, rem_info->plane[i].offset,
				 rem_info->plane[i].width, rem_info->plane[i].height,
				 rem_info->plane[i].stride, st, sg);
	}

	i915_sg_trim(st);

	return st;

err_sg_alloc:
	kfree(st);
err_st_alloc:

	DRM_DEBUG_DRIVER("Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n",
			 obj->base.size, rem_info->plane[0].width, rem_info->plane[0].height, size);

	return ERR_PTR(ret);
}

3818
static noinline struct sg_table *
3819 3820 3821 3822
intel_partial_pages(const struct i915_ggtt_view *view,
		    struct drm_i915_gem_object *obj)
{
	struct sg_table *st;
3823
	struct scatterlist *sg, *iter;
3824
	unsigned int count = view->partial.size;
3825
	unsigned int offset;
3826 3827 3828 3829 3830 3831
	int ret = -ENOMEM;

	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (!st)
		goto err_st_alloc;

3832
	ret = sg_alloc_table(st, count, GFP_KERNEL);
3833 3834 3835
	if (ret)
		goto err_sg_alloc;

3836
	iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset);
3837 3838
	GEM_BUG_ON(!iter);

3839 3840
	sg = st->sgl;
	st->nents = 0;
3841 3842
	do {
		unsigned int len;
3843

3844 3845 3846 3847 3848 3849
		len = min(iter->length - (offset << PAGE_SHIFT),
			  count << PAGE_SHIFT);
		sg_set_page(sg, NULL, len, 0);
		sg_dma_address(sg) =
			sg_dma_address(iter) + (offset << PAGE_SHIFT);
		sg_dma_len(sg) = len;
3850 3851

		st->nents++;
3852 3853 3854
		count -= len >> PAGE_SHIFT;
		if (count == 0) {
			sg_mark_end(sg);
3855 3856
			i915_sg_trim(st); /* Drop any unused tail entries. */

3857 3858
			return st;
		}
3859

3860 3861 3862 3863
		sg = __sg_next(sg);
		iter = __sg_next(iter);
		offset = 0;
	} while (1);
3864 3865 3866 3867 3868 3869 3870

err_sg_alloc:
	kfree(st);
err_st_alloc:
	return ERR_PTR(ret);
}

3871
static int
3872
i915_get_ggtt_vma_pages(struct i915_vma *vma)
3873
{
3874
	int ret;
3875

3876 3877 3878 3879 3880 3881 3882
	/* The vma->pages are only valid within the lifespan of the borrowed
	 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so
	 * must be the vma->pages. A simple rule is that vma->pages must only
	 * be accessed when the obj->mm.pages are pinned.
	 */
	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));

3883
	switch (vma->ggtt_view.type) {
3884 3885 3886
	default:
		GEM_BUG_ON(vma->ggtt_view.type);
		/* fall through */
3887 3888
	case I915_GGTT_VIEW_NORMAL:
		vma->pages = vma->obj->mm.pages;
3889 3890
		return 0;

3891
	case I915_GGTT_VIEW_ROTATED:
3892
		vma->pages =
3893 3894 3895
			intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
		break;

3896 3897 3898 3899 3900
	case I915_GGTT_VIEW_REMAPPED:
		vma->pages =
			intel_remap_pages(&vma->ggtt_view.remapped, vma->obj);
		break;

3901
	case I915_GGTT_VIEW_PARTIAL:
3902
		vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
3903 3904
		break;
	}
3905

3906
	ret = 0;
3907
	if (IS_ERR(vma->pages)) {
3908 3909
		ret = PTR_ERR(vma->pages);
		vma->pages = NULL;
3910 3911
		DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
			  vma->ggtt_view.type, ret);
3912
	}
3913
	return ret;
3914 3915
}

3916 3917
/**
 * i915_gem_gtt_reserve - reserve a node in an address_space (GTT)
3918 3919 3920 3921 3922 3923 3924 3925 3926 3927
 * @vm: the &struct i915_address_space
 * @node: the &struct drm_mm_node (typically i915_vma.mode)
 * @size: how much space to allocate inside the GTT,
 *        must be #I915_GTT_PAGE_SIZE aligned
 * @offset: where to insert inside the GTT,
 *          must be #I915_GTT_MIN_ALIGNMENT aligned, and the node
 *          (@offset + @size) must fit within the address space
 * @color: color to apply to node, if this node is not from a VMA,
 *         color must be #I915_COLOR_UNEVICTABLE
 * @flags: control search and eviction behaviour
3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951
 *
 * i915_gem_gtt_reserve() tries to insert the @node at the exact @offset inside
 * the address space (using @size and @color). If the @node does not fit, it
 * tries to evict any overlapping nodes from the GTT, including any
 * neighbouring nodes if the colors do not match (to ensure guard pages between
 * differing domains). See i915_gem_evict_for_node() for the gory details
 * on the eviction algorithm. #PIN_NONBLOCK may used to prevent waiting on
 * evicting active overlapping objects, and any overlapping node that is pinned
 * or marked as unevictable will also result in failure.
 *
 * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
 * asked to wait for eviction and interrupted.
 */
int i915_gem_gtt_reserve(struct i915_address_space *vm,
			 struct drm_mm_node *node,
			 u64 size, u64 offset, unsigned long color,
			 unsigned int flags)
{
	int err;

	GEM_BUG_ON(!size);
	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
	GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT));
	GEM_BUG_ON(range_overflows(offset, size, vm->total));
3952
	GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->vm);
3953
	GEM_BUG_ON(drm_mm_node_allocated(node));
3954 3955 3956 3957 3958 3959 3960 3961 3962

	node->size = size;
	node->start = offset;
	node->color = color;

	err = drm_mm_reserve_node(&vm->mm, node);
	if (err != -ENOSPC)
		return err;

3963 3964 3965
	if (flags & PIN_NOEVICT)
		return -ENOSPC;

3966 3967 3968 3969 3970 3971 3972
	err = i915_gem_evict_for_node(vm, node, flags);
	if (err == 0)
		err = drm_mm_reserve_node(&vm->mm, node);

	return err;
}

3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997
static u64 random_offset(u64 start, u64 end, u64 len, u64 align)
{
	u64 range, addr;

	GEM_BUG_ON(range_overflows(start, len, end));
	GEM_BUG_ON(round_up(start, align) > round_down(end - len, align));

	range = round_down(end - len, align) - round_up(start, align);
	if (range) {
		if (sizeof(unsigned long) == sizeof(u64)) {
			addr = get_random_long();
		} else {
			addr = get_random_int();
			if (range > U32_MAX) {
				addr <<= 32;
				addr |= get_random_int();
			}
		}
		div64_u64_rem(addr, range, &addr);
		start += addr;
	}

	return round_up(start, align);
}

3998 3999
/**
 * i915_gem_gtt_insert - insert a node into an address_space (GTT)
4000 4001 4002 4003 4004 4005 4006 4007 4008
 * @vm: the &struct i915_address_space
 * @node: the &struct drm_mm_node (typically i915_vma.node)
 * @size: how much space to allocate inside the GTT,
 *        must be #I915_GTT_PAGE_SIZE aligned
 * @alignment: required alignment of starting offset, may be 0 but
 *             if specified, this must be a power-of-two and at least
 *             #I915_GTT_MIN_ALIGNMENT
 * @color: color to apply to node
 * @start: start of any range restriction inside GTT (0 for all),
4009
 *         must be #I915_GTT_PAGE_SIZE aligned
4010 4011 4012
 * @end: end of any range restriction inside GTT (U64_MAX for all),
 *       must be #I915_GTT_PAGE_SIZE aligned if not U64_MAX
 * @flags: control search and eviction behaviour
4013 4014 4015 4016 4017 4018
 *
 * i915_gem_gtt_insert() first searches for an available hole into which
 * is can insert the node. The hole address is aligned to @alignment and
 * its @size must then fit entirely within the [@start, @end] bounds. The
 * nodes on either side of the hole must match @color, or else a guard page
 * will be inserted between the two nodes (or the node evicted). If no
4019 4020
 * suitable hole is found, first a victim is randomly selected and tested
 * for eviction, otherwise then the LRU list of objects within the GTT
4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036
 * is scanned to find the first set of replacement nodes to create the hole.
 * Those old overlapping nodes are evicted from the GTT (and so must be
 * rebound before any future use). Any node that is currently pinned cannot
 * be evicted (see i915_vma_pin()). Similar if the node's VMA is currently
 * active and #PIN_NONBLOCK is specified, that node is also skipped when
 * searching for an eviction candidate. See i915_gem_evict_something() for
 * the gory details on the eviction algorithm.
 *
 * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
 * asked to wait for eviction and interrupted.
 */
int i915_gem_gtt_insert(struct i915_address_space *vm,
			struct drm_mm_node *node,
			u64 size, u64 alignment, unsigned long color,
			u64 start, u64 end, unsigned int flags)
{
4037
	enum drm_mm_insert_mode mode;
4038
	u64 offset;
4039 4040 4041 4042 4043 4044 4045 4046 4047 4048
	int err;

	lockdep_assert_held(&vm->i915->drm.struct_mutex);
	GEM_BUG_ON(!size);
	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
	GEM_BUG_ON(alignment && !is_power_of_2(alignment));
	GEM_BUG_ON(alignment && !IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
	GEM_BUG_ON(start >= end);
	GEM_BUG_ON(start > 0  && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
	GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
4049
	GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->vm);
4050
	GEM_BUG_ON(drm_mm_node_allocated(node));
4051 4052 4053 4054 4055 4056 4057

	if (unlikely(range_overflows(start, size, end)))
		return -ENOSPC;

	if (unlikely(round_up(start, alignment) > round_down(end - size, alignment)))
		return -ENOSPC;

4058 4059
	mode = DRM_MM_INSERT_BEST;
	if (flags & PIN_HIGH)
4060
		mode = DRM_MM_INSERT_HIGHEST;
4061 4062
	if (flags & PIN_MAPPABLE)
		mode = DRM_MM_INSERT_LOW;
4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073

	/* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
	 * so we know that we always have a minimum alignment of 4096.
	 * The drm_mm range manager is optimised to return results
	 * with zero alignment, so where possible use the optimal
	 * path.
	 */
	BUILD_BUG_ON(I915_GTT_MIN_ALIGNMENT > I915_GTT_PAGE_SIZE);
	if (alignment <= I915_GTT_MIN_ALIGNMENT)
		alignment = 0;

4074 4075 4076
	err = drm_mm_insert_node_in_range(&vm->mm, node,
					  size, alignment, color,
					  start, end, mode);
4077 4078 4079
	if (err != -ENOSPC)
		return err;

4080 4081 4082 4083 4084 4085 4086 4087 4088
	if (mode & DRM_MM_INSERT_ONCE) {
		err = drm_mm_insert_node_in_range(&vm->mm, node,
						  size, alignment, color,
						  start, end,
						  DRM_MM_INSERT_BEST);
		if (err != -ENOSPC)
			return err;
	}

4089 4090 4091
	if (flags & PIN_NOEVICT)
		return -ENOSPC;

4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120
	/* No free space, pick a slot at random.
	 *
	 * There is a pathological case here using a GTT shared between
	 * mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt):
	 *
	 *    |<-- 256 MiB aperture -->||<-- 1792 MiB unmappable -->|
	 *         (64k objects)             (448k objects)
	 *
	 * Now imagine that the eviction LRU is ordered top-down (just because
	 * pathology meets real life), and that we need to evict an object to
	 * make room inside the aperture. The eviction scan then has to walk
	 * the 448k list before it finds one within range. And now imagine that
	 * it has to search for a new hole between every byte inside the memcpy,
	 * for several simultaneous clients.
	 *
	 * On a full-ppgtt system, if we have run out of available space, there
	 * will be lots and lots of objects in the eviction list! Again,
	 * searching that LRU list may be slow if we are also applying any
	 * range restrictions (e.g. restriction to low 4GiB) and so, for
	 * simplicity and similarilty between different GTT, try the single
	 * random replacement first.
	 */
	offset = random_offset(start, end,
			       size, alignment ?: I915_GTT_MIN_ALIGNMENT);
	err = i915_gem_gtt_reserve(vm, node, size, offset, color, flags);
	if (err != -ENOSPC)
		return err;

	/* Randomly selected placement is pinned, do a search */
4121 4122 4123 4124 4125
	err = i915_gem_evict_something(vm, size, alignment, color,
				       start, end, flags);
	if (err)
		return err;

4126 4127 4128
	return drm_mm_insert_node_in_range(&vm->mm, node,
					   size, alignment, color,
					   start, end, DRM_MM_INSERT_EVICT);
4129
}
4130 4131 4132

#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/mock_gtt.c"
4133
#include "selftests/i915_gem_gtt.c"
4134
#endif