i915_gem_gtt.c 79.2 KB
Newer Older
1 2
/*
 * Copyright © 2010 Daniel Vetter
3
 * Copyright © 2011-2014 Intel Corporation
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

26
#include <linux/seq_file.h>
27 28
#include <drm/drmP.h>
#include <drm/i915_drm.h>
29
#include "i915_drv.h"
30
#include "i915_vgpu.h"
31 32 33
#include "i915_trace.h"
#include "intel_drv.h"

34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
/**
 * DOC: Global GTT views
 *
 * Background and previous state
 *
 * Historically objects could exists (be bound) in global GTT space only as
 * singular instances with a view representing all of the object's backing pages
 * in a linear fashion. This view will be called a normal view.
 *
 * To support multiple views of the same object, where the number of mapped
 * pages is not equal to the backing store, or where the layout of the pages
 * is not linear, concept of a GGTT view was added.
 *
 * One example of an alternative view is a stereo display driven by a single
 * image. In this case we would have a framebuffer looking like this
 * (2x2 pages):
 *
 *    12
 *    34
 *
 * Above would represent a normal GGTT view as normally mapped for GPU or CPU
 * rendering. In contrast, fed to the display engine would be an alternative
 * view which could look something like this:
 *
 *   1212
 *   3434
 *
 * In this example both the size and layout of pages in the alternative view is
 * different from the normal view.
 *
 * Implementation and usage
 *
 * GGTT views are implemented using VMAs and are distinguished via enum
 * i915_ggtt_view_type and struct i915_ggtt_view.
 *
 * A new flavour of core GEM functions which work with GGTT bound objects were
70 71 72
 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
 * renaming  in large amounts of code. They take the struct i915_ggtt_view
 * parameter encapsulating all metadata required to implement a view.
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
 *
 * As a helper for callers which are only interested in the normal view,
 * globally const i915_ggtt_view_normal singleton instance exists. All old core
 * GEM API functions, the ones not taking the view parameter, are operating on,
 * or with the normal GGTT view.
 *
 * Code wanting to add or use a new GGTT view needs to:
 *
 * 1. Add a new enum with a suitable name.
 * 2. Extend the metadata in the i915_ggtt_view structure if required.
 * 3. Add support to i915_get_vma_pages().
 *
 * New views are required to build a scatter-gather table from within the
 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
 * exists for the lifetime of an VMA.
 *
 * Core API is designed to have copy semantics which means that passed in
 * struct i915_ggtt_view does not need to be persistent (left around after
 * calling the core API functions).
 *
 */

95 96 97
static int
i915_get_ggtt_vma_pages(struct i915_vma *vma);

98
const struct i915_ggtt_view i915_ggtt_view_normal;
99 100 101
const struct i915_ggtt_view i915_ggtt_view_rotated = {
        .type = I915_GGTT_VIEW_ROTATED
};
102

103 104
static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
{
105 106 107 108 109 110
	bool has_aliasing_ppgtt;
	bool has_full_ppgtt;

	has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
	has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;

111 112 113
	if (intel_vgpu_active(dev))
		has_full_ppgtt = false; /* emulation is too hard */

114 115 116 117 118 119
	/*
	 * We don't allow disabling PPGTT for gen9+ as it's a requirement for
	 * execlists, the sole mechanism available to submit work.
	 */
	if (INTEL_INFO(dev)->gen < 9 &&
	    (enable_ppgtt == 0 || !has_aliasing_ppgtt))
120 121 122 123 124
		return 0;

	if (enable_ppgtt == 1)
		return 1;

125
	if (enable_ppgtt == 2 && has_full_ppgtt)
126 127
		return 2;

128 129 130 131
#ifdef CONFIG_INTEL_IOMMU
	/* Disable ppgtt on SNB if VT-d is on. */
	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
		DRM_INFO("Disabling PPGTT because VT-d is on\n");
132
		return 0;
133 134 135
	}
#endif

136
	/* Early VLV doesn't have this */
137 138
	if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
	    dev->pdev->revision < 0xb) {
139 140 141 142
		DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
		return 0;
	}

143 144 145 146
	if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists)
		return 2;
	else
		return has_aliasing_ppgtt ? 1 : 0;
147 148
}

149 150 151
static int ppgtt_bind_vma(struct i915_vma *vma,
			  enum i915_cache_level cache_level,
			  u32 unused)
152 153 154 155 156 157 158 159 160
{
	u32 pte_flags = 0;

	/* Currently applicable only to VLV */
	if (vma->obj->gt_ro)
		pte_flags |= PTE_READ_ONLY;

	vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
				cache_level, pte_flags);
161 162

	return 0;
163 164 165 166 167 168 169 170 171
}

static void ppgtt_unbind_vma(struct i915_vma *vma)
{
	vma->vm->clear_range(vma->vm,
			     vma->node.start,
			     vma->obj->base.size,
			     true);
}
172

173 174 175
static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
				  enum i915_cache_level level,
				  bool valid)
B
Ben Widawsky 已提交
176
{
177
	gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
B
Ben Widawsky 已提交
178
	pte |= addr;
179 180 181

	switch (level) {
	case I915_CACHE_NONE:
B
Ben Widawsky 已提交
182
		pte |= PPAT_UNCACHED_INDEX;
183 184 185 186 187 188 189 190 191
		break;
	case I915_CACHE_WT:
		pte |= PPAT_DISPLAY_ELLC_INDEX;
		break;
	default:
		pte |= PPAT_CACHED_INDEX;
		break;
	}

B
Ben Widawsky 已提交
192 193 194
	return pte;
}

195 196
static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
				  const enum i915_cache_level level)
B
Ben Widawsky 已提交
197
{
198
	gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
B
Ben Widawsky 已提交
199 200 201 202 203 204 205 206
	pde |= addr;
	if (level != I915_CACHE_NONE)
		pde |= PPAT_CACHED_PDE_INDEX;
	else
		pde |= PPAT_UNCACHED_INDEX;
	return pde;
}

207 208 209
static gen6_pte_t snb_pte_encode(dma_addr_t addr,
				 enum i915_cache_level level,
				 bool valid, u32 unused)
210
{
211
	gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
212
	pte |= GEN6_PTE_ADDR_ENCODE(addr);
213 214

	switch (level) {
215 216 217 218 219 220 221 222
	case I915_CACHE_L3_LLC:
	case I915_CACHE_LLC:
		pte |= GEN6_PTE_CACHE_LLC;
		break;
	case I915_CACHE_NONE:
		pte |= GEN6_PTE_UNCACHED;
		break;
	default:
223
		MISSING_CASE(level);
224 225 226 227 228
	}

	return pte;
}

229 230 231
static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
				 enum i915_cache_level level,
				 bool valid, u32 unused)
232
{
233
	gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
234 235 236 237 238
	pte |= GEN6_PTE_ADDR_ENCODE(addr);

	switch (level) {
	case I915_CACHE_L3_LLC:
		pte |= GEN7_PTE_CACHE_L3_LLC;
239 240 241 242 243
		break;
	case I915_CACHE_LLC:
		pte |= GEN6_PTE_CACHE_LLC;
		break;
	case I915_CACHE_NONE:
244
		pte |= GEN6_PTE_UNCACHED;
245 246
		break;
	default:
247
		MISSING_CASE(level);
248 249
	}

250 251 252
	return pte;
}

253 254 255
static gen6_pte_t byt_pte_encode(dma_addr_t addr,
				 enum i915_cache_level level,
				 bool valid, u32 flags)
256
{
257
	gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
258 259
	pte |= GEN6_PTE_ADDR_ENCODE(addr);

260 261
	if (!(flags & PTE_READ_ONLY))
		pte |= BYT_PTE_WRITEABLE;
262 263 264 265 266 267 268

	if (level != I915_CACHE_NONE)
		pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;

	return pte;
}

269 270 271
static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
				 enum i915_cache_level level,
				 bool valid, u32 unused)
272
{
273
	gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
274
	pte |= HSW_PTE_ADDR_ENCODE(addr);
275 276

	if (level != I915_CACHE_NONE)
277
		pte |= HSW_WB_LLC_AGE3;
278 279 280 281

	return pte;
}

282 283 284
static gen6_pte_t iris_pte_encode(dma_addr_t addr,
				  enum i915_cache_level level,
				  bool valid, u32 unused)
285
{
286
	gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
287 288
	pte |= HSW_PTE_ADDR_ENCODE(addr);

289 290 291 292
	switch (level) {
	case I915_CACHE_NONE:
		break;
	case I915_CACHE_WT:
293
		pte |= HSW_WT_ELLC_LLC_AGE3;
294 295
		break;
	default:
296
		pte |= HSW_WB_ELLC_LLC_AGE3;
297 298
		break;
	}
299 300 301 302

	return pte;
}

303 304
static int __setup_page_dma(struct drm_device *dev,
			    struct i915_page_dma *p, gfp_t flags)
305 306 307
{
	struct device *device = &dev->pdev->dev;

308
	p->page = alloc_page(flags);
309 310
	if (!p->page)
		return -ENOMEM;
311

312 313
	p->daddr = dma_map_page(device,
				p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
314

315 316 317 318
	if (dma_mapping_error(device, p->daddr)) {
		__free_page(p->page);
		return -EINVAL;
	}
319 320

	return 0;
321 322
}

323 324 325 326 327
static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
{
	return __setup_page_dma(dev, p, GFP_KERNEL);
}

328
static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
329
{
330
	if (WARN_ON(!p->page))
331
		return;
332

333 334 335 336 337
	dma_unmap_page(&dev->pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL);
	__free_page(p->page);
	memset(p, 0, sizeof(*p));
}

338
static void *kmap_page_dma(struct i915_page_dma *p)
339
{
340 341
	return kmap_atomic(p->page);
}
342

343 344 345 346 347
/* We use the flushing unmap only with ppgtt structures:
 * page directories, page tables and scratch pages.
 */
static void kunmap_page_dma(struct drm_device *dev, void *vaddr)
{
348 349 350 351 352 353 354 355 356
	/* There are only few exceptions for gen >=6. chv and bxt.
	 * And we are not sure about the latter so play safe for now.
	 */
	if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
		drm_clflush_virt_range(vaddr, PAGE_SIZE);

	kunmap_atomic(vaddr);
}

357
#define kmap_px(px) kmap_page_dma(px_base(px))
358 359
#define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr))

360 361 362 363 364
#define setup_px(dev, px) setup_page_dma((dev), px_base(px))
#define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px))
#define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v))
#define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v))

365 366 367 368 369 370 371 372 373 374 375 376
static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p,
			  const uint64_t val)
{
	int i;
	uint64_t * const vaddr = kmap_page_dma(p);

	for (i = 0; i < 512; i++)
		vaddr[i] = val;

	kunmap_page_dma(dev, vaddr);
}

377 378 379 380 381 382 383 384 385 386
static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p,
			     const uint32_t val32)
{
	uint64_t v = val32;

	v = v << 32 | val32;

	fill_page_dma(dev, p, v);
}

387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
static struct i915_page_scratch *alloc_scratch_page(struct drm_device *dev)
{
	struct i915_page_scratch *sp;
	int ret;

	sp = kzalloc(sizeof(*sp), GFP_KERNEL);
	if (sp == NULL)
		return ERR_PTR(-ENOMEM);

	ret = __setup_page_dma(dev, px_base(sp), GFP_DMA32 | __GFP_ZERO);
	if (ret) {
		kfree(sp);
		return ERR_PTR(ret);
	}

	set_pages_uc(px_page(sp), 1);

	return sp;
}

static void free_scratch_page(struct drm_device *dev,
			      struct i915_page_scratch *sp)
{
	set_pages_wb(px_page(sp), 1);

	cleanup_px(dev, sp);
	kfree(sp);
}

416
static struct i915_page_table *alloc_pt(struct drm_device *dev)
417
{
418
	struct i915_page_table *pt;
419 420 421
	const size_t count = INTEL_INFO(dev)->gen >= 8 ?
		GEN8_PTES : GEN6_PTES;
	int ret = -ENOMEM;
422 423 424 425 426

	pt = kzalloc(sizeof(*pt), GFP_KERNEL);
	if (!pt)
		return ERR_PTR(-ENOMEM);

427 428 429 430 431 432
	pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
				GFP_KERNEL);

	if (!pt->used_ptes)
		goto fail_bitmap;

433
	ret = setup_px(dev, pt);
434
	if (ret)
435
		goto fail_page_m;
436 437

	return pt;
438

439
fail_page_m:
440 441 442 443 444
	kfree(pt->used_ptes);
fail_bitmap:
	kfree(pt);

	return ERR_PTR(ret);
445 446
}

447
static void free_pt(struct drm_device *dev, struct i915_page_table *pt)
448
{
449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
	cleanup_px(dev, pt);
	kfree(pt->used_ptes);
	kfree(pt);
}

static void gen8_initialize_pt(struct i915_address_space *vm,
			       struct i915_page_table *pt)
{
	gen8_pte_t scratch_pte;

	scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
				      I915_CACHE_LLC, true);

	fill_px(vm->dev, pt, scratch_pte);
}

static void gen6_initialize_pt(struct i915_address_space *vm,
			       struct i915_page_table *pt)
{
	gen6_pte_t scratch_pte;

	WARN_ON(px_dma(vm->scratch_page) == 0);

	scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
				     I915_CACHE_LLC, true, 0);

	fill32_px(vm->dev, pt, scratch_pte);
476 477
}

478
static struct i915_page_directory *alloc_pd(struct drm_device *dev)
479
{
480
	struct i915_page_directory *pd;
481
	int ret = -ENOMEM;
482 483 484 485 486

	pd = kzalloc(sizeof(*pd), GFP_KERNEL);
	if (!pd)
		return ERR_PTR(-ENOMEM);

487 488 489
	pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES),
				sizeof(*pd->used_pdes), GFP_KERNEL);
	if (!pd->used_pdes)
490
		goto fail_bitmap;
491

492
	ret = setup_px(dev, pd);
493
	if (ret)
494
		goto fail_page_m;
495

496
	return pd;
497

498
fail_page_m:
499
	kfree(pd->used_pdes);
500
fail_bitmap:
501 502 503
	kfree(pd);

	return ERR_PTR(ret);
504 505
}

506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
static void free_pd(struct drm_device *dev, struct i915_page_directory *pd)
{
	if (px_page(pd)) {
		cleanup_px(dev, pd);
		kfree(pd->used_pdes);
		kfree(pd);
	}
}

static void gen8_initialize_pd(struct i915_address_space *vm,
			       struct i915_page_directory *pd)
{
	gen8_pde_t scratch_pde;

	scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC);

	fill_px(vm->dev, pd, scratch_pde);
}

525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
static int __pdp_init(struct drm_device *dev,
		      struct i915_page_directory_pointer *pdp)
{
	size_t pdpes = I915_PDPES_PER_PDP(dev);

	pdp->used_pdpes = kcalloc(BITS_TO_LONGS(pdpes),
				  sizeof(unsigned long),
				  GFP_KERNEL);
	if (!pdp->used_pdpes)
		return -ENOMEM;

	pdp->page_directory = kcalloc(pdpes, sizeof(*pdp->page_directory),
				      GFP_KERNEL);
	if (!pdp->page_directory) {
		kfree(pdp->used_pdpes);
		/* the PDP might be the statically allocated top level. Keep it
		 * as clean as possible */
		pdp->used_pdpes = NULL;
		return -ENOMEM;
	}

	return 0;
}

static void __pdp_fini(struct i915_page_directory_pointer *pdp)
{
	kfree(pdp->used_pdpes);
	kfree(pdp->page_directory);
	pdp->page_directory = NULL;
}

static void free_pdp(struct drm_device *dev,
		     struct i915_page_directory_pointer *pdp)
{
	__pdp_fini(pdp);
}

562
/* Broadwell Page Directory Pointer Descriptors */
563
static int gen8_write_pdp(struct drm_i915_gem_request *req,
564 565
			  unsigned entry,
			  dma_addr_t addr)
566
{
567
	struct intel_engine_cs *ring = req->ring;
568 569 570 571
	int ret;

	BUG_ON(entry >= 4);

572
	ret = intel_ring_begin(req, 6);
573 574 575 576 577
	if (ret)
		return ret;

	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
	intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
578
	intel_ring_emit(ring, upper_32_bits(addr));
579 580
	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
	intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
581
	intel_ring_emit(ring, lower_32_bits(addr));
582 583 584 585 586
	intel_ring_advance(ring);

	return 0;
}

587
static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
588
			  struct drm_i915_gem_request *req)
589
{
590
	int i, ret;
591

592
	for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
593 594
		const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);

595
		ret = gen8_write_pdp(req, i, pd_daddr);
596 597
		if (ret)
			return ret;
598
	}
B
Ben Widawsky 已提交
599

600
	return 0;
601 602
}

603 604 605 606 607
static void gen8_ppgtt_clear_pte_range(struct i915_address_space *vm,
				       struct i915_page_directory_pointer *pdp,
				       uint64_t start,
				       uint64_t length,
				       gen8_pte_t scratch_pte)
608 609 610
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
611
	gen8_pte_t *pt_vaddr;
612 613 614
	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
615
	unsigned num_entries = length >> PAGE_SHIFT;
616 617
	unsigned last_pte, i;

618 619
	if (WARN_ON(!pdp))
		return;
620 621

	while (num_entries) {
622 623
		struct i915_page_directory *pd;
		struct i915_page_table *pt;
624

625
		if (WARN_ON(!pdp->page_directory[pdpe]))
626
			break;
627

628
		pd = pdp->page_directory[pdpe];
629 630

		if (WARN_ON(!pd->page_table[pde]))
631
			break;
632 633 634

		pt = pd->page_table[pde];

635
		if (WARN_ON(!px_page(pt)))
636
			break;
637

638
		last_pte = pte + num_entries;
639 640
		if (last_pte > GEN8_PTES)
			last_pte = GEN8_PTES;
641

642
		pt_vaddr = kmap_px(pt);
643

644
		for (i = pte; i < last_pte; i++) {
645
			pt_vaddr[i] = scratch_pte;
646 647
			num_entries--;
		}
648

649
		kunmap_px(ppgtt, pt);
650

651
		pte = 0;
652
		if (++pde == I915_PDES) {
653 654 655
			pdpe++;
			pde = 0;
		}
656 657 658
	}
}

659 660 661 662
static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
				   uint64_t start,
				   uint64_t length,
				   bool use_scratch)
663 664 665
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
666
	struct i915_page_directory_pointer *pdp = &ppgtt->pdp; /* FIXME: 48b */
667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682

	gen8_pte_t scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
						 I915_CACHE_LLC, use_scratch);

	gen8_ppgtt_clear_pte_range(vm, pdp, start, length, scratch_pte);
}

static void
gen8_ppgtt_insert_pte_entries(struct i915_address_space *vm,
			      struct i915_page_directory_pointer *pdp,
			      struct sg_table *pages,
			      uint64_t start,
			      enum i915_cache_level cache_level)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
683
	gen8_pte_t *pt_vaddr;
684 685 686
	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
687 688
	struct sg_page_iter sg_iter;

689
	pt_vaddr = NULL;
690

691
	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
692
		if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES))
693 694
			break;

B
Ben Widawsky 已提交
695
		if (pt_vaddr == NULL) {
696
			struct i915_page_directory *pd = pdp->page_directory[pdpe];
697
			struct i915_page_table *pt = pd->page_table[pde];
698
			pt_vaddr = kmap_px(pt);
B
Ben Widawsky 已提交
699
		}
700

701
		pt_vaddr[pte] =
702 703
			gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
					cache_level, true);
704
		if (++pte == GEN8_PTES) {
705
			kunmap_px(ppgtt, pt_vaddr);
706
			pt_vaddr = NULL;
707
			if (++pde == I915_PDES) {
708 709 710 711
				pdpe++;
				pde = 0;
			}
			pte = 0;
712 713
		}
	}
714 715 716

	if (pt_vaddr)
		kunmap_px(ppgtt, pt_vaddr);
717 718
}

719 720 721 722 723 724 725 726 727 728 729 730 731
static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
				      struct sg_table *pages,
				      uint64_t start,
				      enum i915_cache_level cache_level,
				      u32 unused)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
	struct i915_page_directory_pointer *pdp = &ppgtt->pdp; /* FIXME: 48b */

	gen8_ppgtt_insert_pte_entries(vm, pdp, pages, start, cache_level);
}

732 733
static void gen8_free_page_tables(struct drm_device *dev,
				  struct i915_page_directory *pd)
734 735 736
{
	int i;

737
	if (!px_page(pd))
738 739
		return;

740
	for_each_set_bit(i, pd->used_pdes, I915_PDES) {
741 742
		if (WARN_ON(!pd->page_table[i]))
			continue;
743

744
		free_pt(dev, pd->page_table[i]);
745 746
		pd->page_table[i] = NULL;
	}
B
Ben Widawsky 已提交
747 748
}

749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784
static int gen8_init_scratch(struct i915_address_space *vm)
{
	struct drm_device *dev = vm->dev;

	vm->scratch_page = alloc_scratch_page(dev);
	if (IS_ERR(vm->scratch_page))
		return PTR_ERR(vm->scratch_page);

	vm->scratch_pt = alloc_pt(dev);
	if (IS_ERR(vm->scratch_pt)) {
		free_scratch_page(dev, vm->scratch_page);
		return PTR_ERR(vm->scratch_pt);
	}

	vm->scratch_pd = alloc_pd(dev);
	if (IS_ERR(vm->scratch_pd)) {
		free_pt(dev, vm->scratch_pt);
		free_scratch_page(dev, vm->scratch_page);
		return PTR_ERR(vm->scratch_pd);
	}

	gen8_initialize_pt(vm, vm->scratch_pt);
	gen8_initialize_pd(vm, vm->scratch_pd);

	return 0;
}

static void gen8_free_scratch(struct i915_address_space *vm)
{
	struct drm_device *dev = vm->dev;

	free_pd(dev, vm->scratch_pd);
	free_pt(dev, vm->scratch_pt);
	free_scratch_page(dev, vm->scratch_page);
}

785
static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
786
{
787 788
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
789 790
	struct i915_page_directory_pointer *pdp = &ppgtt->pdp; /* FIXME: 48b */
	struct drm_device *dev = ppgtt->base.dev;
791 792
	int i;

793 794
	for_each_set_bit(i, pdp->used_pdpes, I915_PDPES_PER_PDP(dev)) {
		if (WARN_ON(!pdp->page_directory[i]))
795 796
			continue;

797 798
		gen8_free_page_tables(dev, pdp->page_directory[i]);
		free_pd(dev, pdp->page_directory[i]);
799
	}
800

801 802
	free_pdp(dev, pdp);

803
	gen8_free_scratch(vm);
804 805
}

806 807
/**
 * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range.
808 809
 * @vm:	Master vm structure.
 * @pd:	Page directory for this address range.
810
 * @start:	Starting virtual address to begin allocations.
811
 * @length:	Size of the allocations.
812 813 814 815 816 817 818 819 820 821 822 823
 * @new_pts:	Bitmap set by function with new allocations. Likely used by the
 *		caller to free on error.
 *
 * Allocate the required number of page tables. Extremely similar to
 * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by
 * the page directory boundary (instead of the page directory pointer). That
 * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is
 * possible, and likely that the caller will need to use multiple calls of this
 * function to achieve the appropriate allocation.
 *
 * Return: 0 if success; negative error code otherwise.
 */
824
static int gen8_ppgtt_alloc_pagetabs(struct i915_address_space *vm,
825
				     struct i915_page_directory *pd,
826
				     uint64_t start,
827 828
				     uint64_t length,
				     unsigned long *new_pts)
829
{
830
	struct drm_device *dev = vm->dev;
831
	struct i915_page_table *pt;
832 833
	uint64_t temp;
	uint32_t pde;
834

835 836
	gen8_for_each_pde(pt, pd, start, length, temp, pde) {
		/* Don't reallocate page tables */
837
		if (test_bit(pde, pd->used_pdes)) {
838
			/* Scratch is never allocated this way */
839
			WARN_ON(pt == vm->scratch_pt);
840 841 842
			continue;
		}

843
		pt = alloc_pt(dev);
844
		if (IS_ERR(pt))
845 846
			goto unwind_out;

847
		gen8_initialize_pt(vm, pt);
848
		pd->page_table[pde] = pt;
849
		__set_bit(pde, new_pts);
850
		trace_i915_page_table_entry_alloc(vm, pde, start, GEN8_PDE_SHIFT);
851 852
	}

853
	return 0;
854 855

unwind_out:
856
	for_each_set_bit(pde, new_pts, I915_PDES)
857
		free_pt(dev, pd->page_table[pde]);
858

B
Ben Widawsky 已提交
859
	return -ENOMEM;
860 861
}

862 863
/**
 * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range.
864
 * @vm:	Master vm structure.
865 866
 * @pdp:	Page directory pointer for this address range.
 * @start:	Starting virtual address to begin allocations.
867 868
 * @length:	Size of the allocations.
 * @new_pds:	Bitmap set by function with new allocations. Likely used by the
869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884
 *		caller to free on error.
 *
 * Allocate the required number of page directories starting at the pde index of
 * @start, and ending at the pde index @start + @length. This function will skip
 * over already allocated page directories within the range, and only allocate
 * new ones, setting the appropriate pointer within the pdp as well as the
 * correct position in the bitmap @new_pds.
 *
 * The function will only allocate the pages within the range for a give page
 * directory pointer. In other words, if @start + @length straddles a virtually
 * addressed PDP boundary (512GB for 4k pages), there will be more allocations
 * required by the caller, This is not currently possible, and the BUG in the
 * code will prevent it.
 *
 * Return: 0 if success; negative error code otherwise.
 */
885 886 887 888 889 890
static int
gen8_ppgtt_alloc_page_directories(struct i915_address_space *vm,
				  struct i915_page_directory_pointer *pdp,
				  uint64_t start,
				  uint64_t length,
				  unsigned long *new_pds)
891
{
892
	struct drm_device *dev = vm->dev;
893
	struct i915_page_directory *pd;
894 895
	uint64_t temp;
	uint32_t pdpe;
896
	uint32_t pdpes = I915_PDPES_PER_PDP(dev);
897

898
	WARN_ON(!bitmap_empty(new_pds, pdpes));
899 900

	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
901
		if (test_bit(pdpe, pdp->used_pdpes))
902
			continue;
903

904
		pd = alloc_pd(dev);
905
		if (IS_ERR(pd))
B
Ben Widawsky 已提交
906
			goto unwind_out;
907

908
		gen8_initialize_pd(vm, pd);
909
		pdp->page_directory[pdpe] = pd;
910
		__set_bit(pdpe, new_pds);
911
		trace_i915_page_directory_entry_alloc(vm, pdpe, start, GEN8_PDPE_SHIFT);
B
Ben Widawsky 已提交
912 913
	}

914
	return 0;
B
Ben Widawsky 已提交
915 916

unwind_out:
917
	for_each_set_bit(pdpe, new_pds, pdpes)
918
		free_pd(dev, pdp->page_directory[pdpe]);
B
Ben Widawsky 已提交
919 920

	return -ENOMEM;
921 922
}

923
static void
924 925
free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts,
		       uint32_t pdpes)
926 927 928
{
	int i;

929
	for (i = 0; i < pdpes; i++)
930 931 932 933 934 935 936 937 938 939
		kfree(new_pts[i]);
	kfree(new_pts);
	kfree(new_pds);
}

/* Fills in the page directory bitmap, and the array of page tables bitmap. Both
 * of these are based on the number of PDPEs in the system.
 */
static
int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
940 941
					 unsigned long ***new_pts,
					 uint32_t pdpes)
942 943 944 945 946
{
	int i;
	unsigned long *pds;
	unsigned long **pts;

947
	pds = kcalloc(BITS_TO_LONGS(pdpes), sizeof(unsigned long), GFP_KERNEL);
948 949 950
	if (!pds)
		return -ENOMEM;

951
	pts = kcalloc(pdpes, sizeof(unsigned long *), GFP_KERNEL);
952 953 954 955 956
	if (!pts) {
		kfree(pds);
		return -ENOMEM;
	}

957
	for (i = 0; i < pdpes; i++) {
958 959 960 961 962 963 964 965 966 967 968 969
		pts[i] = kcalloc(BITS_TO_LONGS(I915_PDES),
				 sizeof(unsigned long), GFP_KERNEL);
		if (!pts[i])
			goto err_out;
	}

	*new_pds = pds;
	*new_pts = pts;

	return 0;

err_out:
970
	free_gen8_temp_bitmaps(pds, pts, pdpes);
971 972 973
	return -ENOMEM;
}

974 975 976 977 978 979 980 981 982 983
/* PDE TLBs are a pain to invalidate on GEN8+. When we modify
 * the page table structures, we mark them dirty so that
 * context switching/execlist queuing code takes extra steps
 * to ensure that tlbs are flushed.
 */
static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
{
	ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask;
}

984
static int gen8_alloc_va_range(struct i915_address_space *vm,
985
			       uint64_t start, uint64_t length)
986
{
987 988
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
989
	unsigned long *new_page_dirs, **new_page_tables;
990 991
	struct drm_device *dev = vm->dev;
	struct i915_page_directory_pointer *pdp = &ppgtt->pdp; /* FIXME: 48b */
992
	struct i915_page_directory *pd;
993 994
	const uint64_t orig_start = start;
	const uint64_t orig_length = length;
995 996
	uint64_t temp;
	uint32_t pdpe;
997
	uint32_t pdpes = I915_PDPES_PER_PDP(dev);
998 999
	int ret;

1000 1001 1002 1003
	/* Wrap is never okay since we can only represent 48b, and we don't
	 * actually use the other side of the canonical address space.
	 */
	if (WARN_ON(start + length < start))
1004 1005
		return -ENODEV;

1006
	if (WARN_ON(start + length > vm->total))
1007
		return -ENODEV;
1008

1009
	ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables, pdpes);
1010 1011 1012
	if (ret)
		return ret;

1013
	/* Do the allocations first so we can easily bail out */
1014 1015
	ret = gen8_ppgtt_alloc_page_directories(vm, pdp, start, length,
						new_page_dirs);
1016
	if (ret) {
1017
		free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
1018 1019 1020 1021
		return ret;
	}

	/* For every page directory referenced, allocate page tables */
1022 1023
	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
		ret = gen8_ppgtt_alloc_pagetabs(vm, pd, start, length,
1024
						new_page_tables[pdpe]);
1025 1026 1027 1028
		if (ret)
			goto err_out;
	}

1029 1030 1031
	start = orig_start;
	length = orig_length;

1032 1033
	/* Allocations have completed successfully, so set the bitmaps, and do
	 * the mappings. */
1034
	gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
1035
		gen8_pde_t *const page_directory = kmap_px(pd);
1036
		struct i915_page_table *pt;
1037
		uint64_t pd_len = length;
1038 1039 1040
		uint64_t pd_start = start;
		uint32_t pde;

1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055
		/* Every pd should be allocated, we just did that above. */
		WARN_ON(!pd);

		gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {
			/* Same reasoning as pd */
			WARN_ON(!pt);
			WARN_ON(!pd_len);
			WARN_ON(!gen8_pte_count(pd_start, pd_len));

			/* Set our used ptes within the page table */
			bitmap_set(pt->used_ptes,
				   gen8_pte_index(pd_start),
				   gen8_pte_count(pd_start, pd_len));

			/* Our pde is now pointing to the pagetable, pt */
1056
			__set_bit(pde, pd->used_pdes);
1057 1058

			/* Map the PDE to the page table */
1059 1060
			page_directory[pde] = gen8_pde_encode(px_dma(pt),
							      I915_CACHE_LLC);
1061 1062 1063 1064
			trace_i915_page_table_entry_map(&ppgtt->base, pde, pt,
							gen8_pte_index(start),
							gen8_pte_count(start, length),
							GEN8_PTES);
1065 1066 1067

			/* NB: We haven't yet mapped ptes to pages. At this
			 * point we're still relying on insert_entries() */
1068
		}
1069

1070
		kunmap_px(ppgtt, page_directory);
1071
		__set_bit(pdpe, pdp->used_pdpes);
1072 1073
	}

1074
	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
1075
	mark_tlbs_dirty(ppgtt);
B
Ben Widawsky 已提交
1076
	return 0;
1077

B
Ben Widawsky 已提交
1078
err_out:
1079 1080
	while (pdpe--) {
		for_each_set_bit(temp, new_page_tables[pdpe], I915_PDES)
1081
			free_pt(dev, pdp->page_directory[pdpe]->page_table[temp]);
1082 1083
	}

1084
	for_each_set_bit(pdpe, new_page_dirs, pdpes)
1085
		free_pd(dev, pdp->page_directory[pdpe]);
1086

1087
	free_gen8_temp_bitmaps(new_page_dirs, new_page_tables, pdpes);
1088
	mark_tlbs_dirty(ppgtt);
1089 1090 1091
	return ret;
}

1092
/*
1093 1094 1095 1096
 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
 * with a net effect resembling a 2-level page table in normal x86 terms. Each
 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
 * space.
B
Ben Widawsky 已提交
1097
 *
1098
 */
1099
static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
B
Ben Widawsky 已提交
1100
{
1101
	int ret;
1102

1103 1104 1105
	ret = gen8_init_scratch(&ppgtt->base);
	if (ret)
		return ret;
1106

1107
	ppgtt->base.start = 0;
1108
	ppgtt->base.total = 1ULL << 32;
1109 1110 1111 1112 1113 1114 1115
	if (IS_ENABLED(CONFIG_X86_32))
		/* While we have a proliferation of size_t variables
		 * we cannot represent the full ppgtt size on 32bit,
		 * so limit it to the same size as the GGTT (currently
		 * 2GiB).
		 */
		ppgtt->base.total = to_i915(ppgtt->base.dev)->gtt.base.total;
1116
	ppgtt->base.cleanup = gen8_ppgtt_cleanup;
1117
	ppgtt->base.allocate_va_range = gen8_alloc_va_range;
1118
	ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
1119
	ppgtt->base.clear_range = gen8_ppgtt_clear_range;
1120 1121
	ppgtt->base.unbind_vma = ppgtt_unbind_vma;
	ppgtt->base.bind_vma = ppgtt_bind_vma;
1122 1123 1124

	ppgtt->switch_mm = gen8_mm_switch;

1125 1126 1127 1128 1129
	ret = __pdp_init(false, &ppgtt->pdp);

	if (ret)
		goto free_scratch;

1130
	return 0;
1131 1132 1133 1134

free_scratch:
	gen8_free_scratch(&ppgtt->base);
	return ret;
1135 1136
}

B
Ben Widawsky 已提交
1137 1138 1139
static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
{
	struct i915_address_space *vm = &ppgtt->base;
1140
	struct i915_page_table *unused;
1141
	gen6_pte_t scratch_pte;
B
Ben Widawsky 已提交
1142
	uint32_t pd_entry;
1143 1144
	uint32_t  pte, pde, temp;
	uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
B
Ben Widawsky 已提交
1145

1146 1147
	scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
				     I915_CACHE_LLC, true, 0);
B
Ben Widawsky 已提交
1148

1149
	gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) {
B
Ben Widawsky 已提交
1150
		u32 expected;
1151
		gen6_pte_t *pt_vaddr;
1152
		const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
1153
		pd_entry = readl(ppgtt->pd_addr + pde);
B
Ben Widawsky 已提交
1154 1155 1156 1157 1158 1159 1160 1161 1162
		expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);

		if (pd_entry != expected)
			seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
				   pde,
				   pd_entry,
				   expected);
		seq_printf(m, "\tPDE: %x\n", pd_entry);

1163 1164
		pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]);

1165
		for (pte = 0; pte < GEN6_PTES; pte+=4) {
B
Ben Widawsky 已提交
1166
			unsigned long va =
1167
				(pde * PAGE_SIZE * GEN6_PTES) +
B
Ben Widawsky 已提交
1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
				(pte * PAGE_SIZE);
			int i;
			bool found = false;
			for (i = 0; i < 4; i++)
				if (pt_vaddr[pte + i] != scratch_pte)
					found = true;
			if (!found)
				continue;

			seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
			for (i = 0; i < 4; i++) {
				if (pt_vaddr[pte + i] != scratch_pte)
					seq_printf(m, " %08x", pt_vaddr[pte + i]);
				else
					seq_puts(m, "  SCRATCH ");
			}
			seq_puts(m, "\n");
		}
1186
		kunmap_px(ppgtt, pt_vaddr);
B
Ben Widawsky 已提交
1187 1188 1189
	}
}

1190
/* Write pde (index) from the page directory @pd to the page table @pt */
1191 1192
static void gen6_write_pde(struct i915_page_directory *pd,
			    const int pde, struct i915_page_table *pt)
B
Ben Widawsky 已提交
1193
{
1194 1195 1196 1197
	/* Caller needs to make sure the write completes if necessary */
	struct i915_hw_ppgtt *ppgtt =
		container_of(pd, struct i915_hw_ppgtt, pd);
	u32 pd_entry;
B
Ben Widawsky 已提交
1198

1199
	pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt));
1200
	pd_entry |= GEN6_PDE_VALID;
B
Ben Widawsky 已提交
1201

1202 1203
	writel(pd_entry, ppgtt->pd_addr + pde);
}
B
Ben Widawsky 已提交
1204

1205 1206 1207
/* Write all the page tables found in the ppgtt structure to incrementing page
 * directories. */
static void gen6_write_page_range(struct drm_i915_private *dev_priv,
1208
				  struct i915_page_directory *pd,
1209 1210
				  uint32_t start, uint32_t length)
{
1211
	struct i915_page_table *pt;
1212 1213 1214 1215 1216 1217 1218 1219
	uint32_t pde, temp;

	gen6_for_each_pde(pt, pd, start, length, temp, pde)
		gen6_write_pde(pd, pde, pt);

	/* Make sure write is complete before other code can use this page
	 * table. Also require for WC mapped PTEs */
	readl(dev_priv->gtt.gsm);
B
Ben Widawsky 已提交
1220 1221
}

1222
static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
B
Ben Widawsky 已提交
1223
{
1224
	BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);
1225

1226
	return (ppgtt->pd.base.ggtt_offset / 64) << 16;
1227 1228
}

1229
static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
1230
			 struct drm_i915_gem_request *req)
1231
{
1232
	struct intel_engine_cs *ring = req->ring;
1233 1234 1235
	int ret;

	/* NB: TLBs must be flushed and invalidated before a switch */
1236
	ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1237 1238 1239
	if (ret)
		return ret;

1240
	ret = intel_ring_begin(req, 6);
1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254
	if (ret)
		return ret;

	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
	intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
	intel_ring_emit(ring, PP_DIR_DCLV_2G);
	intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
	intel_ring_emit(ring, get_pd_offset(ppgtt));
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);

	return 0;
}

1255
static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt,
1256
			  struct drm_i915_gem_request *req)
1257
{
1258
	struct intel_engine_cs *ring = req->ring;
1259 1260 1261 1262 1263 1264 1265
	struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);

	I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
	I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
	return 0;
}

1266
static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
1267
			  struct drm_i915_gem_request *req)
1268
{
1269
	struct intel_engine_cs *ring = req->ring;
1270 1271 1272
	int ret;

	/* NB: TLBs must be flushed and invalidated before a switch */
1273
	ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1274 1275 1276
	if (ret)
		return ret;

1277
	ret = intel_ring_begin(req, 6);
1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
	if (ret)
		return ret;

	intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
	intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
	intel_ring_emit(ring, PP_DIR_DCLV_2G);
	intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
	intel_ring_emit(ring, get_pd_offset(ppgtt));
	intel_ring_emit(ring, MI_NOOP);
	intel_ring_advance(ring);

1289 1290
	/* XXX: RCS is the only one to auto invalidate the TLBs? */
	if (ring->id != RCS) {
1291
		ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
1292 1293 1294 1295
		if (ret)
			return ret;
	}

1296 1297 1298
	return 0;
}

1299
static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
1300
			  struct drm_i915_gem_request *req)
1301
{
1302
	struct intel_engine_cs *ring = req->ring;
1303 1304 1305
	struct drm_device *dev = ppgtt->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;

1306

1307 1308 1309 1310 1311 1312 1313 1314
	I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
	I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));

	POSTING_READ(RING_PP_DIR_DCLV(ring));

	return 0;
}

1315
static void gen8_ppgtt_enable(struct drm_device *dev)
1316 1317
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1318
	struct intel_engine_cs *ring;
1319
	int j;
B
Ben Widawsky 已提交
1320

1321 1322 1323 1324 1325
	for_each_ring(ring, dev_priv, j) {
		I915_WRITE(RING_MODE_GEN7(ring),
			   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
	}
}
B
Ben Widawsky 已提交
1326

1327
static void gen7_ppgtt_enable(struct drm_device *dev)
B
Ben Widawsky 已提交
1328
{
1329
	struct drm_i915_private *dev_priv = dev->dev_private;
1330
	struct intel_engine_cs *ring;
1331
	uint32_t ecochk, ecobits;
B
Ben Widawsky 已提交
1332
	int i;
B
Ben Widawsky 已提交
1333

1334 1335
	ecobits = I915_READ(GAC_ECO_BITS);
	I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1336

1337 1338 1339 1340 1341 1342 1343 1344
	ecochk = I915_READ(GAM_ECOCHK);
	if (IS_HASWELL(dev)) {
		ecochk |= ECOCHK_PPGTT_WB_HSW;
	} else {
		ecochk |= ECOCHK_PPGTT_LLC_IVB;
		ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
	}
	I915_WRITE(GAM_ECOCHK, ecochk);
1345

1346
	for_each_ring(ring, dev_priv, i) {
B
Ben Widawsky 已提交
1347
		/* GFX_MODE is per-ring on gen7+ */
1348 1349
		I915_WRITE(RING_MODE_GEN7(ring),
			   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
B
Ben Widawsky 已提交
1350
	}
1351
}
B
Ben Widawsky 已提交
1352

1353
static void gen6_ppgtt_enable(struct drm_device *dev)
1354
{
1355
	struct drm_i915_private *dev_priv = dev->dev_private;
1356
	uint32_t ecochk, gab_ctl, ecobits;
1357

1358 1359 1360
	ecobits = I915_READ(GAC_ECO_BITS);
	I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
		   ECOBITS_PPGTT_CACHE64B);
B
Ben Widawsky 已提交
1361

1362 1363 1364 1365 1366 1367 1368
	gab_ctl = I915_READ(GAB_CTL);
	I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);

	ecochk = I915_READ(GAM_ECOCHK);
	I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);

	I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
B
Ben Widawsky 已提交
1369 1370
}

1371
/* PPGTT support for Sandybdrige/Gen6 and later */
1372
static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
1373 1374
				   uint64_t start,
				   uint64_t length,
1375
				   bool use_scratch)
1376
{
1377 1378
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
1379
	gen6_pte_t *pt_vaddr, scratch_pte;
1380 1381
	unsigned first_entry = start >> PAGE_SHIFT;
	unsigned num_entries = length >> PAGE_SHIFT;
1382 1383
	unsigned act_pt = first_entry / GEN6_PTES;
	unsigned first_pte = first_entry % GEN6_PTES;
1384
	unsigned last_pte, i;
1385

1386 1387
	scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
				     I915_CACHE_LLC, true, 0);
1388

1389 1390
	while (num_entries) {
		last_pte = first_pte + num_entries;
1391 1392
		if (last_pte > GEN6_PTES)
			last_pte = GEN6_PTES;
1393

1394
		pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1395

1396 1397
		for (i = first_pte; i < last_pte; i++)
			pt_vaddr[i] = scratch_pte;
1398

1399
		kunmap_px(ppgtt, pt_vaddr);
1400

1401 1402
		num_entries -= last_pte - first_pte;
		first_pte = 0;
1403
		act_pt++;
1404
	}
1405 1406
}

1407
static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
D
Daniel Vetter 已提交
1408
				      struct sg_table *pages,
1409
				      uint64_t start,
1410
				      enum i915_cache_level cache_level, u32 flags)
D
Daniel Vetter 已提交
1411
{
1412 1413
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
1414
	gen6_pte_t *pt_vaddr;
1415
	unsigned first_entry = start >> PAGE_SHIFT;
1416 1417
	unsigned act_pt = first_entry / GEN6_PTES;
	unsigned act_pte = first_entry % GEN6_PTES;
1418 1419
	struct sg_page_iter sg_iter;

1420
	pt_vaddr = NULL;
1421
	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
1422
		if (pt_vaddr == NULL)
1423
			pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
1424

1425 1426
		pt_vaddr[act_pte] =
			vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
1427 1428
				       cache_level, true, flags);

1429
		if (++act_pte == GEN6_PTES) {
1430
			kunmap_px(ppgtt, pt_vaddr);
1431
			pt_vaddr = NULL;
1432
			act_pt++;
1433
			act_pte = 0;
D
Daniel Vetter 已提交
1434 1435
		}
	}
1436
	if (pt_vaddr)
1437
		kunmap_px(ppgtt, pt_vaddr);
D
Daniel Vetter 已提交
1438 1439
}

1440
static int gen6_alloc_va_range(struct i915_address_space *vm,
1441
			       uint64_t start_in, uint64_t length_in)
1442
{
1443 1444 1445
	DECLARE_BITMAP(new_page_tables, I915_PDES);
	struct drm_device *dev = vm->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
1446 1447
	struct i915_hw_ppgtt *ppgtt =
				container_of(vm, struct i915_hw_ppgtt, base);
1448
	struct i915_page_table *pt;
1449
	uint32_t start, length, start_save, length_save;
1450
	uint32_t pde, temp;
1451 1452
	int ret;

1453 1454 1455 1456 1457
	if (WARN_ON(start_in + length_in > ppgtt->base.total))
		return -ENODEV;

	start = start_save = start_in;
	length = length_save = length_in;
1458 1459 1460 1461 1462 1463 1464 1465 1466

	bitmap_zero(new_page_tables, I915_PDES);

	/* The allocation is done in two stages so that we can bail out with
	 * minimal amount of pain. The first stage finds new page tables that
	 * need allocation. The second stage marks use ptes within the page
	 * tables.
	 */
	gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
1467
		if (pt != vm->scratch_pt) {
1468 1469 1470 1471 1472 1473 1474
			WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES));
			continue;
		}

		/* We've already allocated a page table */
		WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));

1475
		pt = alloc_pt(dev);
1476 1477 1478 1479 1480 1481 1482 1483
		if (IS_ERR(pt)) {
			ret = PTR_ERR(pt);
			goto unwind_out;
		}

		gen6_initialize_pt(vm, pt);

		ppgtt->pd.page_table[pde] = pt;
1484
		__set_bit(pde, new_page_tables);
1485
		trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT);
1486 1487 1488 1489
	}

	start = start_save;
	length = length_save;
1490 1491 1492 1493 1494 1495 1496 1497

	gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
		DECLARE_BITMAP(tmp_bitmap, GEN6_PTES);

		bitmap_zero(tmp_bitmap, GEN6_PTES);
		bitmap_set(tmp_bitmap, gen6_pte_index(start),
			   gen6_pte_count(start, length));

1498
		if (__test_and_clear_bit(pde, new_page_tables))
1499 1500
			gen6_write_pde(&ppgtt->pd, pde, pt);

1501 1502 1503 1504
		trace_i915_page_table_entry_map(vm, pde, pt,
					 gen6_pte_index(start),
					 gen6_pte_count(start, length),
					 GEN6_PTES);
1505
		bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes,
1506 1507 1508
				GEN6_PTES);
	}

1509 1510 1511 1512 1513 1514
	WARN_ON(!bitmap_empty(new_page_tables, I915_PDES));

	/* Make sure write is complete before other code can use this page
	 * table. Also require for WC mapped PTEs */
	readl(dev_priv->gtt.gsm);

1515
	mark_tlbs_dirty(ppgtt);
1516
	return 0;
1517 1518 1519

unwind_out:
	for_each_set_bit(pde, new_page_tables, I915_PDES) {
1520
		struct i915_page_table *pt = ppgtt->pd.page_table[pde];
1521

1522
		ppgtt->pd.page_table[pde] = vm->scratch_pt;
1523
		free_pt(vm->dev, pt);
1524 1525 1526 1527
	}

	mark_tlbs_dirty(ppgtt);
	return ret;
1528 1529
}

1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556
static int gen6_init_scratch(struct i915_address_space *vm)
{
	struct drm_device *dev = vm->dev;

	vm->scratch_page = alloc_scratch_page(dev);
	if (IS_ERR(vm->scratch_page))
		return PTR_ERR(vm->scratch_page);

	vm->scratch_pt = alloc_pt(dev);
	if (IS_ERR(vm->scratch_pt)) {
		free_scratch_page(dev, vm->scratch_page);
		return PTR_ERR(vm->scratch_pt);
	}

	gen6_initialize_pt(vm, vm->scratch_pt);

	return 0;
}

static void gen6_free_scratch(struct i915_address_space *vm)
{
	struct drm_device *dev = vm->dev;

	free_pt(dev, vm->scratch_pt);
	free_scratch_page(dev, vm->scratch_page);
}

1557
static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
1558
{
1559 1560
	struct i915_hw_ppgtt *ppgtt =
		container_of(vm, struct i915_hw_ppgtt, base);
1561 1562
	struct i915_page_table *pt;
	uint32_t pde;
1563

1564 1565
	drm_mm_remove_node(&ppgtt->node);

1566
	gen6_for_all_pdes(pt, ppgtt, pde) {
1567
		if (pt != vm->scratch_pt)
1568
			free_pt(ppgtt->base.dev, pt);
1569
	}
1570

1571
	gen6_free_scratch(vm);
1572 1573
}

1574
static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
1575
{
1576
	struct i915_address_space *vm = &ppgtt->base;
1577
	struct drm_device *dev = ppgtt->base.dev;
1578
	struct drm_i915_private *dev_priv = dev->dev_private;
1579
	bool retried = false;
1580
	int ret;
1581

B
Ben Widawsky 已提交
1582 1583 1584 1585 1586
	/* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
	 * allocator works in address space sizes, so it's multiplied by page
	 * size. We allocate at the top of the GTT to avoid fragmentation.
	 */
	BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
1587

1588 1589 1590
	ret = gen6_init_scratch(vm);
	if (ret)
		return ret;
1591

1592
alloc:
B
Ben Widawsky 已提交
1593 1594 1595 1596
	ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
						  &ppgtt->node, GEN6_PD_SIZE,
						  GEN6_PD_ALIGN, 0,
						  0, dev_priv->gtt.base.total,
1597
						  DRM_MM_TOPDOWN);
1598 1599 1600
	if (ret == -ENOSPC && !retried) {
		ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
					       GEN6_PD_SIZE, GEN6_PD_ALIGN,
1601 1602 1603
					       I915_CACHE_NONE,
					       0, dev_priv->gtt.base.total,
					       0);
1604
		if (ret)
1605
			goto err_out;
1606 1607 1608 1609

		retried = true;
		goto alloc;
	}
B
Ben Widawsky 已提交
1610

1611
	if (ret)
1612 1613
		goto err_out;

1614

B
Ben Widawsky 已提交
1615 1616
	if (ppgtt->node.start < dev_priv->gtt.mappable_end)
		DRM_DEBUG("Forced to use aperture for PDEs\n");
1617

1618
	return 0;
1619 1620

err_out:
1621
	gen6_free_scratch(vm);
1622
	return ret;
1623 1624 1625 1626
}

static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
{
1627
	return gen6_ppgtt_allocate_page_directories(ppgtt);
1628
}
1629

1630 1631 1632
static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
				  uint64_t start, uint64_t length)
{
1633
	struct i915_page_table *unused;
1634
	uint32_t pde, temp;
1635

1636
	gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde)
1637
		ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
1638 1639
}

1640
static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655
{
	struct drm_device *dev = ppgtt->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
	if (IS_GEN6(dev)) {
		ppgtt->switch_mm = gen6_mm_switch;
	} else if (IS_HASWELL(dev)) {
		ppgtt->switch_mm = hsw_mm_switch;
	} else if (IS_GEN7(dev)) {
		ppgtt->switch_mm = gen7_mm_switch;
	} else
		BUG();

1656 1657 1658
	if (intel_vgpu_active(dev))
		ppgtt->switch_mm = vgpu_mm_switch;

1659 1660 1661 1662
	ret = gen6_ppgtt_alloc(ppgtt);
	if (ret)
		return ret;

1663
	ppgtt->base.allocate_va_range = gen6_alloc_va_range;
1664 1665
	ppgtt->base.clear_range = gen6_ppgtt_clear_range;
	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
1666 1667
	ppgtt->base.unbind_vma = ppgtt_unbind_vma;
	ppgtt->base.bind_vma = ppgtt_bind_vma;
1668 1669
	ppgtt->base.cleanup = gen6_ppgtt_cleanup;
	ppgtt->base.start = 0;
1670
	ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
B
Ben Widawsky 已提交
1671
	ppgtt->debug_dump = gen6_dump_ppgtt;
1672

1673
	ppgtt->pd.base.ggtt_offset =
1674
		ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
1675

1676
	ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm +
1677
		ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);
1678

1679
	gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
1680

1681 1682
	gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);

1683
	DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
1684 1685
			 ppgtt->node.size >> 20,
			 ppgtt->node.start / PAGE_SIZE);
1686

1687
	DRM_DEBUG("Adding PPGTT at offset %x\n",
1688
		  ppgtt->pd.base.ggtt_offset << 10);
1689

1690
	return 0;
1691 1692
}

1693
static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
1694
{
1695
	ppgtt->base.dev = dev;
1696

B
Ben Widawsky 已提交
1697
	if (INTEL_INFO(dev)->gen < 8)
1698
		return gen6_ppgtt_init(ppgtt);
B
Ben Widawsky 已提交
1699
	else
1700
		return gen8_ppgtt_init(ppgtt);
1701
}
1702

1703 1704 1705 1706
int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret = 0;
B
Ben Widawsky 已提交
1707

1708
	ret = __hw_ppgtt_init(dev, ppgtt);
1709
	if (ret == 0) {
B
Ben Widawsky 已提交
1710
		kref_init(&ppgtt->ref);
1711 1712
		drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
			    ppgtt->base.total);
1713
		i915_init_vm(dev_priv, &ppgtt->base);
1714
	}
1715 1716 1717 1718

	return ret;
}

1719 1720
int i915_ppgtt_init_hw(struct drm_device *dev)
{
1721 1722 1723 1724 1725 1726
	/* In the case of execlists, PPGTT is enabled by the context descriptor
	 * and the PDPs are contained within the context itself.  We don't
	 * need to do anything here. */
	if (i915.enable_execlists)
		return 0;

1727 1728 1729 1730 1731 1732 1733 1734 1735 1736
	if (!USES_PPGTT(dev))
		return 0;

	if (IS_GEN6(dev))
		gen6_ppgtt_enable(dev);
	else if (IS_GEN7(dev))
		gen7_ppgtt_enable(dev);
	else if (INTEL_INFO(dev)->gen >= 8)
		gen8_ppgtt_enable(dev);
	else
1737
		MISSING_CASE(INTEL_INFO(dev)->gen);
1738

1739 1740
	return 0;
}
1741

1742
int i915_ppgtt_init_ring(struct drm_i915_gem_request *req)
1743
{
1744
	struct drm_i915_private *dev_priv = req->ring->dev->dev_private;
1745 1746 1747 1748 1749 1750 1751 1752
	struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;

	if (i915.enable_execlists)
		return 0;

	if (!ppgtt)
		return 0;

1753
	return ppgtt->switch_mm(ppgtt, req);
1754
}
1755

1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773
struct i915_hw_ppgtt *
i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
{
	struct i915_hw_ppgtt *ppgtt;
	int ret;

	ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
	if (!ppgtt)
		return ERR_PTR(-ENOMEM);

	ret = i915_ppgtt_init(dev, ppgtt);
	if (ret) {
		kfree(ppgtt);
		return ERR_PTR(ret);
	}

	ppgtt->file_priv = fpriv;

1774 1775
	trace_i915_ppgtt_create(&ppgtt->base);

1776 1777 1778
	return ppgtt;
}

1779 1780 1781 1782 1783
void  i915_ppgtt_release(struct kref *kref)
{
	struct i915_hw_ppgtt *ppgtt =
		container_of(kref, struct i915_hw_ppgtt, ref);

1784 1785
	trace_i915_ppgtt_release(&ppgtt->base);

1786 1787 1788 1789
	/* vmas should already be unbound */
	WARN_ON(!list_empty(&ppgtt->base.active_list));
	WARN_ON(!list_empty(&ppgtt->base.inactive_list));

1790 1791 1792
	list_del(&ppgtt->base.global_link);
	drm_mm_takedown(&ppgtt->base.mm);

1793 1794 1795
	ppgtt->base.cleanup(&ppgtt->base);
	kfree(ppgtt);
}
1796

1797 1798 1799 1800
extern int intel_iommu_gfx_mapped;
/* Certain Gen5 chipsets require require idling the GPU before
 * unmapping anything from the GTT when VT-d is enabled.
 */
1801
static bool needs_idle_maps(struct drm_device *dev)
1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812
{
#ifdef CONFIG_INTEL_IOMMU
	/* Query intel_iommu to see if we need the workaround. Presumably that
	 * was loaded first.
	 */
	if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
		return true;
#endif
	return false;
}

B
Ben Widawsky 已提交
1813 1814 1815 1816
static bool do_idling(struct drm_i915_private *dev_priv)
{
	bool ret = dev_priv->mm.interruptible;

1817
	if (unlikely(dev_priv->gtt.do_idle_maps)) {
B
Ben Widawsky 已提交
1818
		dev_priv->mm.interruptible = false;
1819
		if (i915_gpu_idle(dev_priv->dev)) {
B
Ben Widawsky 已提交
1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830
			DRM_ERROR("Couldn't idle GPU\n");
			/* Wait a bit, in hopes it avoids the hang */
			udelay(10);
		}
	}

	return ret;
}

static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
{
1831
	if (unlikely(dev_priv->gtt.do_idle_maps))
B
Ben Widawsky 已提交
1832 1833 1834
		dev_priv->mm.interruptible = interruptible;
}

1835 1836 1837
void i915_check_and_clear_faults(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
1838
	struct intel_engine_cs *ring;
1839 1840 1841 1842 1843 1844 1845 1846 1847 1848
	int i;

	if (INTEL_INFO(dev)->gen < 6)
		return;

	for_each_ring(ring, dev_priv, i) {
		u32 fault_reg;
		fault_reg = I915_READ(RING_FAULT_REG(ring));
		if (fault_reg & RING_FAULT_VALID) {
			DRM_DEBUG_DRIVER("Unexpected fault\n"
1849
					 "\tAddr: 0x%08lx\n"
1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863
					 "\tAddress space: %s\n"
					 "\tSource ID: %d\n"
					 "\tType: %d\n",
					 fault_reg & PAGE_MASK,
					 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
					 RING_FAULT_SRCID(fault_reg),
					 RING_FAULT_FAULT_TYPE(fault_reg));
			I915_WRITE(RING_FAULT_REG(ring),
				   fault_reg & ~RING_FAULT_VALID);
		}
	}
	POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
}

1864 1865 1866 1867 1868 1869 1870 1871 1872 1873
static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
{
	if (INTEL_INFO(dev_priv->dev)->gen < 6) {
		intel_gtt_chipset_flush();
	} else {
		I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
		POSTING_READ(GFX_FLSH_CNTL_GEN6);
	}
}

1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886
void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	/* Don't bother messing with faults pre GEN6 as we have little
	 * documentation supporting that it's a good idea.
	 */
	if (INTEL_INFO(dev)->gen < 6)
		return;

	i915_check_and_clear_faults(dev);

	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1887 1888
				       dev_priv->gtt.base.start,
				       dev_priv->gtt.base.total,
1889
				       true);
1890 1891

	i915_ggtt_flush(dev_priv);
1892 1893
}

1894
int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
1895
{
1896 1897 1898 1899 1900 1901
	if (!dma_map_sg(&obj->base.dev->pdev->dev,
			obj->pages->sgl, obj->pages->nents,
			PCI_DMA_BIDIRECTIONAL))
		return -ENOSPC;

	return 0;
1902 1903
}

1904
static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
B
Ben Widawsky 已提交
1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915
{
#ifdef writeq
	writeq(pte, addr);
#else
	iowrite32((u32)pte, addr);
	iowrite32(pte >> 32, addr + 4);
#endif
}

static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
				     struct sg_table *st,
1916
				     uint64_t start,
1917
				     enum i915_cache_level level, u32 unused)
B
Ben Widawsky 已提交
1918 1919
{
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
1920
	unsigned first_entry = start >> PAGE_SHIFT;
1921 1922
	gen8_pte_t __iomem *gtt_entries =
		(gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
B
Ben Widawsky 已提交
1923 1924
	int i = 0;
	struct sg_page_iter sg_iter;
1925
	dma_addr_t addr = 0; /* shut up gcc */
B
Ben Widawsky 已提交
1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953

	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
		addr = sg_dma_address(sg_iter.sg) +
			(sg_iter.sg_pgoffset << PAGE_SHIFT);
		gen8_set_pte(&gtt_entries[i],
			     gen8_pte_encode(addr, level, true));
		i++;
	}

	/*
	 * XXX: This serves as a posting read to make sure that the PTE has
	 * actually been updated. There is some concern that even though
	 * registers and PTEs are within the same BAR that they are potentially
	 * of NUMA access patterns. Therefore, even with the way we assume
	 * hardware should work, we must keep this posting read for paranoia.
	 */
	if (i != 0)
		WARN_ON(readq(&gtt_entries[i-1])
			!= gen8_pte_encode(addr, level, true));

	/* This next bit makes the above posting read even more important. We
	 * want to flush the TLBs only after we're certain all the PTE updates
	 * have finished.
	 */
	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
	POSTING_READ(GFX_FLSH_CNTL_GEN6);
}

1954 1955 1956 1957 1958 1959
/*
 * Binds an object into the global gtt with the specified cache level. The object
 * will be accessible to the GPU via commands whose operands reference offsets
 * within the global GTT as well as accessible by the GPU through the GMADR
 * mapped BAR (dev_priv->mm.gtt->gtt).
 */
1960
static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
1961
				     struct sg_table *st,
1962
				     uint64_t start,
1963
				     enum i915_cache_level level, u32 flags)
1964
{
1965
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
1966
	unsigned first_entry = start >> PAGE_SHIFT;
1967 1968
	gen6_pte_t __iomem *gtt_entries =
		(gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1969 1970
	int i = 0;
	struct sg_page_iter sg_iter;
1971
	dma_addr_t addr = 0;
1972

1973
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1974
		addr = sg_page_iter_dma_address(&sg_iter);
1975
		iowrite32(vm->pte_encode(addr, level, true, flags), &gtt_entries[i]);
1976
		i++;
1977 1978 1979 1980 1981 1982 1983 1984
	}

	/* XXX: This serves as a posting read to make sure that the PTE has
	 * actually been updated. There is some concern that even though
	 * registers and PTEs are within the same BAR that they are potentially
	 * of NUMA access patterns. Therefore, even with the way we assume
	 * hardware should work, we must keep this posting read for paranoia.
	 */
1985 1986 1987 1988
	if (i != 0) {
		unsigned long gtt = readl(&gtt_entries[i-1]);
		WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
	}
1989 1990 1991 1992 1993 1994 1995

	/* This next bit makes the above posting read even more important. We
	 * want to flush the TLBs only after we're certain all the PTE updates
	 * have finished.
	 */
	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
	POSTING_READ(GFX_FLSH_CNTL_GEN6);
1996 1997
}

B
Ben Widawsky 已提交
1998
static void gen8_ggtt_clear_range(struct i915_address_space *vm,
1999 2000
				  uint64_t start,
				  uint64_t length,
B
Ben Widawsky 已提交
2001 2002 2003
				  bool use_scratch)
{
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
2004 2005
	unsigned first_entry = start >> PAGE_SHIFT;
	unsigned num_entries = length >> PAGE_SHIFT;
2006 2007
	gen8_pte_t scratch_pte, __iomem *gtt_base =
		(gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
B
Ben Widawsky 已提交
2008 2009 2010 2011 2012 2013 2014 2015
	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
	int i;

	if (WARN(num_entries > max_entries,
		 "First entry = %d; Num entries = %d (max=%d)\n",
		 first_entry, num_entries, max_entries))
		num_entries = max_entries;

2016
	scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
B
Ben Widawsky 已提交
2017 2018 2019 2020 2021 2022 2023
				      I915_CACHE_LLC,
				      use_scratch);
	for (i = 0; i < num_entries; i++)
		gen8_set_pte(&gtt_base[i], scratch_pte);
	readl(gtt_base);
}

2024
static void gen6_ggtt_clear_range(struct i915_address_space *vm,
2025 2026
				  uint64_t start,
				  uint64_t length,
2027
				  bool use_scratch)
2028
{
2029
	struct drm_i915_private *dev_priv = vm->dev->dev_private;
2030 2031
	unsigned first_entry = start >> PAGE_SHIFT;
	unsigned num_entries = length >> PAGE_SHIFT;
2032 2033
	gen6_pte_t scratch_pte, __iomem *gtt_base =
		(gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
2034
	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
2035 2036 2037 2038 2039 2040 2041
	int i;

	if (WARN(num_entries > max_entries,
		 "First entry = %d; Num entries = %d (max=%d)\n",
		 first_entry, num_entries, max_entries))
		num_entries = max_entries;

2042 2043
	scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
				     I915_CACHE_LLC, use_scratch, 0);
2044

2045 2046 2047 2048 2049
	for (i = 0; i < num_entries; i++)
		iowrite32(scratch_pte, &gtt_base[i]);
	readl(gtt_base);
}

2050 2051 2052 2053
static void i915_ggtt_insert_entries(struct i915_address_space *vm,
				     struct sg_table *pages,
				     uint64_t start,
				     enum i915_cache_level cache_level, u32 unused)
2054 2055 2056 2057
{
	unsigned int flags = (cache_level == I915_CACHE_NONE) ?
		AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;

2058
	intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
2059

2060 2061
}

2062
static void i915_ggtt_clear_range(struct i915_address_space *vm,
2063 2064
				  uint64_t start,
				  uint64_t length,
2065
				  bool unused)
2066
{
2067 2068
	unsigned first_entry = start >> PAGE_SHIFT;
	unsigned num_entries = length >> PAGE_SHIFT;
2069 2070 2071
	intel_gtt_clear_range(first_entry, num_entries);
}

2072 2073 2074
static int ggtt_bind_vma(struct i915_vma *vma,
			 enum i915_cache_level cache_level,
			 u32 flags)
2075
{
2076
	struct drm_device *dev = vma->vm->dev;
2077
	struct drm_i915_private *dev_priv = dev->dev_private;
2078
	struct drm_i915_gem_object *obj = vma->obj;
2079
	struct sg_table *pages = obj->pages;
2080
	u32 pte_flags = 0;
2081 2082 2083 2084 2085 2086
	int ret;

	ret = i915_get_ggtt_vma_pages(vma);
	if (ret)
		return ret;
	pages = vma->ggtt_view.pages;
2087

2088 2089
	/* Currently applicable only to VLV */
	if (obj->gt_ro)
2090
		pte_flags |= PTE_READ_ONLY;
2091

2092

2093
	if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
2094 2095 2096
		vma->vm->insert_entries(vma->vm, pages,
					vma->node.start,
					cache_level, pte_flags);
2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107

		/* Note the inconsistency here is due to absence of the
		 * aliasing ppgtt on gen4 and earlier. Though we always
		 * request PIN_USER for execbuffer (translated to LOCAL_BIND),
		 * without the appgtt, we cannot honour that request and so
		 * must substitute it with a global binding. Since we do this
		 * behind the upper layers back, we need to explicitly set
		 * the bound flag ourselves.
		 */
		vma->bound |= GLOBAL_BIND;

2108
	}
2109

2110
	if (dev_priv->mm.aliasing_ppgtt && flags & LOCAL_BIND) {
2111
		struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
2112
		appgtt->base.insert_entries(&appgtt->base, pages,
2113
					    vma->node.start,
2114
					    cache_level, pte_flags);
2115
	}
2116 2117

	return 0;
2118 2119
}

2120
static void ggtt_unbind_vma(struct i915_vma *vma)
2121
{
2122
	struct drm_device *dev = vma->vm->dev;
2123
	struct drm_i915_private *dev_priv = dev->dev_private;
2124
	struct drm_i915_gem_object *obj = vma->obj;
2125 2126 2127
	const uint64_t size = min_t(uint64_t,
				    obj->base.size,
				    vma->node.size);
2128

2129
	if (vma->bound & GLOBAL_BIND) {
2130 2131
		vma->vm->clear_range(vma->vm,
				     vma->node.start,
2132
				     size,
2133 2134
				     true);
	}
2135

2136
	if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) {
2137
		struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
2138

2139
		appgtt->base.clear_range(&appgtt->base,
2140
					 vma->node.start,
2141
					 size,
2142 2143
					 true);
	}
2144 2145 2146
}

void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
2147
{
B
Ben Widawsky 已提交
2148 2149 2150 2151 2152 2153
	struct drm_device *dev = obj->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	bool interruptible;

	interruptible = do_idling(dev_priv);

2154 2155
	dma_unmap_sg(&dev->pdev->dev, obj->pages->sgl, obj->pages->nents,
		     PCI_DMA_BIDIRECTIONAL);
B
Ben Widawsky 已提交
2156 2157

	undo_idling(dev_priv, interruptible);
2158
}
2159

2160 2161
static void i915_gtt_color_adjust(struct drm_mm_node *node,
				  unsigned long color,
2162 2163
				  u64 *start,
				  u64 *end)
2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175
{
	if (node->color != color)
		*start += 4096;

	if (!list_empty(&node->node_list)) {
		node = list_entry(node->node_list.next,
				  struct drm_mm_node,
				  node_list);
		if (node->allocated && node->color != color)
			*end -= 4096;
	}
}
B
Ben Widawsky 已提交
2176

D
Daniel Vetter 已提交
2177 2178 2179 2180
static int i915_gem_setup_global_gtt(struct drm_device *dev,
				     unsigned long start,
				     unsigned long mappable_end,
				     unsigned long end)
2181
{
2182 2183 2184 2185 2186 2187 2188 2189 2190
	/* Let GEM Manage all of the aperture.
	 *
	 * However, leave one page at the end still bound to the scratch page.
	 * There are a number of places where the hardware apparently prefetches
	 * past the end of the object, and we've seen multiple hangs with the
	 * GPU head pointer stuck in a batchbuffer bound at the last page of the
	 * aperture.  One page should be enough to keep any prefetching inside
	 * of the aperture.
	 */
2191 2192
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
2193 2194 2195
	struct drm_mm_node *entry;
	struct drm_i915_gem_object *obj;
	unsigned long hole_start, hole_end;
2196
	int ret;
2197

2198 2199
	BUG_ON(mappable_end > end);

2200
	/* Subtract the guard page ... */
2201
	drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
2202 2203 2204 2205 2206 2207 2208 2209 2210 2211

	dev_priv->gtt.base.start = start;
	dev_priv->gtt.base.total = end - start;

	if (intel_vgpu_active(dev)) {
		ret = intel_vgt_balloon(dev);
		if (ret)
			return ret;
	}

2212
	if (!HAS_LLC(dev))
2213
		dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
2214

2215
	/* Mark any preallocated objects as occupied */
2216
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
2217
		struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
2218

B
Ben Widawsky 已提交
2219
		DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
2220 2221 2222
			      i915_gem_obj_ggtt_offset(obj), obj->base.size);

		WARN_ON(i915_gem_obj_ggtt_bound(obj));
2223
		ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
2224 2225 2226 2227
		if (ret) {
			DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
			return ret;
		}
2228
		vma->bound |= GLOBAL_BIND;
2229 2230 2231
	}

	/* Clear any non-preallocated blocks */
2232
	drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
2233 2234
		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
			      hole_start, hole_end);
2235 2236
		ggtt_vm->clear_range(ggtt_vm, hole_start,
				     hole_end - hole_start, true);
2237 2238 2239
	}

	/* And finally clear the reserved guard page */
2240
	ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
2241

2242 2243 2244 2245 2246 2247 2248
	if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
		struct i915_hw_ppgtt *ppgtt;

		ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
		if (!ppgtt)
			return -ENOMEM;

2249 2250 2251 2252 2253 2254 2255 2256 2257 2258
		ret = __hw_ppgtt_init(dev, ppgtt);
		if (ret) {
			ppgtt->base.cleanup(&ppgtt->base);
			kfree(ppgtt);
			return ret;
		}

		if (ppgtt->base.allocate_va_range)
			ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0,
							    ppgtt->base.total);
2259
		if (ret) {
2260
			ppgtt->base.cleanup(&ppgtt->base);
2261
			kfree(ppgtt);
2262
			return ret;
2263
		}
2264

2265 2266 2267 2268 2269
		ppgtt->base.clear_range(&ppgtt->base,
					ppgtt->base.start,
					ppgtt->base.total,
					true);

2270 2271 2272
		dev_priv->mm.aliasing_ppgtt = ppgtt;
	}

2273
	return 0;
2274 2275
}

2276 2277 2278
void i915_gem_init_global_gtt(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2279
	u64 gtt_size, mappable_size;
2280

2281
	gtt_size = dev_priv->gtt.base.total;
2282
	mappable_size = dev_priv->gtt.mappable_end;
2283

2284
	i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
2285 2286
}

2287 2288 2289 2290 2291
void i915_global_gtt_cleanup(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct i915_address_space *vm = &dev_priv->gtt.base;

2292 2293 2294 2295 2296 2297
	if (dev_priv->mm.aliasing_ppgtt) {
		struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;

		ppgtt->base.cleanup(&ppgtt->base);
	}

2298
	if (drm_mm_initialized(&vm->mm)) {
2299 2300 2301
		if (intel_vgpu_active(dev))
			intel_vgt_deballoon();

2302 2303 2304 2305 2306 2307
		drm_mm_takedown(&vm->mm);
		list_del(&vm->global_link);
	}

	vm->cleanup(vm);
}
2308

2309
static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
2310 2311 2312 2313 2314 2315
{
	snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
	snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
	return snb_gmch_ctl << 20;
}

2316
static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
2317 2318 2319 2320 2321
{
	bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
	bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
	if (bdw_gmch_ctl)
		bdw_gmch_ctl = 1 << bdw_gmch_ctl;
2322 2323 2324 2325 2326 2327 2328

#ifdef CONFIG_X86_32
	/* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
	if (bdw_gmch_ctl > 4)
		bdw_gmch_ctl = 4;
#endif

2329 2330 2331
	return bdw_gmch_ctl << 20;
}

2332
static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
2333 2334 2335 2336 2337 2338 2339 2340 2341 2342
{
	gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
	gmch_ctrl &= SNB_GMCH_GGMS_MASK;

	if (gmch_ctrl)
		return 1 << (20 + gmch_ctrl);

	return 0;
}

2343
static size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
2344 2345 2346 2347 2348 2349
{
	snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
	snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
	return snb_gmch_ctl << 25; /* 32 MB units */
}

2350
static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
2351 2352 2353 2354 2355 2356
{
	bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
	bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
	return bdw_gmch_ctl << 25; /* 32 MB units */
}

2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374
static size_t chv_get_stolen_size(u16 gmch_ctrl)
{
	gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
	gmch_ctrl &= SNB_GMCH_GMS_MASK;

	/*
	 * 0x0  to 0x10: 32MB increments starting at 0MB
	 * 0x11 to 0x16: 4MB increments starting at 8MB
	 * 0x17 to 0x1d: 4MB increments start at 36MB
	 */
	if (gmch_ctrl < 0x11)
		return gmch_ctrl << 25;
	else if (gmch_ctrl < 0x17)
		return (gmch_ctrl - 0x11 + 2) << 22;
	else
		return (gmch_ctrl - 0x17 + 9) << 22;
}

2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386
static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
{
	gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
	gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;

	if (gen9_gmch_ctl < 0xf0)
		return gen9_gmch_ctl << 25; /* 32 MB units */
	else
		/* 4MB increments starting at 0xf0 for 4MB */
		return (gen9_gmch_ctl - 0xf0 + 1) << 22;
}

B
Ben Widawsky 已提交
2387 2388 2389 2390
static int ggtt_probe_common(struct drm_device *dev,
			     size_t gtt_size)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2391
	struct i915_page_scratch *scratch_page;
2392
	phys_addr_t gtt_phys_addr;
B
Ben Widawsky 已提交
2393 2394

	/* For Modern GENs the PTEs and register space are split in the BAR */
2395
	gtt_phys_addr = pci_resource_start(dev->pdev, 0) +
B
Ben Widawsky 已提交
2396 2397
		(pci_resource_len(dev->pdev, 0) / 2);

I
Imre Deak 已提交
2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408
	/*
	 * On BXT writes larger than 64 bit to the GTT pagetable range will be
	 * dropped. For WC mappings in general we have 64 byte burst writes
	 * when the WC buffer is flushed, so we can't use it, but have to
	 * resort to an uncached mapping. The WC issue is easily caught by the
	 * readback check when writing GTT PTE entries.
	 */
	if (IS_BROXTON(dev))
		dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size);
	else
		dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);
B
Ben Widawsky 已提交
2409 2410 2411 2412 2413
	if (!dev_priv->gtt.gsm) {
		DRM_ERROR("Failed to map the gtt page table\n");
		return -ENOMEM;
	}

2414 2415
	scratch_page = alloc_scratch_page(dev);
	if (IS_ERR(scratch_page)) {
B
Ben Widawsky 已提交
2416 2417 2418
		DRM_ERROR("Scratch setup failed\n");
		/* iounmap will also get called at remove, but meh */
		iounmap(dev_priv->gtt.gsm);
2419
		return PTR_ERR(scratch_page);
B
Ben Widawsky 已提交
2420 2421
	}

2422 2423 2424
	dev_priv->gtt.base.scratch_page = scratch_page;

	return 0;
B
Ben Widawsky 已提交
2425 2426
}

B
Ben Widawsky 已提交
2427 2428 2429
/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
 * bits. When using advanced contexts each context stores its own PAT, but
 * writing this data shouldn't be harmful even in those cases. */
2430
static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
B
Ben Widawsky 已提交
2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
{
	uint64_t pat;

	pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
	      GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
	      GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
	      GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
	      GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
	      GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
	      GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
	      GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));

2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458
	if (!USES_PPGTT(dev_priv->dev))
		/* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
		 * so RTL will always use the value corresponding to
		 * pat_sel = 000".
		 * So let's disable cache for GGTT to avoid screen corruptions.
		 * MOCS still can be used though.
		 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
		 * before this patch, i.e. the same uncached + snooping access
		 * like on gen6/7 seems to be in effect.
		 * - So this just fixes blitter/render access. Again it looks
		 * like it's not just uncached access, but uncached + snooping.
		 * So we can still hold onto all our assumptions wrt cpu
		 * clflushing on LLC machines.
		 */
		pat = GEN8_PPAT(0, GEN8_PPAT_UC);

B
Ben Widawsky 已提交
2459 2460 2461 2462 2463 2464
	/* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
	 * write would work. */
	I915_WRITE(GEN8_PRIVATE_PAT, pat);
	I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
}

2465 2466 2467 2468 2469 2470 2471 2472 2473 2474
static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
{
	uint64_t pat;

	/*
	 * Map WB on BDW to snooped on CHV.
	 *
	 * Only the snoop bit has meaning for CHV, the rest is
	 * ignored.
	 *
2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485
	 * The hardware will never snoop for certain types of accesses:
	 * - CPU GTT (GMADR->GGTT->no snoop->memory)
	 * - PPGTT page tables
	 * - some other special cycles
	 *
	 * As with BDW, we also need to consider the following for GT accesses:
	 * "For GGTT, there is NO pat_sel[2:0] from the entry,
	 * so RTL will always use the value corresponding to
	 * pat_sel = 000".
	 * Which means we must set the snoop bit in PAT entry 0
	 * in order to keep the global status page working.
2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499
	 */
	pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(1, 0) |
	      GEN8_PPAT(2, 0) |
	      GEN8_PPAT(3, 0) |
	      GEN8_PPAT(4, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(5, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(6, CHV_PPAT_SNOOP) |
	      GEN8_PPAT(7, CHV_PPAT_SNOOP);

	I915_WRITE(GEN8_PRIVATE_PAT, pat);
	I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
}

B
Ben Widawsky 已提交
2500
static int gen8_gmch_probe(struct drm_device *dev,
2501
			   u64 *gtt_total,
B
Ben Widawsky 已提交
2502 2503
			   size_t *stolen,
			   phys_addr_t *mappable_base,
2504
			   u64 *mappable_end)
B
Ben Widawsky 已提交
2505 2506
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2507
	u64 gtt_size;
B
Ben Widawsky 已提交
2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519
	u16 snb_gmch_ctl;
	int ret;

	/* TODO: We're not aware of mappable constraints on gen8 yet */
	*mappable_base = pci_resource_start(dev->pdev, 2);
	*mappable_end = pci_resource_len(dev->pdev, 2);

	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));

	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);

2520 2521 2522 2523
	if (INTEL_INFO(dev)->gen >= 9) {
		*stolen = gen9_get_stolen_size(snb_gmch_ctl);
		gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
	} else if (IS_CHERRYVIEW(dev)) {
2524 2525 2526 2527 2528 2529
		*stolen = chv_get_stolen_size(snb_gmch_ctl);
		gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
	} else {
		*stolen = gen8_get_stolen_size(snb_gmch_ctl);
		gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
	}
B
Ben Widawsky 已提交
2530

2531
	*gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
B
Ben Widawsky 已提交
2532

S
Sumit Singh 已提交
2533
	if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
2534 2535 2536
		chv_setup_private_ppat(dev_priv);
	else
		bdw_setup_private_ppat(dev_priv);
B
Ben Widawsky 已提交
2537

B
Ben Widawsky 已提交
2538 2539
	ret = ggtt_probe_common(dev, gtt_size);

B
Ben Widawsky 已提交
2540 2541
	dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
	dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
2542 2543
	dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
	dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
B
Ben Widawsky 已提交
2544 2545 2546 2547

	return ret;
}

2548
static int gen6_gmch_probe(struct drm_device *dev,
2549
			   u64 *gtt_total,
2550 2551
			   size_t *stolen,
			   phys_addr_t *mappable_base,
2552
			   u64 *mappable_end)
2553 2554
{
	struct drm_i915_private *dev_priv = dev->dev_private;
2555
	unsigned int gtt_size;
2556 2557 2558
	u16 snb_gmch_ctl;
	int ret;

2559 2560 2561
	*mappable_base = pci_resource_start(dev->pdev, 2);
	*mappable_end = pci_resource_len(dev->pdev, 2);

2562 2563
	/* 64/512MB is the current min/max we actually know of, but this is just
	 * a coarse sanity check.
2564
	 */
2565
	if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
2566
		DRM_ERROR("Unknown GMADR size (%llx)\n",
2567 2568
			  dev_priv->gtt.mappable_end);
		return -ENXIO;
2569 2570 2571 2572 2573 2574
	}

	if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
		pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
	pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);

2575
	*stolen = gen6_get_stolen_size(snb_gmch_ctl);
2576

B
Ben Widawsky 已提交
2577
	gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
2578
	*gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
2579

B
Ben Widawsky 已提交
2580
	ret = ggtt_probe_common(dev, gtt_size);
2581

2582 2583
	dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
	dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
2584 2585
	dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
	dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
2586

2587 2588 2589
	return ret;
}

2590
static void gen6_gmch_remove(struct i915_address_space *vm)
2591
{
2592 2593

	struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
2594

2595
	iounmap(gtt->gsm);
2596
	free_scratch_page(vm->dev, vm->scratch_page);
2597
}
2598 2599

static int i915_gmch_probe(struct drm_device *dev,
2600
			   u64 *gtt_total,
2601 2602
			   size_t *stolen,
			   phys_addr_t *mappable_base,
2603
			   u64 *mappable_end)
2604 2605 2606 2607 2608 2609 2610 2611 2612 2613
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
	if (!ret) {
		DRM_ERROR("failed to set up gmch\n");
		return -EIO;
	}

2614
	intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
2615 2616

	dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
2617
	dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
2618
	dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
2619 2620
	dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
	dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
2621

2622 2623 2624
	if (unlikely(dev_priv->gtt.do_idle_maps))
		DRM_INFO("applying Ironlake quirks for intel_iommu\n");

2625 2626 2627
	return 0;
}

2628
static void i915_gmch_remove(struct i915_address_space *vm)
2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639
{
	intel_gmch_remove();
}

int i915_gem_gtt_init(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct i915_gtt *gtt = &dev_priv->gtt;
	int ret;

	if (INTEL_INFO(dev)->gen <= 5) {
2640
		gtt->gtt_probe = i915_gmch_probe;
2641
		gtt->base.cleanup = i915_gmch_remove;
B
Ben Widawsky 已提交
2642
	} else if (INTEL_INFO(dev)->gen < 8) {
2643
		gtt->gtt_probe = gen6_gmch_probe;
2644
		gtt->base.cleanup = gen6_gmch_remove;
2645
		if (IS_HASWELL(dev) && dev_priv->ellc_size)
2646
			gtt->base.pte_encode = iris_pte_encode;
2647
		else if (IS_HASWELL(dev))
2648
			gtt->base.pte_encode = hsw_pte_encode;
2649
		else if (IS_VALLEYVIEW(dev))
2650
			gtt->base.pte_encode = byt_pte_encode;
2651 2652
		else if (INTEL_INFO(dev)->gen >= 7)
			gtt->base.pte_encode = ivb_pte_encode;
2653
		else
2654
			gtt->base.pte_encode = snb_pte_encode;
B
Ben Widawsky 已提交
2655 2656 2657
	} else {
		dev_priv->gtt.gtt_probe = gen8_gmch_probe;
		dev_priv->gtt.base.cleanup = gen6_gmch_remove;
2658 2659
	}

2660 2661
	gtt->base.dev = dev;

2662
	ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
2663
			     &gtt->mappable_base, &gtt->mappable_end);
2664
	if (ret)
2665 2666 2667
		return ret;

	/* GMADR is the PCI mmio aperture into the global GTT. */
2668
	DRM_INFO("Memory usable by graphics device = %lluM\n",
2669
		 gtt->base.total >> 20);
2670
	DRM_DEBUG_DRIVER("GMADR size = %lldM\n", gtt->mappable_end >> 20);
2671
	DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
2672 2673 2674 2675
#ifdef CONFIG_INTEL_IOMMU
	if (intel_iommu_gfx_mapped)
		DRM_INFO("VT-d active for gfx access\n");
#endif
2676 2677 2678 2679 2680 2681 2682 2683
	/*
	 * i915.enable_ppgtt is read-only, so do an early pass to validate the
	 * user's requested state against the hardware/driver capabilities.  We
	 * do this now so that we can print out any log messages once rather
	 * than every time we check intel_enable_ppgtt().
	 */
	i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
	DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
2684 2685 2686

	return 0;
}
2687

2688 2689 2690 2691 2692
void i915_gem_restore_gtt_mappings(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_gem_object *obj;
	struct i915_address_space *vm;
2693 2694
	struct i915_vma *vma;
	bool flush;
2695 2696 2697 2698 2699 2700 2701 2702 2703

	i915_check_and_clear_faults(dev);

	/* First fill our portion of the GTT with scratch pages */
	dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
				       dev_priv->gtt.base.start,
				       dev_priv->gtt.base.total,
				       true);

2704 2705
	/* Cache flush objects bound into GGTT and rebind them. */
	vm = &dev_priv->gtt.base;
2706
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
2707 2708 2709 2710
		flush = false;
		list_for_each_entry(vma, &obj->vma_list, vma_link) {
			if (vma->vm != vm)
				continue;
2711

2712 2713
			WARN_ON(i915_vma_bind(vma, obj->cache_level,
					      PIN_UPDATE));
2714

2715 2716 2717 2718 2719 2720
			flush = true;
		}

		if (flush)
			i915_gem_clflush_object(obj, obj->pin_display);
	}
2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749

	if (INTEL_INFO(dev)->gen >= 8) {
		if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
			chv_setup_private_ppat(dev_priv);
		else
			bdw_setup_private_ppat(dev_priv);

		return;
	}

	if (USES_PPGTT(dev)) {
		list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
			/* TODO: Perhaps it shouldn't be gen6 specific */

			struct i915_hw_ppgtt *ppgtt =
					container_of(vm, struct i915_hw_ppgtt,
						     base);

			if (i915_is_ggtt(vm))
				ppgtt = dev_priv->mm.aliasing_ppgtt;

			gen6_write_page_range(dev_priv, &ppgtt->pd,
					      0, ppgtt->base.total);
		}
	}

	i915_ggtt_flush(dev_priv);
}

2750 2751 2752 2753
static struct i915_vma *
__i915_gem_vma_create(struct drm_i915_gem_object *obj,
		      struct i915_address_space *vm,
		      const struct i915_ggtt_view *ggtt_view)
2754
{
2755
	struct i915_vma *vma;
2756

2757 2758
	if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
		return ERR_PTR(-EINVAL);
2759 2760

	vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL);
2761 2762
	if (vma == NULL)
		return ERR_PTR(-ENOMEM);
2763

2764 2765 2766 2767 2768 2769
	INIT_LIST_HEAD(&vma->vma_link);
	INIT_LIST_HEAD(&vma->mm_list);
	INIT_LIST_HEAD(&vma->exec_list);
	vma->vm = vm;
	vma->obj = obj;

2770
	if (i915_is_ggtt(vm))
2771
		vma->ggtt_view = *ggtt_view;
2772

2773 2774
	list_add_tail(&vma->vma_link, &obj->vma_list);
	if (!i915_is_ggtt(vm))
2775
		i915_ppgtt_get(i915_vm_to_ppgtt(vm));
2776 2777 2778 2779 2780

	return vma;
}

struct i915_vma *
2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795
i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
				  struct i915_address_space *vm)
{
	struct i915_vma *vma;

	vma = i915_gem_obj_to_vma(obj, vm);
	if (!vma)
		vma = __i915_gem_vma_create(obj, vm,
					    i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL);

	return vma;
}

struct i915_vma *
i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj,
2796
				       const struct i915_ggtt_view *view)
2797
{
2798
	struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
2799 2800
	struct i915_vma *vma;

2801 2802 2803 2804 2805 2806 2807 2808
	if (WARN_ON(!view))
		return ERR_PTR(-EINVAL);

	vma = i915_gem_obj_to_ggtt_view(obj, view);

	if (IS_ERR(vma))
		return vma;

2809
	if (!vma)
2810
		vma = __i915_gem_vma_create(obj, ggtt, view);
2811 2812

	return vma;
2813

2814
}
2815

2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847
static void
rotate_pages(dma_addr_t *in, unsigned int width, unsigned int height,
	     struct sg_table *st)
{
	unsigned int column, row;
	unsigned int src_idx;
	struct scatterlist *sg = st->sgl;

	st->nents = 0;

	for (column = 0; column < width; column++) {
		src_idx = width * (height - 1) + column;
		for (row = 0; row < height; row++) {
			st->nents++;
			/* We don't need the pages, but need to initialize
			 * the entries so the sg list can be happily traversed.
			 * The only thing we need are DMA addresses.
			 */
			sg_set_page(sg, NULL, PAGE_SIZE, 0);
			sg_dma_address(sg) = in[src_idx];
			sg_dma_len(sg) = PAGE_SIZE;
			sg = sg_next(sg);
			src_idx -= width;
		}
	}
}

static struct sg_table *
intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view,
			  struct drm_i915_gem_object *obj)
{
	struct intel_rotation_info *rot_info = &ggtt_view->rotation_info;
2848
	unsigned int size_pages = rot_info->size >> PAGE_SHIFT;
2849 2850 2851 2852
	struct sg_page_iter sg_iter;
	unsigned long i;
	dma_addr_t *page_addr_list;
	struct sg_table *st;
2853
	int ret = -ENOMEM;
2854 2855

	/* Allocate a temporary list of source pages for random access. */
2856 2857
	page_addr_list = drm_malloc_ab(obj->base.size / PAGE_SIZE,
				       sizeof(dma_addr_t));
2858 2859 2860 2861 2862 2863 2864 2865
	if (!page_addr_list)
		return ERR_PTR(ret);

	/* Allocate target SG list. */
	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (!st)
		goto err_st_alloc;

2866
	ret = sg_alloc_table(st, size_pages, GFP_KERNEL);
2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877
	if (ret)
		goto err_sg_alloc;

	/* Populate source page list from the object. */
	i = 0;
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
		page_addr_list[i] = sg_page_iter_dma_address(&sg_iter);
		i++;
	}

	/* Rotate the pages. */
2878 2879 2880
	rotate_pages(page_addr_list,
		     rot_info->width_pages, rot_info->height_pages,
		     st);
2881 2882

	DRM_DEBUG_KMS(
2883
		      "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages).\n",
2884
		      obj->base.size, rot_info->pitch, rot_info->height,
2885 2886
		      rot_info->pixel_format, rot_info->width_pages,
		      rot_info->height_pages, size_pages);
2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897

	drm_free_large(page_addr_list);

	return st;

err_sg_alloc:
	kfree(st);
err_st_alloc:
	drm_free_large(page_addr_list);

	DRM_DEBUG_KMS(
2898
		      "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages)\n",
2899
		      obj->base.size, ret, rot_info->pitch, rot_info->height,
2900 2901
		      rot_info->pixel_format, rot_info->width_pages,
		      rot_info->height_pages, size_pages);
2902 2903
	return ERR_PTR(ret);
}
2904

2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945
static struct sg_table *
intel_partial_pages(const struct i915_ggtt_view *view,
		    struct drm_i915_gem_object *obj)
{
	struct sg_table *st;
	struct scatterlist *sg;
	struct sg_page_iter obj_sg_iter;
	int ret = -ENOMEM;

	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (!st)
		goto err_st_alloc;

	ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL);
	if (ret)
		goto err_sg_alloc;

	sg = st->sgl;
	st->nents = 0;
	for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents,
		view->params.partial.offset)
	{
		if (st->nents >= view->params.partial.size)
			break;

		sg_set_page(sg, NULL, PAGE_SIZE, 0);
		sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter);
		sg_dma_len(sg) = PAGE_SIZE;

		sg = sg_next(sg);
		st->nents++;
	}

	return st;

err_sg_alloc:
	kfree(st);
err_st_alloc:
	return ERR_PTR(ret);
}

2946
static int
2947
i915_get_ggtt_vma_pages(struct i915_vma *vma)
2948
{
2949 2950
	int ret = 0;

2951 2952 2953 2954 2955
	if (vma->ggtt_view.pages)
		return 0;

	if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
		vma->ggtt_view.pages = vma->obj->pages;
2956 2957 2958
	else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
		vma->ggtt_view.pages =
			intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj);
2959 2960 2961
	else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL)
		vma->ggtt_view.pages =
			intel_partial_pages(&vma->ggtt_view, vma->obj);
2962 2963 2964 2965 2966
	else
		WARN_ONCE(1, "GGTT view %u not implemented!\n",
			  vma->ggtt_view.type);

	if (!vma->ggtt_view.pages) {
2967
		DRM_ERROR("Failed to get pages for GGTT view type %u!\n",
2968
			  vma->ggtt_view.type);
2969 2970 2971 2972 2973 2974
		ret = -EINVAL;
	} else if (IS_ERR(vma->ggtt_view.pages)) {
		ret = PTR_ERR(vma->ggtt_view.pages);
		vma->ggtt_view.pages = NULL;
		DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
			  vma->ggtt_view.type, ret);
2975 2976
	}

2977
	return ret;
2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992
}

/**
 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
 * @vma: VMA to map
 * @cache_level: mapping cache level
 * @flags: flags like global or local mapping
 *
 * DMA addresses are taken from the scatter-gather table of this object (or of
 * this VMA in case of non-default GGTT views) and PTE entries set up.
 * Note that DMA addresses are also the only part of the SG table we care about.
 */
int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
		  u32 flags)
{
2993 2994
	int ret;
	u32 bind_flags;
2995

2996 2997
	if (WARN_ON(flags == 0))
		return -EINVAL;
2998

2999
	bind_flags = 0;
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009
	if (flags & PIN_GLOBAL)
		bind_flags |= GLOBAL_BIND;
	if (flags & PIN_USER)
		bind_flags |= LOCAL_BIND;

	if (flags & PIN_UPDATE)
		bind_flags |= vma->bound;
	else
		bind_flags &= ~vma->bound;

3010 3011 3012 3013 3014 3015 3016 3017 3018
	if (bind_flags == 0)
		return 0;

	if (vma->bound == 0 && vma->vm->allocate_va_range) {
		trace_i915_va_alloc(vma->vm,
				    vma->node.start,
				    vma->node.size,
				    VM_TO_TRACE_NAME(vma->vm));

3019 3020
		/* XXX: i915_vma_pin() will fix this +- hack */
		vma->pin_count++;
3021 3022 3023
		ret = vma->vm->allocate_va_range(vma->vm,
						 vma->node.start,
						 vma->node.size);
3024
		vma->pin_count--;
3025 3026 3027 3028 3029
		if (ret)
			return ret;
	}

	ret = vma->vm->bind_vma(vma, cache_level, bind_flags);
3030 3031
	if (ret)
		return ret;
3032 3033

	vma->bound |= bind_flags;
3034 3035 3036

	return 0;
}
3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048

/**
 * i915_ggtt_view_size - Get the size of a GGTT view.
 * @obj: Object the view is of.
 * @view: The view in question.
 *
 * @return The size of the GGTT view in bytes.
 */
size_t
i915_ggtt_view_size(struct drm_i915_gem_object *obj,
		    const struct i915_ggtt_view *view)
{
3049
	if (view->type == I915_GGTT_VIEW_NORMAL) {
3050
		return obj->base.size;
3051 3052
	} else if (view->type == I915_GGTT_VIEW_ROTATED) {
		return view->rotation_info.size;
3053 3054
	} else if (view->type == I915_GGTT_VIEW_PARTIAL) {
		return view->params.partial.size << PAGE_SHIFT;
3055 3056 3057 3058 3059
	} else {
		WARN_ONCE(1, "GGTT view %u not implemented!\n", view->type);
		return obj->base.size;
	}
}