i915_gem.c 130.4 KB
Newer Older
1
/*
2
 * Copyright © 2008-2015 Intel Corporation
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

28
#include <drm/drmP.h>
29
#include <drm/drm_vma_manager.h>
30
#include <drm/i915_drm.h>
31
#include "i915_drv.h"
32
#include "i915_vgpu.h"
C
Chris Wilson 已提交
33
#include "i915_trace.h"
34
#include "intel_drv.h"
35
#include <linux/shmem_fs.h>
36
#include <linux/slab.h>
37
#include <linux/swap.h>
J
Jesse Barnes 已提交
38
#include <linux/pci.h>
39
#include <linux/dma-buf.h>
40

41 42
#define RQ_BUG_ON(expr)

43
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
44
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
45
static void
46 47 48
i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
static void
i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
49

50 51 52 53 54 55
static bool cpu_cache_is_coherent(struct drm_device *dev,
				  enum i915_cache_level level)
{
	return HAS_LLC(dev) || level != I915_CACHE_NONE;
}

56 57 58 59 60 61 62 63
static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
{
	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		return true;

	return obj->pin_display;
}

64 65 66 67
/* some bookkeeping */
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
				  size_t size)
{
68
	spin_lock(&dev_priv->mm.object_stat_lock);
69 70
	dev_priv->mm.object_count++;
	dev_priv->mm.object_memory += size;
71
	spin_unlock(&dev_priv->mm.object_stat_lock);
72 73 74 75 76
}

static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
				     size_t size)
{
77
	spin_lock(&dev_priv->mm.object_stat_lock);
78 79
	dev_priv->mm.object_count--;
	dev_priv->mm.object_memory -= size;
80
	spin_unlock(&dev_priv->mm.object_stat_lock);
81 82
}

83
static int
84
i915_gem_wait_for_error(struct i915_gpu_error *error)
85 86 87
{
	int ret;

88 89
#define EXIT_COND (!i915_reset_in_progress(error) || \
		   i915_terminally_wedged(error))
90
	if (EXIT_COND)
91 92
		return 0;

93 94 95 96 97
	/*
	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
	 * userspace. If it takes that long something really bad is going on and
	 * we should simply try to bail out and fail as gracefully as possible.
	 */
98 99 100
	ret = wait_event_interruptible_timeout(error->reset_queue,
					       EXIT_COND,
					       10*HZ);
101 102 103 104
	if (ret == 0) {
		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		return -EIO;
	} else if (ret < 0) {
105
		return ret;
106
	}
107
#undef EXIT_COND
108

109
	return 0;
110 111
}

112
int i915_mutex_lock_interruptible(struct drm_device *dev)
113
{
114
	struct drm_i915_private *dev_priv = dev->dev_private;
115 116
	int ret;

117
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
118 119 120 121 122 123 124
	if (ret)
		return ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

125
	WARN_ON(i915_verify_lists(dev));
126 127
	return 0;
}
128

129 130
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
131
			    struct drm_file *file)
132
{
133
	struct drm_i915_private *dev_priv = dev->dev_private;
134
	struct drm_i915_gem_get_aperture *args = data;
135 136
	struct i915_gtt *ggtt = &dev_priv->gtt;
	struct i915_vma *vma;
137
	size_t pinned;
138

139
	pinned = 0;
140
	mutex_lock(&dev->struct_mutex);
141 142 143 144 145 146
	list_for_each_entry(vma, &ggtt->base.active_list, mm_list)
		if (vma->pin_count)
			pinned += vma->node.size;
	list_for_each_entry(vma, &ggtt->base.inactive_list, mm_list)
		if (vma->pin_count)
			pinned += vma->node.size;
147
	mutex_unlock(&dev->struct_mutex);
148

149
	args->aper_size = dev_priv->gtt.base.total;
150
	args->aper_available_size = args->aper_size - pinned;
151

152 153 154
	return 0;
}

155 156
static int
i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
157
{
158 159 160 161 162
	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
	char *vaddr = obj->phys_handle->vaddr;
	struct sg_table *st;
	struct scatterlist *sg;
	int i;
163

164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
	if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
		return -EINVAL;

	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
		struct page *page;
		char *src;

		page = shmem_read_mapping_page(mapping, i);
		if (IS_ERR(page))
			return PTR_ERR(page);

		src = kmap_atomic(page);
		memcpy(vaddr, src, PAGE_SIZE);
		drm_clflush_virt_range(vaddr, PAGE_SIZE);
		kunmap_atomic(src);

		page_cache_release(page);
		vaddr += PAGE_SIZE;
	}

	i915_gem_chipset_flush(obj->base.dev);

	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (st == NULL)
		return -ENOMEM;

	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
		kfree(st);
		return -ENOMEM;
	}

	sg = st->sgl;
	sg->offset = 0;
	sg->length = obj->base.size;
198

199 200 201 202 203 204 205 206 207 208 209 210 211
	sg_dma_address(sg) = obj->phys_handle->busaddr;
	sg_dma_len(sg) = obj->base.size;

	obj->pages = st;
	return 0;
}

static void
i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
{
	int ret;

	BUG_ON(obj->madv == __I915_MADV_PURGED);
212

213 214 215 216 217 218 219 220 221 222 223 224 225
	ret = i915_gem_object_set_to_cpu_domain(obj, true);
	if (ret) {
		/* In the event of a disaster, abandon all caches and
		 * hope for the best.
		 */
		WARN_ON(ret != -EIO);
		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	}

	if (obj->madv == I915_MADV_DONTNEED)
		obj->dirty = 0;

	if (obj->dirty) {
226
		struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
227
		char *vaddr = obj->phys_handle->vaddr;
228 229 230
		int i;

		for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
231 232 233 234 235 236 237 238 239 240 241 242 243 244
			struct page *page;
			char *dst;

			page = shmem_read_mapping_page(mapping, i);
			if (IS_ERR(page))
				continue;

			dst = kmap_atomic(page);
			drm_clflush_virt_range(vaddr, PAGE_SIZE);
			memcpy(dst, vaddr, PAGE_SIZE);
			kunmap_atomic(dst);

			set_page_dirty(page);
			if (obj->madv == I915_MADV_WILLNEED)
245
				mark_page_accessed(page);
246
			page_cache_release(page);
247 248
			vaddr += PAGE_SIZE;
		}
249
		obj->dirty = 0;
250 251
	}

252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
	sg_free_table(obj->pages);
	kfree(obj->pages);
}

static void
i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
{
	drm_pci_free(obj->base.dev, obj->phys_handle);
}

static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
	.get_pages = i915_gem_object_get_pages_phys,
	.put_pages = i915_gem_object_put_pages_phys,
	.release = i915_gem_object_release_phys,
};

static int
drop_pages(struct drm_i915_gem_object *obj)
{
	struct i915_vma *vma, *next;
	int ret;

	drm_gem_object_reference(&obj->base);
	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link)
		if (i915_vma_unbind(vma))
			break;

	ret = i915_gem_object_put_pages(obj);
	drm_gem_object_unreference(&obj->base);

	return ret;
283 284 285 286 287 288 289
}

int
i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
			    int align)
{
	drm_dma_handle_t *phys;
290
	int ret;
291 292 293 294 295 296 297 298 299 300 301 302 303 304

	if (obj->phys_handle) {
		if ((unsigned long)obj->phys_handle->vaddr & (align -1))
			return -EBUSY;

		return 0;
	}

	if (obj->madv != I915_MADV_WILLNEED)
		return -EFAULT;

	if (obj->base.filp == NULL)
		return -EINVAL;

305 306 307 308
	ret = drop_pages(obj);
	if (ret)
		return ret;

309 310 311 312 313 314
	/* create a new object */
	phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
	if (!phys)
		return -ENOMEM;

	obj->phys_handle = phys;
315 316 317
	obj->ops = &i915_gem_phys_ops;

	return i915_gem_object_get_pages(obj);
318 319 320 321 322 323 324 325 326 327
}

static int
i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pwrite *args,
		     struct drm_file *file_priv)
{
	struct drm_device *dev = obj->base.dev;
	void *vaddr = obj->phys_handle->vaddr + args->offset;
	char __user *user_data = to_user_ptr(args->data_ptr);
328
	int ret = 0;
329 330 331 332 333 334 335

	/* We manually control the domain here and pretend that it
	 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
	 */
	ret = i915_gem_object_wait_rendering(obj, false);
	if (ret)
		return ret;
336

337
	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
338 339 340 341 342 343 344 345 346 347
	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
		unsigned long unwritten;

		/* The physical object once assigned is fixed for the lifetime
		 * of the obj, so we can safely drop the lock and continue
		 * to access vaddr.
		 */
		mutex_unlock(&dev->struct_mutex);
		unwritten = copy_from_user(vaddr, user_data, args->size);
		mutex_lock(&dev->struct_mutex);
348 349 350 351
		if (unwritten) {
			ret = -EFAULT;
			goto out;
		}
352 353
	}

354
	drm_clflush_virt_range(vaddr, args->size);
355
	i915_gem_chipset_flush(dev);
356 357

out:
358
	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
359
	return ret;
360 361
}

362 363 364
void *i915_gem_object_alloc(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
365
	return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
366 367 368 369 370
}

void i915_gem_object_free(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
371
	kmem_cache_free(dev_priv->objects, obj);
372 373
}

374 375 376 377 378
static int
i915_gem_create(struct drm_file *file,
		struct drm_device *dev,
		uint64_t size,
		uint32_t *handle_p)
379
{
380
	struct drm_i915_gem_object *obj;
381 382
	int ret;
	u32 handle;
383

384
	size = roundup(size, PAGE_SIZE);
385 386
	if (size == 0)
		return -EINVAL;
387 388

	/* Allocate the new object */
389
	obj = i915_gem_alloc_object(dev, size);
390 391 392
	if (obj == NULL)
		return -ENOMEM;

393
	ret = drm_gem_handle_create(file, &obj->base, &handle);
394
	/* drop reference from allocate - handle holds it now */
395 396 397
	drm_gem_object_unreference_unlocked(&obj->base);
	if (ret)
		return ret;
398

399
	*handle_p = handle;
400 401 402
	return 0;
}

403 404 405 406 407 408
int
i915_gem_dumb_create(struct drm_file *file,
		     struct drm_device *dev,
		     struct drm_mode_create_dumb *args)
{
	/* have to work out size/pitch and return them */
409
	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
410 411
	args->size = args->pitch * args->height;
	return i915_gem_create(file, dev,
412
			       args->size, &args->handle);
413 414 415 416 417 418 419 420 421 422
}

/**
 * Creates a new mm object and returns a handle to it.
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file)
{
	struct drm_i915_gem_create *args = data;
423

424
	return i915_gem_create(file, dev,
425
			       args->size, &args->handle);
426 427
}

428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
static inline int
__copy_to_user_swizzled(char __user *cpu_vaddr,
			const char *gpu_vaddr, int gpu_offset,
			int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_to_user(cpu_vaddr + cpu_offset,
				     gpu_vaddr + swizzled_gpu_offset,
				     this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

454
static inline int
455 456
__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
			  const char __user *cpu_vaddr,
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
			  int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
				       cpu_vaddr + cpu_offset,
				       this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
/*
 * Pins the specified object's pages and synchronizes the object with
 * GPU accesses. Sets needs_clflush to non-zero if the caller should
 * flush the object from the CPU cache.
 */
int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
				    int *needs_clflush)
{
	int ret;

	*needs_clflush = 0;

	if (!obj->base.filp)
		return -EINVAL;

	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		/* If we're not in the cpu read domain, set ourself into the gtt
		 * read domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will dirty the data
		 * anyway again before the next pread happens. */
		*needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
							obj->cache_level);
		ret = i915_gem_object_wait_rendering(obj, true);
		if (ret)
			return ret;
	}

	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

	i915_gem_object_pin_pages(obj);

	return ret;
}

516 517 518
/* Per-page copy function for the shmem pread fastpath.
 * Flushes invalid cachelines before reading the target if
 * needs_clflush is set. */
519
static int
520 521 522 523 524 525 526
shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

527
	if (unlikely(page_do_bit17_swizzling))
528 529 530 531 532 533 534 535 536 537 538
		return -EINVAL;

	vaddr = kmap_atomic(page);
	if (needs_clflush)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_to_user_inatomic(user_data,
				      vaddr + shmem_page_offset,
				      page_length);
	kunmap_atomic(vaddr);

539
	return ret ? -EFAULT : 0;
540 541
}

542 543 544 545
static void
shmem_clflush_swizzled_range(char *addr, unsigned long length,
			     bool swizzled)
{
546
	if (unlikely(swizzled)) {
547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563
		unsigned long start = (unsigned long) addr;
		unsigned long end = (unsigned long) addr + length;

		/* For swizzling simply ensure that we always flush both
		 * channels. Lame, but simple and it works. Swizzled
		 * pwrite/pread is far from a hotpath - current userspace
		 * doesn't use it at all. */
		start = round_down(start, 128);
		end = round_up(end, 128);

		drm_clflush_virt_range((void *)start, end - start);
	} else {
		drm_clflush_virt_range(addr, length);
	}

}

564 565 566 567 568 569 570 571 572 573 574 575
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
static int
shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

	vaddr = kmap(page);
	if (needs_clflush)
576 577 578
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
579 580 581 582 583 584 585 586 587 588 589

	if (page_do_bit17_swizzling)
		ret = __copy_to_user_swizzled(user_data,
					      vaddr, shmem_page_offset,
					      page_length);
	else
		ret = __copy_to_user(user_data,
				     vaddr + shmem_page_offset,
				     page_length);
	kunmap(page);

590
	return ret ? - EFAULT : 0;
591 592
}

593
static int
594 595 596 597
i915_gem_shmem_pread(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pread *args,
		     struct drm_file *file)
598
{
599
	char __user *user_data;
600
	ssize_t remain;
601
	loff_t offset;
602
	int shmem_page_offset, page_length, ret = 0;
603
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
604
	int prefaulted = 0;
605
	int needs_clflush = 0;
606
	struct sg_page_iter sg_iter;
607

V
Ville Syrjälä 已提交
608
	user_data = to_user_ptr(args->data_ptr);
609 610
	remain = args->size;

611
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
612

613
	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
614 615 616
	if (ret)
		return ret;

617
	offset = args->offset;
618

619 620
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
621
		struct page *page = sg_page_iter_page(&sg_iter);
622 623 624 625

		if (remain <= 0)
			break;

626 627 628 629 630
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
631
		shmem_page_offset = offset_in_page(offset);
632 633 634 635
		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

636 637 638
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

639 640 641 642 643
		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
		if (ret == 0)
			goto next_page;
644 645 646

		mutex_unlock(&dev->struct_mutex);

647
		if (likely(!i915.prefault_disable) && !prefaulted) {
648
			ret = fault_in_multipages_writeable(user_data, remain);
649 650 651 652 653 654 655
			/* Userspace is tricking us, but we've already clobbered
			 * its pages with the prefault and promised to write the
			 * data up to the first fault. Hence ignore any errors
			 * and just continue. */
			(void)ret;
			prefaulted = 1;
		}
656

657 658 659
		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
660

661
		mutex_lock(&dev->struct_mutex);
662 663

		if (ret)
664 665
			goto out;

666
next_page:
667
		remain -= page_length;
668
		user_data += page_length;
669 670 671
		offset += page_length;
	}

672
out:
673 674
	i915_gem_object_unpin_pages(obj);

675 676 677
	return ret;
}

678 679 680 681 682 683 684
/**
 * Reads data from the object referenced by handle.
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
685
		     struct drm_file *file)
686 687
{
	struct drm_i915_gem_pread *args = data;
688
	struct drm_i915_gem_object *obj;
689
	int ret = 0;
690

691 692 693 694
	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_WRITE,
V
Ville Syrjälä 已提交
695
		       to_user_ptr(args->data_ptr),
696 697 698
		       args->size))
		return -EFAULT;

699
	ret = i915_mutex_lock_interruptible(dev);
700
	if (ret)
701
		return ret;
702

703
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
704
	if (&obj->base == NULL) {
705 706
		ret = -ENOENT;
		goto unlock;
707
	}
708

709
	/* Bounds check source.  */
710 711
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
C
Chris Wilson 已提交
712
		ret = -EINVAL;
713
		goto out;
C
Chris Wilson 已提交
714 715
	}

716 717 718 719 720 721 722 723
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

C
Chris Wilson 已提交
724 725
	trace_i915_gem_object_pread(obj, args->offset, args->size);

726
	ret = i915_gem_shmem_pread(dev, obj, args, file);
727

728
out:
729
	drm_gem_object_unreference(&obj->base);
730
unlock:
731
	mutex_unlock(&dev->struct_mutex);
732
	return ret;
733 734
}

735 736
/* This is the fast write path which cannot handle
 * page faults in the source data
737
 */
738 739 740 741 742 743

static inline int
fast_user_write(struct io_mapping *mapping,
		loff_t page_base, int page_offset,
		char __user *user_data,
		int length)
744
{
745 746
	void __iomem *vaddr_atomic;
	void *vaddr;
747
	unsigned long unwritten;
748

P
Peter Zijlstra 已提交
749
	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
750 751 752
	/* We can use the cpu mem copy function because this is X86. */
	vaddr = (void __force*)vaddr_atomic + page_offset;
	unwritten = __copy_from_user_inatomic_nocache(vaddr,
753
						      user_data, length);
P
Peter Zijlstra 已提交
754
	io_mapping_unmap_atomic(vaddr_atomic);
755
	return unwritten;
756 757
}

758 759 760 761
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
 */
762
static int
763 764
i915_gem_gtt_pwrite_fast(struct drm_device *dev,
			 struct drm_i915_gem_object *obj,
765
			 struct drm_i915_gem_pwrite *args,
766
			 struct drm_file *file)
767
{
768
	struct drm_i915_private *dev_priv = dev->dev_private;
769
	ssize_t remain;
770
	loff_t offset, page_base;
771
	char __user *user_data;
D
Daniel Vetter 已提交
772 773
	int page_offset, page_length, ret;

774
	ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
D
Daniel Vetter 已提交
775 776 777 778 779 780 781 782 783 784
	if (ret)
		goto out;

	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		goto out_unpin;

	ret = i915_gem_object_put_fence(obj);
	if (ret)
		goto out_unpin;
785

V
Ville Syrjälä 已提交
786
	user_data = to_user_ptr(args->data_ptr);
787 788
	remain = args->size;

789
	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
790

791
	intel_fb_obj_invalidate(obj, ORIGIN_GTT);
792

793 794 795
	while (remain > 0) {
		/* Operation in this page
		 *
796 797 798
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
799
		 */
800 801
		page_base = offset & PAGE_MASK;
		page_offset = offset_in_page(offset);
802 803 804 805 806
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		/* If we get a fault while copying data, then (presumably) our
807 808
		 * source page isn't available.  Return the error and we'll
		 * retry in the slow path.
809
		 */
B
Ben Widawsky 已提交
810
		if (fast_user_write(dev_priv->gtt.mappable, page_base,
D
Daniel Vetter 已提交
811 812
				    page_offset, user_data, page_length)) {
			ret = -EFAULT;
813
			goto out_flush;
D
Daniel Vetter 已提交
814
		}
815

816 817 818
		remain -= page_length;
		user_data += page_length;
		offset += page_length;
819 820
	}

821
out_flush:
822
	intel_fb_obj_flush(obj, false, ORIGIN_GTT);
D
Daniel Vetter 已提交
823
out_unpin:
B
Ben Widawsky 已提交
824
	i915_gem_object_ggtt_unpin(obj);
D
Daniel Vetter 已提交
825
out:
826
	return ret;
827 828
}

829 830 831 832
/* Per-page copy function for the shmem pwrite fastpath.
 * Flushes invalid cachelines before writing to the target if
 * needs_clflush_before is set and flushes out any written cachelines after
 * writing if needs_clflush is set. */
833
static int
834 835 836 837 838
shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
839
{
840
	char *vaddr;
841
	int ret;
842

843
	if (unlikely(page_do_bit17_swizzling))
844
		return -EINVAL;
845

846 847 848 849
	vaddr = kmap_atomic(page);
	if (needs_clflush_before)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
850 851
	ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
					user_data, page_length);
852 853 854 855
	if (needs_clflush_after)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	kunmap_atomic(vaddr);
856

857
	return ret ? -EFAULT : 0;
858 859
}

860 861
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
862
static int
863 864 865 866 867
shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
868
{
869 870
	char *vaddr;
	int ret;
871

872
	vaddr = kmap(page);
873
	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
874 875 876
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
877 878
	if (page_do_bit17_swizzling)
		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
879 880
						user_data,
						page_length);
881 882 883 884 885
	else
		ret = __copy_from_user(vaddr + shmem_page_offset,
				       user_data,
				       page_length);
	if (needs_clflush_after)
886 887 888
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
889
	kunmap(page);
890

891
	return ret ? -EFAULT : 0;
892 893 894
}

static int
895 896 897 898
i915_gem_shmem_pwrite(struct drm_device *dev,
		      struct drm_i915_gem_object *obj,
		      struct drm_i915_gem_pwrite *args,
		      struct drm_file *file)
899 900
{
	ssize_t remain;
901 902
	loff_t offset;
	char __user *user_data;
903
	int shmem_page_offset, page_length, ret = 0;
904
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
905
	int hit_slowpath = 0;
906 907
	int needs_clflush_after = 0;
	int needs_clflush_before = 0;
908
	struct sg_page_iter sg_iter;
909

V
Ville Syrjälä 已提交
910
	user_data = to_user_ptr(args->data_ptr);
911 912
	remain = args->size;

913
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
914

915 916 917 918 919
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		/* If we're not in the cpu write domain, set ourself into the gtt
		 * write domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will use the data
		 * right away and we therefore have to clflush anyway. */
920
		needs_clflush_after = cpu_write_needs_clflush(obj);
921 922 923
		ret = i915_gem_object_wait_rendering(obj, false);
		if (ret)
			return ret;
924
	}
925 926 927 928 929
	/* Same trick applies to invalidate partially written cachelines read
	 * before writing. */
	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		needs_clflush_before =
			!cpu_cache_is_coherent(dev, obj->cache_level);
930

931 932 933 934
	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

935
	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
936

937 938
	i915_gem_object_pin_pages(obj);

939
	offset = args->offset;
940
	obj->dirty = 1;
941

942 943
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
944
		struct page *page = sg_page_iter_page(&sg_iter);
945
		int partial_cacheline_write;
946

947 948 949
		if (remain <= 0)
			break;

950 951 952 953 954
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
955
		shmem_page_offset = offset_in_page(offset);
956 957 958 959 960

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

961 962 963 964 965 966 967
		/* If we don't overwrite a cacheline completely we need to be
		 * careful to have up-to-date data by first clflushing. Don't
		 * overcomplicate things and flush the entire patch. */
		partial_cacheline_write = needs_clflush_before &&
			((shmem_page_offset | page_length)
				& (boot_cpu_data.x86_clflush_size - 1));

968 969 970
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

971 972 973 974 975 976
		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
		if (ret == 0)
			goto next_page;
977 978 979

		hit_slowpath = 1;
		mutex_unlock(&dev->struct_mutex);
980 981 982 983
		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
984

985
		mutex_lock(&dev->struct_mutex);
986 987

		if (ret)
988 989
			goto out;

990
next_page:
991
		remain -= page_length;
992
		user_data += page_length;
993
		offset += page_length;
994 995
	}

996
out:
997 998
	i915_gem_object_unpin_pages(obj);

999
	if (hit_slowpath) {
1000 1001 1002 1003 1004 1005 1006
		/*
		 * Fixup: Flush cpu caches in case we didn't flush the dirty
		 * cachelines in-line while writing and the object moved
		 * out of the cpu write domain while we've dropped the lock.
		 */
		if (!needs_clflush_after &&
		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1007
			if (i915_gem_clflush_object(obj, obj->pin_display))
1008
				needs_clflush_after = true;
1009
		}
1010
	}
1011

1012
	if (needs_clflush_after)
1013
		i915_gem_chipset_flush(dev);
1014 1015
	else
		obj->cache_dirty = true;
1016

1017
	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1018
	return ret;
1019 1020 1021 1022 1023 1024 1025 1026 1027
}

/**
 * Writes data to the object referenced by handle.
 *
 * On error, the contents of the buffer that were to be modified are undefined.
 */
int
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1028
		      struct drm_file *file)
1029
{
1030
	struct drm_i915_private *dev_priv = dev->dev_private;
1031
	struct drm_i915_gem_pwrite *args = data;
1032
	struct drm_i915_gem_object *obj;
1033 1034 1035 1036 1037 1038
	int ret;

	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_READ,
V
Ville Syrjälä 已提交
1039
		       to_user_ptr(args->data_ptr),
1040 1041 1042
		       args->size))
		return -EFAULT;

1043
	if (likely(!i915.prefault_disable)) {
1044 1045 1046 1047 1048
		ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
						   args->size);
		if (ret)
			return -EFAULT;
	}
1049

1050 1051
	intel_runtime_pm_get(dev_priv);

1052
	ret = i915_mutex_lock_interruptible(dev);
1053
	if (ret)
1054
		goto put_rpm;
1055

1056
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1057
	if (&obj->base == NULL) {
1058 1059
		ret = -ENOENT;
		goto unlock;
1060
	}
1061

1062
	/* Bounds check destination. */
1063 1064
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
C
Chris Wilson 已提交
1065
		ret = -EINVAL;
1066
		goto out;
C
Chris Wilson 已提交
1067 1068
	}

1069 1070 1071 1072 1073 1074 1075 1076
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

C
Chris Wilson 已提交
1077 1078
	trace_i915_gem_object_pwrite(obj, args->offset, args->size);

D
Daniel Vetter 已提交
1079
	ret = -EFAULT;
1080 1081 1082 1083 1084 1085
	/* We can only do the GTT pwrite on untiled buffers, as otherwise
	 * it would end up going through the fenced access, and we'll get
	 * different detiling behavior between reading and writing.
	 * pread/pwrite currently are reading and writing from the CPU
	 * perspective, requiring manual detiling by the client.
	 */
1086 1087 1088
	if (obj->tiling_mode == I915_TILING_NONE &&
	    obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
	    cpu_write_needs_clflush(obj)) {
1089
		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
D
Daniel Vetter 已提交
1090 1091 1092
		/* Note that the gtt paths might fail with non-page-backed user
		 * pointers (e.g. gtt mappings when moving data between
		 * textures). Fallback to the shmem path in that case. */
1093
	}
1094

1095 1096 1097 1098 1099 1100
	if (ret == -EFAULT || ret == -ENOSPC) {
		if (obj->phys_handle)
			ret = i915_gem_phys_pwrite(obj, args, file);
		else
			ret = i915_gem_shmem_pwrite(dev, obj, args, file);
	}
1101

1102
out:
1103
	drm_gem_object_unreference(&obj->base);
1104
unlock:
1105
	mutex_unlock(&dev->struct_mutex);
1106 1107 1108
put_rpm:
	intel_runtime_pm_put(dev_priv);

1109 1110 1111
	return ret;
}

1112
int
1113
i915_gem_check_wedge(struct i915_gpu_error *error,
1114 1115
		     bool interruptible)
{
1116
	if (i915_reset_in_progress(error)) {
1117 1118 1119 1120 1121
		/* Non-interruptible callers can't handle -EAGAIN, hence return
		 * -EIO unconditionally for these. */
		if (!interruptible)
			return -EIO;

1122 1123
		/* Recovery complete, but the reset failed ... */
		if (i915_terminally_wedged(error))
1124 1125
			return -EIO;

1126 1127 1128 1129 1130 1131 1132
		/*
		 * Check if GPU Reset is in progress - we need intel_ring_begin
		 * to work properly to reinit the hw state while the gpu is
		 * still marked as reset-in-progress. Handle this with a flag.
		 */
		if (!error->reload_in_reset)
			return -EAGAIN;
1133 1134 1135 1136 1137
	}

	return 0;
}

1138 1139 1140 1141 1142 1143
static void fake_irq(unsigned long data)
{
	wake_up_process((struct task_struct *)data);
}

static bool missed_irq(struct drm_i915_private *dev_priv,
1144
		       struct intel_engine_cs *ring)
1145 1146 1147 1148
{
	return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
}

D
Daniel Vetter 已提交
1149
static int __i915_spin_request(struct drm_i915_gem_request *req)
1150
{
1151 1152
	unsigned long timeout;

D
Daniel Vetter 已提交
1153
	if (i915_gem_request_get_ring(req)->irq_refcount)
1154 1155 1156 1157
		return -EBUSY;

	timeout = jiffies + 1;
	while (!need_resched()) {
D
Daniel Vetter 已提交
1158
		if (i915_gem_request_completed(req, true))
1159 1160 1161 1162
			return 0;

		if (time_after_eq(jiffies, timeout))
			break;
1163

1164 1165
		cpu_relax_lowlatency();
	}
D
Daniel Vetter 已提交
1166
	if (i915_gem_request_completed(req, false))
1167 1168 1169
		return 0;

	return -EAGAIN;
1170 1171
}

1172
/**
1173 1174 1175
 * __i915_wait_request - wait until execution of request has finished
 * @req: duh!
 * @reset_counter: reset sequence associated with the given request
1176 1177 1178
 * @interruptible: do an interruptible wait (normally yes)
 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
 *
1179 1180 1181 1182 1183 1184 1185
 * Note: It is of utmost importance that the passed in seqno and reset_counter
 * values have been read by the caller in an smp safe manner. Where read-side
 * locks are involved, it is sufficient to read the reset_counter before
 * unlocking the lock that protects the seqno. For lockless tricks, the
 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
 * inserted.
 *
1186
 * Returns 0 if the request was found within the alloted time. Else returns the
1187 1188
 * errno with remaining time filled in timeout argument.
 */
1189
int __i915_wait_request(struct drm_i915_gem_request *req,
1190
			unsigned reset_counter,
1191
			bool interruptible,
1192
			s64 *timeout,
1193
			struct intel_rps_client *rps)
1194
{
1195
	struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
1196
	struct drm_device *dev = ring->dev;
1197
	struct drm_i915_private *dev_priv = dev->dev_private;
1198 1199
	const bool irq_test_in_progress =
		ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
1200
	DEFINE_WAIT(wait);
1201
	unsigned long timeout_expire;
1202
	s64 before, now;
1203 1204
	int ret;

1205
	WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
1206

1207 1208 1209
	if (list_empty(&req->list))
		return 0;

1210
	if (i915_gem_request_completed(req, true))
1211 1212
		return 0;

1213 1214
	timeout_expire = timeout ?
		jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0;
1215

1216
	if (INTEL_INFO(dev_priv)->gen >= 6)
1217
		gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
1218

1219
	/* Record current time in case interrupted by signal, or wedged */
1220
	trace_i915_gem_request_wait_begin(req);
1221
	before = ktime_get_raw_ns();
1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232

	/* Optimistic spin for the next jiffie before touching IRQs */
	ret = __i915_spin_request(req);
	if (ret == 0)
		goto out;

	if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) {
		ret = -ENODEV;
		goto out;
	}

1233 1234
	for (;;) {
		struct timer_list timer;
1235

1236 1237
		prepare_to_wait(&ring->irq_queue, &wait,
				interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE);
1238

1239 1240
		/* We need to check whether any gpu reset happened in between
		 * the caller grabbing the seqno and now ... */
1241 1242 1243 1244 1245 1246 1247 1248
		if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
			/* ... but upgrade the -EAGAIN to an -EIO if the gpu
			 * is truely gone. */
			ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
			if (ret == 0)
				ret = -EAGAIN;
			break;
		}
1249

1250
		if (i915_gem_request_completed(req, false)) {
1251 1252 1253
			ret = 0;
			break;
		}
1254

1255 1256 1257 1258 1259
		if (interruptible && signal_pending(current)) {
			ret = -ERESTARTSYS;
			break;
		}

1260
		if (timeout && time_after_eq(jiffies, timeout_expire)) {
1261 1262 1263 1264 1265 1266
			ret = -ETIME;
			break;
		}

		timer.function = NULL;
		if (timeout || missed_irq(dev_priv, ring)) {
1267 1268
			unsigned long expire;

1269
			setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1270
			expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
1271 1272 1273
			mod_timer(&timer, expire);
		}

1274
		io_schedule();
1275 1276 1277 1278 1279 1280

		if (timer.function) {
			del_singleshot_timer_sync(&timer);
			destroy_timer_on_stack(&timer);
		}
	}
1281 1282
	if (!irq_test_in_progress)
		ring->irq_put(ring);
1283 1284

	finish_wait(&ring->irq_queue, &wait);
1285

1286 1287 1288 1289
out:
	now = ktime_get_raw_ns();
	trace_i915_gem_request_wait_end(req);

1290
	if (timeout) {
1291 1292 1293
		s64 tres = *timeout - (now - before);

		*timeout = tres < 0 ? 0 : tres;
1294 1295 1296 1297 1298 1299 1300 1301 1302 1303

		/*
		 * Apparently ktime isn't accurate enough and occasionally has a
		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
		 * things up to make the test happy. We allow up to 1 jiffy.
		 *
		 * This is a regrssion from the timespec->ktime conversion.
		 */
		if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
			*timeout = 0;
1304 1305
	}

1306
	return ret;
1307 1308
}

1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
				   struct drm_file *file)
{
	struct drm_i915_private *dev_private;
	struct drm_i915_file_private *file_priv;

	WARN_ON(!req || !file || req->file_priv);

	if (!req || !file)
		return -EINVAL;

	if (req->file_priv)
		return -EINVAL;

	dev_private = req->ring->dev->dev_private;
	file_priv = file->driver_priv;

	spin_lock(&file_priv->mm.lock);
	req->file_priv = file_priv;
	list_add_tail(&req->client_list, &file_priv->mm.request_list);
	spin_unlock(&file_priv->mm.lock);

	req->pid = get_pid(task_pid(current));

	return 0;
}

1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
static inline void
i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
{
	struct drm_i915_file_private *file_priv = request->file_priv;

	if (!file_priv)
		return;

	spin_lock(&file_priv->mm.lock);
	list_del(&request->client_list);
	request->file_priv = NULL;
	spin_unlock(&file_priv->mm.lock);
1348 1349 1350

	put_pid(request->pid);
	request->pid = NULL;
1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
}

static void i915_gem_request_retire(struct drm_i915_gem_request *request)
{
	trace_i915_gem_request_retire(request);

	/* We know the GPU must have read the request to have
	 * sent us the seqno + interrupt, so use the position
	 * of tail of the request to update the last known position
	 * of the GPU head.
	 *
	 * Note this requires that we are always called in request
	 * completion order.
	 */
	request->ringbuf->last_retired_head = request->postfix;

	list_del_init(&request->list);
	i915_gem_request_remove_from_client(request);

	i915_gem_request_unreference(request);
}

static void
__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
{
	struct intel_engine_cs *engine = req->ring;
	struct drm_i915_gem_request *tmp;

	lockdep_assert_held(&engine->dev->struct_mutex);

	if (list_empty(&req->list))
		return;

	do {
		tmp = list_first_entry(&engine->request_list,
				       typeof(*tmp), list);

		i915_gem_request_retire(tmp);
	} while (tmp != req);

	WARN_ON(i915_verify_lists(engine->dev));
}

1394
/**
1395
 * Waits for a request to be signaled, and cleans up the
1396 1397 1398
 * request and object lists appropriately for that event.
 */
int
1399
i915_wait_request(struct drm_i915_gem_request *req)
1400
{
1401 1402 1403
	struct drm_device *dev;
	struct drm_i915_private *dev_priv;
	bool interruptible;
1404 1405
	int ret;

1406 1407 1408 1409 1410 1411
	BUG_ON(req == NULL);

	dev = req->ring->dev;
	dev_priv = dev->dev_private;
	interruptible = dev_priv->mm.interruptible;

1412 1413
	BUG_ON(!mutex_is_locked(&dev->struct_mutex));

1414
	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1415 1416 1417
	if (ret)
		return ret;

1418 1419
	ret = __i915_wait_request(req,
				  atomic_read(&dev_priv->gpu_error.reset_counter),
1420
				  interruptible, NULL, NULL);
1421 1422
	if (ret)
		return ret;
1423

1424
	__i915_gem_request_retire__upto(req);
1425 1426 1427
	return 0;
}

1428 1429 1430 1431
/**
 * Ensures that all rendering to the object has completed and the object is
 * safe to unbind from the GTT or access from the CPU.
 */
1432
int
1433 1434 1435
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
			       bool readonly)
{
1436
	int ret, i;
1437

1438
	if (!obj->active)
1439 1440
		return 0;

1441 1442 1443 1444 1445
	if (readonly) {
		if (obj->last_write_req != NULL) {
			ret = i915_wait_request(obj->last_write_req);
			if (ret)
				return ret;
1446

1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481
			i = obj->last_write_req->ring->id;
			if (obj->last_read_req[i] == obj->last_write_req)
				i915_gem_object_retire__read(obj, i);
			else
				i915_gem_object_retire__write(obj);
		}
	} else {
		for (i = 0; i < I915_NUM_RINGS; i++) {
			if (obj->last_read_req[i] == NULL)
				continue;

			ret = i915_wait_request(obj->last_read_req[i]);
			if (ret)
				return ret;

			i915_gem_object_retire__read(obj, i);
		}
		RQ_BUG_ON(obj->active);
	}

	return 0;
}

static void
i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
			       struct drm_i915_gem_request *req)
{
	int ring = req->ring->id;

	if (obj->last_read_req[ring] == req)
		i915_gem_object_retire__read(obj, ring);
	else if (obj->last_write_req == req)
		i915_gem_object_retire__write(obj);

	__i915_gem_request_retire__upto(req);
1482 1483
}

1484 1485 1486 1487 1488
/* A nonblocking variant of the above wait. This is a highly dangerous routine
 * as the object state may change during this call.
 */
static __must_check int
i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1489
					    struct intel_rps_client *rps,
1490 1491 1492 1493
					    bool readonly)
{
	struct drm_device *dev = obj->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
1494
	struct drm_i915_gem_request *requests[I915_NUM_RINGS];
1495
	unsigned reset_counter;
1496
	int ret, i, n = 0;
1497 1498 1499 1500

	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
	BUG_ON(!dev_priv->mm.interruptible);

1501
	if (!obj->active)
1502 1503
		return 0;

1504
	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1505 1506 1507
	if (ret)
		return ret;

1508
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529

	if (readonly) {
		struct drm_i915_gem_request *req;

		req = obj->last_write_req;
		if (req == NULL)
			return 0;

		requests[n++] = i915_gem_request_reference(req);
	} else {
		for (i = 0; i < I915_NUM_RINGS; i++) {
			struct drm_i915_gem_request *req;

			req = obj->last_read_req[i];
			if (req == NULL)
				continue;

			requests[n++] = i915_gem_request_reference(req);
		}
	}

1530
	mutex_unlock(&dev->struct_mutex);
1531 1532
	for (i = 0; ret == 0 && i < n; i++)
		ret = __i915_wait_request(requests[i], reset_counter, true,
1533
					  NULL, rps);
1534 1535
	mutex_lock(&dev->struct_mutex);

1536 1537 1538 1539 1540 1541 1542
	for (i = 0; i < n; i++) {
		if (ret == 0)
			i915_gem_object_retire_request(obj, requests[i]);
		i915_gem_request_unreference(requests[i]);
	}

	return ret;
1543 1544
}

1545 1546 1547 1548 1549 1550
static struct intel_rps_client *to_rps_client(struct drm_file *file)
{
	struct drm_i915_file_private *fpriv = file->driver_priv;
	return &fpriv->rps;
}

1551
/**
1552 1553
 * Called when user space prepares to use an object with the CPU, either
 * through the mmap ioctl's mapping or a GTT mapping.
1554 1555 1556
 */
int
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1557
			  struct drm_file *file)
1558 1559
{
	struct drm_i915_gem_set_domain *args = data;
1560
	struct drm_i915_gem_object *obj;
1561 1562
	uint32_t read_domains = args->read_domains;
	uint32_t write_domain = args->write_domain;
1563 1564
	int ret;

1565
	/* Only handle setting domains to types used by the CPU. */
1566
	if (write_domain & I915_GEM_GPU_DOMAINS)
1567 1568
		return -EINVAL;

1569
	if (read_domains & I915_GEM_GPU_DOMAINS)
1570 1571 1572 1573 1574 1575 1576 1577
		return -EINVAL;

	/* Having something in the write domain implies it's in the read
	 * domain, and only that read domain.  Enforce that in the request.
	 */
	if (write_domain != 0 && read_domains != write_domain)
		return -EINVAL;

1578
	ret = i915_mutex_lock_interruptible(dev);
1579
	if (ret)
1580
		return ret;
1581

1582
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1583
	if (&obj->base == NULL) {
1584 1585
		ret = -ENOENT;
		goto unlock;
1586
	}
1587

1588 1589 1590 1591
	/* Try to flush the object off the GPU without holding the lock.
	 * We will repeat the flush holding the lock in the normal manner
	 * to catch cases where we are gazumped.
	 */
1592
	ret = i915_gem_object_wait_rendering__nonblocking(obj,
1593
							  to_rps_client(file),
1594
							  !write_domain);
1595 1596 1597
	if (ret)
		goto unref;

1598
	if (read_domains & I915_GEM_DOMAIN_GTT)
1599
		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1600
	else
1601
		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1602

1603 1604 1605 1606 1607
	if (write_domain != 0)
		intel_fb_obj_invalidate(obj,
					write_domain == I915_GEM_DOMAIN_GTT ?
					ORIGIN_GTT : ORIGIN_CPU);

1608
unref:
1609
	drm_gem_object_unreference(&obj->base);
1610
unlock:
1611 1612 1613 1614 1615 1616 1617 1618 1619
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Called when user space has done writes to this buffer
 */
int
i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1620
			 struct drm_file *file)
1621 1622
{
	struct drm_i915_gem_sw_finish *args = data;
1623
	struct drm_i915_gem_object *obj;
1624 1625
	int ret = 0;

1626
	ret = i915_mutex_lock_interruptible(dev);
1627
	if (ret)
1628
		return ret;
1629

1630
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1631
	if (&obj->base == NULL) {
1632 1633
		ret = -ENOENT;
		goto unlock;
1634 1635 1636
	}

	/* Pinned buffers may be scanout, so flush the cache */
1637
	if (obj->pin_display)
1638
		i915_gem_object_flush_cpu_write_domain(obj);
1639

1640
	drm_gem_object_unreference(&obj->base);
1641
unlock:
1642 1643 1644 1645 1646 1647 1648 1649 1650 1651
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Maps the contents of an object, returning the address it is mapped
 * into.
 *
 * While the mapping holds a reference on the contents of the object, it doesn't
 * imply a ref on the object itself.
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
 *
 * IMPORTANT:
 *
 * DRM driver writers who look a this function as an example for how to do GEM
 * mmap support, please don't implement mmap support like here. The modern way
 * to implement DRM mmap support is with an mmap offset ioctl (like
 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
 * That way debug tooling like valgrind will understand what's going on, hiding
 * the mmap call in a driver private ioctl will break that. The i915 driver only
 * does cpu mmaps this way because we didn't know better.
1662 1663 1664
 */
int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1665
		    struct drm_file *file)
1666 1667 1668 1669 1670
{
	struct drm_i915_gem_mmap *args = data;
	struct drm_gem_object *obj;
	unsigned long addr;

1671 1672 1673 1674 1675 1676
	if (args->flags & ~(I915_MMAP_WC))
		return -EINVAL;

	if (args->flags & I915_MMAP_WC && !cpu_has_pat)
		return -ENODEV;

1677
	obj = drm_gem_object_lookup(dev, file, args->handle);
1678
	if (obj == NULL)
1679
		return -ENOENT;
1680

1681 1682 1683 1684 1685 1686 1687 1688
	/* prime objects have no backing filp to GEM mmap
	 * pages from.
	 */
	if (!obj->filp) {
		drm_gem_object_unreference_unlocked(obj);
		return -EINVAL;
	}

1689
	addr = vm_mmap(obj->filp, 0, args->size,
1690 1691
		       PROT_READ | PROT_WRITE, MAP_SHARED,
		       args->offset);
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704
	if (args->flags & I915_MMAP_WC) {
		struct mm_struct *mm = current->mm;
		struct vm_area_struct *vma;

		down_write(&mm->mmap_sem);
		vma = find_vma(mm, addr);
		if (vma)
			vma->vm_page_prot =
				pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
		else
			addr = -ENOMEM;
		up_write(&mm->mmap_sem);
	}
1705
	drm_gem_object_unreference_unlocked(obj);
1706 1707 1708 1709 1710 1711 1712 1713
	if (IS_ERR((void *)addr))
		return addr;

	args->addr_ptr = (uint64_t) addr;

	return 0;
}

1714 1715
/**
 * i915_gem_fault - fault a page into the GTT
1716 1717
 * @vma: VMA in question
 * @vmf: fault info
1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731
 *
 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
 * from userspace.  The fault handler takes care of binding the object to
 * the GTT (if needed), allocating and programming a fence register (again,
 * only if needed based on whether the old reg is still valid or the object
 * is tiled) and inserting a new PTE into the faulting process.
 *
 * Note that the faulting process may involve evicting existing objects
 * from the GTT and/or fence registers to make room.  So performance may
 * suffer if the GTT working set is large or there are few fence registers
 * left.
 */
int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
1732 1733
	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
	struct drm_device *dev = obj->base.dev;
1734
	struct drm_i915_private *dev_priv = dev->dev_private;
1735
	struct i915_ggtt_view view = i915_ggtt_view_normal;
1736 1737 1738
	pgoff_t page_offset;
	unsigned long pfn;
	int ret = 0;
1739
	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1740

1741 1742
	intel_runtime_pm_get(dev_priv);

1743 1744 1745 1746
	/* We don't use vmf->pgoff since that has the fake offset */
	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
		PAGE_SHIFT;

1747 1748 1749
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto out;
1750

C
Chris Wilson 已提交
1751 1752
	trace_i915_gem_object_fault(obj, page_offset, true, write);

1753 1754 1755 1756 1757 1758 1759 1760 1761
	/* Try to flush the object off the GPU first without holding the lock.
	 * Upon reacquiring the lock, we will perform our sanity checks and then
	 * repeat the flush holding the lock in the normal manner to catch cases
	 * where we are gazumped.
	 */
	ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
	if (ret)
		goto unlock;

1762 1763
	/* Access to snoopable pages through the GTT is incoherent. */
	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1764
		ret = -EFAULT;
1765 1766 1767
		goto unlock;
	}

1768
	/* Use a partial view if the object is bigger than the aperture. */
1769 1770
	if (obj->base.size >= dev_priv->gtt.mappable_end &&
	    obj->tiling_mode == I915_TILING_NONE) {
1771
		static const unsigned int chunk_size = 256; // 1 MiB
1772

1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784
		memset(&view, 0, sizeof(view));
		view.type = I915_GGTT_VIEW_PARTIAL;
		view.params.partial.offset = rounddown(page_offset, chunk_size);
		view.params.partial.size =
			min_t(unsigned int,
			      chunk_size,
			      (vma->vm_end - vma->vm_start)/PAGE_SIZE -
			      view.params.partial.offset);
	}

	/* Now pin it into the GTT if needed */
	ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
1785 1786
	if (ret)
		goto unlock;
1787

1788 1789 1790
	ret = i915_gem_object_set_to_gtt_domain(obj, write);
	if (ret)
		goto unpin;
1791

1792
	ret = i915_gem_object_get_fence(obj);
1793
	if (ret)
1794
		goto unpin;
1795

1796
	/* Finally, remap it using the new GTT offset */
1797 1798
	pfn = dev_priv->gtt.mappable_base +
		i915_gem_obj_ggtt_offset_view(obj, &view);
1799
	pfn >>= PAGE_SHIFT;
1800

1801 1802 1803 1804 1805 1806 1807 1808 1809
	if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
		/* Overriding existing pages in partial view does not cause
		 * us any trouble as TLBs are still valid because the fault
		 * is due to userspace losing part of the mapping or never
		 * having accessed it before (at this partials' range).
		 */
		unsigned long base = vma->vm_start +
				     (view.params.partial.offset << PAGE_SHIFT);
		unsigned int i;
1810

1811 1812
		for (i = 0; i < view.params.partial.size; i++) {
			ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
1813 1814 1815 1816 1817
			if (ret)
				break;
		}

		obj->fault_mappable = true;
1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838
	} else {
		if (!obj->fault_mappable) {
			unsigned long size = min_t(unsigned long,
						   vma->vm_end - vma->vm_start,
						   obj->base.size);
			int i;

			for (i = 0; i < size >> PAGE_SHIFT; i++) {
				ret = vm_insert_pfn(vma,
						    (unsigned long)vma->vm_start + i * PAGE_SIZE,
						    pfn + i);
				if (ret)
					break;
			}

			obj->fault_mappable = true;
		} else
			ret = vm_insert_pfn(vma,
					    (unsigned long)vmf->virtual_address,
					    pfn + page_offset);
	}
1839
unpin:
1840
	i915_gem_object_ggtt_unpin_view(obj, &view);
1841
unlock:
1842
	mutex_unlock(&dev->struct_mutex);
1843
out:
1844
	switch (ret) {
1845
	case -EIO:
1846 1847 1848 1849 1850 1851 1852
		/*
		 * We eat errors when the gpu is terminally wedged to avoid
		 * userspace unduly crashing (gl has no provisions for mmaps to
		 * fail). But any other -EIO isn't ours (e.g. swap in failure)
		 * and so needs to be reported.
		 */
		if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1853 1854 1855
			ret = VM_FAULT_SIGBUS;
			break;
		}
1856
	case -EAGAIN:
D
Daniel Vetter 已提交
1857 1858 1859 1860
		/*
		 * EAGAIN means the gpu is hung and we'll wait for the error
		 * handler to reset everything when re-faulting in
		 * i915_mutex_lock_interruptible.
1861
		 */
1862 1863
	case 0:
	case -ERESTARTSYS:
1864
	case -EINTR:
1865 1866 1867 1868 1869
	case -EBUSY:
		/*
		 * EBUSY is ok: this just means that another thread
		 * already did the job.
		 */
1870 1871
		ret = VM_FAULT_NOPAGE;
		break;
1872
	case -ENOMEM:
1873 1874
		ret = VM_FAULT_OOM;
		break;
1875
	case -ENOSPC:
1876
	case -EFAULT:
1877 1878
		ret = VM_FAULT_SIGBUS;
		break;
1879
	default:
1880
		WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1881 1882
		ret = VM_FAULT_SIGBUS;
		break;
1883
	}
1884 1885 1886

	intel_runtime_pm_put(dev_priv);
	return ret;
1887 1888
}

1889 1890 1891 1892
/**
 * i915_gem_release_mmap - remove physical page mappings
 * @obj: obj in question
 *
1893
 * Preserve the reservation of the mmapping with the DRM core code, but
1894 1895 1896 1897 1898 1899 1900 1901 1902
 * relinquish ownership of the pages back to the system.
 *
 * It is vital that we remove the page mapping if we have mapped a tiled
 * object through the GTT and then lose the fence register due to
 * resource pressure. Similarly if the object has been moved out of the
 * aperture, than pages mapped into userspace must be revoked. Removing the
 * mapping will then trigger a page fault on the next user access, allowing
 * fixup by i915_gem_fault().
 */
1903
void
1904
i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1905
{
1906 1907
	if (!obj->fault_mappable)
		return;
1908

1909 1910
	drm_vma_node_unmap(&obj->base.vma_node,
			   obj->base.dev->anon_inode->i_mapping);
1911
	obj->fault_mappable = false;
1912 1913
}

1914 1915 1916 1917 1918 1919 1920 1921 1922
void
i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
{
	struct drm_i915_gem_object *obj;

	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
		i915_gem_release_mmap(obj);
}

1923
uint32_t
1924
i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1925
{
1926
	uint32_t gtt_size;
1927 1928

	if (INTEL_INFO(dev)->gen >= 4 ||
1929 1930
	    tiling_mode == I915_TILING_NONE)
		return size;
1931 1932 1933

	/* Previous chips need a power-of-two fence region when tiling */
	if (INTEL_INFO(dev)->gen == 3)
1934
		gtt_size = 1024*1024;
1935
	else
1936
		gtt_size = 512*1024;
1937

1938 1939
	while (gtt_size < size)
		gtt_size <<= 1;
1940

1941
	return gtt_size;
1942 1943
}

1944 1945 1946 1947 1948
/**
 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 * @obj: object to check
 *
 * Return the required GTT alignment for an object, taking into account
1949
 * potential fence register mapping.
1950
 */
1951 1952 1953
uint32_t
i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
			   int tiling_mode, bool fenced)
1954 1955 1956 1957 1958
{
	/*
	 * Minimum alignment is 4k (GTT page size), but might be greater
	 * if a fence register is needed for the object.
	 */
1959
	if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1960
	    tiling_mode == I915_TILING_NONE)
1961 1962
		return 4096;

1963 1964 1965 1966
	/*
	 * Previous chips need to be aligned to the size of the smallest
	 * fence register that can contain the object.
	 */
1967
	return i915_gem_get_gtt_size(dev, size, tiling_mode);
1968 1969
}

1970 1971 1972 1973 1974
static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	int ret;

1975
	if (drm_vma_node_has_offset(&obj->base.vma_node))
1976 1977
		return 0;

1978 1979
	dev_priv->mm.shrinker_no_lock_stealing = true;

1980 1981
	ret = drm_gem_create_mmap_offset(&obj->base);
	if (ret != -ENOSPC)
1982
		goto out;
1983 1984 1985 1986 1987 1988 1989 1990

	/* Badly fragmented mmap space? The only way we can recover
	 * space is by destroying unwanted objects. We can't randomly release
	 * mmap_offsets as userspace expects them to be persistent for the
	 * lifetime of the objects. The closest we can is to release the
	 * offsets on purgeable objects by truncating it and marking it purged,
	 * which prevents userspace from ever using that object again.
	 */
1991 1992 1993 1994 1995
	i915_gem_shrink(dev_priv,
			obj->base.size >> PAGE_SHIFT,
			I915_SHRINK_BOUND |
			I915_SHRINK_UNBOUND |
			I915_SHRINK_PURGEABLE);
1996 1997
	ret = drm_gem_create_mmap_offset(&obj->base);
	if (ret != -ENOSPC)
1998
		goto out;
1999 2000

	i915_gem_shrink_all(dev_priv);
2001 2002 2003 2004 2005
	ret = drm_gem_create_mmap_offset(&obj->base);
out:
	dev_priv->mm.shrinker_no_lock_stealing = false;

	return ret;
2006 2007 2008 2009 2010 2011 2012
}

static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
{
	drm_gem_free_mmap_offset(&obj->base);
}

2013
int
2014 2015
i915_gem_mmap_gtt(struct drm_file *file,
		  struct drm_device *dev,
2016
		  uint32_t handle,
2017
		  uint64_t *offset)
2018
{
2019
	struct drm_i915_gem_object *obj;
2020 2021
	int ret;

2022
	ret = i915_mutex_lock_interruptible(dev);
2023
	if (ret)
2024
		return ret;
2025

2026
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
2027
	if (&obj->base == NULL) {
2028 2029 2030
		ret = -ENOENT;
		goto unlock;
	}
2031

2032
	if (obj->madv != I915_MADV_WILLNEED) {
2033
		DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
2034
		ret = -EFAULT;
2035
		goto out;
2036 2037
	}

2038 2039 2040
	ret = i915_gem_object_create_mmap_offset(obj);
	if (ret)
		goto out;
2041

2042
	*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2043

2044
out:
2045
	drm_gem_object_unreference(&obj->base);
2046
unlock:
2047
	mutex_unlock(&dev->struct_mutex);
2048
	return ret;
2049 2050
}

2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071
/**
 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 * @dev: DRM device
 * @data: GTT mapping ioctl data
 * @file: GEM object info
 *
 * Simply returns the fake offset to userspace so it can mmap it.
 * The mmap call will end up in drm_gem_mmap(), which will set things
 * up so we can get faults in the handler above.
 *
 * The fault handler will take care of binding the object into the GTT
 * (since it may have been evicted to make room for something), allocating
 * a fence register, and mapping the appropriate aperture address into
 * userspace.
 */
int
i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file)
{
	struct drm_i915_gem_mmap_gtt *args = data;

2072
	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2073 2074
}

D
Daniel Vetter 已提交
2075 2076 2077
/* Immediately discard the backing storage */
static void
i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2078
{
2079
	i915_gem_object_free_mmap_offset(obj);
2080

2081 2082
	if (obj->base.filp == NULL)
		return;
2083

D
Daniel Vetter 已提交
2084 2085 2086 2087 2088
	/* Our goal here is to return as much of the memory as
	 * is possible back to the system as we are called from OOM.
	 * To do this we must instruct the shmfs to drop all of its
	 * backing pages, *now*.
	 */
2089
	shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
D
Daniel Vetter 已提交
2090 2091
	obj->madv = __I915_MADV_PURGED;
}
2092

2093 2094 2095
/* Try to discard unwanted pages */
static void
i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
D
Daniel Vetter 已提交
2096
{
2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110
	struct address_space *mapping;

	switch (obj->madv) {
	case I915_MADV_DONTNEED:
		i915_gem_object_truncate(obj);
	case __I915_MADV_PURGED:
		return;
	}

	if (obj->base.filp == NULL)
		return;

	mapping = file_inode(obj->base.filp)->i_mapping,
	invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2111 2112
}

2113
static void
2114
i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2115
{
2116 2117
	struct sg_page_iter sg_iter;
	int ret;
2118

2119
	BUG_ON(obj->madv == __I915_MADV_PURGED);
2120

C
Chris Wilson 已提交
2121 2122 2123 2124 2125 2126
	ret = i915_gem_object_set_to_cpu_domain(obj, true);
	if (ret) {
		/* In the event of a disaster, abandon all caches and
		 * hope for the best.
		 */
		WARN_ON(ret != -EIO);
2127
		i915_gem_clflush_object(obj, true);
C
Chris Wilson 已提交
2128 2129 2130
		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	}

I
Imre Deak 已提交
2131 2132
	i915_gem_gtt_finish_object(obj);

2133
	if (i915_gem_object_needs_bit17_swizzle(obj))
2134 2135
		i915_gem_object_save_bit_17_swizzle(obj);

2136 2137
	if (obj->madv == I915_MADV_DONTNEED)
		obj->dirty = 0;
2138

2139
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
2140
		struct page *page = sg_page_iter_page(&sg_iter);
2141

2142
		if (obj->dirty)
2143
			set_page_dirty(page);
2144

2145
		if (obj->madv == I915_MADV_WILLNEED)
2146
			mark_page_accessed(page);
2147

2148
		page_cache_release(page);
2149
	}
2150
	obj->dirty = 0;
2151

2152 2153
	sg_free_table(obj->pages);
	kfree(obj->pages);
2154
}
C
Chris Wilson 已提交
2155

2156
int
2157 2158 2159 2160
i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
{
	const struct drm_i915_gem_object_ops *ops = obj->ops;

2161
	if (obj->pages == NULL)
2162 2163
		return 0;

2164 2165 2166
	if (obj->pages_pin_count)
		return -EBUSY;

2167
	BUG_ON(i915_gem_obj_bound_any(obj));
B
Ben Widawsky 已提交
2168

2169 2170 2171
	/* ->put_pages might need to allocate memory for the bit17 swizzle
	 * array, hence protect them from being reaped by removing them from gtt
	 * lists early. */
2172
	list_del(&obj->global_list);
2173

2174
	ops->put_pages(obj);
2175
	obj->pages = NULL;
2176

2177
	i915_gem_object_invalidate(obj);
C
Chris Wilson 已提交
2178 2179 2180 2181

	return 0;
}

2182
static int
C
Chris Wilson 已提交
2183
i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2184
{
C
Chris Wilson 已提交
2185
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2186 2187
	int page_count, i;
	struct address_space *mapping;
2188 2189
	struct sg_table *st;
	struct scatterlist *sg;
2190
	struct sg_page_iter sg_iter;
2191
	struct page *page;
2192
	unsigned long last_pfn = 0;	/* suppress gcc warning */
I
Imre Deak 已提交
2193
	int ret;
C
Chris Wilson 已提交
2194
	gfp_t gfp;
2195

C
Chris Wilson 已提交
2196 2197 2198 2199 2200 2201 2202
	/* Assert that the object is not currently in any GPU domain. As it
	 * wasn't in the GTT, there shouldn't be any way it could have been in
	 * a GPU cache
	 */
	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);

2203 2204 2205 2206
	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (st == NULL)
		return -ENOMEM;

2207
	page_count = obj->base.size / PAGE_SIZE;
2208 2209
	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		kfree(st);
2210
		return -ENOMEM;
2211
	}
2212

2213 2214 2215 2216 2217
	/* Get the list of pages out of our struct file.  They'll be pinned
	 * at this point until we release them.
	 *
	 * Fail silently without starting the shrinker
	 */
A
Al Viro 已提交
2218
	mapping = file_inode(obj->base.filp)->i_mapping;
C
Chris Wilson 已提交
2219
	gfp = mapping_gfp_mask(mapping);
2220
	gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
C
Chris Wilson 已提交
2221
	gfp &= ~(__GFP_IO | __GFP_WAIT);
2222 2223 2224
	sg = st->sgl;
	st->nents = 0;
	for (i = 0; i < page_count; i++) {
C
Chris Wilson 已提交
2225 2226
		page = shmem_read_mapping_page_gfp(mapping, i, gfp);
		if (IS_ERR(page)) {
2227 2228 2229 2230 2231
			i915_gem_shrink(dev_priv,
					page_count,
					I915_SHRINK_BOUND |
					I915_SHRINK_UNBOUND |
					I915_SHRINK_PURGEABLE);
C
Chris Wilson 已提交
2232 2233 2234 2235 2236 2237 2238 2239
			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
		}
		if (IS_ERR(page)) {
			/* We've tried hard to allocate the memory by reaping
			 * our own buffer, now let the real VM do its job and
			 * go down in flames if truly OOM.
			 */
			i915_gem_shrink_all(dev_priv);
2240
			page = shmem_read_mapping_page(mapping, i);
I
Imre Deak 已提交
2241 2242
			if (IS_ERR(page)) {
				ret = PTR_ERR(page);
C
Chris Wilson 已提交
2243
				goto err_pages;
I
Imre Deak 已提交
2244
			}
C
Chris Wilson 已提交
2245
		}
2246 2247 2248 2249 2250 2251 2252 2253
#ifdef CONFIG_SWIOTLB
		if (swiotlb_nr_tbl()) {
			st->nents++;
			sg_set_page(sg, page, PAGE_SIZE, 0);
			sg = sg_next(sg);
			continue;
		}
#endif
2254 2255 2256 2257 2258 2259 2260 2261 2262
		if (!i || page_to_pfn(page) != last_pfn + 1) {
			if (i)
				sg = sg_next(sg);
			st->nents++;
			sg_set_page(sg, page, PAGE_SIZE, 0);
		} else {
			sg->length += PAGE_SIZE;
		}
		last_pfn = page_to_pfn(page);
2263 2264 2265

		/* Check that the i965g/gm workaround works. */
		WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2266
	}
2267 2268 2269 2270
#ifdef CONFIG_SWIOTLB
	if (!swiotlb_nr_tbl())
#endif
		sg_mark_end(sg);
2271 2272
	obj->pages = st;

I
Imre Deak 已提交
2273 2274 2275 2276
	ret = i915_gem_gtt_prepare_object(obj);
	if (ret)
		goto err_pages;

2277
	if (i915_gem_object_needs_bit17_swizzle(obj))
2278 2279
		i915_gem_object_do_bit_17_swizzle(obj);

2280 2281 2282 2283
	if (obj->tiling_mode != I915_TILING_NONE &&
	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
		i915_gem_object_pin_pages(obj);

2284 2285 2286
	return 0;

err_pages:
2287 2288
	sg_mark_end(sg);
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
2289
		page_cache_release(sg_page_iter_page(&sg_iter));
2290 2291
	sg_free_table(st);
	kfree(st);
2292 2293 2294 2295 2296 2297 2298 2299 2300

	/* shmemfs first checks if there is enough memory to allocate the page
	 * and reports ENOSPC should there be insufficient, along with the usual
	 * ENOMEM for a genuine allocation failure.
	 *
	 * We use ENOSPC in our driver to mean that we have run out of aperture
	 * space and so want to translate the error from shmemfs back to our
	 * usual understanding of ENOMEM.
	 */
I
Imre Deak 已提交
2301 2302 2303 2304
	if (ret == -ENOSPC)
		ret = -ENOMEM;

	return ret;
2305 2306
}

2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320
/* Ensure that the associated pages are gathered from the backing storage
 * and pinned into our object. i915_gem_object_get_pages() may be called
 * multiple times before they are released by a single call to
 * i915_gem_object_put_pages() - once the pages are no longer referenced
 * either as a result of memory pressure (reaping pages under the shrinker)
 * or as the object is itself released.
 */
int
i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	const struct drm_i915_gem_object_ops *ops = obj->ops;
	int ret;

2321
	if (obj->pages)
2322 2323
		return 0;

2324
	if (obj->madv != I915_MADV_WILLNEED) {
2325
		DRM_DEBUG("Attempting to obtain a purgeable object\n");
2326
		return -EFAULT;
2327 2328
	}

2329 2330
	BUG_ON(obj->pages_pin_count);

2331 2332 2333 2334
	ret = ops->get_pages(obj);
	if (ret)
		return ret;

2335
	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2336 2337 2338 2339

	obj->get_page.sg = obj->pages->sgl;
	obj->get_page.last = 0;

2340
	return 0;
2341 2342
}

2343
void i915_vma_move_to_active(struct i915_vma *vma,
2344
			     struct drm_i915_gem_request *req)
2345
{
2346
	struct drm_i915_gem_object *obj = vma->obj;
2347 2348 2349
	struct intel_engine_cs *ring;

	ring = i915_gem_request_get_ring(req);
2350 2351

	/* Add a reference if we're newly entering the active list. */
2352
	if (obj->active == 0)
2353
		drm_gem_object_reference(&obj->base);
2354
	obj->active |= intel_ring_flag(ring);
2355

2356
	list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
2357
	i915_gem_request_assign(&obj->last_read_req[ring->id], req);
2358

2359
	list_move_tail(&vma->mm_list, &vma->vm->active_list);
2360 2361
}

2362 2363
static void
i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
B
Ben Widawsky 已提交
2364
{
2365 2366 2367 2368
	RQ_BUG_ON(obj->last_write_req == NULL);
	RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring)));

	i915_gem_request_assign(&obj->last_write_req, NULL);
2369
	intel_fb_obj_flush(obj, true, ORIGIN_CS);
B
Ben Widawsky 已提交
2370 2371
}

2372
static void
2373
i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
2374
{
2375
	struct i915_vma *vma;
2376

2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388
	RQ_BUG_ON(obj->last_read_req[ring] == NULL);
	RQ_BUG_ON(!(obj->active & (1 << ring)));

	list_del_init(&obj->ring_list[ring]);
	i915_gem_request_assign(&obj->last_read_req[ring], NULL);

	if (obj->last_write_req && obj->last_write_req->ring->id == ring)
		i915_gem_object_retire__write(obj);

	obj->active &= ~(1 << ring);
	if (obj->active)
		return;
2389

2390 2391 2392 2393 2394 2395 2396
	/* Bump our place on the bound list to keep it roughly in LRU order
	 * so that we don't steal from recently used but inactive objects
	 * (unless we are forced to ofc!)
	 */
	list_move_tail(&obj->global_list,
		       &to_i915(obj->base.dev)->mm.bound_list);

2397 2398 2399
	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		if (!list_empty(&vma->mm_list))
			list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
2400
	}
2401

2402
	i915_gem_request_assign(&obj->last_fenced_req, NULL);
2403
	drm_gem_object_unreference(&obj->base);
2404 2405
}

2406
static int
2407
i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2408
{
2409
	struct drm_i915_private *dev_priv = dev->dev_private;
2410
	struct intel_engine_cs *ring;
2411
	int ret, i, j;
2412

2413
	/* Carefully retire all requests without writing to the rings */
2414
	for_each_ring(ring, dev_priv, i) {
2415 2416 2417
		ret = intel_ring_idle(ring);
		if (ret)
			return ret;
2418 2419
	}
	i915_gem_retire_requests(dev);
2420 2421

	/* Finally reset hw state */
2422
	for_each_ring(ring, dev_priv, i) {
2423
		intel_ring_init_seqno(ring, seqno);
2424

2425 2426
		for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
			ring->semaphore.sync_seqno[j] = 0;
2427
	}
2428

2429
	return 0;
2430 2431
}

2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457
int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	if (seqno == 0)
		return -EINVAL;

	/* HWS page needs to be set less than what we
	 * will inject to ring
	 */
	ret = i915_gem_init_seqno(dev, seqno - 1);
	if (ret)
		return ret;

	/* Carefully set the last_seqno value so that wrap
	 * detection still works
	 */
	dev_priv->next_seqno = seqno;
	dev_priv->last_seqno = seqno - 1;
	if (dev_priv->last_seqno == 0)
		dev_priv->last_seqno--;

	return 0;
}

2458 2459
int
i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2460
{
2461 2462 2463 2464
	struct drm_i915_private *dev_priv = dev->dev_private;

	/* reserve 0 for non-seqno */
	if (dev_priv->next_seqno == 0) {
2465
		int ret = i915_gem_init_seqno(dev, 0);
2466 2467
		if (ret)
			return ret;
2468

2469 2470
		dev_priv->next_seqno = 1;
	}
2471

2472
	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2473
	return 0;
2474 2475
}

2476 2477 2478 2479 2480
/*
 * NB: This function is not allowed to fail. Doing so would mean the the
 * request is not being tracked for completion but the work itself is
 * going to happen on the hardware. This would be a Bad Thing(tm).
 */
2481
void __i915_add_request(struct drm_i915_gem_request *request,
2482 2483
			struct drm_i915_gem_object *obj,
			bool flush_caches)
2484
{
2485 2486
	struct intel_engine_cs *ring;
	struct drm_i915_private *dev_priv;
2487
	struct intel_ringbuffer *ringbuf;
2488
	u32 request_start;
2489 2490
	int ret;

2491
	if (WARN_ON(request == NULL))
2492
		return;
2493

2494 2495 2496 2497
	ring = request->ring;
	dev_priv = ring->dev->dev_private;
	ringbuf = request->ringbuf;

2498 2499 2500 2501 2502 2503 2504
	/*
	 * To ensure that this call will not fail, space for its emissions
	 * should already have been reserved in the ring buffer. Let the ring
	 * know that it is time to use that space up.
	 */
	intel_ring_reserved_space_use(ringbuf);

2505
	request_start = intel_ring_get_tail(ringbuf);
2506 2507 2508 2509 2510 2511 2512
	/*
	 * Emit any outstanding flushes - execbuf can fail to emit the flush
	 * after having emitted the batchbuffer command. Hence we need to fix
	 * things up similar to emitting the lazy request. The difference here
	 * is that the flush _must_ happen before the next request, no matter
	 * what.
	 */
2513 2514
	if (flush_caches) {
		if (i915.enable_execlists)
2515
			ret = logical_ring_flush_all_caches(request);
2516
		else
2517
			ret = intel_ring_flush_all_caches(request);
2518 2519 2520
		/* Not allowed to fail! */
		WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
	}
2521

2522 2523 2524 2525 2526
	/* Record the position of the start of the request so that
	 * should we detect the updated seqno part-way through the
	 * GPU processing the request, we never over-estimate the
	 * position of the head.
	 */
2527
	request->postfix = intel_ring_get_tail(ringbuf);
2528

2529
	if (i915.enable_execlists)
2530
		ret = ring->emit_request(request);
2531
	else {
2532
		ret = ring->add_request(request);
2533 2534

		request->tail = intel_ring_get_tail(ringbuf);
2535
	}
2536 2537
	/* Not allowed to fail! */
	WARN(ret, "emit|add_request failed: %d!\n", ret);
2538

2539 2540 2541 2542 2543 2544 2545 2546
	request->head = request_start;

	/* Whilst this request exists, batch_obj will be on the
	 * active_list, and so will hold the active reference. Only when this
	 * request is retired will the the batch_obj be moved onto the
	 * inactive_list and lose its active reference. Hence we do not need
	 * to explicitly hold another reference here.
	 */
2547
	request->batch_obj = obj;
2548

2549
	request->emitted_jiffies = jiffies;
2550
	ring->last_submitted_seqno = request->seqno;
2551
	list_add_tail(&request->list, &ring->request_list);
2552

2553
	trace_i915_gem_request_add(request);
C
Chris Wilson 已提交
2554

2555
	i915_queue_hangcheck(ring->dev);
2556

2557 2558 2559 2560
	queue_delayed_work(dev_priv->wq,
			   &dev_priv->mm.retire_work,
			   round_jiffies_up_relative(HZ));
	intel_mark_busy(dev_priv->dev);
2561

2562 2563
	/* Sanity check that the reserved size was large enough. */
	intel_ring_reserved_space_end(ringbuf);
2564 2565
}

2566
static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
2567
				   const struct intel_context *ctx)
2568
{
2569
	unsigned long elapsed;
2570

2571 2572 2573
	elapsed = get_seconds() - ctx->hang_stats.guilty_ts;

	if (ctx->hang_stats.banned)
2574 2575
		return true;

2576 2577
	if (ctx->hang_stats.ban_period_seconds &&
	    elapsed <= ctx->hang_stats.ban_period_seconds) {
2578
		if (!i915_gem_context_is_default(ctx)) {
2579
			DRM_DEBUG("context hanging too fast, banning!\n");
2580
			return true;
2581 2582 2583
		} else if (i915_stop_ring_allow_ban(dev_priv)) {
			if (i915_stop_ring_allow_warn(dev_priv))
				DRM_ERROR("gpu hanging too fast, banning!\n");
2584
			return true;
2585
		}
2586 2587 2588 2589 2590
	}

	return false;
}

2591
static void i915_set_reset_status(struct drm_i915_private *dev_priv,
2592
				  struct intel_context *ctx,
2593
				  const bool guilty)
2594
{
2595 2596 2597 2598
	struct i915_ctx_hang_stats *hs;

	if (WARN_ON(!ctx))
		return;
2599

2600 2601 2602
	hs = &ctx->hang_stats;

	if (guilty) {
2603
		hs->banned = i915_context_is_banned(dev_priv, ctx);
2604 2605 2606 2607
		hs->batch_active++;
		hs->guilty_ts = get_seconds();
	} else {
		hs->batch_pending++;
2608 2609 2610
	}
}

2611 2612 2613 2614 2615 2616
void i915_gem_request_free(struct kref *req_ref)
{
	struct drm_i915_gem_request *req = container_of(req_ref,
						 typeof(*req), ref);
	struct intel_context *ctx = req->ctx;

2617 2618 2619
	if (req->file_priv)
		i915_gem_request_remove_from_client(req);

2620 2621
	if (ctx) {
		if (i915.enable_execlists) {
2622 2623
			if (ctx != req->ring->default_context)
				intel_lr_context_unpin(req);
2624
		}
2625

2626 2627
		i915_gem_context_unreference(ctx);
	}
2628

2629
	kmem_cache_free(req->i915->requests, req);
2630 2631
}

2632
int i915_gem_request_alloc(struct intel_engine_cs *ring,
2633 2634
			   struct intel_context *ctx,
			   struct drm_i915_gem_request **req_out)
2635
{
2636
	struct drm_i915_private *dev_priv = to_i915(ring->dev);
D
Daniel Vetter 已提交
2637
	struct drm_i915_gem_request *req;
2638 2639
	int ret;

2640 2641 2642
	if (!req_out)
		return -EINVAL;

2643
	*req_out = NULL;
2644

D
Daniel Vetter 已提交
2645 2646
	req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
	if (req == NULL)
2647 2648
		return -ENOMEM;

D
Daniel Vetter 已提交
2649
	ret = i915_gem_get_seqno(ring->dev, &req->seqno);
2650 2651
	if (ret)
		goto err;
2652

2653 2654
	kref_init(&req->ref);
	req->i915 = dev_priv;
D
Daniel Vetter 已提交
2655
	req->ring = ring;
2656 2657
	req->ctx  = ctx;
	i915_gem_context_reference(req->ctx);
2658 2659

	if (i915.enable_execlists)
2660
		ret = intel_logical_ring_alloc_request_extras(req);
2661
	else
D
Daniel Vetter 已提交
2662
		ret = intel_ring_alloc_request_extras(req);
2663 2664
	if (ret) {
		i915_gem_context_unreference(req->ctx);
2665
		goto err;
2666
	}
2667

2668 2669 2670 2671 2672 2673 2674
	/*
	 * Reserve space in the ring buffer for all the commands required to
	 * eventually emit this request. This is to guarantee that the
	 * i915_add_request() call can't fail. Note that the reserve may need
	 * to be redone if the request is not actually submitted straight
	 * away, e.g. because a GPU scheduler has deferred it.
	 */
2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687
	if (i915.enable_execlists)
		ret = intel_logical_ring_reserve_space(req);
	else
		ret = intel_ring_reserve_space(req);
	if (ret) {
		/*
		 * At this point, the request is fully allocated even if not
		 * fully prepared. Thus it can be cleaned up using the proper
		 * free code.
		 */
		i915_gem_request_cancel(req);
		return ret;
	}
2688

2689
	*req_out = req;
2690
	return 0;
2691 2692 2693 2694

err:
	kmem_cache_free(dev_priv->requests, req);
	return ret;
2695 2696
}

2697 2698 2699 2700 2701 2702 2703
void i915_gem_request_cancel(struct drm_i915_gem_request *req)
{
	intel_ring_reserved_space_cancel(req->ringbuf);

	i915_gem_request_unreference(req);
}

2704
struct drm_i915_gem_request *
2705
i915_gem_find_active_request(struct intel_engine_cs *ring)
2706
{
2707 2708 2709
	struct drm_i915_gem_request *request;

	list_for_each_entry(request, &ring->request_list, list) {
2710
		if (i915_gem_request_completed(request, false))
2711
			continue;
2712

2713
		return request;
2714
	}
2715 2716 2717 2718 2719

	return NULL;
}

static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
2720
				       struct intel_engine_cs *ring)
2721 2722 2723 2724
{
	struct drm_i915_gem_request *request;
	bool ring_hung;

2725
	request = i915_gem_find_active_request(ring);
2726 2727 2728 2729 2730 2731

	if (request == NULL)
		return;

	ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;

2732
	i915_set_reset_status(dev_priv, request->ctx, ring_hung);
2733 2734

	list_for_each_entry_continue(request, &ring->request_list, list)
2735
		i915_set_reset_status(dev_priv, request->ctx, false);
2736
}
2737

2738
static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
2739
					struct intel_engine_cs *ring)
2740
{
2741
	while (!list_empty(&ring->active_list)) {
2742
		struct drm_i915_gem_object *obj;
2743

2744 2745
		obj = list_first_entry(&ring->active_list,
				       struct drm_i915_gem_object,
2746
				       ring_list[ring->id]);
2747

2748
		i915_gem_object_retire__read(obj, ring->id);
2749
	}
2750

2751 2752 2753 2754 2755 2756
	/*
	 * Clear the execlists queue up before freeing the requests, as those
	 * are the ones that keep the context and ringbuffer backing objects
	 * pinned in place.
	 */
	while (!list_empty(&ring->execlist_queue)) {
2757
		struct drm_i915_gem_request *submit_req;
2758 2759

		submit_req = list_first_entry(&ring->execlist_queue,
2760
				struct drm_i915_gem_request,
2761 2762
				execlist_link);
		list_del(&submit_req->execlist_link);
2763 2764

		if (submit_req->ctx != ring->default_context)
2765
			intel_lr_context_unpin(submit_req);
2766

2767
		i915_gem_request_unreference(submit_req);
2768 2769
	}

2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783
	/*
	 * We must free the requests after all the corresponding objects have
	 * been moved off active lists. Which is the same order as the normal
	 * retire_requests function does. This is important if object hold
	 * implicit references on things like e.g. ppgtt address spaces through
	 * the request.
	 */
	while (!list_empty(&ring->request_list)) {
		struct drm_i915_gem_request *request;

		request = list_first_entry(&ring->request_list,
					   struct drm_i915_gem_request,
					   list);

2784
		i915_gem_request_retire(request);
2785
	}
2786 2787
}

2788
void i915_gem_reset(struct drm_device *dev)
2789
{
2790
	struct drm_i915_private *dev_priv = dev->dev_private;
2791
	struct intel_engine_cs *ring;
2792
	int i;
2793

2794 2795 2796 2797 2798 2799 2800 2801
	/*
	 * Before we free the objects from the requests, we need to inspect
	 * them for finding the guilty party. As the requests only borrow
	 * their reference to the objects, the inspection must be done first.
	 */
	for_each_ring(ring, dev_priv, i)
		i915_gem_reset_ring_status(dev_priv, ring);

2802
	for_each_ring(ring, dev_priv, i)
2803
		i915_gem_reset_ring_cleanup(dev_priv, ring);
2804

2805 2806
	i915_gem_context_reset(dev);

2807
	i915_gem_restore_fences(dev);
2808 2809

	WARN_ON(i915_verify_lists(dev));
2810 2811 2812 2813 2814
}

/**
 * This function clears the request list as sequence numbers are passed.
 */
2815
void
2816
i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2817
{
C
Chris Wilson 已提交
2818
	WARN_ON(i915_verify_lists(ring->dev));
2819

2820 2821 2822 2823
	/* Retire requests first as we use it above for the early return.
	 * If we retire requests last, we may use a later seqno and so clear
	 * the requests lists without clearing the active list, leading to
	 * confusion.
2824
	 */
2825
	while (!list_empty(&ring->request_list)) {
2826 2827
		struct drm_i915_gem_request *request;

2828
		request = list_first_entry(&ring->request_list,
2829 2830 2831
					   struct drm_i915_gem_request,
					   list);

2832
		if (!i915_gem_request_completed(request, true))
2833 2834
			break;

2835
		i915_gem_request_retire(request);
2836
	}
2837

2838 2839 2840 2841 2842 2843 2844 2845 2846
	/* Move any buffers on the active list that are no longer referenced
	 * by the ringbuffer to the flushing/inactive lists as appropriate,
	 * before we free the context associated with the requests.
	 */
	while (!list_empty(&ring->active_list)) {
		struct drm_i915_gem_object *obj;

		obj = list_first_entry(&ring->active_list,
				      struct drm_i915_gem_object,
2847
				      ring_list[ring->id]);
2848

2849
		if (!list_empty(&obj->last_read_req[ring->id]->list))
2850 2851
			break;

2852
		i915_gem_object_retire__read(obj, ring->id);
2853 2854
	}

2855 2856
	if (unlikely(ring->trace_irq_req &&
		     i915_gem_request_completed(ring->trace_irq_req, true))) {
2857
		ring->irq_put(ring);
2858
		i915_gem_request_assign(&ring->trace_irq_req, NULL);
2859
	}
2860

C
Chris Wilson 已提交
2861
	WARN_ON(i915_verify_lists(ring->dev));
2862 2863
}

2864
bool
2865 2866
i915_gem_retire_requests(struct drm_device *dev)
{
2867
	struct drm_i915_private *dev_priv = dev->dev_private;
2868
	struct intel_engine_cs *ring;
2869
	bool idle = true;
2870
	int i;
2871

2872
	for_each_ring(ring, dev_priv, i) {
2873
		i915_gem_retire_requests_ring(ring);
2874
		idle &= list_empty(&ring->request_list);
2875 2876 2877 2878 2879 2880 2881 2882 2883
		if (i915.enable_execlists) {
			unsigned long flags;

			spin_lock_irqsave(&ring->execlist_lock, flags);
			idle &= list_empty(&ring->execlist_queue);
			spin_unlock_irqrestore(&ring->execlist_lock, flags);

			intel_execlists_retire_requests(ring);
		}
2884 2885 2886 2887 2888 2889 2890 2891
	}

	if (idle)
		mod_delayed_work(dev_priv->wq,
				   &dev_priv->mm.idle_work,
				   msecs_to_jiffies(100));

	return idle;
2892 2893
}

2894
static void
2895 2896
i915_gem_retire_work_handler(struct work_struct *work)
{
2897 2898 2899
	struct drm_i915_private *dev_priv =
		container_of(work, typeof(*dev_priv), mm.retire_work.work);
	struct drm_device *dev = dev_priv->dev;
2900
	bool idle;
2901

2902
	/* Come back later if the device is busy... */
2903 2904 2905 2906
	idle = false;
	if (mutex_trylock(&dev->struct_mutex)) {
		idle = i915_gem_retire_requests(dev);
		mutex_unlock(&dev->struct_mutex);
2907
	}
2908
	if (!idle)
2909 2910
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
				   round_jiffies_up_relative(HZ));
2911
}
2912

2913 2914 2915 2916 2917
static void
i915_gem_idle_work_handler(struct work_struct *work)
{
	struct drm_i915_private *dev_priv =
		container_of(work, typeof(*dev_priv), mm.idle_work.work);
2918
	struct drm_device *dev = dev_priv->dev;
2919 2920
	struct intel_engine_cs *ring;
	int i;
2921

2922 2923 2924
	for_each_ring(ring, dev_priv, i)
		if (!list_empty(&ring->request_list))
			return;
2925 2926 2927 2928 2929 2930 2931 2932 2933

	intel_mark_idle(dev);

	if (mutex_trylock(&dev->struct_mutex)) {
		struct intel_engine_cs *ring;
		int i;

		for_each_ring(ring, dev_priv, i)
			i915_gem_batch_pool_fini(&ring->batch_pool);
2934

2935 2936
		mutex_unlock(&dev->struct_mutex);
	}
2937 2938
}

2939 2940 2941 2942 2943 2944 2945 2946
/**
 * Ensures that an object will eventually get non-busy by flushing any required
 * write domains, emitting any outstanding lazy request and retiring and
 * completed requests.
 */
static int
i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
{
2947
	int i;
2948 2949 2950

	if (!obj->active)
		return 0;
2951

2952 2953
	for (i = 0; i < I915_NUM_RINGS; i++) {
		struct drm_i915_gem_request *req;
2954

2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966
		req = obj->last_read_req[i];
		if (req == NULL)
			continue;

		if (list_empty(&req->list))
			goto retire;

		if (i915_gem_request_completed(req, true)) {
			__i915_gem_request_retire__upto(req);
retire:
			i915_gem_object_retire__read(obj, i);
		}
2967 2968 2969 2970 2971
	}

	return 0;
}

2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996
/**
 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
 * @DRM_IOCTL_ARGS: standard ioctl arguments
 *
 * Returns 0 if successful, else an error is returned with the remaining time in
 * the timeout parameter.
 *  -ETIME: object is still busy after timeout
 *  -ERESTARTSYS: signal interrupted the wait
 *  -ENONENT: object doesn't exist
 * Also possible, but rare:
 *  -EAGAIN: GPU wedged
 *  -ENOMEM: damn
 *  -ENODEV: Internal IRQ fail
 *  -E?: The add request failed
 *
 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
 * non-zero timeout parameter the wait ioctl will wait for the given number of
 * nanoseconds on an object becoming unbusy. Since the wait itself does so
 * without holding struct_mutex the object may become re-busied before this
 * function completes. A similar but shorter * race condition exists in the busy
 * ioctl
 */
int
i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
2997
	struct drm_i915_private *dev_priv = dev->dev_private;
2998 2999
	struct drm_i915_gem_wait *args = data;
	struct drm_i915_gem_object *obj;
3000
	struct drm_i915_gem_request *req[I915_NUM_RINGS];
3001
	unsigned reset_counter;
3002 3003
	int i, n = 0;
	int ret;
3004

3005 3006 3007
	if (args->flags != 0)
		return -EINVAL;

3008 3009 3010 3011 3012 3013 3014 3015 3016 3017
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
	if (&obj->base == NULL) {
		mutex_unlock(&dev->struct_mutex);
		return -ENOENT;
	}

3018 3019
	/* Need to make sure the object gets inactive eventually. */
	ret = i915_gem_object_flush_active(obj);
3020 3021 3022
	if (ret)
		goto out;

3023
	if (!obj->active)
3024
		goto out;
3025 3026

	/* Do this after OLR check to make sure we make forward progress polling
3027
	 * on this IOCTL with a timeout == 0 (like busy ioctl)
3028
	 */
3029
	if (args->timeout_ns == 0) {
3030 3031 3032 3033 3034
		ret = -ETIME;
		goto out;
	}

	drm_gem_object_unreference(&obj->base);
3035
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3036 3037 3038 3039 3040 3041 3042 3043

	for (i = 0; i < I915_NUM_RINGS; i++) {
		if (obj->last_read_req[i] == NULL)
			continue;

		req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
	}

3044 3045
	mutex_unlock(&dev->struct_mutex);

3046 3047 3048 3049 3050 3051 3052
	for (i = 0; i < n; i++) {
		if (ret == 0)
			ret = __i915_wait_request(req[i], reset_counter, true,
						  args->timeout_ns > 0 ? &args->timeout_ns : NULL,
						  file->driver_priv);
		i915_gem_request_unreference__unlocked(req[i]);
	}
3053
	return ret;
3054 3055 3056 3057 3058 3059 3060

out:
	drm_gem_object_unreference(&obj->base);
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

3061 3062 3063
static int
__i915_gem_object_sync(struct drm_i915_gem_object *obj,
		       struct intel_engine_cs *to,
3064 3065
		       struct drm_i915_gem_request *from_req,
		       struct drm_i915_gem_request **to_req)
3066 3067 3068 3069
{
	struct intel_engine_cs *from;
	int ret;

3070
	from = i915_gem_request_get_ring(from_req);
3071 3072 3073
	if (to == from)
		return 0;

3074
	if (i915_gem_request_completed(from_req, true))
3075 3076 3077
		return 0;

	if (!i915_semaphore_is_enabled(obj->base.dev)) {
3078
		struct drm_i915_private *i915 = to_i915(obj->base.dev);
3079
		ret = __i915_wait_request(from_req,
3080 3081 3082 3083
					  atomic_read(&i915->gpu_error.reset_counter),
					  i915->mm.interruptible,
					  NULL,
					  &i915->rps.semaphores);
3084 3085 3086
		if (ret)
			return ret;

3087
		i915_gem_object_retire_request(obj, from_req);
3088 3089
	} else {
		int idx = intel_ring_sync_index(from, to);
3090 3091 3092
		u32 seqno = i915_gem_request_get_seqno(from_req);

		WARN_ON(!to_req);
3093 3094 3095 3096

		if (seqno <= from->semaphore.sync_seqno[idx])
			return 0;

3097 3098 3099 3100 3101 3102
		if (*to_req == NULL) {
			ret = i915_gem_request_alloc(to, to->default_context, to_req);
			if (ret)
				return ret;
		}

3103 3104
		trace_i915_gem_ring_sync_to(*to_req, from, from_req);
		ret = to->semaphore.sync_to(*to_req, from, seqno);
3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118
		if (ret)
			return ret;

		/* We use last_read_req because sync_to()
		 * might have just caused seqno wrap under
		 * the radar.
		 */
		from->semaphore.sync_seqno[idx] =
			i915_gem_request_get_seqno(obj->last_read_req[from->id]);
	}

	return 0;
}

3119 3120 3121 3122 3123
/**
 * i915_gem_object_sync - sync an object to a ring.
 *
 * @obj: object which may be in use on another ring.
 * @to: ring we wish to use the object on. May be NULL.
3124 3125 3126
 * @to_req: request we wish to use the object for. See below.
 *          This will be allocated and returned if a request is
 *          required but not passed in.
3127 3128 3129
 *
 * This code is meant to abstract object synchronization with the GPU.
 * Calling with NULL implies synchronizing the object with the CPU
3130
 * rather than a particular GPU ring. Conceptually we serialise writes
3131
 * between engines inside the GPU. We only allow one engine to write
3132 3133 3134 3135 3136 3137 3138 3139 3140
 * into a buffer at any time, but multiple readers. To ensure each has
 * a coherent view of memory, we must:
 *
 * - If there is an outstanding write request to the object, the new
 *   request must wait for it to complete (either CPU or in hw, requests
 *   on the same ring will be naturally ordered).
 *
 * - If we are a write request (pending_write_domain is set), the new
 *   request must wait for outstanding read requests to complete.
3141
 *
3142 3143 3144 3145 3146 3147 3148 3149 3150 3151
 * For CPU synchronisation (NULL to) no request is required. For syncing with
 * rings to_req must be non-NULL. However, a request does not have to be
 * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
 * request will be allocated automatically and returned through *to_req. Note
 * that it is not guaranteed that commands will be emitted (because the system
 * might already be idle). Hence there is no need to create a request that
 * might never have any work submitted. Note further that if a request is
 * returned in *to_req, it is the responsibility of the caller to submit
 * that request (after potentially adding more work to it).
 *
3152 3153
 * Returns 0 if successful, else propagates up the lower layer error.
 */
3154 3155
int
i915_gem_object_sync(struct drm_i915_gem_object *obj,
3156 3157
		     struct intel_engine_cs *to,
		     struct drm_i915_gem_request **to_req)
3158
{
3159 3160 3161
	const bool readonly = obj->base.pending_write_domain == 0;
	struct drm_i915_gem_request *req[I915_NUM_RINGS];
	int ret, i, n;
3162

3163
	if (!obj->active)
3164 3165
		return 0;

3166 3167
	if (to == NULL)
		return i915_gem_object_wait_rendering(obj, readonly);
3168

3169 3170 3171 3172 3173 3174 3175 3176 3177 3178
	n = 0;
	if (readonly) {
		if (obj->last_write_req)
			req[n++] = obj->last_write_req;
	} else {
		for (i = 0; i < I915_NUM_RINGS; i++)
			if (obj->last_read_req[i])
				req[n++] = obj->last_read_req[i];
	}
	for (i = 0; i < n; i++) {
3179
		ret = __i915_gem_object_sync(obj, to, req[i], to_req);
3180 3181 3182
		if (ret)
			return ret;
	}
3183

3184
	return 0;
3185 3186
}

3187 3188 3189 3190 3191 3192 3193
static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
{
	u32 old_write_domain, old_read_domains;

	/* Force a pagefault for domain tracking on next user access */
	i915_gem_release_mmap(obj);

3194 3195 3196
	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		return;

3197 3198 3199
	/* Wait for any direct GTT access to complete */
	mb();

3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210
	old_read_domains = obj->base.read_domains;
	old_write_domain = obj->base.write_domain;

	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;

	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);
}

3211
int i915_vma_unbind(struct i915_vma *vma)
3212
{
3213
	struct drm_i915_gem_object *obj = vma->obj;
3214
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3215
	int ret;
3216

3217
	if (list_empty(&vma->vma_link))
3218 3219
		return 0;

3220 3221 3222 3223
	if (!drm_mm_node_allocated(&vma->node)) {
		i915_gem_vma_destroy(vma);
		return 0;
	}
3224

B
Ben Widawsky 已提交
3225
	if (vma->pin_count)
3226
		return -EBUSY;
3227

3228 3229
	BUG_ON(obj->pages == NULL);

3230
	ret = i915_gem_object_wait_rendering(obj, false);
3231
	if (ret)
3232 3233
		return ret;

3234 3235
	if (i915_is_ggtt(vma->vm) &&
	    vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3236
		i915_gem_object_finish_gtt(obj);
3237

3238 3239 3240 3241 3242
		/* release the fence reg _after_ flushing */
		ret = i915_gem_object_put_fence(obj);
		if (ret)
			return ret;
	}
3243

3244
	trace_i915_vma_unbind(vma);
C
Chris Wilson 已提交
3245

3246
	vma->vm->unbind_vma(vma);
3247
	vma->bound = 0;
3248

3249
	list_del_init(&vma->mm_list);
3250 3251 3252 3253 3254 3255 3256
	if (i915_is_ggtt(vma->vm)) {
		if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
			obj->map_and_fenceable = false;
		} else if (vma->ggtt_view.pages) {
			sg_free_table(vma->ggtt_view.pages);
			kfree(vma->ggtt_view.pages);
		}
3257
		vma->ggtt_view.pages = NULL;
3258
	}
3259

B
Ben Widawsky 已提交
3260 3261 3262 3263
	drm_mm_remove_node(&vma->node);
	i915_gem_vma_destroy(vma);

	/* Since the unbound list is global, only move to that list if
3264
	 * no more VMAs exist. */
I
Imre Deak 已提交
3265
	if (list_empty(&obj->vma_list))
B
Ben Widawsky 已提交
3266
		list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3267

3268 3269 3270 3271 3272 3273
	/* And finally now the object is completely decoupled from this vma,
	 * we can drop its hold on the backing storage and allow it to be
	 * reaped by the shrinker.
	 */
	i915_gem_object_unpin_pages(obj);

3274
	return 0;
3275 3276
}

3277
int i915_gpu_idle(struct drm_device *dev)
3278
{
3279
	struct drm_i915_private *dev_priv = dev->dev_private;
3280
	struct intel_engine_cs *ring;
3281
	int ret, i;
3282 3283

	/* Flush everything onto the inactive list. */
3284
	for_each_ring(ring, dev_priv, i) {
3285
		if (!i915.enable_execlists) {
3286 3287 3288
			struct drm_i915_gem_request *req;

			ret = i915_gem_request_alloc(ring, ring->default_context, &req);
3289 3290
			if (ret)
				return ret;
3291

3292
			ret = i915_switch_context(req);
3293 3294 3295 3296 3297
			if (ret) {
				i915_gem_request_cancel(req);
				return ret;
			}

3298
			i915_add_request_no_flush(req);
3299
		}
3300

3301
		ret = intel_ring_idle(ring);
3302 3303 3304
		if (ret)
			return ret;
	}
3305

3306
	WARN_ON(i915_verify_lists(dev));
3307
	return 0;
3308 3309
}

3310
static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3311 3312
				     unsigned long cache_level)
{
3313
	struct drm_mm_node *gtt_space = &vma->node;
3314 3315
	struct drm_mm_node *other;

3316 3317 3318 3319 3320 3321
	/*
	 * On some machines we have to be careful when putting differing types
	 * of snoopable memory together to avoid the prefetcher crossing memory
	 * domains and dying. During vm initialisation, we decide whether or not
	 * these constraints apply and set the drm_mm.color_adjust
	 * appropriately.
3322
	 */
3323
	if (vma->vm->mm.color_adjust == NULL)
3324 3325
		return true;

3326
	if (!drm_mm_node_allocated(gtt_space))
3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342
		return true;

	if (list_empty(&gtt_space->node_list))
		return true;

	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
	if (other->allocated && !other->hole_follows && other->color != cache_level)
		return false;

	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		return false;

	return true;
}

3343
/**
3344 3345
 * Finds free space in the GTT aperture and binds the object or a view of it
 * there.
3346
 */
3347
static struct i915_vma *
3348 3349
i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
			   struct i915_address_space *vm,
3350
			   const struct i915_ggtt_view *ggtt_view,
3351
			   unsigned alignment,
3352
			   uint64_t flags)
3353
{
3354
	struct drm_device *dev = obj->base.dev;
3355
	struct drm_i915_private *dev_priv = dev->dev_private;
3356
	u32 fence_alignment, unfenced_alignment;
3357 3358
	u32 search_flag, alloc_flag;
	u64 start, end;
3359
	u64 size, fence_size;
B
Ben Widawsky 已提交
3360
	struct i915_vma *vma;
3361
	int ret;
3362

3363 3364 3365 3366 3367
	if (i915_is_ggtt(vm)) {
		u32 view_size;

		if (WARN_ON(!ggtt_view))
			return ERR_PTR(-EINVAL);
3368

3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397
		view_size = i915_ggtt_view_size(obj, ggtt_view);

		fence_size = i915_gem_get_gtt_size(dev,
						   view_size,
						   obj->tiling_mode);
		fence_alignment = i915_gem_get_gtt_alignment(dev,
							     view_size,
							     obj->tiling_mode,
							     true);
		unfenced_alignment = i915_gem_get_gtt_alignment(dev,
								view_size,
								obj->tiling_mode,
								false);
		size = flags & PIN_MAPPABLE ? fence_size : view_size;
	} else {
		fence_size = i915_gem_get_gtt_size(dev,
						   obj->base.size,
						   obj->tiling_mode);
		fence_alignment = i915_gem_get_gtt_alignment(dev,
							     obj->base.size,
							     obj->tiling_mode,
							     true);
		unfenced_alignment =
			i915_gem_get_gtt_alignment(dev,
						   obj->base.size,
						   obj->tiling_mode,
						   false);
		size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
	}
3398

3399 3400 3401 3402 3403 3404 3405
	start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
	end = vm->total;
	if (flags & PIN_MAPPABLE)
		end = min_t(u64, end, dev_priv->gtt.mappable_end);
	if (flags & PIN_ZONE_4G)
		end = min_t(u64, end, (1ULL << 32));

3406
	if (alignment == 0)
3407
		alignment = flags & PIN_MAPPABLE ? fence_alignment :
3408
						unfenced_alignment;
3409
	if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3410 3411 3412
		DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
			  ggtt_view ? ggtt_view->type : 0,
			  alignment);
3413
		return ERR_PTR(-EINVAL);
3414 3415
	}

3416 3417 3418
	/* If binding the object/GGTT view requires more space than the entire
	 * aperture has, reject it early before evicting everything in a vain
	 * attempt to find space.
3419
	 */
3420
	if (size > end) {
3421
		DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
3422 3423
			  ggtt_view ? ggtt_view->type : 0,
			  size,
3424
			  flags & PIN_MAPPABLE ? "mappable" : "total",
3425
			  end);
3426
		return ERR_PTR(-E2BIG);
3427 3428
	}

3429
	ret = i915_gem_object_get_pages(obj);
C
Chris Wilson 已提交
3430
	if (ret)
3431
		return ERR_PTR(ret);
C
Chris Wilson 已提交
3432

3433 3434
	i915_gem_object_pin_pages(obj);

3435 3436 3437
	vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
			  i915_gem_obj_lookup_or_create_vma(obj, vm);

3438
	if (IS_ERR(vma))
3439
		goto err_unpin;
B
Ben Widawsky 已提交
3440

3441 3442 3443 3444 3445 3446 3447 3448
	if (flags & PIN_HIGH) {
		search_flag = DRM_MM_SEARCH_BELOW;
		alloc_flag = DRM_MM_CREATE_TOP;
	} else {
		search_flag = DRM_MM_SEARCH_DEFAULT;
		alloc_flag = DRM_MM_CREATE_DEFAULT;
	}

3449
search_free:
3450
	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3451
						  size, alignment,
3452 3453
						  obj->cache_level,
						  start, end,
3454 3455
						  search_flag,
						  alloc_flag);
3456
	if (ret) {
3457
		ret = i915_gem_evict_something(dev, vm, size, alignment,
3458 3459 3460
					       obj->cache_level,
					       start, end,
					       flags);
3461 3462
		if (ret == 0)
			goto search_free;
3463

3464
		goto err_free_vma;
3465
	}
3466
	if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
B
Ben Widawsky 已提交
3467
		ret = -EINVAL;
3468
		goto err_remove_node;
3469 3470
	}

3471
	trace_i915_vma_bind(vma, flags);
3472
	ret = i915_vma_bind(vma, obj->cache_level, flags);
3473
	if (ret)
I
Imre Deak 已提交
3474
		goto err_remove_node;
3475

3476
	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
B
Ben Widawsky 已提交
3477
	list_add_tail(&vma->mm_list, &vm->inactive_list);
3478

3479
	return vma;
B
Ben Widawsky 已提交
3480

3481
err_remove_node:
3482
	drm_mm_remove_node(&vma->node);
3483
err_free_vma:
B
Ben Widawsky 已提交
3484
	i915_gem_vma_destroy(vma);
3485
	vma = ERR_PTR(ret);
3486
err_unpin:
B
Ben Widawsky 已提交
3487
	i915_gem_object_unpin_pages(obj);
3488
	return vma;
3489 3490
}

3491
bool
3492 3493
i915_gem_clflush_object(struct drm_i915_gem_object *obj,
			bool force)
3494 3495 3496 3497 3498
{
	/* If we don't have a page list set up, then we're not pinned
	 * to GPU, and we can ignore the cache flush because it'll happen
	 * again at bind time.
	 */
3499
	if (obj->pages == NULL)
3500
		return false;
3501

3502 3503 3504 3505
	/*
	 * Stolen memory is always coherent with the GPU as it is explicitly
	 * marked as wc by the system, or the system is cache-coherent.
	 */
3506
	if (obj->stolen || obj->phys_handle)
3507
		return false;
3508

3509 3510 3511 3512 3513 3514 3515 3516
	/* If the GPU is snooping the contents of the CPU cache,
	 * we do not need to manually clear the CPU cache lines.  However,
	 * the caches are only snooped when the render cache is
	 * flushed/invalidated.  As we always have to emit invalidations
	 * and flushes when moving into and out of the RENDER domain, correct
	 * snooping behaviour occurs naturally as the result of our domain
	 * tracking.
	 */
3517 3518
	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
		obj->cache_dirty = true;
3519
		return false;
3520
	}
3521

C
Chris Wilson 已提交
3522
	trace_i915_gem_object_clflush(obj);
3523
	drm_clflush_sg(obj->pages);
3524
	obj->cache_dirty = false;
3525 3526

	return true;
3527 3528 3529 3530
}

/** Flushes the GTT write domain for the object if it's dirty. */
static void
3531
i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3532
{
C
Chris Wilson 已提交
3533 3534
	uint32_t old_write_domain;

3535
	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3536 3537
		return;

3538
	/* No actual flushing is required for the GTT write domain.  Writes
3539 3540
	 * to it immediately go to main memory as far as we know, so there's
	 * no chipset flush.  It also doesn't land in render cache.
3541 3542 3543 3544
	 *
	 * However, we do have to enforce the order so that all writes through
	 * the GTT land before any writes to the device, such as updates to
	 * the GATT itself.
3545
	 */
3546 3547
	wmb();

3548 3549
	old_write_domain = obj->base.write_domain;
	obj->base.write_domain = 0;
C
Chris Wilson 已提交
3550

3551
	intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3552

C
Chris Wilson 已提交
3553
	trace_i915_gem_object_change_domain(obj,
3554
					    obj->base.read_domains,
C
Chris Wilson 已提交
3555
					    old_write_domain);
3556 3557 3558 3559
}

/** Flushes the CPU write domain for the object if it's dirty. */
static void
3560
i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3561
{
C
Chris Wilson 已提交
3562
	uint32_t old_write_domain;
3563

3564
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3565 3566
		return;

3567
	if (i915_gem_clflush_object(obj, obj->pin_display))
3568 3569
		i915_gem_chipset_flush(obj->base.dev);

3570 3571
	old_write_domain = obj->base.write_domain;
	obj->base.write_domain = 0;
C
Chris Wilson 已提交
3572

3573
	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3574

C
Chris Wilson 已提交
3575
	trace_i915_gem_object_change_domain(obj,
3576
					    obj->base.read_domains,
C
Chris Wilson 已提交
3577
					    old_write_domain);
3578 3579
}

3580 3581 3582 3583 3584 3585
/**
 * Moves a single object to the GTT read, and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
J
Jesse Barnes 已提交
3586
int
3587
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3588
{
C
Chris Wilson 已提交
3589
	uint32_t old_write_domain, old_read_domains;
3590
	struct i915_vma *vma;
3591
	int ret;
3592

3593 3594 3595
	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		return 0;

3596
	ret = i915_gem_object_wait_rendering(obj, !write);
3597 3598 3599
	if (ret)
		return ret;

3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611
	/* Flush and acquire obj->pages so that we are coherent through
	 * direct access in memory with previous cached writes through
	 * shmemfs and that our cache domain tracking remains valid.
	 * For example, if the obj->filp was moved to swap without us
	 * being notified and releasing the pages, we would mistakenly
	 * continue to assume that the obj remained out of the CPU cached
	 * domain.
	 */
	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

3612
	i915_gem_object_flush_cpu_write_domain(obj);
C
Chris Wilson 已提交
3613

3614 3615 3616 3617 3618 3619 3620
	/* Serialise direct access to this object with the barriers for
	 * coherent writes from the GPU, by effectively invalidating the
	 * GTT domain upon first access.
	 */
	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		mb();

3621 3622
	old_write_domain = obj->base.write_domain;
	old_read_domains = obj->base.read_domains;
C
Chris Wilson 已提交
3623

3624 3625 3626
	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3627 3628
	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3629
	if (write) {
3630 3631 3632
		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		obj->dirty = 1;
3633 3634
	}

C
Chris Wilson 已提交
3635 3636 3637 3638
	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);

3639
	/* And bump the LRU for this access */
3640 3641
	vma = i915_gem_obj_to_ggtt(obj);
	if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
3642
		list_move_tail(&vma->mm_list,
3643
			       &to_i915(obj->base.dev)->gtt.base.inactive_list);
3644

3645 3646 3647
	return 0;
}

3648 3649 3650
int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
				    enum i915_cache_level cache_level)
{
3651
	struct drm_device *dev = obj->base.dev;
3652
	struct i915_vma *vma, *next;
3653
	int ret = 0;
3654 3655

	if (obj->cache_level == cache_level)
3656
		goto out;
3657

B
Ben Widawsky 已提交
3658
	if (i915_gem_obj_is_pinned(obj)) {
3659 3660 3661 3662
		DRM_DEBUG("can not change the cache level of pinned objects\n");
		return -EBUSY;
	}

3663
	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
3664
		if (!i915_gem_valid_gtt_space(vma, cache_level)) {
3665
			ret = i915_vma_unbind(vma);
3666 3667 3668
			if (ret)
				return ret;
		}
3669 3670
	}

3671
	if (i915_gem_obj_bound_any(obj)) {
3672
		ret = i915_gem_object_wait_rendering(obj, false);
3673 3674 3675 3676 3677 3678 3679 3680 3681
		if (ret)
			return ret;

		i915_gem_object_finish_gtt(obj);

		/* Before SandyBridge, you could not use tiling or fence
		 * registers with snooped memory, so relinquish any fences
		 * currently pointing to our region in the aperture.
		 */
3682
		if (INTEL_INFO(dev)->gen < 6) {
3683 3684 3685 3686 3687
			ret = i915_gem_object_put_fence(obj);
			if (ret)
				return ret;
		}

3688
		list_for_each_entry(vma, &obj->vma_list, vma_link)
3689 3690
			if (drm_mm_node_allocated(&vma->node)) {
				ret = i915_vma_bind(vma, cache_level,
3691
						    PIN_UPDATE);
3692 3693 3694
				if (ret)
					return ret;
			}
3695 3696
	}

3697 3698 3699 3700
	list_for_each_entry(vma, &obj->vma_list, vma_link)
		vma->node.color = cache_level;
	obj->cache_level = cache_level;

3701
out:
3702 3703 3704 3705 3706
	if (obj->cache_dirty &&
	    obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
	    cpu_write_needs_clflush(obj)) {
		if (i915_gem_clflush_object(obj, true))
			i915_gem_chipset_flush(obj->base.dev);
3707 3708 3709 3710 3711
	}

	return 0;
}

B
Ben Widawsky 已提交
3712 3713
int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3714
{
B
Ben Widawsky 已提交
3715
	struct drm_i915_gem_caching *args = data;
3716 3717 3718
	struct drm_i915_gem_object *obj;

	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3719 3720
	if (&obj->base == NULL)
		return -ENOENT;
3721

3722 3723 3724 3725 3726 3727
	switch (obj->cache_level) {
	case I915_CACHE_LLC:
	case I915_CACHE_L3_LLC:
		args->caching = I915_CACHING_CACHED;
		break;

3728 3729 3730 3731
	case I915_CACHE_WT:
		args->caching = I915_CACHING_DISPLAY;
		break;

3732 3733 3734 3735
	default:
		args->caching = I915_CACHING_NONE;
		break;
	}
3736

3737 3738
	drm_gem_object_unreference_unlocked(&obj->base);
	return 0;
3739 3740
}

B
Ben Widawsky 已提交
3741 3742
int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3743
{
B
Ben Widawsky 已提交
3744
	struct drm_i915_gem_caching *args = data;
3745 3746 3747 3748
	struct drm_i915_gem_object *obj;
	enum i915_cache_level level;
	int ret;

B
Ben Widawsky 已提交
3749 3750
	switch (args->caching) {
	case I915_CACHING_NONE:
3751 3752
		level = I915_CACHE_NONE;
		break;
B
Ben Widawsky 已提交
3753
	case I915_CACHING_CACHED:
3754 3755 3756 3757 3758 3759 3760 3761 3762
		/*
		 * Due to a HW issue on BXT A stepping, GPU stores via a
		 * snooped mapping may leave stale data in a corresponding CPU
		 * cacheline, whereas normally such cachelines would get
		 * invalidated.
		 */
		if (IS_BROXTON(dev) && INTEL_REVID(dev) < BXT_REVID_B0)
			return -ENODEV;

3763 3764
		level = I915_CACHE_LLC;
		break;
3765 3766 3767
	case I915_CACHING_DISPLAY:
		level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
		break;
3768 3769 3770 3771
	default:
		return -EINVAL;
	}

B
Ben Widawsky 已提交
3772 3773 3774 3775
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
	if (&obj->base == NULL) {
		ret = -ENOENT;
		goto unlock;
	}

	ret = i915_gem_object_set_cache_level(obj, level);

	drm_gem_object_unreference(&obj->base);
unlock:
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

3790
/*
3791 3792 3793
 * Prepare buffer for display plane (scanout, cursors, etc).
 * Can be called from an uninterruptible phase (modesetting) and allows
 * any flushes to be pipelined (for pageflips).
3794 3795
 */
int
3796 3797
i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
				     u32 alignment,
3798
				     struct intel_engine_cs *pipelined,
3799
				     struct drm_i915_gem_request **pipelined_request,
3800
				     const struct i915_ggtt_view *view)
3801
{
3802
	u32 old_read_domains, old_write_domain;
3803 3804
	int ret;

3805
	ret = i915_gem_object_sync(obj, pipelined, pipelined_request);
3806 3807
	if (ret)
		return ret;
3808

3809 3810 3811
	/* Mark the pin_display early so that we account for the
	 * display coherency whilst setting up the cache domains.
	 */
3812
	obj->pin_display++;
3813

3814 3815 3816 3817 3818 3819 3820 3821 3822
	/* The display engine is not coherent with the LLC cache on gen6.  As
	 * a result, we make sure that the pinning that is about to occur is
	 * done with uncached PTEs. This is lowest common denominator for all
	 * chipsets.
	 *
	 * However for gen6+, we could do better by using the GFDT bit instead
	 * of uncaching, which would allow us to flush all the LLC-cached data
	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
	 */
3823 3824
	ret = i915_gem_object_set_cache_level(obj,
					      HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3825
	if (ret)
3826
		goto err_unpin_display;
3827

3828 3829 3830 3831
	/* As the user may map the buffer once pinned in the display plane
	 * (e.g. libkms for the bootup splash), we have to ensure that we
	 * always use map_and_fenceable for all scanout buffers.
	 */
3832 3833 3834
	ret = i915_gem_object_ggtt_pin(obj, view, alignment,
				       view->type == I915_GGTT_VIEW_NORMAL ?
				       PIN_MAPPABLE : 0);
3835
	if (ret)
3836
		goto err_unpin_display;
3837

3838
	i915_gem_object_flush_cpu_write_domain(obj);
3839

3840
	old_write_domain = obj->base.write_domain;
3841
	old_read_domains = obj->base.read_domains;
3842 3843 3844 3845

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3846
	obj->base.write_domain = 0;
3847
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3848 3849 3850

	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
3851
					    old_write_domain);
3852 3853

	return 0;
3854 3855

err_unpin_display:
3856
	obj->pin_display--;
3857 3858 3859 3860
	return ret;
}

void
3861 3862
i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
					 const struct i915_ggtt_view *view)
3863
{
3864 3865 3866
	if (WARN_ON(obj->pin_display == 0))
		return;

3867 3868
	i915_gem_object_ggtt_unpin_view(obj, view);

3869
	obj->pin_display--;
3870 3871
}

3872 3873 3874 3875 3876 3877
/**
 * Moves a single object to the CPU read, and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
3878
int
3879
i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3880
{
C
Chris Wilson 已提交
3881
	uint32_t old_write_domain, old_read_domains;
3882 3883
	int ret;

3884 3885 3886
	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		return 0;

3887
	ret = i915_gem_object_wait_rendering(obj, !write);
3888 3889 3890
	if (ret)
		return ret;

3891
	i915_gem_object_flush_gtt_write_domain(obj);
3892

3893 3894
	old_write_domain = obj->base.write_domain;
	old_read_domains = obj->base.read_domains;
C
Chris Wilson 已提交
3895

3896
	/* Flush the CPU cache if it's still invalid. */
3897
	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3898
		i915_gem_clflush_object(obj, false);
3899

3900
		obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3901 3902 3903 3904 3905
	}

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3906
	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3907 3908 3909 3910 3911

	/* If we're writing through the CPU, then the GPU read domains will
	 * need to be invalidated at next use.
	 */
	if (write) {
3912 3913
		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3914
	}
3915

C
Chris Wilson 已提交
3916 3917 3918 3919
	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);

3920 3921 3922
	return 0;
}

3923 3924 3925
/* Throttle our rendering by waiting until the ring has completed our requests
 * emitted over 20 msec ago.
 *
3926 3927 3928 3929
 * Note that if we were to use the current jiffies each time around the loop,
 * we wouldn't escape the function with any frames outstanding if the time to
 * render a frame was over 20ms.
 *
3930 3931 3932
 * This should get us reasonable parallelism between CPU and GPU but also
 * relatively low latency when blocking on a particular request to finish.
 */
3933
static int
3934
i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3935
{
3936 3937
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_file_private *file_priv = file->driver_priv;
3938
	unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
3939
	struct drm_i915_gem_request *request, *target = NULL;
3940
	unsigned reset_counter;
3941
	int ret;
3942

3943 3944 3945 3946 3947 3948 3949
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
	if (ret)
		return ret;

	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
	if (ret)
		return ret;
3950

3951
	spin_lock(&file_priv->mm.lock);
3952
	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3953 3954
		if (time_after_eq(request->emitted_jiffies, recent_enough))
			break;
3955

3956 3957 3958 3959 3960 3961 3962
		/*
		 * Note that the request might not have been submitted yet.
		 * In which case emitted_jiffies will be zero.
		 */
		if (!request->emitted_jiffies)
			continue;

3963
		target = request;
3964
	}
3965
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3966 3967
	if (target)
		i915_gem_request_reference(target);
3968
	spin_unlock(&file_priv->mm.lock);
3969

3970
	if (target == NULL)
3971
		return 0;
3972

3973
	ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
3974 3975
	if (ret == 0)
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3976

3977
	i915_gem_request_unreference__unlocked(target);
3978

3979 3980 3981
	return ret;
}

3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000
static bool
i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
{
	struct drm_i915_gem_object *obj = vma->obj;

	if (alignment &&
	    vma->node.start & (alignment - 1))
		return true;

	if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
		return true;

	if (flags & PIN_OFFSET_BIAS &&
	    vma->node.start < (flags & PIN_OFFSET_MASK))
		return true;

	return false;
}

4001 4002 4003 4004 4005 4006
static int
i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
		       struct i915_address_space *vm,
		       const struct i915_ggtt_view *ggtt_view,
		       uint32_t alignment,
		       uint64_t flags)
4007
{
4008
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4009
	struct i915_vma *vma;
4010
	unsigned bound;
4011 4012
	int ret;

4013 4014 4015
	if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
		return -ENODEV;

4016
	if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
4017
		return -EINVAL;
4018

4019 4020 4021
	if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
		return -EINVAL;

4022 4023 4024 4025 4026 4027 4028 4029 4030
	if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
		return -EINVAL;

	vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
			  i915_gem_obj_to_vma(obj, vm);

	if (IS_ERR(vma))
		return PTR_ERR(vma);

4031
	if (vma) {
B
Ben Widawsky 已提交
4032 4033 4034
		if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
			return -EBUSY;

4035
		if (i915_vma_misplaced(vma, alignment, flags)) {
B
Ben Widawsky 已提交
4036
			WARN(vma->pin_count,
4037
			     "bo is already pinned in %s with incorrect alignment:"
4038
			     " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
4039
			     " obj->map_and_fenceable=%d\n",
4040
			     ggtt_view ? "ggtt" : "ppgtt",
4041 4042
			     upper_32_bits(vma->node.start),
			     lower_32_bits(vma->node.start),
4043
			     alignment,
4044
			     !!(flags & PIN_MAPPABLE),
4045
			     obj->map_and_fenceable);
4046
			ret = i915_vma_unbind(vma);
4047 4048
			if (ret)
				return ret;
4049 4050

			vma = NULL;
4051 4052 4053
		}
	}

4054
	bound = vma ? vma->bound : 0;
4055
	if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
4056 4057
		vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
						 flags);
4058 4059
		if (IS_ERR(vma))
			return PTR_ERR(vma);
4060 4061
	} else {
		ret = i915_vma_bind(vma, obj->cache_level, flags);
4062 4063 4064
		if (ret)
			return ret;
	}
4065

4066 4067
	if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
	    (bound ^ vma->bound) & GLOBAL_BIND) {
4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081
		bool mappable, fenceable;
		u32 fence_size, fence_alignment;

		fence_size = i915_gem_get_gtt_size(obj->base.dev,
						   obj->base.size,
						   obj->tiling_mode);
		fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
							     obj->base.size,
							     obj->tiling_mode,
							     true);

		fenceable = (vma->node.size == fence_size &&
			     (vma->node.start & (fence_alignment - 1)) == 0);

4082
		mappable = (vma->node.start + fence_size <=
4083 4084 4085 4086
			    dev_priv->gtt.mappable_end);

		obj->map_and_fenceable = mappable && fenceable;

4087 4088
		WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
	}
4089

4090
	vma->pin_count++;
4091 4092 4093
	return 0;
}

4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114
int
i915_gem_object_pin(struct drm_i915_gem_object *obj,
		    struct i915_address_space *vm,
		    uint32_t alignment,
		    uint64_t flags)
{
	return i915_gem_object_do_pin(obj, vm,
				      i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
				      alignment, flags);
}

int
i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
			 const struct i915_ggtt_view *view,
			 uint32_t alignment,
			 uint64_t flags)
{
	if (WARN_ONCE(!view, "no view specified"))
		return -EINVAL;

	return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view,
4115
				      alignment, flags | PIN_GLOBAL);
4116 4117
}

4118
void
4119 4120
i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
				const struct i915_ggtt_view *view)
4121
{
4122
	struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
4123

B
Ben Widawsky 已提交
4124
	BUG_ON(!vma);
4125
	WARN_ON(vma->pin_count == 0);
4126
	WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
B
Ben Widawsky 已提交
4127

4128
	--vma->pin_count;
4129 4130 4131 4132
}

int
i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4133
		    struct drm_file *file)
4134 4135
{
	struct drm_i915_gem_busy *args = data;
4136
	struct drm_i915_gem_object *obj;
4137 4138
	int ret;

4139
	ret = i915_mutex_lock_interruptible(dev);
4140
	if (ret)
4141
		return ret;
4142

4143
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4144
	if (&obj->base == NULL) {
4145 4146
		ret = -ENOENT;
		goto unlock;
4147
	}
4148

4149 4150 4151 4152
	/* Count all active objects as busy, even if they are currently not used
	 * by the gpu. Users of this interface expect objects to eventually
	 * become non-busy without any further actions, therefore emit any
	 * necessary flushes here.
4153
	 */
4154
	ret = i915_gem_object_flush_active(obj);
4155 4156
	if (ret)
		goto unref;
4157

4158 4159 4160 4161
	BUILD_BUG_ON(I915_NUM_RINGS > 16);
	args->busy = obj->active << 16;
	if (obj->last_write_req)
		args->busy |= obj->last_write_req->ring->id;
4162

4163
unref:
4164
	drm_gem_object_unreference(&obj->base);
4165
unlock:
4166
	mutex_unlock(&dev->struct_mutex);
4167
	return ret;
4168 4169 4170 4171 4172 4173
}

int
i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file_priv)
{
4174
	return i915_gem_ring_throttle(dev, file_priv);
4175 4176
}

4177 4178 4179 4180
int
i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
4181
	struct drm_i915_private *dev_priv = dev->dev_private;
4182
	struct drm_i915_gem_madvise *args = data;
4183
	struct drm_i915_gem_object *obj;
4184
	int ret;
4185 4186 4187 4188 4189 4190 4191 4192 4193

	switch (args->madv) {
	case I915_MADV_DONTNEED:
	case I915_MADV_WILLNEED:
	    break;
	default:
	    return -EINVAL;
	}

4194 4195 4196 4197
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

4198
	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4199
	if (&obj->base == NULL) {
4200 4201
		ret = -ENOENT;
		goto unlock;
4202 4203
	}

B
Ben Widawsky 已提交
4204
	if (i915_gem_obj_is_pinned(obj)) {
4205 4206
		ret = -EINVAL;
		goto out;
4207 4208
	}

4209 4210 4211 4212 4213 4214 4215 4216 4217
	if (obj->pages &&
	    obj->tiling_mode != I915_TILING_NONE &&
	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
		if (obj->madv == I915_MADV_WILLNEED)
			i915_gem_object_unpin_pages(obj);
		if (args->madv == I915_MADV_WILLNEED)
			i915_gem_object_pin_pages(obj);
	}

4218 4219
	if (obj->madv != __I915_MADV_PURGED)
		obj->madv = args->madv;
4220

C
Chris Wilson 已提交
4221
	/* if the object is no longer attached, discard its backing storage */
4222
	if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
4223 4224
		i915_gem_object_truncate(obj);

4225
	args->retained = obj->madv != __I915_MADV_PURGED;
C
Chris Wilson 已提交
4226

4227
out:
4228
	drm_gem_object_unreference(&obj->base);
4229
unlock:
4230
	mutex_unlock(&dev->struct_mutex);
4231
	return ret;
4232 4233
}

4234 4235
void i915_gem_object_init(struct drm_i915_gem_object *obj,
			  const struct drm_i915_gem_object_ops *ops)
4236
{
4237 4238
	int i;

4239
	INIT_LIST_HEAD(&obj->global_list);
4240 4241
	for (i = 0; i < I915_NUM_RINGS; i++)
		INIT_LIST_HEAD(&obj->ring_list[i]);
4242
	INIT_LIST_HEAD(&obj->obj_exec_link);
B
Ben Widawsky 已提交
4243
	INIT_LIST_HEAD(&obj->vma_list);
4244
	INIT_LIST_HEAD(&obj->batch_pool_link);
4245

4246 4247
	obj->ops = ops;

4248 4249 4250 4251 4252 4253
	obj->fence_reg = I915_FENCE_REG_NONE;
	obj->madv = I915_MADV_WILLNEED;

	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
}

4254 4255 4256 4257 4258
static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
	.get_pages = i915_gem_object_get_pages_gtt,
	.put_pages = i915_gem_object_put_pages_gtt,
};

4259 4260
struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
						  size_t size)
4261
{
4262
	struct drm_i915_gem_object *obj;
4263
	struct address_space *mapping;
D
Daniel Vetter 已提交
4264
	gfp_t mask;
4265

4266
	obj = i915_gem_object_alloc(dev);
4267 4268
	if (obj == NULL)
		return NULL;
4269

4270
	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4271
		i915_gem_object_free(obj);
4272 4273
		return NULL;
	}
4274

4275 4276 4277 4278 4279 4280 4281
	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
	if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
		/* 965gm cannot relocate objects above 4GiB. */
		mask &= ~__GFP_HIGHMEM;
		mask |= __GFP_DMA32;
	}

A
Al Viro 已提交
4282
	mapping = file_inode(obj->base.filp)->i_mapping;
4283
	mapping_set_gfp_mask(mapping, mask);
4284

4285
	i915_gem_object_init(obj, &i915_gem_object_ops);
4286

4287 4288
	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4289

4290 4291
	if (HAS_LLC(dev)) {
		/* On some devices, we can have the GPU use the LLC (the CPU
4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306
		 * cache) for about a 10% performance improvement
		 * compared to uncached.  Graphics requests other than
		 * display scanout are coherent with the CPU in
		 * accessing this cache.  This means in this mode we
		 * don't need to clflush on the CPU side, and on the
		 * GPU side we only need to flush internal caches to
		 * get data visible to the CPU.
		 *
		 * However, we maintain the display planes as UC, and so
		 * need to rebind when first used as such.
		 */
		obj->cache_level = I915_CACHE_LLC;
	} else
		obj->cache_level = I915_CACHE_NONE;

4307 4308
	trace_i915_gem_object_create(obj);

4309
	return obj;
4310 4311
}

4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335
static bool discard_backing_storage(struct drm_i915_gem_object *obj)
{
	/* If we are the last user of the backing storage (be it shmemfs
	 * pages or stolen etc), we know that the pages are going to be
	 * immediately released. In this case, we can then skip copying
	 * back the contents from the GPU.
	 */

	if (obj->madv != I915_MADV_WILLNEED)
		return false;

	if (obj->base.filp == NULL)
		return true;

	/* At first glance, this looks racy, but then again so would be
	 * userspace racing mmap against close. However, the first external
	 * reference to the filp can only be obtained through the
	 * i915_gem_mmap_ioctl() which safeguards us against the user
	 * acquiring such a reference whilst we are in the middle of
	 * freeing the object.
	 */
	return atomic_long_read(&obj->base.filp->f_count) == 1;
}

4336
void i915_gem_free_object(struct drm_gem_object *gem_obj)
4337
{
4338
	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4339
	struct drm_device *dev = obj->base.dev;
4340
	struct drm_i915_private *dev_priv = dev->dev_private;
4341
	struct i915_vma *vma, *next;
4342

4343 4344
	intel_runtime_pm_get(dev_priv);

4345 4346
	trace_i915_gem_object_destroy(obj);

4347
	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
B
Ben Widawsky 已提交
4348 4349 4350 4351
		int ret;

		vma->pin_count = 0;
		ret = i915_vma_unbind(vma);
4352 4353
		if (WARN_ON(ret == -ERESTARTSYS)) {
			bool was_interruptible;
4354

4355 4356
			was_interruptible = dev_priv->mm.interruptible;
			dev_priv->mm.interruptible = false;
4357

4358
			WARN_ON(i915_vma_unbind(vma));
4359

4360 4361
			dev_priv->mm.interruptible = was_interruptible;
		}
4362 4363
	}

B
Ben Widawsky 已提交
4364 4365 4366 4367 4368
	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
	 * before progressing. */
	if (obj->stolen)
		i915_gem_object_unpin_pages(obj);

4369 4370
	WARN_ON(obj->frontbuffer_bits);

4371 4372 4373 4374 4375
	if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
	    obj->tiling_mode != I915_TILING_NONE)
		i915_gem_object_unpin_pages(obj);

B
Ben Widawsky 已提交
4376 4377
	if (WARN_ON(obj->pages_pin_count))
		obj->pages_pin_count = 0;
4378
	if (discard_backing_storage(obj))
4379
		obj->madv = I915_MADV_DONTNEED;
4380
	i915_gem_object_put_pages(obj);
4381
	i915_gem_object_free_mmap_offset(obj);
4382

4383 4384
	BUG_ON(obj->pages);

4385 4386
	if (obj->base.import_attach)
		drm_prime_gem_destroy(&obj->base, NULL);
4387

4388 4389 4390
	if (obj->ops->release)
		obj->ops->release(obj);

4391 4392
	drm_gem_object_release(&obj->base);
	i915_gem_info_remove_obj(dev_priv, obj->base.size);
4393

4394
	kfree(obj->bit_17);
4395
	i915_gem_object_free(obj);
4396 4397

	intel_runtime_pm_put(dev_priv);
4398 4399
}

4400 4401
struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
				     struct i915_address_space *vm)
4402 4403
{
	struct i915_vma *vma;
4404 4405 4406 4407 4408
	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		if (i915_is_ggtt(vma->vm) &&
		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
			continue;
		if (vma->vm == vm)
4409
			return vma;
4410 4411 4412 4413 4414 4415 4416 4417 4418
	}
	return NULL;
}

struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
					   const struct i915_ggtt_view *view)
{
	struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
	struct i915_vma *vma;
4419

4420 4421 4422 4423
	if (WARN_ONCE(!view, "no view specified"))
		return ERR_PTR(-EINVAL);

	list_for_each_entry(vma, &obj->vma_list, vma_link)
4424 4425
		if (vma->vm == ggtt &&
		    i915_ggtt_view_equal(&vma->ggtt_view, view))
4426
			return vma;
4427 4428 4429
	return NULL;
}

B
Ben Widawsky 已提交
4430 4431
void i915_gem_vma_destroy(struct i915_vma *vma)
{
4432
	struct i915_address_space *vm = NULL;
B
Ben Widawsky 已提交
4433
	WARN_ON(vma->node.allocated);
4434 4435 4436 4437 4438

	/* Keep the vma as a placeholder in the execbuffer reservation lists */
	if (!list_empty(&vma->exec_list))
		return;

4439 4440
	vm = vma->vm;

4441 4442
	if (!i915_is_ggtt(vm))
		i915_ppgtt_put(i915_vm_to_ppgtt(vm));
4443

4444
	list_del(&vma->vma_link);
4445

4446
	kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
B
Ben Widawsky 已提交
4447 4448
}

4449 4450 4451 4452
static void
i915_gem_stop_ringbuffers(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
4453
	struct intel_engine_cs *ring;
4454 4455 4456
	int i;

	for_each_ring(ring, dev_priv, i)
4457
		dev_priv->gt.stop_ring(ring);
4458 4459
}

4460
int
4461
i915_gem_suspend(struct drm_device *dev)
4462
{
4463
	struct drm_i915_private *dev_priv = dev->dev_private;
4464
	int ret = 0;
4465

4466
	mutex_lock(&dev->struct_mutex);
4467
	ret = i915_gpu_idle(dev);
4468
	if (ret)
4469
		goto err;
4470

4471
	i915_gem_retire_requests(dev);
4472

4473
	i915_gem_stop_ringbuffers(dev);
4474 4475
	mutex_unlock(&dev->struct_mutex);

4476
	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4477
	cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4478
	flush_delayed_work(&dev_priv->mm.idle_work);
4479

4480 4481 4482 4483 4484
	/* Assert that we sucessfully flushed all the work and
	 * reset the GPU back to its idle, low power state.
	 */
	WARN_ON(dev_priv->mm.busy);

4485
	return 0;
4486 4487 4488 4489

err:
	mutex_unlock(&dev->struct_mutex);
	return ret;
4490 4491
}

4492
int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
B
Ben Widawsky 已提交
4493
{
4494
	struct intel_engine_cs *ring = req->ring;
4495
	struct drm_device *dev = ring->dev;
4496
	struct drm_i915_private *dev_priv = dev->dev_private;
4497 4498
	u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
	u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
4499
	int i, ret;
B
Ben Widawsky 已提交
4500

4501
	if (!HAS_L3_DPF(dev) || !remap_info)
4502
		return 0;
B
Ben Widawsky 已提交
4503

4504
	ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
4505 4506
	if (ret)
		return ret;
B
Ben Widawsky 已提交
4507

4508 4509 4510 4511 4512
	/*
	 * Note: We do not worry about the concurrent register cacheline hang
	 * here because no other code should access these registers other than
	 * at initialization time.
	 */
B
Ben Widawsky 已提交
4513
	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
4514 4515 4516
		intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
		intel_ring_emit(ring, reg_base + i);
		intel_ring_emit(ring, remap_info[i/4]);
B
Ben Widawsky 已提交
4517 4518
	}

4519
	intel_ring_advance(ring);
B
Ben Widawsky 已提交
4520

4521
	return ret;
B
Ben Widawsky 已提交
4522 4523
}

4524 4525
void i915_gem_init_swizzling(struct drm_device *dev)
{
4526
	struct drm_i915_private *dev_priv = dev->dev_private;
4527

4528
	if (INTEL_INFO(dev)->gen < 5 ||
4529 4530 4531 4532 4533 4534
	    dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		return;

	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
				 DISP_TILE_SURFACE_SWIZZLING);

4535 4536 4537
	if (IS_GEN5(dev))
		return;

4538 4539
	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
	if (IS_GEN6(dev))
4540
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4541
	else if (IS_GEN7(dev))
4542
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
B
Ben Widawsky 已提交
4543 4544
	else if (IS_GEN8(dev))
		I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4545 4546
	else
		BUG();
4547
}
D
Daniel Vetter 已提交
4548

4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564
static bool
intel_enable_blt(struct drm_device *dev)
{
	if (!HAS_BLT(dev))
		return false;

	/* The blitter was dysfunctional on early prototypes */
	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		DRM_INFO("BLT not supported on this pre-production hardware;"
			 " graphics performance will be degraded.\n");
		return false;
	}

	return true;
}

4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591
static void init_unused_ring(struct drm_device *dev, u32 base)
{
	struct drm_i915_private *dev_priv = dev->dev_private;

	I915_WRITE(RING_CTL(base), 0);
	I915_WRITE(RING_HEAD(base), 0);
	I915_WRITE(RING_TAIL(base), 0);
	I915_WRITE(RING_START(base), 0);
}

static void init_unused_rings(struct drm_device *dev)
{
	if (IS_I830(dev)) {
		init_unused_ring(dev, PRB1_BASE);
		init_unused_ring(dev, SRB0_BASE);
		init_unused_ring(dev, SRB1_BASE);
		init_unused_ring(dev, SRB2_BASE);
		init_unused_ring(dev, SRB3_BASE);
	} else if (IS_GEN2(dev)) {
		init_unused_ring(dev, SRB0_BASE);
		init_unused_ring(dev, SRB1_BASE);
	} else if (IS_GEN3(dev)) {
		init_unused_ring(dev, PRB1_BASE);
		init_unused_ring(dev, PRB2_BASE);
	}
}

4592
int i915_gem_init_rings(struct drm_device *dev)
4593
{
4594
	struct drm_i915_private *dev_priv = dev->dev_private;
4595
	int ret;
4596

4597
	ret = intel_init_render_ring_buffer(dev);
4598
	if (ret)
4599
		return ret;
4600 4601

	if (HAS_BSD(dev)) {
4602
		ret = intel_init_bsd_ring_buffer(dev);
4603 4604
		if (ret)
			goto cleanup_render_ring;
4605
	}
4606

4607
	if (intel_enable_blt(dev)) {
4608 4609 4610 4611 4612
		ret = intel_init_blt_ring_buffer(dev);
		if (ret)
			goto cleanup_bsd_ring;
	}

B
Ben Widawsky 已提交
4613 4614 4615 4616 4617 4618
	if (HAS_VEBOX(dev)) {
		ret = intel_init_vebox_ring_buffer(dev);
		if (ret)
			goto cleanup_blt_ring;
	}

4619 4620 4621 4622 4623
	if (HAS_BSD2(dev)) {
		ret = intel_init_bsd2_ring_buffer(dev);
		if (ret)
			goto cleanup_vebox_ring;
	}
B
Ben Widawsky 已提交
4624

4625 4626
	return 0;

B
Ben Widawsky 已提交
4627 4628
cleanup_vebox_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641
cleanup_blt_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
cleanup_bsd_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
cleanup_render_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);

	return ret;
}

int
i915_gem_init_hw(struct drm_device *dev)
{
4642
	struct drm_i915_private *dev_priv = dev->dev_private;
D
Daniel Vetter 已提交
4643
	struct intel_engine_cs *ring;
4644
	int ret, i, j;
4645 4646 4647 4648

	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		return -EIO;

4649 4650 4651
	/* Double layer security blanket, see i915_gem_init() */
	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);

B
Ben Widawsky 已提交
4652
	if (dev_priv->ellc_size)
4653
		I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4654

4655 4656 4657
	if (IS_HASWELL(dev))
		I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
			   LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4658

4659
	if (HAS_PCH_NOP(dev)) {
4660 4661 4662 4663 4664 4665 4666 4667 4668
		if (IS_IVYBRIDGE(dev)) {
			u32 temp = I915_READ(GEN7_MSG_CTL);
			temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
			I915_WRITE(GEN7_MSG_CTL, temp);
		} else if (INTEL_INFO(dev)->gen >= 7) {
			u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
			temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
			I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
		}
4669 4670
	}

4671 4672
	i915_gem_init_swizzling(dev);

4673 4674 4675 4676 4677 4678 4679 4680
	/*
	 * At least 830 can leave some of the unused rings
	 * "active" (ie. head != tail) after resume which
	 * will prevent c3 entry. Makes sure all unused rings
	 * are totally idle.
	 */
	init_unused_rings(dev);

4681 4682
	BUG_ON(!dev_priv->ring[RCS].default_context);

4683 4684 4685 4686 4687 4688 4689
	ret = i915_ppgtt_init_hw(dev);
	if (ret) {
		DRM_ERROR("PPGTT enable HW failed %d\n", ret);
		goto out;
	}

	/* Need to do basic initialisation of all rings first: */
D
Daniel Vetter 已提交
4690 4691 4692
	for_each_ring(ring, dev_priv, i) {
		ret = ring->init_hw(ring);
		if (ret)
4693
			goto out;
D
Daniel Vetter 已提交
4694
	}
4695

4696
	/* We can't enable contexts until all firmware is loaded */
4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712
	if (HAS_GUC_UCODE(dev)) {
		ret = intel_guc_ucode_load(dev);
		if (ret) {
			/*
			 * If we got an error and GuC submission is enabled, map
			 * the error to -EIO so the GPU will be declared wedged.
			 * OTOH, if we didn't intend to use the GuC anyway, just
			 * discard the error and carry on.
			 */
			DRM_ERROR("Failed to initialize GuC, error %d%s\n", ret,
				  i915.enable_guc_submission ? "" :
				  " (ignored)");
			ret = i915.enable_guc_submission ? -EIO : 0;
			if (ret)
				goto out;
		}
4713 4714
	}

4715 4716 4717 4718 4719 4720 4721 4722
	/*
	 * Increment the next seqno by 0x100 so we have a visible break
	 * on re-initialisation
	 */
	ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
	if (ret)
		goto out;

4723 4724
	/* Now it is safe to go back round and do everything else: */
	for_each_ring(ring, dev_priv, i) {
4725 4726
		struct drm_i915_gem_request *req;

4727 4728
		WARN_ON(!ring->default_context);

4729 4730 4731 4732 4733 4734
		ret = i915_gem_request_alloc(ring, ring->default_context, &req);
		if (ret) {
			i915_gem_cleanup_ringbuffer(dev);
			goto out;
		}

4735 4736
		if (ring->id == RCS) {
			for (j = 0; j < NUM_L3_SLICES(dev); j++)
4737
				i915_gem_l3_remap(req, j);
4738
		}
4739

4740
		ret = i915_ppgtt_init_ring(req);
4741 4742
		if (ret && ret != -EIO) {
			DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret);
4743
			i915_gem_request_cancel(req);
4744 4745 4746
			i915_gem_cleanup_ringbuffer(dev);
			goto out;
		}
4747

4748
		ret = i915_gem_context_enable(req);
4749 4750
		if (ret && ret != -EIO) {
			DRM_ERROR("Context enable ring #%d failed %d\n", i, ret);
4751
			i915_gem_request_cancel(req);
4752 4753 4754
			i915_gem_cleanup_ringbuffer(dev);
			goto out;
		}
4755

4756
		i915_add_request_no_flush(req);
4757
	}
D
Daniel Vetter 已提交
4758

4759 4760
out:
	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4761
	return ret;
4762 4763
}

4764 4765 4766 4767 4768
int i915_gem_init(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

4769 4770 4771
	i915.enable_execlists = intel_sanitize_enable_execlists(dev,
			i915.enable_execlists);

4772
	mutex_lock(&dev->struct_mutex);
4773 4774 4775

	if (IS_VALLEYVIEW(dev)) {
		/* VLVA0 (potential hack), BIOS isn't actually waking us */
4776 4777 4778
		I915_WRITE(VLV_GTLC_WAKE_CTRL, VLV_GTLC_ALLOWWAKEREQ);
		if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) &
			      VLV_GTLC_ALLOWWAKEACK), 10))
4779 4780 4781
			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
	}

4782
	if (!i915.enable_execlists) {
4783
		dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
4784 4785 4786
		dev_priv->gt.init_rings = i915_gem_init_rings;
		dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
		dev_priv->gt.stop_ring = intel_stop_ring_buffer;
4787
	} else {
4788
		dev_priv->gt.execbuf_submit = intel_execlists_submission;
4789 4790 4791
		dev_priv->gt.init_rings = intel_logical_rings_init;
		dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
		dev_priv->gt.stop_ring = intel_logical_ring_stop;
4792 4793
	}

4794 4795 4796 4797 4798 4799 4800 4801
	/* This is just a security blanket to placate dragons.
	 * On some systems, we very sporadically observe that the first TLBs
	 * used by the CS may be stale, despite us poking the TLB reset. If
	 * we hold the forcewake during initialisation these problems
	 * just magically go away.
	 */
	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);

4802
	ret = i915_gem_init_userptr(dev);
4803 4804
	if (ret)
		goto out_unlock;
4805

4806
	i915_gem_init_global_gtt(dev);
4807

4808
	ret = i915_gem_context_init(dev);
4809 4810
	if (ret)
		goto out_unlock;
4811

D
Daniel Vetter 已提交
4812 4813
	ret = dev_priv->gt.init_rings(dev);
	if (ret)
4814
		goto out_unlock;
4815

4816
	ret = i915_gem_init_hw(dev);
4817 4818 4819 4820 4821 4822
	if (ret == -EIO) {
		/* Allow ring initialisation to fail by marking the GPU as
		 * wedged. But we only want to do this where the GPU is angry,
		 * for all other failure, such as an allocation failure, bail.
		 */
		DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4823
		atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
4824
		ret = 0;
4825
	}
4826 4827

out_unlock:
4828
	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4829
	mutex_unlock(&dev->struct_mutex);
4830

4831
	return ret;
4832 4833
}

4834 4835 4836
void
i915_gem_cleanup_ringbuffer(struct drm_device *dev)
{
4837
	struct drm_i915_private *dev_priv = dev->dev_private;
4838
	struct intel_engine_cs *ring;
4839
	int i;
4840

4841
	for_each_ring(ring, dev_priv, i)
4842
		dev_priv->gt.cleanup_ring(ring);
4843 4844 4845 4846 4847 4848 4849 4850

    if (i915.enable_execlists)
            /*
             * Neither the BIOS, ourselves or any other kernel
             * expects the system to be in execlists mode on startup,
             * so we need to reset the GPU back to legacy mode.
             */
            intel_gpu_reset(dev);
4851 4852
}

4853
static void
4854
init_ring_lists(struct intel_engine_cs *ring)
4855 4856 4857 4858 4859
{
	INIT_LIST_HEAD(&ring->active_list);
	INIT_LIST_HEAD(&ring->request_list);
}

4860 4861 4862
void
i915_gem_load(struct drm_device *dev)
{
4863
	struct drm_i915_private *dev_priv = dev->dev_private;
4864 4865
	int i;

4866
	dev_priv->objects =
4867 4868 4869 4870
		kmem_cache_create("i915_gem_object",
				  sizeof(struct drm_i915_gem_object), 0,
				  SLAB_HWCACHE_ALIGN,
				  NULL);
4871 4872 4873 4874 4875
	dev_priv->vmas =
		kmem_cache_create("i915_gem_vma",
				  sizeof(struct i915_vma), 0,
				  SLAB_HWCACHE_ALIGN,
				  NULL);
4876 4877 4878 4879 4880
	dev_priv->requests =
		kmem_cache_create("i915_gem_request",
				  sizeof(struct drm_i915_gem_request), 0,
				  SLAB_HWCACHE_ALIGN,
				  NULL);
4881

B
Ben Widawsky 已提交
4882
	INIT_LIST_HEAD(&dev_priv->vm_list);
4883
	INIT_LIST_HEAD(&dev_priv->context_list);
C
Chris Wilson 已提交
4884 4885
	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4886
	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4887 4888
	for (i = 0; i < I915_NUM_RINGS; i++)
		init_ring_lists(&dev_priv->ring[i]);
4889
	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4890
		INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4891 4892
	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
			  i915_gem_retire_work_handler);
4893 4894
	INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
			  i915_gem_idle_work_handler);
4895
	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4896

4897 4898
	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;

4899 4900 4901
	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		dev_priv->num_fence_regs = 32;
	else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4902 4903 4904 4905
		dev_priv->num_fence_regs = 16;
	else
		dev_priv->num_fence_regs = 8;

4906 4907 4908 4909
	if (intel_vgpu_active(dev))
		dev_priv->num_fence_regs =
				I915_READ(vgtif_reg(avail_rs.fence_num));

4910 4911 4912 4913 4914 4915 4916 4917
	/*
	 * Set initial sequence number for requests.
	 * Using this number allows the wraparound to happen early,
	 * catching any obvious problems.
	 */
	dev_priv->next_seqno = ((u32)~0 - 0x1100);
	dev_priv->last_seqno = ((u32)~0 - 0x1101);

4918
	/* Initialize fence registers to zero */
4919 4920
	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
	i915_gem_restore_fences(dev);
4921

4922
	i915_gem_detect_bit_6_swizzle(dev);
4923
	init_waitqueue_head(&dev_priv->pending_flip_queue);
4924

4925 4926
	dev_priv->mm.interruptible = true;

4927
	i915_gem_shrinker_init(dev_priv);
4928 4929

	mutex_init(&dev_priv->fb_tracking.lock);
4930
}
4931

4932
void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4933
{
4934
	struct drm_i915_file_private *file_priv = file->driver_priv;
4935 4936 4937 4938 4939

	/* Clean up our request list when the client is going away, so that
	 * later retire_requests won't dereference our soon-to-be-gone
	 * file_priv.
	 */
4940
	spin_lock(&file_priv->mm.lock);
4941 4942 4943 4944 4945 4946 4947 4948 4949
	while (!list_empty(&file_priv->mm.request_list)) {
		struct drm_i915_gem_request *request;

		request = list_first_entry(&file_priv->mm.request_list,
					   struct drm_i915_gem_request,
					   client_list);
		list_del(&request->client_list);
		request->file_priv = NULL;
	}
4950
	spin_unlock(&file_priv->mm.lock);
4951

4952
	if (!list_empty(&file_priv->rps.link)) {
4953
		spin_lock(&to_i915(dev)->rps.client_lock);
4954
		list_del(&file_priv->rps.link);
4955
		spin_unlock(&to_i915(dev)->rps.client_lock);
4956
	}
4957 4958 4959 4960 4961
}

int i915_gem_open(struct drm_device *dev, struct drm_file *file)
{
	struct drm_i915_file_private *file_priv;
4962
	int ret;
4963 4964 4965 4966 4967 4968 4969 4970 4971

	DRM_DEBUG_DRIVER("\n");

	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
	if (!file_priv)
		return -ENOMEM;

	file->driver_priv = file_priv;
	file_priv->dev_priv = dev->dev_private;
4972
	file_priv->file = file;
4973
	INIT_LIST_HEAD(&file_priv->rps.link);
4974 4975 4976 4977

	spin_lock_init(&file_priv->mm.lock);
	INIT_LIST_HEAD(&file_priv->mm.request_list);

4978 4979 4980
	ret = i915_gem_context_open(dev, file);
	if (ret)
		kfree(file_priv);
4981

4982
	return ret;
4983 4984
}

4985 4986
/**
 * i915_gem_track_fb - update frontbuffer tracking
4987 4988 4989
 * @old: current GEM buffer for the frontbuffer slots
 * @new: new GEM buffer for the frontbuffer slots
 * @frontbuffer_bits: bitmask of frontbuffer slots
4990 4991 4992 4993
 *
 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
 * from @old and setting them in @new. Both @old and @new can be NULL.
 */
4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010
void i915_gem_track_fb(struct drm_i915_gem_object *old,
		       struct drm_i915_gem_object *new,
		       unsigned frontbuffer_bits)
{
	if (old) {
		WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
		WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
		old->frontbuffer_bits &= ~frontbuffer_bits;
	}

	if (new) {
		WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
		WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
		new->frontbuffer_bits |= frontbuffer_bits;
	}
}

5011
/* All the new VM stuff */
5012 5013
u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
			struct i915_address_space *vm)
5014 5015 5016 5017
{
	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
	struct i915_vma *vma;

5018
	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5019 5020

	list_for_each_entry(vma, &o->vma_list, vma_link) {
5021 5022 5023 5024
		if (i915_is_ggtt(vma->vm) &&
		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
			continue;
		if (vma->vm == vm)
5025 5026
			return vma->node.start;
	}
5027

5028 5029
	WARN(1, "%s vma for this object not found.\n",
	     i915_is_ggtt(vm) ? "global" : "ppgtt");
5030 5031 5032
	return -1;
}

5033 5034
u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
				  const struct i915_ggtt_view *view)
5035
{
5036
	struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
5037 5038 5039
	struct i915_vma *vma;

	list_for_each_entry(vma, &o->vma_list, vma_link)
5040 5041
		if (vma->vm == ggtt &&
		    i915_ggtt_view_equal(&vma->ggtt_view, view))
5042 5043
			return vma->node.start;

5044
	WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064
	return -1;
}

bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
			struct i915_address_space *vm)
{
	struct i915_vma *vma;

	list_for_each_entry(vma, &o->vma_list, vma_link) {
		if (i915_is_ggtt(vma->vm) &&
		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
			continue;
		if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
			return true;
	}

	return false;
}

bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
5065
				  const struct i915_ggtt_view *view)
5066 5067 5068 5069 5070 5071
{
	struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
	struct i915_vma *vma;

	list_for_each_entry(vma, &o->vma_list, vma_link)
		if (vma->vm == ggtt &&
5072
		    i915_ggtt_view_equal(&vma->ggtt_view, view) &&
5073
		    drm_mm_node_allocated(&vma->node))
5074 5075 5076 5077 5078 5079 5080
			return true;

	return false;
}

bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
{
5081
	struct i915_vma *vma;
5082

5083 5084
	list_for_each_entry(vma, &o->vma_list, vma_link)
		if (drm_mm_node_allocated(&vma->node))
5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095
			return true;

	return false;
}

unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
				struct i915_address_space *vm)
{
	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
	struct i915_vma *vma;

5096
	WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5097 5098 5099

	BUG_ON(list_empty(&o->vma_list));

5100 5101 5102 5103
	list_for_each_entry(vma, &o->vma_list, vma_link) {
		if (i915_is_ggtt(vma->vm) &&
		    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
			continue;
5104 5105
		if (vma->vm == vm)
			return vma->node.size;
5106
	}
5107 5108 5109
	return 0;
}

5110
bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
5111 5112
{
	struct i915_vma *vma;
5113
	list_for_each_entry(vma, &obj->vma_list, vma_link)
5114 5115
		if (vma->pin_count > 0)
			return true;
5116

5117
	return false;
5118
}
5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158

/* Allocate a new GEM object and fill it with the supplied data */
struct drm_i915_gem_object *
i915_gem_object_create_from_data(struct drm_device *dev,
			         const void *data, size_t size)
{
	struct drm_i915_gem_object *obj;
	struct sg_table *sg;
	size_t bytes;
	int ret;

	obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE));
	if (IS_ERR_OR_NULL(obj))
		return obj;

	ret = i915_gem_object_set_to_cpu_domain(obj, true);
	if (ret)
		goto fail;

	ret = i915_gem_object_get_pages(obj);
	if (ret)
		goto fail;

	i915_gem_object_pin_pages(obj);
	sg = obj->pages;
	bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
	i915_gem_object_unpin_pages(obj);

	if (WARN_ON(bytes != size)) {
		DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
		ret = -EFAULT;
		goto fail;
	}

	return obj;

fail:
	drm_gem_object_unreference(&obj->base);
	return ERR_PTR(ret);
}