i915_gem.c 122.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright © 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

28
#include <drm/drmP.h>
29
#include <drm/drm_vma_manager.h>
30
#include <drm/i915_drm.h>
31
#include "i915_drv.h"
C
Chris Wilson 已提交
32
#include "i915_trace.h"
33
#include "intel_drv.h"
34
#include <linux/shmem_fs.h>
35
#include <linux/slab.h>
36
#include <linux/swap.h>
J
Jesse Barnes 已提交
37
#include <linux/pci.h>
38
#include <linux/dma-buf.h>
39

40
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
41 42
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
						   bool force);
43 44 45 46 47 48
static __must_check int
i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
			   struct i915_address_space *vm,
			   unsigned alignment,
			   bool map_and_fenceable,
			   bool nonblocking);
49 50
static int i915_gem_phys_pwrite(struct drm_device *dev,
				struct drm_i915_gem_object *obj,
51
				struct drm_i915_gem_pwrite *args,
52
				struct drm_file *file);
53

54 55 56 57 58 59
static void i915_gem_write_fence(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj);
static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
					 struct drm_i915_fence_reg *fence,
					 bool enable);

60
static int i915_gem_inactive_shrink(struct shrinker *shrinker,
61
				    struct shrink_control *sc);
C
Chris Wilson 已提交
62 63
static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
64
static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
65

66 67 68 69 70 71
static bool cpu_cache_is_coherent(struct drm_device *dev,
				  enum i915_cache_level level)
{
	return HAS_LLC(dev) || level != I915_CACHE_NONE;
}

72 73 74 75 76 77 78 79
static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
{
	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
		return true;

	return obj->pin_display;
}

80 81 82 83 84 85 86 87
static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
{
	if (obj->tiling_mode)
		i915_gem_release_mmap(obj);

	/* As we do not have an associated fence register, we will force
	 * a tiling change if we ever need to acquire one.
	 */
88
	obj->fence_dirty = false;
89 90 91
	obj->fence_reg = I915_FENCE_REG_NONE;
}

92 93 94 95
/* some bookkeeping */
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
				  size_t size)
{
96
	spin_lock(&dev_priv->mm.object_stat_lock);
97 98
	dev_priv->mm.object_count++;
	dev_priv->mm.object_memory += size;
99
	spin_unlock(&dev_priv->mm.object_stat_lock);
100 101 102 103 104
}

static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
				     size_t size)
{
105
	spin_lock(&dev_priv->mm.object_stat_lock);
106 107
	dev_priv->mm.object_count--;
	dev_priv->mm.object_memory -= size;
108
	spin_unlock(&dev_priv->mm.object_stat_lock);
109 110
}

111
static int
112
i915_gem_wait_for_error(struct i915_gpu_error *error)
113 114 115
{
	int ret;

116 117
#define EXIT_COND (!i915_reset_in_progress(error) || \
		   i915_terminally_wedged(error))
118
	if (EXIT_COND)
119 120
		return 0;

121 122 123 124 125
	/*
	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
	 * userspace. If it takes that long something really bad is going on and
	 * we should simply try to bail out and fail as gracefully as possible.
	 */
126 127 128
	ret = wait_event_interruptible_timeout(error->reset_queue,
					       EXIT_COND,
					       10*HZ);
129 130 131 132
	if (ret == 0) {
		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		return -EIO;
	} else if (ret < 0) {
133
		return ret;
134
	}
135
#undef EXIT_COND
136

137
	return 0;
138 139
}

140
int i915_mutex_lock_interruptible(struct drm_device *dev)
141
{
142
	struct drm_i915_private *dev_priv = dev->dev_private;
143 144
	int ret;

145
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
146 147 148 149 150 151 152
	if (ret)
		return ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

153
	WARN_ON(i915_verify_lists(dev));
154 155
	return 0;
}
156

157
static inline bool
158
i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
159
{
160
	return i915_gem_obj_bound_any(obj) && !obj->active;
161 162
}

J
Jesse Barnes 已提交
163 164
int
i915_gem_init_ioctl(struct drm_device *dev, void *data,
165
		    struct drm_file *file)
J
Jesse Barnes 已提交
166
{
167
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
168
	struct drm_i915_gem_init *args = data;
169

170 171 172
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return -ENODEV;

173 174 175
	if (args->gtt_start >= args->gtt_end ||
	    (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
		return -EINVAL;
J
Jesse Barnes 已提交
176

177 178 179 180
	/* GEM with user mode setting was never supported on ilk and later. */
	if (INTEL_INFO(dev)->gen >= 5)
		return -ENODEV;

J
Jesse Barnes 已提交
181
	mutex_lock(&dev->struct_mutex);
182 183
	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
				  args->gtt_end);
184
	dev_priv->gtt.mappable_end = args->gtt_end;
185 186
	mutex_unlock(&dev->struct_mutex);

187
	return 0;
188 189
}

190 191
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
192
			    struct drm_file *file)
193
{
194
	struct drm_i915_private *dev_priv = dev->dev_private;
195
	struct drm_i915_gem_get_aperture *args = data;
196 197
	struct drm_i915_gem_object *obj;
	size_t pinned;
198

199
	pinned = 0;
200
	mutex_lock(&dev->struct_mutex);
201
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
202
		if (obj->pin_count)
203
			pinned += i915_gem_obj_ggtt_size(obj);
204
	mutex_unlock(&dev->struct_mutex);
205

206
	args->aper_size = dev_priv->gtt.base.total;
207
	args->aper_available_size = args->aper_size - pinned;
208

209 210 211
	return 0;
}

212 213 214
void *i915_gem_object_alloc(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
215
	return kmem_cache_zalloc(dev_priv->slab, GFP_KERNEL);
216 217 218 219 220 221 222 223
}

void i915_gem_object_free(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	kmem_cache_free(dev_priv->slab, obj);
}

224 225 226 227 228
static int
i915_gem_create(struct drm_file *file,
		struct drm_device *dev,
		uint64_t size,
		uint32_t *handle_p)
229
{
230
	struct drm_i915_gem_object *obj;
231 232
	int ret;
	u32 handle;
233

234
	size = roundup(size, PAGE_SIZE);
235 236
	if (size == 0)
		return -EINVAL;
237 238

	/* Allocate the new object */
239
	obj = i915_gem_alloc_object(dev, size);
240 241 242
	if (obj == NULL)
		return -ENOMEM;

243
	ret = drm_gem_handle_create(file, &obj->base, &handle);
244
	/* drop reference from allocate - handle holds it now */
245 246 247
	drm_gem_object_unreference_unlocked(&obj->base);
	if (ret)
		return ret;
248

249
	*handle_p = handle;
250 251 252
	return 0;
}

253 254 255 256 257 258
int
i915_gem_dumb_create(struct drm_file *file,
		     struct drm_device *dev,
		     struct drm_mode_create_dumb *args)
{
	/* have to work out size/pitch and return them */
259
	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
260 261 262 263 264 265 266 267 268 269 270 271 272
	args->size = args->pitch * args->height;
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

/**
 * Creates a new mm object and returns a handle to it.
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file)
{
	struct drm_i915_gem_create *args = data;
273

274 275 276 277
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
static inline int
__copy_to_user_swizzled(char __user *cpu_vaddr,
			const char *gpu_vaddr, int gpu_offset,
			int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_to_user(cpu_vaddr + cpu_offset,
				     gpu_vaddr + swizzled_gpu_offset,
				     this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

304
static inline int
305 306
__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
			  const char __user *cpu_vaddr,
307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
			  int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
				       cpu_vaddr + cpu_offset,
				       this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

330 331 332
/* Per-page copy function for the shmem pread fastpath.
 * Flushes invalid cachelines before reading the target if
 * needs_clflush is set. */
333
static int
334 335 336 337 338 339 340
shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

341
	if (unlikely(page_do_bit17_swizzling))
342 343 344 345 346 347 348 349 350 351 352
		return -EINVAL;

	vaddr = kmap_atomic(page);
	if (needs_clflush)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_to_user_inatomic(user_data,
				      vaddr + shmem_page_offset,
				      page_length);
	kunmap_atomic(vaddr);

353
	return ret ? -EFAULT : 0;
354 355
}

356 357 358 359
static void
shmem_clflush_swizzled_range(char *addr, unsigned long length,
			     bool swizzled)
{
360
	if (unlikely(swizzled)) {
361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
		unsigned long start = (unsigned long) addr;
		unsigned long end = (unsigned long) addr + length;

		/* For swizzling simply ensure that we always flush both
		 * channels. Lame, but simple and it works. Swizzled
		 * pwrite/pread is far from a hotpath - current userspace
		 * doesn't use it at all. */
		start = round_down(start, 128);
		end = round_up(end, 128);

		drm_clflush_virt_range((void *)start, end - start);
	} else {
		drm_clflush_virt_range(addr, length);
	}

}

378 379 380 381 382 383 384 385 386 387 388 389
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
static int
shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

	vaddr = kmap(page);
	if (needs_clflush)
390 391 392
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
393 394 395 396 397 398 399 400 401 402 403

	if (page_do_bit17_swizzling)
		ret = __copy_to_user_swizzled(user_data,
					      vaddr, shmem_page_offset,
					      page_length);
	else
		ret = __copy_to_user(user_data,
				     vaddr + shmem_page_offset,
				     page_length);
	kunmap(page);

404
	return ret ? - EFAULT : 0;
405 406
}

407
static int
408 409 410 411
i915_gem_shmem_pread(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pread *args,
		     struct drm_file *file)
412
{
413
	char __user *user_data;
414
	ssize_t remain;
415
	loff_t offset;
416
	int shmem_page_offset, page_length, ret = 0;
417
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
418
	int prefaulted = 0;
419
	int needs_clflush = 0;
420
	struct sg_page_iter sg_iter;
421

V
Ville Syrjälä 已提交
422
	user_data = to_user_ptr(args->data_ptr);
423 424
	remain = args->size;

425
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
426

427 428 429 430 431
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		/* If we're not in the cpu read domain, set ourself into the gtt
		 * read domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will dirty the data
		 * anyway again before the next pread happens. */
432
		needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
433
		if (i915_gem_obj_bound_any(obj)) {
C
Chris Wilson 已提交
434 435 436 437
			ret = i915_gem_object_set_to_gtt_domain(obj, false);
			if (ret)
				return ret;
		}
438
	}
439

440 441 442 443 444 445
	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

	i915_gem_object_pin_pages(obj);

446
	offset = args->offset;
447

448 449
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
450
		struct page *page = sg_page_iter_page(&sg_iter);
451 452 453 454

		if (remain <= 0)
			break;

455 456 457 458 459
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
460
		shmem_page_offset = offset_in_page(offset);
461 462 463 464
		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

465 466 467
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

468 469 470 471 472
		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
		if (ret == 0)
			goto next_page;
473 474 475

		mutex_unlock(&dev->struct_mutex);

476
		if (likely(!i915_prefault_disable) && !prefaulted) {
477
			ret = fault_in_multipages_writeable(user_data, remain);
478 479 480 481 482 483 484
			/* Userspace is tricking us, but we've already clobbered
			 * its pages with the prefault and promised to write the
			 * data up to the first fault. Hence ignore any errors
			 * and just continue. */
			(void)ret;
			prefaulted = 1;
		}
485

486 487 488
		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
489

490
		mutex_lock(&dev->struct_mutex);
491

492
next_page:
493 494
		mark_page_accessed(page);

495
		if (ret)
496 497
			goto out;

498
		remain -= page_length;
499
		user_data += page_length;
500 501 502
		offset += page_length;
	}

503
out:
504 505
	i915_gem_object_unpin_pages(obj);

506 507 508
	return ret;
}

509 510 511 512 513 514 515
/**
 * Reads data from the object referenced by handle.
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
516
		     struct drm_file *file)
517 518
{
	struct drm_i915_gem_pread *args = data;
519
	struct drm_i915_gem_object *obj;
520
	int ret = 0;
521

522 523 524 525
	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_WRITE,
V
Ville Syrjälä 已提交
526
		       to_user_ptr(args->data_ptr),
527 528 529
		       args->size))
		return -EFAULT;

530
	ret = i915_mutex_lock_interruptible(dev);
531
	if (ret)
532
		return ret;
533

534
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
535
	if (&obj->base == NULL) {
536 537
		ret = -ENOENT;
		goto unlock;
538
	}
539

540
	/* Bounds check source.  */
541 542
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
C
Chris Wilson 已提交
543
		ret = -EINVAL;
544
		goto out;
C
Chris Wilson 已提交
545 546
	}

547 548 549 550 551 552 553 554
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

C
Chris Wilson 已提交
555 556
	trace_i915_gem_object_pread(obj, args->offset, args->size);

557
	ret = i915_gem_shmem_pread(dev, obj, args, file);
558

559
out:
560
	drm_gem_object_unreference(&obj->base);
561
unlock:
562
	mutex_unlock(&dev->struct_mutex);
563
	return ret;
564 565
}

566 567
/* This is the fast write path which cannot handle
 * page faults in the source data
568
 */
569 570 571 572 573 574

static inline int
fast_user_write(struct io_mapping *mapping,
		loff_t page_base, int page_offset,
		char __user *user_data,
		int length)
575
{
576 577
	void __iomem *vaddr_atomic;
	void *vaddr;
578
	unsigned long unwritten;
579

P
Peter Zijlstra 已提交
580
	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
581 582 583
	/* We can use the cpu mem copy function because this is X86. */
	vaddr = (void __force*)vaddr_atomic + page_offset;
	unwritten = __copy_from_user_inatomic_nocache(vaddr,
584
						      user_data, length);
P
Peter Zijlstra 已提交
585
	io_mapping_unmap_atomic(vaddr_atomic);
586
	return unwritten;
587 588
}

589 590 591 592
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
 */
593
static int
594 595
i915_gem_gtt_pwrite_fast(struct drm_device *dev,
			 struct drm_i915_gem_object *obj,
596
			 struct drm_i915_gem_pwrite *args,
597
			 struct drm_file *file)
598
{
599
	drm_i915_private_t *dev_priv = dev->dev_private;
600
	ssize_t remain;
601
	loff_t offset, page_base;
602
	char __user *user_data;
D
Daniel Vetter 已提交
603 604
	int page_offset, page_length, ret;

B
Ben Widawsky 已提交
605
	ret = i915_gem_obj_ggtt_pin(obj, 0, true, true);
D
Daniel Vetter 已提交
606 607 608 609 610 611 612 613 614 615
	if (ret)
		goto out;

	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		goto out_unpin;

	ret = i915_gem_object_put_fence(obj);
	if (ret)
		goto out_unpin;
616

V
Ville Syrjälä 已提交
617
	user_data = to_user_ptr(args->data_ptr);
618 619
	remain = args->size;

620
	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
621 622 623 624

	while (remain > 0) {
		/* Operation in this page
		 *
625 626 627
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
628
		 */
629 630
		page_base = offset & PAGE_MASK;
		page_offset = offset_in_page(offset);
631 632 633 634 635
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		/* If we get a fault while copying data, then (presumably) our
636 637
		 * source page isn't available.  Return the error and we'll
		 * retry in the slow path.
638
		 */
B
Ben Widawsky 已提交
639
		if (fast_user_write(dev_priv->gtt.mappable, page_base,
D
Daniel Vetter 已提交
640 641 642 643
				    page_offset, user_data, page_length)) {
			ret = -EFAULT;
			goto out_unpin;
		}
644

645 646 647
		remain -= page_length;
		user_data += page_length;
		offset += page_length;
648 649
	}

D
Daniel Vetter 已提交
650 651 652
out_unpin:
	i915_gem_object_unpin(obj);
out:
653
	return ret;
654 655
}

656 657 658 659
/* Per-page copy function for the shmem pwrite fastpath.
 * Flushes invalid cachelines before writing to the target if
 * needs_clflush_before is set and flushes out any written cachelines after
 * writing if needs_clflush is set. */
660
static int
661 662 663 664 665
shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
666
{
667
	char *vaddr;
668
	int ret;
669

670
	if (unlikely(page_do_bit17_swizzling))
671
		return -EINVAL;
672

673 674 675 676 677 678 679 680 681 682 683
	vaddr = kmap_atomic(page);
	if (needs_clflush_before)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
						user_data,
						page_length);
	if (needs_clflush_after)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	kunmap_atomic(vaddr);
684

685
	return ret ? -EFAULT : 0;
686 687
}

688 689
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
690
static int
691 692 693 694 695
shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
696
{
697 698
	char *vaddr;
	int ret;
699

700
	vaddr = kmap(page);
701
	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
702 703 704
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
705 706
	if (page_do_bit17_swizzling)
		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
707 708
						user_data,
						page_length);
709 710 711 712 713
	else
		ret = __copy_from_user(vaddr + shmem_page_offset,
				       user_data,
				       page_length);
	if (needs_clflush_after)
714 715 716
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
717
	kunmap(page);
718

719
	return ret ? -EFAULT : 0;
720 721 722
}

static int
723 724 725 726
i915_gem_shmem_pwrite(struct drm_device *dev,
		      struct drm_i915_gem_object *obj,
		      struct drm_i915_gem_pwrite *args,
		      struct drm_file *file)
727 728
{
	ssize_t remain;
729 730
	loff_t offset;
	char __user *user_data;
731
	int shmem_page_offset, page_length, ret = 0;
732
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
733
	int hit_slowpath = 0;
734 735
	int needs_clflush_after = 0;
	int needs_clflush_before = 0;
736
	struct sg_page_iter sg_iter;
737

V
Ville Syrjälä 已提交
738
	user_data = to_user_ptr(args->data_ptr);
739 740
	remain = args->size;

741
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
742

743 744 745 746 747
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		/* If we're not in the cpu write domain, set ourself into the gtt
		 * write domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will use the data
		 * right away and we therefore have to clflush anyway. */
748
		needs_clflush_after = cpu_write_needs_clflush(obj);
749
		if (i915_gem_obj_bound_any(obj)) {
C
Chris Wilson 已提交
750 751 752 753
			ret = i915_gem_object_set_to_gtt_domain(obj, true);
			if (ret)
				return ret;
		}
754
	}
755 756 757 758 759
	/* Same trick applies to invalidate partially written cachelines read
	 * before writing. */
	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
		needs_clflush_before =
			!cpu_cache_is_coherent(dev, obj->cache_level);
760

761 762 763 764 765 766
	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

	i915_gem_object_pin_pages(obj);

767
	offset = args->offset;
768
	obj->dirty = 1;
769

770 771
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
772
		struct page *page = sg_page_iter_page(&sg_iter);
773
		int partial_cacheline_write;
774

775 776 777
		if (remain <= 0)
			break;

778 779 780 781 782
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
783
		shmem_page_offset = offset_in_page(offset);
784 785 786 787 788

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

789 790 791 792 793 794 795
		/* If we don't overwrite a cacheline completely we need to be
		 * careful to have up-to-date data by first clflushing. Don't
		 * overcomplicate things and flush the entire patch. */
		partial_cacheline_write = needs_clflush_before &&
			((shmem_page_offset | page_length)
				& (boot_cpu_data.x86_clflush_size - 1));

796 797 798
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

799 800 801 802 803 804
		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
		if (ret == 0)
			goto next_page;
805 806 807

		hit_slowpath = 1;
		mutex_unlock(&dev->struct_mutex);
808 809 810 811
		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
812

813
		mutex_lock(&dev->struct_mutex);
814

815
next_page:
816 817 818
		set_page_dirty(page);
		mark_page_accessed(page);

819
		if (ret)
820 821
			goto out;

822
		remain -= page_length;
823
		user_data += page_length;
824
		offset += page_length;
825 826
	}

827
out:
828 829
	i915_gem_object_unpin_pages(obj);

830
	if (hit_slowpath) {
831 832 833 834 835 836 837
		/*
		 * Fixup: Flush cpu caches in case we didn't flush the dirty
		 * cachelines in-line while writing and the object moved
		 * out of the cpu write domain while we've dropped the lock.
		 */
		if (!needs_clflush_after &&
		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
838 839
			if (i915_gem_clflush_object(obj, obj->pin_display))
				i915_gem_chipset_flush(dev);
840
		}
841
	}
842

843
	if (needs_clflush_after)
844
		i915_gem_chipset_flush(dev);
845

846
	return ret;
847 848 849 850 851 852 853 854 855
}

/**
 * Writes data to the object referenced by handle.
 *
 * On error, the contents of the buffer that were to be modified are undefined.
 */
int
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
856
		      struct drm_file *file)
857 858
{
	struct drm_i915_gem_pwrite *args = data;
859
	struct drm_i915_gem_object *obj;
860 861 862 863 864 865
	int ret;

	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_READ,
V
Ville Syrjälä 已提交
866
		       to_user_ptr(args->data_ptr),
867 868 869
		       args->size))
		return -EFAULT;

870 871 872 873 874 875
	if (likely(!i915_prefault_disable)) {
		ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
						   args->size);
		if (ret)
			return -EFAULT;
	}
876

877
	ret = i915_mutex_lock_interruptible(dev);
878
	if (ret)
879
		return ret;
880

881
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
882
	if (&obj->base == NULL) {
883 884
		ret = -ENOENT;
		goto unlock;
885
	}
886

887
	/* Bounds check destination. */
888 889
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
C
Chris Wilson 已提交
890
		ret = -EINVAL;
891
		goto out;
C
Chris Wilson 已提交
892 893
	}

894 895 896 897 898 899 900 901
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

C
Chris Wilson 已提交
902 903
	trace_i915_gem_object_pwrite(obj, args->offset, args->size);

D
Daniel Vetter 已提交
904
	ret = -EFAULT;
905 906 907 908 909 910
	/* We can only do the GTT pwrite on untiled buffers, as otherwise
	 * it would end up going through the fenced access, and we'll get
	 * different detiling behavior between reading and writing.
	 * pread/pwrite currently are reading and writing from the CPU
	 * perspective, requiring manual detiling by the client.
	 */
911
	if (obj->phys_obj) {
912
		ret = i915_gem_phys_pwrite(dev, obj, args, file);
913 914 915
		goto out;
	}

916 917 918
	if (obj->tiling_mode == I915_TILING_NONE &&
	    obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
	    cpu_write_needs_clflush(obj)) {
919
		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
D
Daniel Vetter 已提交
920 921 922
		/* Note that the gtt paths might fail with non-page-backed user
		 * pointers (e.g. gtt mappings when moving data between
		 * textures). Fallback to the shmem path in that case. */
923
	}
924

925
	if (ret == -EFAULT || ret == -ENOSPC)
D
Daniel Vetter 已提交
926
		ret = i915_gem_shmem_pwrite(dev, obj, args, file);
927

928
out:
929
	drm_gem_object_unreference(&obj->base);
930
unlock:
931
	mutex_unlock(&dev->struct_mutex);
932 933 934
	return ret;
}

935
int
936
i915_gem_check_wedge(struct i915_gpu_error *error,
937 938
		     bool interruptible)
{
939
	if (i915_reset_in_progress(error)) {
940 941 942 943 944
		/* Non-interruptible callers can't handle -EAGAIN, hence return
		 * -EIO unconditionally for these. */
		if (!interruptible)
			return -EIO;

945 946
		/* Recovery complete, but the reset failed ... */
		if (i915_terminally_wedged(error))
947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966
			return -EIO;

		return -EAGAIN;
	}

	return 0;
}

/*
 * Compare seqno against outstanding lazy request. Emit a request if they are
 * equal.
 */
static int
i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
{
	int ret;

	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));

	ret = 0;
967
	if (seqno == ring->outstanding_lazy_seqno)
968
		ret = i915_add_request(ring, NULL);
969 970 971 972 973 974 975 976

	return ret;
}

/**
 * __wait_seqno - wait until execution of seqno has finished
 * @ring: the ring expected to report seqno
 * @seqno: duh!
977
 * @reset_counter: reset sequence associated with the given seqno
978 979 980
 * @interruptible: do an interruptible wait (normally yes)
 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
 *
981 982 983 984 985 986 987
 * Note: It is of utmost importance that the passed in seqno and reset_counter
 * values have been read by the caller in an smp safe manner. Where read-side
 * locks are involved, it is sufficient to read the reset_counter before
 * unlocking the lock that protects the seqno. For lockless tricks, the
 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
 * inserted.
 *
988 989 990 991
 * Returns 0 if the seqno was found within the alloted time. Else returns the
 * errno with remaining time filled in timeout argument.
 */
static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
992
			unsigned reset_counter,
993 994 995 996 997 998 999 1000 1001
			bool interruptible, struct timespec *timeout)
{
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
	struct timespec before, now, wait_time={1,0};
	unsigned long timeout_jiffies;
	long end;
	bool wait_forever = true;
	int ret;

1002 1003
	WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n");

1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		return 0;

	trace_i915_gem_request_wait_begin(ring, seqno);

	if (timeout != NULL) {
		wait_time = *timeout;
		wait_forever = false;
	}

1014
	timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
1015 1016 1017 1018 1019 1020 1021 1022 1023

	if (WARN_ON(!ring->irq_get(ring)))
		return -ENODEV;

	/* Record current time in case interrupted by signal, or wedged * */
	getrawmonotonic(&before);

#define EXIT_COND \
	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
1024 1025
	 i915_reset_in_progress(&dev_priv->gpu_error) || \
	 reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
1026 1027 1028 1029 1030 1031 1032 1033 1034
	do {
		if (interruptible)
			end = wait_event_interruptible_timeout(ring->irq_queue,
							       EXIT_COND,
							       timeout_jiffies);
		else
			end = wait_event_timeout(ring->irq_queue, EXIT_COND,
						 timeout_jiffies);

1035 1036 1037 1038 1039 1040 1041
		/* We need to check whether any gpu reset happened in between
		 * the caller grabbing the seqno and now ... */
		if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
			end = -EAGAIN;

		/* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
		 * gone. */
1042
		ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055
		if (ret)
			end = ret;
	} while (end == 0 && wait_forever);

	getrawmonotonic(&now);

	ring->irq_put(ring);
	trace_i915_gem_request_wait_end(ring, seqno);
#undef EXIT_COND

	if (timeout) {
		struct timespec sleep_time = timespec_sub(now, before);
		*timeout = timespec_sub(*timeout, sleep_time);
1056 1057
		if (!timespec_valid(timeout)) /* i.e. negative time remains */
			set_normalized_timespec(timeout, 0, 0);
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
	}

	switch (end) {
	case -EIO:
	case -EAGAIN: /* Wedged */
	case -ERESTARTSYS: /* Signal */
		return (int)end;
	case 0: /* Timeout */
		return -ETIME;
	default: /* Completed */
		WARN_ON(end < 0); /* We're not aware of other errors */
		return 0;
	}
}

/**
 * Waits for a sequence number to be signaled, and cleans up the
 * request and object lists appropriately for that event.
 */
int
i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
{
	struct drm_device *dev = ring->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	bool interruptible = dev_priv->mm.interruptible;
	int ret;

	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
	BUG_ON(seqno == 0);

1088
	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1089 1090 1091 1092 1093 1094 1095
	if (ret)
		return ret;

	ret = i915_gem_check_olr(ring, seqno);
	if (ret)
		return ret;

1096 1097 1098
	return __wait_seqno(ring, seqno,
			    atomic_read(&dev_priv->gpu_error.reset_counter),
			    interruptible, NULL);
1099 1100
}

1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
static int
i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
				     struct intel_ring_buffer *ring)
{
	i915_gem_retire_requests_ring(ring);

	/* Manually manage the write flush as we may have not yet
	 * retired the buffer.
	 *
	 * Note that the last_write_seqno is always the earlier of
	 * the two (read/write) seqno, so if we haved successfully waited,
	 * we know we have passed the last write.
	 */
	obj->last_write_seqno = 0;
	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;

	return 0;
}

1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139
/**
 * Ensures that all rendering to the object has completed and the object is
 * safe to unbind from the GTT or access from the CPU.
 */
static __must_check int
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
			       bool readonly)
{
	struct intel_ring_buffer *ring = obj->ring;
	u32 seqno;
	int ret;

	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
	if (seqno == 0)
		return 0;

	ret = i915_wait_seqno(ring, seqno);
	if (ret)
		return ret;

1140
	return i915_gem_object_wait_rendering__tail(obj, ring);
1141 1142
}

1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
/* A nonblocking variant of the above wait. This is a highly dangerous routine
 * as the object state may change during this call.
 */
static __must_check int
i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
					    bool readonly)
{
	struct drm_device *dev = obj->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_ring_buffer *ring = obj->ring;
1153
	unsigned reset_counter;
1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
	u32 seqno;
	int ret;

	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
	BUG_ON(!dev_priv->mm.interruptible);

	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
	if (seqno == 0)
		return 0;

1164
	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1165 1166 1167 1168 1169 1170 1171
	if (ret)
		return ret;

	ret = i915_gem_check_olr(ring, seqno);
	if (ret)
		return ret;

1172
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1173
	mutex_unlock(&dev->struct_mutex);
1174
	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
1175
	mutex_lock(&dev->struct_mutex);
1176 1177
	if (ret)
		return ret;
1178

1179
	return i915_gem_object_wait_rendering__tail(obj, ring);
1180 1181
}

1182
/**
1183 1184
 * Called when user space prepares to use an object with the CPU, either
 * through the mmap ioctl's mapping or a GTT mapping.
1185 1186 1187
 */
int
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1188
			  struct drm_file *file)
1189 1190
{
	struct drm_i915_gem_set_domain *args = data;
1191
	struct drm_i915_gem_object *obj;
1192 1193
	uint32_t read_domains = args->read_domains;
	uint32_t write_domain = args->write_domain;
1194 1195
	int ret;

1196
	/* Only handle setting domains to types used by the CPU. */
1197
	if (write_domain & I915_GEM_GPU_DOMAINS)
1198 1199
		return -EINVAL;

1200
	if (read_domains & I915_GEM_GPU_DOMAINS)
1201 1202 1203 1204 1205 1206 1207 1208
		return -EINVAL;

	/* Having something in the write domain implies it's in the read
	 * domain, and only that read domain.  Enforce that in the request.
	 */
	if (write_domain != 0 && read_domains != write_domain)
		return -EINVAL;

1209
	ret = i915_mutex_lock_interruptible(dev);
1210
	if (ret)
1211
		return ret;
1212

1213
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1214
	if (&obj->base == NULL) {
1215 1216
		ret = -ENOENT;
		goto unlock;
1217
	}
1218

1219 1220 1221 1222 1223 1224 1225 1226
	/* Try to flush the object off the GPU without holding the lock.
	 * We will repeat the flush holding the lock in the normal manner
	 * to catch cases where we are gazumped.
	 */
	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
	if (ret)
		goto unref;

1227 1228
	if (read_domains & I915_GEM_DOMAIN_GTT) {
		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1229 1230 1231 1232 1233 1234 1235

		/* Silently promote "you're not bound, there was nothing to do"
		 * to success, since the client was just asking us to
		 * make sure everything was done.
		 */
		if (ret == -EINVAL)
			ret = 0;
1236
	} else {
1237
		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1238 1239
	}

1240
unref:
1241
	drm_gem_object_unreference(&obj->base);
1242
unlock:
1243 1244 1245 1246 1247 1248 1249 1250 1251
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Called when user space has done writes to this buffer
 */
int
i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1252
			 struct drm_file *file)
1253 1254
{
	struct drm_i915_gem_sw_finish *args = data;
1255
	struct drm_i915_gem_object *obj;
1256 1257
	int ret = 0;

1258
	ret = i915_mutex_lock_interruptible(dev);
1259
	if (ret)
1260
		return ret;
1261

1262
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1263
	if (&obj->base == NULL) {
1264 1265
		ret = -ENOENT;
		goto unlock;
1266 1267 1268
	}

	/* Pinned buffers may be scanout, so flush the cache */
1269 1270
	if (obj->pin_display)
		i915_gem_object_flush_cpu_write_domain(obj, true);
1271

1272
	drm_gem_object_unreference(&obj->base);
1273
unlock:
1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Maps the contents of an object, returning the address it is mapped
 * into.
 *
 * While the mapping holds a reference on the contents of the object, it doesn't
 * imply a ref on the object itself.
 */
int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1287
		    struct drm_file *file)
1288 1289 1290 1291 1292
{
	struct drm_i915_gem_mmap *args = data;
	struct drm_gem_object *obj;
	unsigned long addr;

1293
	obj = drm_gem_object_lookup(dev, file, args->handle);
1294
	if (obj == NULL)
1295
		return -ENOENT;
1296

1297 1298 1299 1300 1301 1302 1303 1304
	/* prime objects have no backing filp to GEM mmap
	 * pages from.
	 */
	if (!obj->filp) {
		drm_gem_object_unreference_unlocked(obj);
		return -EINVAL;
	}

1305
	addr = vm_mmap(obj->filp, 0, args->size,
1306 1307
		       PROT_READ | PROT_WRITE, MAP_SHARED,
		       args->offset);
1308
	drm_gem_object_unreference_unlocked(obj);
1309 1310 1311 1312 1313 1314 1315 1316
	if (IS_ERR((void *)addr))
		return addr;

	args->addr_ptr = (uint64_t) addr;

	return 0;
}

1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334
/**
 * i915_gem_fault - fault a page into the GTT
 * vma: VMA in question
 * vmf: fault info
 *
 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
 * from userspace.  The fault handler takes care of binding the object to
 * the GTT (if needed), allocating and programming a fence register (again,
 * only if needed based on whether the old reg is still valid or the object
 * is tiled) and inserting a new PTE into the faulting process.
 *
 * Note that the faulting process may involve evicting existing objects
 * from the GTT and/or fence registers to make room.  So performance may
 * suffer if the GTT working set is large or there are few fence registers
 * left.
 */
int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
1335 1336
	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
	struct drm_device *dev = obj->base.dev;
1337
	drm_i915_private_t *dev_priv = dev->dev_private;
1338 1339 1340
	pgoff_t page_offset;
	unsigned long pfn;
	int ret = 0;
1341
	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1342 1343 1344 1345 1346

	/* We don't use vmf->pgoff since that has the fake offset */
	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
		PAGE_SHIFT;

1347 1348 1349
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto out;
1350

C
Chris Wilson 已提交
1351 1352
	trace_i915_gem_object_fault(obj, page_offset, true, write);

1353 1354 1355 1356 1357 1358
	/* Access to snoopable pages through the GTT is incoherent. */
	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
		ret = -EINVAL;
		goto unlock;
	}

1359
	/* Now bind it into the GTT if needed */
B
Ben Widawsky 已提交
1360
	ret = i915_gem_obj_ggtt_pin(obj,  0, true, false);
1361 1362
	if (ret)
		goto unlock;
1363

1364 1365 1366
	ret = i915_gem_object_set_to_gtt_domain(obj, write);
	if (ret)
		goto unpin;
1367

1368
	ret = i915_gem_object_get_fence(obj);
1369
	if (ret)
1370
		goto unpin;
1371

1372 1373
	obj->fault_mappable = true;

1374 1375 1376
	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
	pfn >>= PAGE_SHIFT;
	pfn += page_offset;
1377 1378 1379

	/* Finally, remap it using the new GTT offset */
	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1380 1381
unpin:
	i915_gem_object_unpin(obj);
1382
unlock:
1383
	mutex_unlock(&dev->struct_mutex);
1384
out:
1385
	switch (ret) {
1386
	case -EIO:
1387 1388 1389
		/* If this -EIO is due to a gpu hang, give the reset code a
		 * chance to clean up the mess. Otherwise return the proper
		 * SIGBUS. */
1390
		if (i915_terminally_wedged(&dev_priv->gpu_error))
1391
			return VM_FAULT_SIGBUS;
1392
	case -EAGAIN:
1393 1394 1395 1396 1397 1398 1399
		/* Give the error handler a chance to run and move the
		 * objects off the GPU active list. Next time we service the
		 * fault, we should be able to transition the page into the
		 * GTT without touching the GPU (and so avoid further
		 * EIO/EGAIN). If the GPU is wedged, then there is no issue
		 * with coherency, just lost writes.
		 */
1400
		set_need_resched();
1401 1402
	case 0:
	case -ERESTARTSYS:
1403
	case -EINTR:
1404 1405 1406 1407 1408
	case -EBUSY:
		/*
		 * EBUSY is ok: this just means that another thread
		 * already did the job.
		 */
1409
		return VM_FAULT_NOPAGE;
1410 1411
	case -ENOMEM:
		return VM_FAULT_OOM;
1412 1413
	case -ENOSPC:
		return VM_FAULT_SIGBUS;
1414
	default:
1415
		WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1416
		return VM_FAULT_SIGBUS;
1417 1418 1419
	}
}

1420 1421 1422 1423
/**
 * i915_gem_release_mmap - remove physical page mappings
 * @obj: obj in question
 *
1424
 * Preserve the reservation of the mmapping with the DRM core code, but
1425 1426 1427 1428 1429 1430 1431 1432 1433
 * relinquish ownership of the pages back to the system.
 *
 * It is vital that we remove the page mapping if we have mapped a tiled
 * object through the GTT and then lose the fence register due to
 * resource pressure. Similarly if the object has been moved out of the
 * aperture, than pages mapped into userspace must be revoked. Removing the
 * mapping will then trigger a page fault on the next user access, allowing
 * fixup by i915_gem_fault().
 */
1434
void
1435
i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1436
{
1437 1438
	if (!obj->fault_mappable)
		return;
1439

1440
	drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
1441
	obj->fault_mappable = false;
1442 1443
}

1444
uint32_t
1445
i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1446
{
1447
	uint32_t gtt_size;
1448 1449

	if (INTEL_INFO(dev)->gen >= 4 ||
1450 1451
	    tiling_mode == I915_TILING_NONE)
		return size;
1452 1453 1454

	/* Previous chips need a power-of-two fence region when tiling */
	if (INTEL_INFO(dev)->gen == 3)
1455
		gtt_size = 1024*1024;
1456
	else
1457
		gtt_size = 512*1024;
1458

1459 1460
	while (gtt_size < size)
		gtt_size <<= 1;
1461

1462
	return gtt_size;
1463 1464
}

1465 1466 1467 1468 1469
/**
 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 * @obj: object to check
 *
 * Return the required GTT alignment for an object, taking into account
1470
 * potential fence register mapping.
1471
 */
1472 1473 1474
uint32_t
i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
			   int tiling_mode, bool fenced)
1475 1476 1477 1478 1479
{
	/*
	 * Minimum alignment is 4k (GTT page size), but might be greater
	 * if a fence register is needed for the object.
	 */
1480
	if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1481
	    tiling_mode == I915_TILING_NONE)
1482 1483
		return 4096;

1484 1485 1486 1487
	/*
	 * Previous chips need to be aligned to the size of the smallest
	 * fence register that can contain the object.
	 */
1488
	return i915_gem_get_gtt_size(dev, size, tiling_mode);
1489 1490
}

1491 1492 1493 1494 1495
static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	int ret;

1496
	if (drm_vma_node_has_offset(&obj->base.vma_node))
1497 1498
		return 0;

1499 1500
	dev_priv->mm.shrinker_no_lock_stealing = true;

1501 1502
	ret = drm_gem_create_mmap_offset(&obj->base);
	if (ret != -ENOSPC)
1503
		goto out;
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514

	/* Badly fragmented mmap space? The only way we can recover
	 * space is by destroying unwanted objects. We can't randomly release
	 * mmap_offsets as userspace expects them to be persistent for the
	 * lifetime of the objects. The closest we can is to release the
	 * offsets on purgeable objects by truncating it and marking it purged,
	 * which prevents userspace from ever using that object again.
	 */
	i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
	ret = drm_gem_create_mmap_offset(&obj->base);
	if (ret != -ENOSPC)
1515
		goto out;
1516 1517

	i915_gem_shrink_all(dev_priv);
1518 1519 1520 1521 1522
	ret = drm_gem_create_mmap_offset(&obj->base);
out:
	dev_priv->mm.shrinker_no_lock_stealing = false;

	return ret;
1523 1524 1525 1526 1527 1528 1529
}

static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
{
	drm_gem_free_mmap_offset(&obj->base);
}

1530
int
1531 1532 1533 1534
i915_gem_mmap_gtt(struct drm_file *file,
		  struct drm_device *dev,
		  uint32_t handle,
		  uint64_t *offset)
1535
{
1536
	struct drm_i915_private *dev_priv = dev->dev_private;
1537
	struct drm_i915_gem_object *obj;
1538 1539
	int ret;

1540
	ret = i915_mutex_lock_interruptible(dev);
1541
	if (ret)
1542
		return ret;
1543

1544
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1545
	if (&obj->base == NULL) {
1546 1547 1548
		ret = -ENOENT;
		goto unlock;
	}
1549

B
Ben Widawsky 已提交
1550
	if (obj->base.size > dev_priv->gtt.mappable_end) {
1551
		ret = -E2BIG;
1552
		goto out;
1553 1554
	}

1555
	if (obj->madv != I915_MADV_WILLNEED) {
1556
		DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1557 1558
		ret = -EINVAL;
		goto out;
1559 1560
	}

1561 1562 1563
	ret = i915_gem_object_create_mmap_offset(obj);
	if (ret)
		goto out;
1564

1565
	*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1566

1567
out:
1568
	drm_gem_object_unreference(&obj->base);
1569
unlock:
1570
	mutex_unlock(&dev->struct_mutex);
1571
	return ret;
1572 1573
}

1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597
/**
 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 * @dev: DRM device
 * @data: GTT mapping ioctl data
 * @file: GEM object info
 *
 * Simply returns the fake offset to userspace so it can mmap it.
 * The mmap call will end up in drm_gem_mmap(), which will set things
 * up so we can get faults in the handler above.
 *
 * The fault handler will take care of binding the object into the GTT
 * (since it may have been evicted to make room for something), allocating
 * a fence register, and mapping the appropriate aperture address into
 * userspace.
 */
int
i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file)
{
	struct drm_i915_gem_mmap_gtt *args = data;

	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
}

D
Daniel Vetter 已提交
1598 1599 1600
/* Immediately discard the backing storage */
static void
i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1601 1602 1603
{
	struct inode *inode;

1604
	i915_gem_object_free_mmap_offset(obj);
1605

1606 1607
	if (obj->base.filp == NULL)
		return;
1608

D
Daniel Vetter 已提交
1609 1610 1611 1612 1613
	/* Our goal here is to return as much of the memory as
	 * is possible back to the system as we are called from OOM.
	 * To do this we must instruct the shmfs to drop all of its
	 * backing pages, *now*.
	 */
A
Al Viro 已提交
1614
	inode = file_inode(obj->base.filp);
D
Daniel Vetter 已提交
1615
	shmem_truncate_range(inode, 0, (loff_t)-1);
1616

D
Daniel Vetter 已提交
1617 1618
	obj->madv = __I915_MADV_PURGED;
}
1619

D
Daniel Vetter 已提交
1620 1621 1622 1623
static inline int
i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
{
	return obj->madv == I915_MADV_DONTNEED;
1624 1625
}

1626
static void
1627
i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1628
{
1629 1630
	struct sg_page_iter sg_iter;
	int ret;
1631

1632
	BUG_ON(obj->madv == __I915_MADV_PURGED);
1633

C
Chris Wilson 已提交
1634 1635 1636 1637 1638 1639
	ret = i915_gem_object_set_to_cpu_domain(obj, true);
	if (ret) {
		/* In the event of a disaster, abandon all caches and
		 * hope for the best.
		 */
		WARN_ON(ret != -EIO);
1640
		i915_gem_clflush_object(obj, true);
C
Chris Wilson 已提交
1641 1642 1643
		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	}

1644
	if (i915_gem_object_needs_bit17_swizzle(obj))
1645 1646
		i915_gem_object_save_bit_17_swizzle(obj);

1647 1648
	if (obj->madv == I915_MADV_DONTNEED)
		obj->dirty = 0;
1649

1650
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1651
		struct page *page = sg_page_iter_page(&sg_iter);
1652

1653
		if (obj->dirty)
1654
			set_page_dirty(page);
1655

1656
		if (obj->madv == I915_MADV_WILLNEED)
1657
			mark_page_accessed(page);
1658

1659
		page_cache_release(page);
1660
	}
1661
	obj->dirty = 0;
1662

1663 1664
	sg_free_table(obj->pages);
	kfree(obj->pages);
1665
}
C
Chris Wilson 已提交
1666

1667
int
1668 1669 1670 1671
i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
{
	const struct drm_i915_gem_object_ops *ops = obj->ops;

1672
	if (obj->pages == NULL)
1673 1674
		return 0;

1675 1676 1677
	if (obj->pages_pin_count)
		return -EBUSY;

1678
	BUG_ON(i915_gem_obj_bound_any(obj));
B
Ben Widawsky 已提交
1679

1680 1681 1682
	/* ->put_pages might need to allocate memory for the bit17 swizzle
	 * array, hence protect them from being reaped by removing them from gtt
	 * lists early. */
1683
	list_del(&obj->global_list);
1684

1685
	ops->put_pages(obj);
1686
	obj->pages = NULL;
1687

C
Chris Wilson 已提交
1688 1689 1690 1691 1692 1693 1694
	if (i915_gem_object_is_purgeable(obj))
		i915_gem_object_truncate(obj);

	return 0;
}

static long
1695 1696
__i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
		  bool purgeable_only)
C
Chris Wilson 已提交
1697 1698 1699 1700 1701 1702
{
	struct drm_i915_gem_object *obj, *next;
	long count = 0;

	list_for_each_entry_safe(obj, next,
				 &dev_priv->mm.unbound_list,
1703
				 global_list) {
1704
		if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
1705
		    i915_gem_object_put_pages(obj) == 0) {
C
Chris Wilson 已提交
1706 1707 1708 1709 1710 1711
			count += obj->base.size >> PAGE_SHIFT;
			if (count >= target)
				return count;
		}
	}

1712 1713 1714
	list_for_each_entry_safe(obj, next, &dev_priv->mm.bound_list,
				 global_list) {
		struct i915_vma *vma, *v;
1715 1716 1717 1718

		if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
			continue;

1719 1720 1721
		list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link)
			if (i915_vma_unbind(vma))
				break;
1722 1723

		if (!i915_gem_object_put_pages(obj)) {
C
Chris Wilson 已提交
1724 1725 1726 1727 1728 1729 1730 1731 1732
			count += obj->base.size >> PAGE_SHIFT;
			if (count >= target)
				return count;
		}
	}

	return count;
}

1733 1734 1735 1736 1737 1738
static long
i915_gem_purge(struct drm_i915_private *dev_priv, long target)
{
	return __i915_gem_shrink(dev_priv, target, true);
}

C
Chris Wilson 已提交
1739 1740 1741 1742 1743 1744 1745
static void
i915_gem_shrink_all(struct drm_i915_private *dev_priv)
{
	struct drm_i915_gem_object *obj, *next;

	i915_gem_evict_everything(dev_priv->dev);

1746 1747
	list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
				 global_list)
1748
		i915_gem_object_put_pages(obj);
D
Daniel Vetter 已提交
1749 1750
}

1751
static int
C
Chris Wilson 已提交
1752
i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
1753
{
C
Chris Wilson 已提交
1754
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1755 1756
	int page_count, i;
	struct address_space *mapping;
1757 1758
	struct sg_table *st;
	struct scatterlist *sg;
1759
	struct sg_page_iter sg_iter;
1760
	struct page *page;
1761
	unsigned long last_pfn = 0;	/* suppress gcc warning */
C
Chris Wilson 已提交
1762
	gfp_t gfp;
1763

C
Chris Wilson 已提交
1764 1765 1766 1767 1768 1769 1770
	/* Assert that the object is not currently in any GPU domain. As it
	 * wasn't in the GTT, there shouldn't be any way it could have been in
	 * a GPU cache
	 */
	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);

1771 1772 1773 1774
	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (st == NULL)
		return -ENOMEM;

1775
	page_count = obj->base.size / PAGE_SIZE;
1776 1777
	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		kfree(st);
1778
		return -ENOMEM;
1779
	}
1780

1781 1782 1783 1784 1785
	/* Get the list of pages out of our struct file.  They'll be pinned
	 * at this point until we release them.
	 *
	 * Fail silently without starting the shrinker
	 */
A
Al Viro 已提交
1786
	mapping = file_inode(obj->base.filp)->i_mapping;
C
Chris Wilson 已提交
1787
	gfp = mapping_gfp_mask(mapping);
1788
	gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
C
Chris Wilson 已提交
1789
	gfp &= ~(__GFP_IO | __GFP_WAIT);
1790 1791 1792
	sg = st->sgl;
	st->nents = 0;
	for (i = 0; i < page_count; i++) {
C
Chris Wilson 已提交
1793 1794 1795 1796 1797 1798 1799 1800 1801 1802
		page = shmem_read_mapping_page_gfp(mapping, i, gfp);
		if (IS_ERR(page)) {
			i915_gem_purge(dev_priv, page_count);
			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
		}
		if (IS_ERR(page)) {
			/* We've tried hard to allocate the memory by reaping
			 * our own buffer, now let the real VM do its job and
			 * go down in flames if truly OOM.
			 */
1803
			gfp &= ~(__GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD);
C
Chris Wilson 已提交
1804 1805 1806 1807 1808 1809 1810
			gfp |= __GFP_IO | __GFP_WAIT;

			i915_gem_shrink_all(dev_priv);
			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
			if (IS_ERR(page))
				goto err_pages;

1811
			gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
C
Chris Wilson 已提交
1812 1813
			gfp &= ~(__GFP_IO | __GFP_WAIT);
		}
1814 1815 1816 1817 1818 1819 1820 1821
#ifdef CONFIG_SWIOTLB
		if (swiotlb_nr_tbl()) {
			st->nents++;
			sg_set_page(sg, page, PAGE_SIZE, 0);
			sg = sg_next(sg);
			continue;
		}
#endif
1822 1823 1824 1825 1826 1827 1828 1829 1830
		if (!i || page_to_pfn(page) != last_pfn + 1) {
			if (i)
				sg = sg_next(sg);
			st->nents++;
			sg_set_page(sg, page, PAGE_SIZE, 0);
		} else {
			sg->length += PAGE_SIZE;
		}
		last_pfn = page_to_pfn(page);
1831
	}
1832 1833 1834 1835
#ifdef CONFIG_SWIOTLB
	if (!swiotlb_nr_tbl())
#endif
		sg_mark_end(sg);
1836 1837
	obj->pages = st;

1838
	if (i915_gem_object_needs_bit17_swizzle(obj))
1839 1840 1841 1842 1843
		i915_gem_object_do_bit_17_swizzle(obj);

	return 0;

err_pages:
1844 1845
	sg_mark_end(sg);
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
1846
		page_cache_release(sg_page_iter_page(&sg_iter));
1847 1848
	sg_free_table(st);
	kfree(st);
1849
	return PTR_ERR(page);
1850 1851
}

1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865
/* Ensure that the associated pages are gathered from the backing storage
 * and pinned into our object. i915_gem_object_get_pages() may be called
 * multiple times before they are released by a single call to
 * i915_gem_object_put_pages() - once the pages are no longer referenced
 * either as a result of memory pressure (reaping pages under the shrinker)
 * or as the object is itself released.
 */
int
i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	const struct drm_i915_gem_object_ops *ops = obj->ops;
	int ret;

1866
	if (obj->pages)
1867 1868
		return 0;

1869 1870 1871 1872 1873
	if (obj->madv != I915_MADV_WILLNEED) {
		DRM_ERROR("Attempting to obtain a purgeable object\n");
		return -EINVAL;
	}

1874 1875
	BUG_ON(obj->pages_pin_count);

1876 1877 1878 1879
	ret = ops->get_pages(obj);
	if (ret)
		return ret;

1880
	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
1881
	return 0;
1882 1883
}

1884
void
1885
i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1886
			       struct intel_ring_buffer *ring)
1887
{
1888
	struct drm_device *dev = obj->base.dev;
1889
	struct drm_i915_private *dev_priv = dev->dev_private;
1890
	u32 seqno = intel_ring_get_seqno(ring);
1891

1892
	BUG_ON(ring == NULL);
1893 1894 1895 1896
	if (obj->ring != ring && obj->last_write_seqno) {
		/* Keep the seqno relative to the current ring */
		obj->last_write_seqno = seqno;
	}
1897
	obj->ring = ring;
1898 1899

	/* Add a reference if we're newly entering the active list. */
1900 1901 1902
	if (!obj->active) {
		drm_gem_object_reference(&obj->base);
		obj->active = 1;
1903
	}
1904

1905
	list_move_tail(&obj->ring_list, &ring->active_list);
1906

1907
	obj->last_read_seqno = seqno;
1908

1909
	if (obj->fenced_gpu_access) {
1910 1911
		obj->last_fenced_seqno = seqno;

1912 1913 1914 1915 1916 1917 1918 1919
		/* Bump MRU to take account of the delayed flush */
		if (obj->fence_reg != I915_FENCE_REG_NONE) {
			struct drm_i915_fence_reg *reg;

			reg = &dev_priv->fence_regs[obj->fence_reg];
			list_move_tail(&reg->lru_list,
				       &dev_priv->mm.fence_list);
		}
1920 1921 1922 1923 1924
	}
}

static void
i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1925
{
B
Ben Widawsky 已提交
1926 1927 1928
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
	struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
1929

1930
	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
1931
	BUG_ON(!obj->active);
1932

B
Ben Widawsky 已提交
1933
	list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list);
1934

1935
	list_del_init(&obj->ring_list);
1936 1937
	obj->ring = NULL;

1938 1939 1940 1941 1942
	obj->last_read_seqno = 0;
	obj->last_write_seqno = 0;
	obj->base.write_domain = 0;

	obj->last_fenced_seqno = 0;
1943 1944 1945 1946 1947 1948
	obj->fenced_gpu_access = false;

	obj->active = 0;
	drm_gem_object_unreference(&obj->base);

	WARN_ON(i915_verify_lists(dev));
1949
}
1950

1951
static int
1952
i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
1953
{
1954 1955 1956
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_ring_buffer *ring;
	int ret, i, j;
1957

1958
	/* Carefully retire all requests without writing to the rings */
1959
	for_each_ring(ring, dev_priv, i) {
1960 1961 1962
		ret = intel_ring_idle(ring);
		if (ret)
			return ret;
1963 1964
	}
	i915_gem_retire_requests(dev);
1965 1966

	/* Finally reset hw state */
1967
	for_each_ring(ring, dev_priv, i) {
1968
		intel_ring_init_seqno(ring, seqno);
1969

1970 1971 1972
		for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
			ring->sync_seqno[j] = 0;
	}
1973

1974
	return 0;
1975 1976
}

1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002
int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	if (seqno == 0)
		return -EINVAL;

	/* HWS page needs to be set less than what we
	 * will inject to ring
	 */
	ret = i915_gem_init_seqno(dev, seqno - 1);
	if (ret)
		return ret;

	/* Carefully set the last_seqno value so that wrap
	 * detection still works
	 */
	dev_priv->next_seqno = seqno;
	dev_priv->last_seqno = seqno - 1;
	if (dev_priv->last_seqno == 0)
		dev_priv->last_seqno--;

	return 0;
}

2003 2004
int
i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2005
{
2006 2007 2008 2009
	struct drm_i915_private *dev_priv = dev->dev_private;

	/* reserve 0 for non-seqno */
	if (dev_priv->next_seqno == 0) {
2010
		int ret = i915_gem_init_seqno(dev, 0);
2011 2012
		if (ret)
			return ret;
2013

2014 2015
		dev_priv->next_seqno = 1;
	}
2016

2017
	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2018
	return 0;
2019 2020
}

2021 2022
int __i915_add_request(struct intel_ring_buffer *ring,
		       struct drm_file *file,
2023
		       struct drm_i915_gem_object *obj,
2024
		       u32 *out_seqno)
2025
{
C
Chris Wilson 已提交
2026
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
2027
	struct drm_i915_gem_request *request;
2028
	u32 request_ring_position, request_start;
2029
	int was_empty;
2030 2031
	int ret;

2032
	request_start = intel_ring_get_tail(ring);
2033 2034 2035 2036 2037 2038 2039
	/*
	 * Emit any outstanding flushes - execbuf can fail to emit the flush
	 * after having emitted the batchbuffer command. Hence we need to fix
	 * things up similar to emitting the lazy request. The difference here
	 * is that the flush _must_ happen before the next request, no matter
	 * what.
	 */
2040 2041 2042
	ret = intel_ring_flush_all_caches(ring);
	if (ret)
		return ret;
2043

2044 2045
	request = ring->preallocated_lazy_request;
	if (WARN_ON(request == NULL))
2046
		return -ENOMEM;
2047

2048 2049 2050 2051 2052 2053 2054
	/* Record the position of the start of the request so that
	 * should we detect the updated seqno part-way through the
	 * GPU processing the request, we never over-estimate the
	 * position of the head.
	 */
	request_ring_position = intel_ring_get_tail(ring);

2055
	ret = ring->add_request(ring);
2056
	if (ret)
2057
		return ret;
2058

2059
	request->seqno = intel_ring_get_seqno(ring);
2060
	request->ring = ring;
2061
	request->head = request_start;
2062
	request->tail = request_ring_position;
2063 2064 2065 2066 2067 2068 2069

	/* Whilst this request exists, batch_obj will be on the
	 * active_list, and so will hold the active reference. Only when this
	 * request is retired will the the batch_obj be moved onto the
	 * inactive_list and lose its active reference. Hence we do not need
	 * to explicitly hold another reference here.
	 */
2070
	request->batch_obj = obj;
2071

2072 2073 2074 2075
	/* Hold a reference to the current context so that we can inspect
	 * it later in case a hangcheck error event fires.
	 */
	request->ctx = ring->last_context;
2076 2077 2078
	if (request->ctx)
		i915_gem_context_reference(request->ctx);

2079
	request->emitted_jiffies = jiffies;
2080 2081
	was_empty = list_empty(&ring->request_list);
	list_add_tail(&request->list, &ring->request_list);
2082
	request->file_priv = NULL;
2083

C
Chris Wilson 已提交
2084 2085 2086
	if (file) {
		struct drm_i915_file_private *file_priv = file->driver_priv;

2087
		spin_lock(&file_priv->mm.lock);
2088
		request->file_priv = file_priv;
2089
		list_add_tail(&request->client_list,
2090
			      &file_priv->mm.request_list);
2091
		spin_unlock(&file_priv->mm.lock);
2092
	}
2093

2094
	trace_i915_gem_request_add(ring, request->seqno);
2095
	ring->outstanding_lazy_seqno = 0;
2096
	ring->preallocated_lazy_request = NULL;
C
Chris Wilson 已提交
2097

2098
	if (!dev_priv->ums.mm_suspended) {
2099 2100
		i915_queue_hangcheck(ring->dev);

2101
		if (was_empty) {
2102
			queue_delayed_work(dev_priv->wq,
2103 2104
					   &dev_priv->mm.retire_work,
					   round_jiffies_up_relative(HZ));
2105 2106
			intel_mark_busy(dev_priv->dev);
		}
B
Ben Gamari 已提交
2107
	}
2108

2109
	if (out_seqno)
2110
		*out_seqno = request->seqno;
2111
	return 0;
2112 2113
}

2114 2115
static inline void
i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2116
{
2117
	struct drm_i915_file_private *file_priv = request->file_priv;
2118

2119 2120
	if (!file_priv)
		return;
C
Chris Wilson 已提交
2121

2122
	spin_lock(&file_priv->mm.lock);
2123 2124 2125 2126
	if (request->file_priv) {
		list_del(&request->client_list);
		request->file_priv = NULL;
	}
2127
	spin_unlock(&file_priv->mm.lock);
2128 2129
}

2130 2131
static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj,
				    struct i915_address_space *vm)
2132
{
2133 2134
	if (acthd >= i915_gem_obj_offset(obj, vm) &&
	    acthd < i915_gem_obj_offset(obj, vm) + obj->base.size)
2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156
		return true;

	return false;
}

static bool i915_head_inside_request(const u32 acthd_unmasked,
				     const u32 request_start,
				     const u32 request_end)
{
	const u32 acthd = acthd_unmasked & HEAD_ADDR;

	if (request_start < request_end) {
		if (acthd >= request_start && acthd < request_end)
			return true;
	} else if (request_start > request_end) {
		if (acthd >= request_start || acthd < request_end)
			return true;
	}

	return false;
}

2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167
static struct i915_address_space *
request_to_vm(struct drm_i915_gem_request *request)
{
	struct drm_i915_private *dev_priv = request->ring->dev->dev_private;
	struct i915_address_space *vm;

	vm = &dev_priv->gtt.base;

	return vm;
}

2168 2169 2170 2171 2172 2173 2174 2175
static bool i915_request_guilty(struct drm_i915_gem_request *request,
				const u32 acthd, bool *inside)
{
	/* There is a possibility that unmasked head address
	 * pointing inside the ring, matches the batch_obj address range.
	 * However this is extremely unlikely.
	 */
	if (request->batch_obj) {
2176 2177
		if (i915_head_inside_object(acthd, request->batch_obj,
					    request_to_vm(request))) {
2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
			*inside = true;
			return true;
		}
	}

	if (i915_head_inside_request(acthd, request->head, request->tail)) {
		*inside = false;
		return true;
	}

	return false;
}

2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205
static bool i915_context_is_banned(const struct i915_ctx_hang_stats *hs)
{
	const unsigned long elapsed = get_seconds() - hs->guilty_ts;

	if (hs->banned)
		return true;

	if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
		DRM_ERROR("context hanging too fast, declaring banned!\n");
		return true;
	}

	return false;
}

2206 2207 2208 2209 2210 2211
static void i915_set_reset_status(struct intel_ring_buffer *ring,
				  struct drm_i915_gem_request *request,
				  u32 acthd)
{
	struct i915_ctx_hang_stats *hs = NULL;
	bool inside, guilty;
2212
	unsigned long offset = 0;
2213 2214 2215 2216

	/* Innocent until proven guilty */
	guilty = false;

2217 2218 2219 2220
	if (request->batch_obj)
		offset = i915_gem_obj_offset(request->batch_obj,
					     request_to_vm(request));

2221
	if (ring->hangcheck.action != HANGCHECK_WAIT &&
2222
	    i915_request_guilty(request, acthd, &inside)) {
2223
		DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
2224 2225
			  ring->name,
			  inside ? "inside" : "flushing",
2226
			  offset,
2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241
			  request->ctx ? request->ctx->id : 0,
			  acthd);

		guilty = true;
	}

	/* If contexts are disabled or this is the default context, use
	 * file_priv->reset_state
	 */
	if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
		hs = &request->ctx->hang_stats;
	else if (request->file_priv)
		hs = &request->file_priv->hang_stats;

	if (hs) {
2242 2243
		if (guilty) {
			hs->banned = i915_context_is_banned(hs);
2244
			hs->batch_active++;
2245 2246
			hs->guilty_ts = get_seconds();
		} else {
2247
			hs->batch_pending++;
2248
		}
2249 2250 2251
	}
}

2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262
static void i915_gem_free_request(struct drm_i915_gem_request *request)
{
	list_del(&request->list);
	i915_gem_request_remove_from_client(request);

	if (request->ctx)
		i915_gem_context_unreference(request->ctx);

	kfree(request);
}

2263 2264
static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
				      struct intel_ring_buffer *ring)
2265
{
2266 2267 2268 2269 2270 2271
	u32 completed_seqno;
	u32 acthd;

	acthd = intel_ring_get_active_head(ring);
	completed_seqno = ring->get_seqno(ring, false);

2272 2273
	while (!list_empty(&ring->request_list)) {
		struct drm_i915_gem_request *request;
2274

2275 2276 2277
		request = list_first_entry(&ring->request_list,
					   struct drm_i915_gem_request,
					   list);
2278

2279 2280 2281
		if (request->seqno > completed_seqno)
			i915_set_reset_status(ring, request, acthd);

2282
		i915_gem_free_request(request);
2283
	}
2284

2285
	while (!list_empty(&ring->active_list)) {
2286
		struct drm_i915_gem_object *obj;
2287

2288 2289 2290
		obj = list_first_entry(&ring->active_list,
				       struct drm_i915_gem_object,
				       ring_list);
2291

2292
		i915_gem_object_move_to_inactive(obj);
2293 2294 2295
	}
}

2296
void i915_gem_restore_fences(struct drm_device *dev)
2297 2298 2299 2300
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int i;

2301
	for (i = 0; i < dev_priv->num_fence_regs; i++) {
2302
		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2303

2304 2305 2306 2307 2308 2309 2310 2311 2312 2313
		/*
		 * Commit delayed tiling changes if we have an object still
		 * attached to the fence, otherwise just clear the fence.
		 */
		if (reg->obj) {
			i915_gem_object_update_fence(reg->obj, reg,
						     reg->obj->tiling_mode);
		} else {
			i915_gem_write_fence(dev, i, NULL);
		}
2314 2315 2316
	}
}

2317
void i915_gem_reset(struct drm_device *dev)
2318
{
2319
	struct drm_i915_private *dev_priv = dev->dev_private;
2320
	struct intel_ring_buffer *ring;
2321
	int i;
2322

2323 2324
	for_each_ring(ring, dev_priv, i)
		i915_gem_reset_ring_lists(dev_priv, ring);
2325

2326
	i915_gem_restore_fences(dev);
2327 2328 2329 2330 2331
}

/**
 * This function clears the request list as sequence numbers are passed.
 */
2332
void
C
Chris Wilson 已提交
2333
i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2334 2335 2336
{
	uint32_t seqno;

C
Chris Wilson 已提交
2337
	if (list_empty(&ring->request_list))
2338 2339
		return;

C
Chris Wilson 已提交
2340
	WARN_ON(i915_verify_lists(ring->dev));
2341

2342
	seqno = ring->get_seqno(ring, true);
2343

2344
	while (!list_empty(&ring->request_list)) {
2345 2346
		struct drm_i915_gem_request *request;

2347
		request = list_first_entry(&ring->request_list,
2348 2349 2350
					   struct drm_i915_gem_request,
					   list);

2351
		if (!i915_seqno_passed(seqno, request->seqno))
2352 2353
			break;

C
Chris Wilson 已提交
2354
		trace_i915_gem_request_retire(ring, request->seqno);
2355 2356 2357 2358 2359 2360
		/* We know the GPU must have read the request to have
		 * sent us the seqno + interrupt, so use the position
		 * of tail of the request to update the last known position
		 * of the GPU head.
		 */
		ring->last_retired_head = request->tail;
2361

2362
		i915_gem_free_request(request);
2363
	}
2364

2365 2366 2367 2368
	/* Move any buffers on the active list that are no longer referenced
	 * by the ringbuffer to the flushing/inactive lists as appropriate.
	 */
	while (!list_empty(&ring->active_list)) {
2369
		struct drm_i915_gem_object *obj;
2370

2371
		obj = list_first_entry(&ring->active_list,
2372 2373
				      struct drm_i915_gem_object,
				      ring_list);
2374

2375
		if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2376
			break;
2377

2378
		i915_gem_object_move_to_inactive(obj);
2379
	}
2380

C
Chris Wilson 已提交
2381 2382
	if (unlikely(ring->trace_irq_seqno &&
		     i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2383
		ring->irq_put(ring);
C
Chris Wilson 已提交
2384
		ring->trace_irq_seqno = 0;
2385
	}
2386

C
Chris Wilson 已提交
2387
	WARN_ON(i915_verify_lists(ring->dev));
2388 2389
}

2390 2391 2392 2393
void
i915_gem_retire_requests(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2394
	struct intel_ring_buffer *ring;
2395
	int i;
2396

2397 2398
	for_each_ring(ring, dev_priv, i)
		i915_gem_retire_requests_ring(ring);
2399 2400
}

2401
static void
2402 2403 2404 2405
i915_gem_retire_work_handler(struct work_struct *work)
{
	drm_i915_private_t *dev_priv;
	struct drm_device *dev;
2406
	struct intel_ring_buffer *ring;
2407 2408
	bool idle;
	int i;
2409 2410 2411 2412 2413

	dev_priv = container_of(work, drm_i915_private_t,
				mm.retire_work.work);
	dev = dev_priv->dev;

2414 2415
	/* Come back later if the device is busy... */
	if (!mutex_trylock(&dev->struct_mutex)) {
2416 2417
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
				   round_jiffies_up_relative(HZ));
2418 2419
		return;
	}
2420

2421
	i915_gem_retire_requests(dev);
2422

2423 2424
	/* Send a periodic flush down the ring so we don't hold onto GEM
	 * objects indefinitely.
2425
	 */
2426
	idle = true;
2427
	for_each_ring(ring, dev_priv, i) {
2428
		if (ring->gpu_caches_dirty)
2429
			i915_add_request(ring, NULL);
2430 2431

		idle &= list_empty(&ring->request_list);
2432 2433
	}

2434
	if (!dev_priv->ums.mm_suspended && !idle)
2435 2436
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
				   round_jiffies_up_relative(HZ));
2437 2438
	if (idle)
		intel_mark_idle(dev);
2439

2440 2441 2442
	mutex_unlock(&dev->struct_mutex);
}

2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453
/**
 * Ensures that an object will eventually get non-busy by flushing any required
 * write domains, emitting any outstanding lazy request and retiring and
 * completed requests.
 */
static int
i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
{
	int ret;

	if (obj->active) {
2454
		ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2455 2456 2457 2458 2459 2460 2461 2462 2463
		if (ret)
			return ret;

		i915_gem_retire_requests_ring(obj->ring);
	}

	return 0;
}

2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488
/**
 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
 * @DRM_IOCTL_ARGS: standard ioctl arguments
 *
 * Returns 0 if successful, else an error is returned with the remaining time in
 * the timeout parameter.
 *  -ETIME: object is still busy after timeout
 *  -ERESTARTSYS: signal interrupted the wait
 *  -ENONENT: object doesn't exist
 * Also possible, but rare:
 *  -EAGAIN: GPU wedged
 *  -ENOMEM: damn
 *  -ENODEV: Internal IRQ fail
 *  -E?: The add request failed
 *
 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
 * non-zero timeout parameter the wait ioctl will wait for the given number of
 * nanoseconds on an object becoming unbusy. Since the wait itself does so
 * without holding struct_mutex the object may become re-busied before this
 * function completes. A similar but shorter * race condition exists in the busy
 * ioctl
 */
int
i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
2489
	drm_i915_private_t *dev_priv = dev->dev_private;
2490 2491 2492
	struct drm_i915_gem_wait *args = data;
	struct drm_i915_gem_object *obj;
	struct intel_ring_buffer *ring = NULL;
2493
	struct timespec timeout_stack, *timeout = NULL;
2494
	unsigned reset_counter;
2495 2496 2497
	u32 seqno = 0;
	int ret = 0;

2498 2499 2500 2501
	if (args->timeout_ns >= 0) {
		timeout_stack = ns_to_timespec(args->timeout_ns);
		timeout = &timeout_stack;
	}
2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512

	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
	if (&obj->base == NULL) {
		mutex_unlock(&dev->struct_mutex);
		return -ENOENT;
	}

2513 2514
	/* Need to make sure the object gets inactive eventually. */
	ret = i915_gem_object_flush_active(obj);
2515 2516 2517 2518
	if (ret)
		goto out;

	if (obj->active) {
2519
		seqno = obj->last_read_seqno;
2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534
		ring = obj->ring;
	}

	if (seqno == 0)
		 goto out;

	/* Do this after OLR check to make sure we make forward progress polling
	 * on this IOCTL with a 0 timeout (like busy ioctl)
	 */
	if (!args->timeout_ns) {
		ret = -ETIME;
		goto out;
	}

	drm_gem_object_unreference(&obj->base);
2535
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2536 2537
	mutex_unlock(&dev->struct_mutex);

2538
	ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
2539
	if (timeout)
2540
		args->timeout_ns = timespec_to_ns(timeout);
2541 2542 2543 2544 2545 2546 2547 2548
	return ret;

out:
	drm_gem_object_unreference(&obj->base);
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560
/**
 * i915_gem_object_sync - sync an object to a ring.
 *
 * @obj: object which may be in use on another ring.
 * @to: ring we wish to use the object on. May be NULL.
 *
 * This code is meant to abstract object synchronization with the GPU.
 * Calling with NULL implies synchronizing the object with the CPU
 * rather than a particular GPU ring.
 *
 * Returns 0 if successful, else propagates up the lower layer error.
 */
2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571
int
i915_gem_object_sync(struct drm_i915_gem_object *obj,
		     struct intel_ring_buffer *to)
{
	struct intel_ring_buffer *from = obj->ring;
	u32 seqno;
	int ret, idx;

	if (from == NULL || to == from)
		return 0;

2572
	if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2573
		return i915_gem_object_wait_rendering(obj, false);
2574 2575 2576

	idx = intel_ring_sync_index(from, to);

2577
	seqno = obj->last_read_seqno;
2578 2579 2580
	if (seqno <= from->sync_seqno[idx])
		return 0;

2581 2582 2583
	ret = i915_gem_check_olr(obj->ring, seqno);
	if (ret)
		return ret;
2584

2585
	ret = to->sync_to(to, from, seqno);
2586
	if (!ret)
2587 2588 2589 2590 2591
		/* We use last_read_seqno because sync_to()
		 * might have just caused seqno wrap under
		 * the radar.
		 */
		from->sync_seqno[idx] = obj->last_read_seqno;
2592

2593
	return ret;
2594 2595
}

2596 2597 2598 2599 2600 2601 2602
static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
{
	u32 old_write_domain, old_read_domains;

	/* Force a pagefault for domain tracking on next user access */
	i915_gem_release_mmap(obj);

2603 2604 2605
	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		return;

2606 2607 2608
	/* Wait for any direct GTT access to complete */
	mb();

2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619
	old_read_domains = obj->base.read_domains;
	old_write_domain = obj->base.write_domain;

	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;

	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);
}

2620
int i915_vma_unbind(struct i915_vma *vma)
2621
{
2622
	struct drm_i915_gem_object *obj = vma->obj;
2623
	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2624
	int ret;
2625

2626 2627 2628
	/* For now we only ever use 1 vma per object */
	WARN_ON(!list_is_singular(&obj->vma_list));

2629
	if (list_empty(&vma->vma_link))
2630 2631
		return 0;

2632 2633 2634 2635 2636
	if (!drm_mm_node_allocated(&vma->node)) {
		i915_gem_vma_destroy(vma);

		return 0;
	}
2637

2638 2639
	if (obj->pin_count)
		return -EBUSY;
2640

2641 2642
	BUG_ON(obj->pages == NULL);

2643
	ret = i915_gem_object_finish_gpu(obj);
2644
	if (ret)
2645 2646 2647 2648 2649 2650
		return ret;
	/* Continue on if we fail due to EIO, the GPU is hung so we
	 * should be safe and we need to cleanup or else we might
	 * cause memory corruption through use-after-free.
	 */

2651
	i915_gem_object_finish_gtt(obj);
2652

2653
	/* release the fence reg _after_ flushing */
2654
	ret = i915_gem_object_put_fence(obj);
2655
	if (ret)
2656
		return ret;
2657

2658
	trace_i915_vma_unbind(vma);
C
Chris Wilson 已提交
2659

2660 2661
	if (obj->has_global_gtt_mapping)
		i915_gem_gtt_unbind_object(obj);
2662 2663 2664 2665
	if (obj->has_aliasing_ppgtt_mapping) {
		i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		obj->has_aliasing_ppgtt_mapping = 0;
	}
2666
	i915_gem_gtt_finish_object(obj);
B
Ben Widawsky 已提交
2667
	i915_gem_object_unpin_pages(obj);
2668

B
Ben Widawsky 已提交
2669
	list_del(&vma->mm_list);
2670
	/* Avoid an unnecessary call to unbind on rebind. */
2671 2672
	if (i915_is_ggtt(vma->vm))
		obj->map_and_fenceable = true;
2673

B
Ben Widawsky 已提交
2674
	drm_mm_remove_node(&vma->node);
2675

B
Ben Widawsky 已提交
2676 2677 2678
	i915_gem_vma_destroy(vma);

	/* Since the unbound list is global, only move to that list if
2679
	 * no more VMAs exist. */
B
Ben Widawsky 已提交
2680 2681
	if (list_empty(&obj->vma_list))
		list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2682

2683
	return 0;
2684 2685
}

2686 2687 2688 2689 2690 2691 2692 2693 2694
/**
 * Unbinds an object from the global GTT aperture.
 */
int
i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	struct i915_address_space *ggtt = &dev_priv->gtt.base;

2695
	if (!i915_gem_obj_ggtt_bound(obj))
2696 2697 2698 2699 2700 2701 2702 2703 2704 2705
		return 0;

	if (obj->pin_count)
		return -EBUSY;

	BUG_ON(obj->pages == NULL);

	return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt));
}

2706
int i915_gpu_idle(struct drm_device *dev)
2707 2708
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2709
	struct intel_ring_buffer *ring;
2710
	int ret, i;
2711 2712

	/* Flush everything onto the inactive list. */
2713
	for_each_ring(ring, dev_priv, i) {
2714 2715 2716 2717
		ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
		if (ret)
			return ret;

2718
		ret = intel_ring_idle(ring);
2719 2720 2721
		if (ret)
			return ret;
	}
2722

2723
	return 0;
2724 2725
}

2726 2727
static void i965_write_fence_reg(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
2728 2729
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2730 2731
	int fence_reg;
	int fence_pitch_shift;
2732

2733 2734 2735 2736 2737 2738 2739 2740
	if (INTEL_INFO(dev)->gen >= 6) {
		fence_reg = FENCE_REG_SANDYBRIDGE_0;
		fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
	} else {
		fence_reg = FENCE_REG_965_0;
		fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
	}

2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754
	fence_reg += reg * 8;

	/* To w/a incoherency with non-atomic 64-bit register updates,
	 * we split the 64-bit update into two 32-bit writes. In order
	 * for a partial fence not to be evaluated between writes, we
	 * precede the update with write to turn off the fence register,
	 * and only enable the fence as the last step.
	 *
	 * For extra levels of paranoia, we make sure each step lands
	 * before applying the next step.
	 */
	I915_WRITE(fence_reg, 0);
	POSTING_READ(fence_reg);

2755
	if (obj) {
2756
		u32 size = i915_gem_obj_ggtt_size(obj);
2757
		uint64_t val;
2758

2759
		val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
2760
				 0xfffff000) << 32;
2761
		val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
2762
		val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
2763 2764 2765
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I965_FENCE_TILING_Y_SHIFT;
		val |= I965_FENCE_REG_VALID;
2766

2767 2768 2769 2770 2771 2772 2773 2774 2775
		I915_WRITE(fence_reg + 4, val >> 32);
		POSTING_READ(fence_reg + 4);

		I915_WRITE(fence_reg + 0, val);
		POSTING_READ(fence_reg);
	} else {
		I915_WRITE(fence_reg + 4, 0);
		POSTING_READ(fence_reg + 4);
	}
2776 2777
}

2778 2779
static void i915_write_fence_reg(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
2780 2781
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2782
	u32 val;
2783

2784
	if (obj) {
2785
		u32 size = i915_gem_obj_ggtt_size(obj);
2786 2787
		int pitch_val;
		int tile_width;
2788

2789
		WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
2790
		     (size & -size) != size ||
2791 2792 2793
		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		     "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		     i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2794

2795 2796 2797 2798 2799 2800 2801 2802 2803
		if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
			tile_width = 128;
		else
			tile_width = 512;

		/* Note: pitch better be a power of two tile widths */
		pitch_val = obj->stride / tile_width;
		pitch_val = ffs(pitch_val) - 1;

2804
		val = i915_gem_obj_ggtt_offset(obj);
2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
		val |= I915_FENCE_SIZE_BITS(size);
		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
		val |= I830_FENCE_REG_VALID;
	} else
		val = 0;

	if (reg < 8)
		reg = FENCE_REG_830_0 + reg * 4;
	else
		reg = FENCE_REG_945_8 + (reg - 8) * 4;

	I915_WRITE(reg, val);
	POSTING_READ(reg);
2820 2821
}

2822 2823
static void i830_write_fence_reg(struct drm_device *dev, int reg,
				struct drm_i915_gem_object *obj)
2824 2825 2826 2827
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	uint32_t val;

2828
	if (obj) {
2829
		u32 size = i915_gem_obj_ggtt_size(obj);
2830
		uint32_t pitch_val;
2831

2832
		WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
2833
		     (size & -size) != size ||
2834 2835 2836
		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		     "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
		     i915_gem_obj_ggtt_offset(obj), size);
2837

2838 2839
		pitch_val = obj->stride / 128;
		pitch_val = ffs(pitch_val) - 1;
2840

2841
		val = i915_gem_obj_ggtt_offset(obj);
2842 2843 2844 2845 2846 2847 2848
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
		val |= I830_FENCE_SIZE_BITS(size);
		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
		val |= I830_FENCE_REG_VALID;
	} else
		val = 0;
2849

2850 2851 2852 2853
	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
	POSTING_READ(FENCE_REG_830_0 + reg * 4);
}

2854 2855 2856 2857 2858
inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
{
	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
}

2859 2860 2861
static void i915_gem_write_fence(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
{
2862 2863 2864 2865 2866 2867 2868 2869
	struct drm_i915_private *dev_priv = dev->dev_private;

	/* Ensure that all CPU reads are completed before installing a fence
	 * and all writes before removing the fence.
	 */
	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		mb();

2870 2871 2872 2873
	WARN(obj && (!obj->stride || !obj->tiling_mode),
	     "bogus fence setup with stride: 0x%x, tiling mode: %i\n",
	     obj->stride, obj->tiling_mode);

2874 2875
	switch (INTEL_INFO(dev)->gen) {
	case 7:
2876
	case 6:
2877 2878 2879 2880
	case 5:
	case 4: i965_write_fence_reg(dev, reg, obj); break;
	case 3: i915_write_fence_reg(dev, reg, obj); break;
	case 2: i830_write_fence_reg(dev, reg, obj); break;
2881
	default: BUG();
2882
	}
2883 2884 2885 2886 2887 2888

	/* And similarly be paranoid that no direct access to this region
	 * is reordered to before the fence is installed.
	 */
	if (i915_gem_object_needs_mb(obj))
		mb();
2889 2890
}

2891 2892 2893 2894 2895 2896 2897 2898 2899 2900
static inline int fence_number(struct drm_i915_private *dev_priv,
			       struct drm_i915_fence_reg *fence)
{
	return fence - dev_priv->fence_regs;
}

static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
					 struct drm_i915_fence_reg *fence,
					 bool enable)
{
2901
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2902 2903 2904
	int reg = fence_number(dev_priv, fence);

	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2905 2906

	if (enable) {
2907
		obj->fence_reg = reg;
2908 2909 2910 2911 2912 2913 2914
		fence->obj = obj;
		list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
	} else {
		obj->fence_reg = I915_FENCE_REG_NONE;
		fence->obj = NULL;
		list_del_init(&fence->lru_list);
	}
2915
	obj->fence_dirty = false;
2916 2917
}

2918
static int
2919
i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
2920
{
2921
	if (obj->last_fenced_seqno) {
2922
		int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2923 2924
		if (ret)
			return ret;
2925 2926 2927 2928

		obj->last_fenced_seqno = 0;
	}

2929
	obj->fenced_gpu_access = false;
2930 2931 2932 2933 2934 2935
	return 0;
}

int
i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
{
2936
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2937
	struct drm_i915_fence_reg *fence;
2938 2939
	int ret;

2940
	ret = i915_gem_object_wait_fence(obj);
2941 2942 2943
	if (ret)
		return ret;

2944 2945
	if (obj->fence_reg == I915_FENCE_REG_NONE)
		return 0;
2946

2947 2948
	fence = &dev_priv->fence_regs[obj->fence_reg];

2949
	i915_gem_object_fence_lost(obj);
2950
	i915_gem_object_update_fence(obj, fence, false);
2951 2952 2953 2954 2955

	return 0;
}

static struct drm_i915_fence_reg *
C
Chris Wilson 已提交
2956
i915_find_fence_reg(struct drm_device *dev)
2957 2958
{
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
2959
	struct drm_i915_fence_reg *reg, *avail;
2960
	int i;
2961 2962

	/* First try to find a free reg */
2963
	avail = NULL;
2964 2965 2966
	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		reg = &dev_priv->fence_regs[i];
		if (!reg->obj)
2967
			return reg;
2968

2969
		if (!reg->pin_count)
2970
			avail = reg;
2971 2972
	}

2973 2974
	if (avail == NULL)
		return NULL;
2975 2976

	/* None available, try to steal one or wait for a user to finish */
2977
	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2978
		if (reg->pin_count)
2979 2980
			continue;

C
Chris Wilson 已提交
2981
		return reg;
2982 2983
	}

C
Chris Wilson 已提交
2984
	return NULL;
2985 2986
}

2987
/**
2988
 * i915_gem_object_get_fence - set up fencing for an object
2989 2990 2991 2992 2993 2994 2995 2996 2997
 * @obj: object to map through a fence reg
 *
 * When mapping objects through the GTT, userspace wants to be able to write
 * to them without having to worry about swizzling if the object is tiled.
 * This function walks the fence regs looking for a free one for @obj,
 * stealing one if it can't find any.
 *
 * It then sets up the reg based on the object's properties: address, pitch
 * and tiling format.
2998 2999
 *
 * For an untiled surface, this removes any existing fence.
3000
 */
3001
int
3002
i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
3003
{
3004
	struct drm_device *dev = obj->base.dev;
J
Jesse Barnes 已提交
3005
	struct drm_i915_private *dev_priv = dev->dev_private;
3006
	bool enable = obj->tiling_mode != I915_TILING_NONE;
3007
	struct drm_i915_fence_reg *reg;
3008
	int ret;
3009

3010 3011 3012
	/* Have we updated the tiling parameters upon the object and so
	 * will need to serialise the write to the associated fence register?
	 */
3013
	if (obj->fence_dirty) {
3014
		ret = i915_gem_object_wait_fence(obj);
3015 3016 3017
		if (ret)
			return ret;
	}
3018

3019
	/* Just update our place in the LRU if our fence is getting reused. */
3020 3021
	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		reg = &dev_priv->fence_regs[obj->fence_reg];
3022
		if (!obj->fence_dirty) {
3023 3024 3025 3026 3027 3028 3029 3030
			list_move_tail(&reg->lru_list,
				       &dev_priv->mm.fence_list);
			return 0;
		}
	} else if (enable) {
		reg = i915_find_fence_reg(dev);
		if (reg == NULL)
			return -EDEADLK;
3031

3032 3033 3034
		if (reg->obj) {
			struct drm_i915_gem_object *old = reg->obj;

3035
			ret = i915_gem_object_wait_fence(old);
3036 3037 3038
			if (ret)
				return ret;

3039
			i915_gem_object_fence_lost(old);
3040
		}
3041
	} else
3042 3043
		return 0;

3044 3045
	i915_gem_object_update_fence(obj, reg, enable);

3046
	return 0;
3047 3048
}

3049 3050 3051 3052 3053 3054 3055 3056
static bool i915_gem_valid_gtt_space(struct drm_device *dev,
				     struct drm_mm_node *gtt_space,
				     unsigned long cache_level)
{
	struct drm_mm_node *other;

	/* On non-LLC machines we have to be careful when putting differing
	 * types of snoopable memory together to avoid the prefetcher
3057
	 * crossing memory domains and dying.
3058 3059 3060 3061
	 */
	if (HAS_LLC(dev))
		return true;

3062
	if (!drm_mm_node_allocated(gtt_space))
3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085
		return true;

	if (list_empty(&gtt_space->node_list))
		return true;

	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
	if (other->allocated && !other->hole_follows && other->color != cache_level)
		return false;

	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		return false;

	return true;
}

static void i915_gem_verify_gtt(struct drm_device *dev)
{
#if WATCH_GTT
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_gem_object *obj;
	int err = 0;

3086
	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3087 3088 3089 3090 3091 3092 3093 3094
		if (obj->gtt_space == NULL) {
			printk(KERN_ERR "object found on GTT list with no space reserved\n");
			err++;
			continue;
		}

		if (obj->cache_level != obj->gtt_space->color) {
			printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
3095 3096
			       i915_gem_obj_ggtt_offset(obj),
			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3097 3098 3099 3100 3101 3102 3103 3104 3105 3106
			       obj->cache_level,
			       obj->gtt_space->color);
			err++;
			continue;
		}

		if (!i915_gem_valid_gtt_space(dev,
					      obj->gtt_space,
					      obj->cache_level)) {
			printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
3107 3108
			       i915_gem_obj_ggtt_offset(obj),
			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3109 3110 3111 3112 3113 3114 3115 3116 3117 3118
			       obj->cache_level);
			err++;
			continue;
		}
	}

	WARN_ON(err);
#endif
}

3119 3120 3121 3122
/**
 * Finds free space in the GTT aperture and binds the object there.
 */
static int
3123 3124 3125 3126 3127
i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
			   struct i915_address_space *vm,
			   unsigned alignment,
			   bool map_and_fenceable,
			   bool nonblocking)
3128
{
3129
	struct drm_device *dev = obj->base.dev;
3130
	drm_i915_private_t *dev_priv = dev->dev_private;
3131
	u32 size, fence_size, fence_alignment, unfenced_alignment;
3132 3133
	size_t gtt_max =
		map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total;
B
Ben Widawsky 已提交
3134
	struct i915_vma *vma;
3135
	int ret;
3136

3137 3138 3139 3140 3141
	fence_size = i915_gem_get_gtt_size(dev,
					   obj->base.size,
					   obj->tiling_mode);
	fence_alignment = i915_gem_get_gtt_alignment(dev,
						     obj->base.size,
3142
						     obj->tiling_mode, true);
3143
	unfenced_alignment =
3144
		i915_gem_get_gtt_alignment(dev,
3145
						    obj->base.size,
3146
						    obj->tiling_mode, false);
3147

3148
	if (alignment == 0)
3149 3150
		alignment = map_and_fenceable ? fence_alignment :
						unfenced_alignment;
3151
	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
3152 3153 3154 3155
		DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		return -EINVAL;
	}

3156
	size = map_and_fenceable ? fence_size : obj->base.size;
3157

3158 3159 3160
	/* If the object is bigger than the entire aperture, reject it early
	 * before evicting everything in a vain attempt to find space.
	 */
3161
	if (obj->base.size > gtt_max) {
3162
		DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
3163 3164
			  obj->base.size,
			  map_and_fenceable ? "mappable" : "total",
3165
			  gtt_max);
3166 3167 3168
		return -E2BIG;
	}

3169
	ret = i915_gem_object_get_pages(obj);
C
Chris Wilson 已提交
3170 3171 3172
	if (ret)
		return ret;

3173 3174
	i915_gem_object_pin_pages(obj);

3175 3176
	BUG_ON(!i915_is_ggtt(vm));

3177
	vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
3178
	if (IS_ERR(vma)) {
3179 3180
		ret = PTR_ERR(vma);
		goto err_unpin;
B
Ben Widawsky 已提交
3181 3182
	}

3183 3184 3185
	/* For now we only ever use 1 vma per object */
	WARN_ON(!list_is_singular(&obj->vma_list));

3186
search_free:
3187
	ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3188
						  size, alignment,
3189 3190
						  obj->cache_level, 0, gtt_max,
						  DRM_MM_SEARCH_DEFAULT);
3191
	if (ret) {
3192
		ret = i915_gem_evict_something(dev, vm, size, alignment,
3193
					       obj->cache_level,
3194 3195
					       map_and_fenceable,
					       nonblocking);
3196 3197
		if (ret == 0)
			goto search_free;
3198

3199
		goto err_free_vma;
3200
	}
B
Ben Widawsky 已提交
3201
	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
3202
					      obj->cache_level))) {
B
Ben Widawsky 已提交
3203
		ret = -EINVAL;
3204
		goto err_remove_node;
3205 3206
	}

3207
	ret = i915_gem_gtt_prepare_object(obj);
B
Ben Widawsky 已提交
3208
	if (ret)
3209
		goto err_remove_node;
3210

3211
	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
B
Ben Widawsky 已提交
3212
	list_add_tail(&vma->mm_list, &vm->inactive_list);
3213

3214 3215
	if (i915_is_ggtt(vm)) {
		bool mappable, fenceable;
3216

3217 3218
		fenceable = (vma->node.size == fence_size &&
			     (vma->node.start & (fence_alignment - 1)) == 0);
3219

3220 3221
		mappable = (vma->node.start + obj->base.size <=
			    dev_priv->gtt.mappable_end);
3222

3223
		obj->map_and_fenceable = mappable && fenceable;
3224
	}
3225

3226
	WARN_ON(map_and_fenceable && !obj->map_and_fenceable);
3227

3228
	trace_i915_vma_bind(vma, map_and_fenceable);
3229
	i915_gem_verify_gtt(dev);
3230
	return 0;
B
Ben Widawsky 已提交
3231

3232
err_remove_node:
3233
	drm_mm_remove_node(&vma->node);
3234
err_free_vma:
B
Ben Widawsky 已提交
3235
	i915_gem_vma_destroy(vma);
3236
err_unpin:
B
Ben Widawsky 已提交
3237 3238
	i915_gem_object_unpin_pages(obj);
	return ret;
3239 3240
}

3241
bool
3242 3243
i915_gem_clflush_object(struct drm_i915_gem_object *obj,
			bool force)
3244 3245 3246 3247 3248
{
	/* If we don't have a page list set up, then we're not pinned
	 * to GPU, and we can ignore the cache flush because it'll happen
	 * again at bind time.
	 */
3249
	if (obj->pages == NULL)
3250
		return false;
3251

3252 3253 3254 3255 3256
	/*
	 * Stolen memory is always coherent with the GPU as it is explicitly
	 * marked as wc by the system, or the system is cache-coherent.
	 */
	if (obj->stolen)
3257
		return false;
3258

3259 3260 3261 3262 3263 3264 3265 3266
	/* If the GPU is snooping the contents of the CPU cache,
	 * we do not need to manually clear the CPU cache lines.  However,
	 * the caches are only snooped when the render cache is
	 * flushed/invalidated.  As we always have to emit invalidations
	 * and flushes when moving into and out of the RENDER domain, correct
	 * snooping behaviour occurs naturally as the result of our domain
	 * tracking.
	 */
3267
	if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
3268
		return false;
3269

C
Chris Wilson 已提交
3270
	trace_i915_gem_object_clflush(obj);
3271
	drm_clflush_sg(obj->pages);
3272 3273

	return true;
3274 3275 3276 3277
}

/** Flushes the GTT write domain for the object if it's dirty. */
static void
3278
i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3279
{
C
Chris Wilson 已提交
3280 3281
	uint32_t old_write_domain;

3282
	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3283 3284
		return;

3285
	/* No actual flushing is required for the GTT write domain.  Writes
3286 3287
	 * to it immediately go to main memory as far as we know, so there's
	 * no chipset flush.  It also doesn't land in render cache.
3288 3289 3290 3291
	 *
	 * However, we do have to enforce the order so that all writes through
	 * the GTT land before any writes to the device, such as updates to
	 * the GATT itself.
3292
	 */
3293 3294
	wmb();

3295 3296
	old_write_domain = obj->base.write_domain;
	obj->base.write_domain = 0;
C
Chris Wilson 已提交
3297 3298

	trace_i915_gem_object_change_domain(obj,
3299
					    obj->base.read_domains,
C
Chris Wilson 已提交
3300
					    old_write_domain);
3301 3302 3303 3304
}

/** Flushes the CPU write domain for the object if it's dirty. */
static void
3305 3306
i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
				       bool force)
3307
{
C
Chris Wilson 已提交
3308
	uint32_t old_write_domain;
3309

3310
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3311 3312
		return;

3313 3314 3315
	if (i915_gem_clflush_object(obj, force))
		i915_gem_chipset_flush(obj->base.dev);

3316 3317
	old_write_domain = obj->base.write_domain;
	obj->base.write_domain = 0;
C
Chris Wilson 已提交
3318 3319

	trace_i915_gem_object_change_domain(obj,
3320
					    obj->base.read_domains,
C
Chris Wilson 已提交
3321
					    old_write_domain);
3322 3323
}

3324 3325 3326 3327 3328 3329
/**
 * Moves a single object to the GTT read, and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
J
Jesse Barnes 已提交
3330
int
3331
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3332
{
3333
	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
C
Chris Wilson 已提交
3334
	uint32_t old_write_domain, old_read_domains;
3335
	int ret;
3336

3337
	/* Not valid to be called on unbound objects. */
3338
	if (!i915_gem_obj_bound_any(obj))
3339 3340
		return -EINVAL;

3341 3342 3343
	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		return 0;

3344
	ret = i915_gem_object_wait_rendering(obj, !write);
3345 3346 3347
	if (ret)
		return ret;

3348
	i915_gem_object_flush_cpu_write_domain(obj, false);
C
Chris Wilson 已提交
3349

3350 3351 3352 3353 3354 3355 3356
	/* Serialise direct access to this object with the barriers for
	 * coherent writes from the GPU, by effectively invalidating the
	 * GTT domain upon first access.
	 */
	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		mb();

3357 3358
	old_write_domain = obj->base.write_domain;
	old_read_domains = obj->base.read_domains;
C
Chris Wilson 已提交
3359

3360 3361 3362
	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3363 3364
	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3365
	if (write) {
3366 3367 3368
		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		obj->dirty = 1;
3369 3370
	}

C
Chris Wilson 已提交
3371 3372 3373 3374
	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);

3375
	/* And bump the LRU for this access */
B
Ben Widawsky 已提交
3376 3377 3378 3379 3380 3381 3382 3383
	if (i915_gem_object_is_inactive(obj)) {
		struct i915_vma *vma = i915_gem_obj_to_vma(obj,
							   &dev_priv->gtt.base);
		if (vma)
			list_move_tail(&vma->mm_list,
				       &dev_priv->gtt.base.inactive_list);

	}
3384

3385 3386 3387
	return 0;
}

3388 3389 3390
int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
				    enum i915_cache_level cache_level)
{
3391 3392
	struct drm_device *dev = obj->base.dev;
	drm_i915_private_t *dev_priv = dev->dev_private;
3393
	struct i915_vma *vma;
3394 3395 3396 3397 3398 3399 3400 3401 3402 3403
	int ret;

	if (obj->cache_level == cache_level)
		return 0;

	if (obj->pin_count) {
		DRM_DEBUG("can not change the cache level of pinned objects\n");
		return -EBUSY;
	}

3404 3405
	list_for_each_entry(vma, &obj->vma_list, vma_link) {
		if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
3406
			ret = i915_vma_unbind(vma);
3407 3408 3409 3410 3411
			if (ret)
				return ret;

			break;
		}
3412 3413
	}

3414
	if (i915_gem_obj_bound_any(obj)) {
3415 3416 3417 3418 3419 3420 3421 3422 3423 3424
		ret = i915_gem_object_finish_gpu(obj);
		if (ret)
			return ret;

		i915_gem_object_finish_gtt(obj);

		/* Before SandyBridge, you could not use tiling or fence
		 * registers with snooped memory, so relinquish any fences
		 * currently pointing to our region in the aperture.
		 */
3425
		if (INTEL_INFO(dev)->gen < 6) {
3426 3427 3428 3429 3430
			ret = i915_gem_object_put_fence(obj);
			if (ret)
				return ret;
		}

3431 3432
		if (obj->has_global_gtt_mapping)
			i915_gem_gtt_bind_object(obj, cache_level);
3433 3434 3435
		if (obj->has_aliasing_ppgtt_mapping)
			i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
					       obj, cache_level);
3436 3437
	}

3438 3439 3440 3441 3442
	list_for_each_entry(vma, &obj->vma_list, vma_link)
		vma->node.color = cache_level;
	obj->cache_level = cache_level;

	if (cpu_write_needs_clflush(obj)) {
3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463
		u32 old_read_domains, old_write_domain;

		/* If we're coming from LLC cached, then we haven't
		 * actually been tracking whether the data is in the
		 * CPU cache or not, since we only allow one bit set
		 * in obj->write_domain and have been skipping the clflushes.
		 * Just set it to the CPU cache for now.
		 */
		WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);

		old_read_domains = obj->base.read_domains;
		old_write_domain = obj->base.write_domain;

		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		obj->base.write_domain = I915_GEM_DOMAIN_CPU;

		trace_i915_gem_object_change_domain(obj,
						    old_read_domains,
						    old_write_domain);
	}

3464
	i915_gem_verify_gtt(dev);
3465 3466 3467
	return 0;
}

B
Ben Widawsky 已提交
3468 3469
int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3470
{
B
Ben Widawsky 已提交
3471
	struct drm_i915_gem_caching *args = data;
3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484
	struct drm_i915_gem_object *obj;
	int ret;

	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
	if (&obj->base == NULL) {
		ret = -ENOENT;
		goto unlock;
	}

3485 3486 3487 3488 3489 3490
	switch (obj->cache_level) {
	case I915_CACHE_LLC:
	case I915_CACHE_L3_LLC:
		args->caching = I915_CACHING_CACHED;
		break;

3491 3492 3493 3494
	case I915_CACHE_WT:
		args->caching = I915_CACHING_DISPLAY;
		break;

3495 3496 3497 3498
	default:
		args->caching = I915_CACHING_NONE;
		break;
	}
3499 3500 3501 3502 3503 3504 3505

	drm_gem_object_unreference(&obj->base);
unlock:
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

B
Ben Widawsky 已提交
3506 3507
int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3508
{
B
Ben Widawsky 已提交
3509
	struct drm_i915_gem_caching *args = data;
3510 3511 3512 3513
	struct drm_i915_gem_object *obj;
	enum i915_cache_level level;
	int ret;

B
Ben Widawsky 已提交
3514 3515
	switch (args->caching) {
	case I915_CACHING_NONE:
3516 3517
		level = I915_CACHE_NONE;
		break;
B
Ben Widawsky 已提交
3518
	case I915_CACHING_CACHED:
3519 3520
		level = I915_CACHE_LLC;
		break;
3521 3522 3523
	case I915_CACHING_DISPLAY:
		level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
		break;
3524 3525 3526 3527
	default:
		return -EINVAL;
	}

B
Ben Widawsky 已提交
3528 3529 3530 3531
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
	if (&obj->base == NULL) {
		ret = -ENOENT;
		goto unlock;
	}

	ret = i915_gem_object_set_cache_level(obj, level);

	drm_gem_object_unreference(&obj->base);
unlock:
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561
static bool is_pin_display(struct drm_i915_gem_object *obj)
{
	/* There are 3 sources that pin objects:
	 *   1. The display engine (scanouts, sprites, cursors);
	 *   2. Reservations for execbuffer;
	 *   3. The user.
	 *
	 * We can ignore reservations as we hold the struct_mutex and
	 * are only called outside of the reservation path.  The user
	 * can only increment pin_count once, and so if after
	 * subtracting the potential reference by the user, any pin_count
	 * remains, it must be due to another use by the display engine.
	 */
	return obj->pin_count - !!obj->user_pin_count;
}

3562
/*
3563 3564 3565
 * Prepare buffer for display plane (scanout, cursors, etc).
 * Can be called from an uninterruptible phase (modesetting) and allows
 * any flushes to be pipelined (for pageflips).
3566 3567
 */
int
3568 3569
i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
				     u32 alignment,
3570
				     struct intel_ring_buffer *pipelined)
3571
{
3572
	u32 old_read_domains, old_write_domain;
3573 3574
	int ret;

3575
	if (pipelined != obj->ring) {
3576 3577
		ret = i915_gem_object_sync(obj, pipelined);
		if (ret)
3578 3579 3580
			return ret;
	}

3581 3582 3583 3584 3585
	/* Mark the pin_display early so that we account for the
	 * display coherency whilst setting up the cache domains.
	 */
	obj->pin_display = true;

3586 3587 3588 3589 3590 3591 3592 3593 3594
	/* The display engine is not coherent with the LLC cache on gen6.  As
	 * a result, we make sure that the pinning that is about to occur is
	 * done with uncached PTEs. This is lowest common denominator for all
	 * chipsets.
	 *
	 * However for gen6+, we could do better by using the GFDT bit instead
	 * of uncaching, which would allow us to flush all the LLC-cached data
	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
	 */
3595 3596
	ret = i915_gem_object_set_cache_level(obj,
					      HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
3597
	if (ret)
3598
		goto err_unpin_display;
3599

3600 3601 3602 3603
	/* As the user may map the buffer once pinned in the display plane
	 * (e.g. libkms for the bootup splash), we have to ensure that we
	 * always use map_and_fenceable for all scanout buffers.
	 */
B
Ben Widawsky 已提交
3604
	ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false);
3605
	if (ret)
3606
		goto err_unpin_display;
3607

3608
	i915_gem_object_flush_cpu_write_domain(obj, true);
3609

3610
	old_write_domain = obj->base.write_domain;
3611
	old_read_domains = obj->base.read_domains;
3612 3613 3614 3615

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3616
	obj->base.write_domain = 0;
3617
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3618 3619 3620

	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
3621
					    old_write_domain);
3622 3623

	return 0;
3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634

err_unpin_display:
	obj->pin_display = is_pin_display(obj);
	return ret;
}

void
i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
{
	i915_gem_object_unpin(obj);
	obj->pin_display = is_pin_display(obj);
3635 3636
}

3637
int
3638
i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3639
{
3640 3641
	int ret;

3642
	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3643 3644
		return 0;

3645
	ret = i915_gem_object_wait_rendering(obj, false);
3646 3647 3648
	if (ret)
		return ret;

3649 3650
	/* Ensure that we invalidate the GPU's caches and TLBs. */
	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3651
	return 0;
3652 3653
}

3654 3655 3656 3657 3658 3659
/**
 * Moves a single object to the CPU read, and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
3660
int
3661
i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3662
{
C
Chris Wilson 已提交
3663
	uint32_t old_write_domain, old_read_domains;
3664 3665
	int ret;

3666 3667 3668
	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		return 0;

3669
	ret = i915_gem_object_wait_rendering(obj, !write);
3670 3671 3672
	if (ret)
		return ret;

3673
	i915_gem_object_flush_gtt_write_domain(obj);
3674

3675 3676
	old_write_domain = obj->base.write_domain;
	old_read_domains = obj->base.read_domains;
C
Chris Wilson 已提交
3677

3678
	/* Flush the CPU cache if it's still invalid. */
3679
	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3680
		i915_gem_clflush_object(obj, false);
3681

3682
		obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3683 3684 3685 3686 3687
	}

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3688
	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3689 3690 3691 3692 3693

	/* If we're writing through the CPU, then the GPU read domains will
	 * need to be invalidated at next use.
	 */
	if (write) {
3694 3695
		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3696
	}
3697

C
Chris Wilson 已提交
3698 3699 3700 3701
	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);

3702 3703 3704
	return 0;
}

3705 3706 3707
/* Throttle our rendering by waiting until the ring has completed our requests
 * emitted over 20 msec ago.
 *
3708 3709 3710 3711
 * Note that if we were to use the current jiffies each time around the loop,
 * we wouldn't escape the function with any frames outstanding if the time to
 * render a frame was over 20ms.
 *
3712 3713 3714
 * This should get us reasonable parallelism between CPU and GPU but also
 * relatively low latency when blocking on a particular request to finish.
 */
3715
static int
3716
i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3717
{
3718 3719
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_file_private *file_priv = file->driver_priv;
3720
	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3721 3722
	struct drm_i915_gem_request *request;
	struct intel_ring_buffer *ring = NULL;
3723
	unsigned reset_counter;
3724 3725
	u32 seqno = 0;
	int ret;
3726

3727 3728 3729 3730 3731 3732 3733
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
	if (ret)
		return ret;

	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
	if (ret)
		return ret;
3734

3735
	spin_lock(&file_priv->mm.lock);
3736
	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3737 3738
		if (time_after_eq(request->emitted_jiffies, recent_enough))
			break;
3739

3740 3741
		ring = request->ring;
		seqno = request->seqno;
3742
	}
3743
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3744
	spin_unlock(&file_priv->mm.lock);
3745

3746 3747
	if (seqno == 0)
		return 0;
3748

3749
	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3750 3751
	if (ret == 0)
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3752 3753 3754 3755

	return ret;
}

3756
int
3757
i915_gem_object_pin(struct drm_i915_gem_object *obj,
B
Ben Widawsky 已提交
3758
		    struct i915_address_space *vm,
3759
		    uint32_t alignment,
3760 3761
		    bool map_and_fenceable,
		    bool nonblocking)
3762
{
3763
	struct i915_vma *vma;
3764 3765
	int ret;

3766 3767
	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		return -EBUSY;
3768

3769 3770 3771 3772 3773 3774 3775
	WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));

	vma = i915_gem_obj_to_vma(obj, vm);

	if (vma) {
		if ((alignment &&
		     vma->node.start & (alignment - 1)) ||
3776 3777
		    (map_and_fenceable && !obj->map_and_fenceable)) {
			WARN(obj->pin_count,
3778
			     "bo is already pinned with incorrect alignment:"
3779
			     " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
3780
			     " obj->map_and_fenceable=%d\n",
3781
			     i915_gem_obj_offset(obj, vm), alignment,
3782
			     map_and_fenceable,
3783
			     obj->map_and_fenceable);
3784
			ret = i915_vma_unbind(vma);
3785 3786 3787 3788 3789
			if (ret)
				return ret;
		}
	}

3790
	if (!i915_gem_obj_bound(obj, vm)) {
3791 3792
		struct drm_i915_private *dev_priv = obj->base.dev->dev_private;

3793 3794 3795
		ret = i915_gem_object_bind_to_vm(obj, vm, alignment,
						 map_and_fenceable,
						 nonblocking);
3796
		if (ret)
3797
			return ret;
3798 3799 3800

		if (!dev_priv->mm.aliasing_ppgtt)
			i915_gem_gtt_bind_object(obj, obj->cache_level);
3801
	}
J
Jesse Barnes 已提交
3802

3803 3804 3805
	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		i915_gem_gtt_bind_object(obj, obj->cache_level);

3806
	obj->pin_count++;
3807
	obj->pin_mappable |= map_and_fenceable;
3808 3809 3810 3811 3812

	return 0;
}

void
3813
i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3814
{
3815
	BUG_ON(obj->pin_count == 0);
3816
	BUG_ON(!i915_gem_obj_bound_any(obj));
3817

3818
	if (--obj->pin_count == 0)
3819
		obj->pin_mappable = false;
3820 3821 3822 3823
}

int
i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3824
		   struct drm_file *file)
3825 3826
{
	struct drm_i915_gem_pin *args = data;
3827
	struct drm_i915_gem_object *obj;
3828 3829
	int ret;

3830 3831 3832
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;
3833

3834
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3835
	if (&obj->base == NULL) {
3836 3837
		ret = -ENOENT;
		goto unlock;
3838 3839
	}

3840
	if (obj->madv != I915_MADV_WILLNEED) {
C
Chris Wilson 已提交
3841
		DRM_ERROR("Attempting to pin a purgeable buffer\n");
3842 3843
		ret = -EINVAL;
		goto out;
3844 3845
	}

3846
	if (obj->pin_filp != NULL && obj->pin_filp != file) {
J
Jesse Barnes 已提交
3847 3848
		DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
			  args->handle);
3849 3850
		ret = -EINVAL;
		goto out;
J
Jesse Barnes 已提交
3851 3852
	}

3853
	if (obj->user_pin_count == 0) {
B
Ben Widawsky 已提交
3854
		ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false);
3855 3856
		if (ret)
			goto out;
3857 3858
	}

3859 3860 3861
	obj->user_pin_count++;
	obj->pin_filp = file;

3862
	args->offset = i915_gem_obj_ggtt_offset(obj);
3863
out:
3864
	drm_gem_object_unreference(&obj->base);
3865
unlock:
3866
	mutex_unlock(&dev->struct_mutex);
3867
	return ret;
3868 3869 3870 3871
}

int
i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3872
		     struct drm_file *file)
3873 3874
{
	struct drm_i915_gem_pin *args = data;
3875
	struct drm_i915_gem_object *obj;
3876
	int ret;
3877

3878 3879 3880
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;
3881

3882
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3883
	if (&obj->base == NULL) {
3884 3885
		ret = -ENOENT;
		goto unlock;
3886
	}
3887

3888
	if (obj->pin_filp != file) {
J
Jesse Barnes 已提交
3889 3890
		DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
			  args->handle);
3891 3892
		ret = -EINVAL;
		goto out;
J
Jesse Barnes 已提交
3893
	}
3894 3895 3896
	obj->user_pin_count--;
	if (obj->user_pin_count == 0) {
		obj->pin_filp = NULL;
J
Jesse Barnes 已提交
3897 3898
		i915_gem_object_unpin(obj);
	}
3899

3900
out:
3901
	drm_gem_object_unreference(&obj->base);
3902
unlock:
3903
	mutex_unlock(&dev->struct_mutex);
3904
	return ret;
3905 3906 3907 3908
}

int
i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3909
		    struct drm_file *file)
3910 3911
{
	struct drm_i915_gem_busy *args = data;
3912
	struct drm_i915_gem_object *obj;
3913 3914
	int ret;

3915
	ret = i915_mutex_lock_interruptible(dev);
3916
	if (ret)
3917
		return ret;
3918

3919
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3920
	if (&obj->base == NULL) {
3921 3922
		ret = -ENOENT;
		goto unlock;
3923
	}
3924

3925 3926 3927 3928
	/* Count all active objects as busy, even if they are currently not used
	 * by the gpu. Users of this interface expect objects to eventually
	 * become non-busy without any further actions, therefore emit any
	 * necessary flushes here.
3929
	 */
3930
	ret = i915_gem_object_flush_active(obj);
3931

3932
	args->busy = obj->active;
3933 3934 3935 3936
	if (obj->ring) {
		BUILD_BUG_ON(I915_NUM_RINGS > 16);
		args->busy |= intel_ring_flag(obj->ring) << 16;
	}
3937

3938
	drm_gem_object_unreference(&obj->base);
3939
unlock:
3940
	mutex_unlock(&dev->struct_mutex);
3941
	return ret;
3942 3943 3944 3945 3946 3947
}

int
i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file_priv)
{
3948
	return i915_gem_ring_throttle(dev, file_priv);
3949 3950
}

3951 3952 3953 3954 3955
int
i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
	struct drm_i915_gem_madvise *args = data;
3956
	struct drm_i915_gem_object *obj;
3957
	int ret;
3958 3959 3960 3961 3962 3963 3964 3965 3966

	switch (args->madv) {
	case I915_MADV_DONTNEED:
	case I915_MADV_WILLNEED:
	    break;
	default:
	    return -EINVAL;
	}

3967 3968 3969 3970
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

3971
	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3972
	if (&obj->base == NULL) {
3973 3974
		ret = -ENOENT;
		goto unlock;
3975 3976
	}

3977
	if (obj->pin_count) {
3978 3979
		ret = -EINVAL;
		goto out;
3980 3981
	}

3982 3983
	if (obj->madv != __I915_MADV_PURGED)
		obj->madv = args->madv;
3984

C
Chris Wilson 已提交
3985 3986
	/* if the object is no longer attached, discard its backing storage */
	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
3987 3988
		i915_gem_object_truncate(obj);

3989
	args->retained = obj->madv != __I915_MADV_PURGED;
C
Chris Wilson 已提交
3990

3991
out:
3992
	drm_gem_object_unreference(&obj->base);
3993
unlock:
3994
	mutex_unlock(&dev->struct_mutex);
3995
	return ret;
3996 3997
}

3998 3999
void i915_gem_object_init(struct drm_i915_gem_object *obj,
			  const struct drm_i915_gem_object_ops *ops)
4000
{
4001
	INIT_LIST_HEAD(&obj->global_list);
4002
	INIT_LIST_HEAD(&obj->ring_list);
4003
	INIT_LIST_HEAD(&obj->obj_exec_link);
B
Ben Widawsky 已提交
4004
	INIT_LIST_HEAD(&obj->vma_list);
4005

4006 4007
	obj->ops = ops;

4008 4009 4010 4011 4012 4013 4014 4015
	obj->fence_reg = I915_FENCE_REG_NONE;
	obj->madv = I915_MADV_WILLNEED;
	/* Avoid an unnecessary call to unbind on the first bind. */
	obj->map_and_fenceable = true;

	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
}

4016 4017 4018 4019 4020
static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
	.get_pages = i915_gem_object_get_pages_gtt,
	.put_pages = i915_gem_object_put_pages_gtt,
};

4021 4022
struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
						  size_t size)
4023
{
4024
	struct drm_i915_gem_object *obj;
4025
	struct address_space *mapping;
D
Daniel Vetter 已提交
4026
	gfp_t mask;
4027

4028
	obj = i915_gem_object_alloc(dev);
4029 4030
	if (obj == NULL)
		return NULL;
4031

4032
	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4033
		i915_gem_object_free(obj);
4034 4035
		return NULL;
	}
4036

4037 4038 4039 4040 4041 4042 4043
	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
	if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
		/* 965gm cannot relocate objects above 4GiB. */
		mask &= ~__GFP_HIGHMEM;
		mask |= __GFP_DMA32;
	}

A
Al Viro 已提交
4044
	mapping = file_inode(obj->base.filp)->i_mapping;
4045
	mapping_set_gfp_mask(mapping, mask);
4046

4047
	i915_gem_object_init(obj, &i915_gem_object_ops);
4048

4049 4050
	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4051

4052 4053
	if (HAS_LLC(dev)) {
		/* On some devices, we can have the GPU use the LLC (the CPU
4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068
		 * cache) for about a 10% performance improvement
		 * compared to uncached.  Graphics requests other than
		 * display scanout are coherent with the CPU in
		 * accessing this cache.  This means in this mode we
		 * don't need to clflush on the CPU side, and on the
		 * GPU side we only need to flush internal caches to
		 * get data visible to the CPU.
		 *
		 * However, we maintain the display planes as UC, and so
		 * need to rebind when first used as such.
		 */
		obj->cache_level = I915_CACHE_LLC;
	} else
		obj->cache_level = I915_CACHE_NONE;

4069 4070
	trace_i915_gem_object_create(obj);

4071
	return obj;
4072 4073 4074 4075 4076
}

int i915_gem_init_object(struct drm_gem_object *obj)
{
	BUG();
4077

4078 4079 4080
	return 0;
}

4081
void i915_gem_free_object(struct drm_gem_object *gem_obj)
4082
{
4083
	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4084
	struct drm_device *dev = obj->base.dev;
4085
	drm_i915_private_t *dev_priv = dev->dev_private;
4086
	struct i915_vma *vma, *next;
4087

4088 4089
	trace_i915_gem_object_destroy(obj);

4090 4091 4092 4093
	if (obj->phys_obj)
		i915_gem_detach_phys_object(dev, obj);

	obj->pin_count = 0;
4094 4095 4096 4097 4098 4099 4100
	/* NB: 0 or 1 elements */
	WARN_ON(!list_empty(&obj->vma_list) &&
		!list_is_singular(&obj->vma_list));
	list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
		int ret = i915_vma_unbind(vma);
		if (WARN_ON(ret == -ERESTARTSYS)) {
			bool was_interruptible;
4101

4102 4103
			was_interruptible = dev_priv->mm.interruptible;
			dev_priv->mm.interruptible = false;
4104

4105
			WARN_ON(i915_vma_unbind(vma));
4106

4107 4108
			dev_priv->mm.interruptible = was_interruptible;
		}
4109 4110
	}

B
Ben Widawsky 已提交
4111 4112 4113 4114 4115
	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
	 * before progressing. */
	if (obj->stolen)
		i915_gem_object_unpin_pages(obj);

B
Ben Widawsky 已提交
4116 4117
	if (WARN_ON(obj->pages_pin_count))
		obj->pages_pin_count = 0;
4118
	i915_gem_object_put_pages(obj);
4119
	i915_gem_object_free_mmap_offset(obj);
4120
	i915_gem_object_release_stolen(obj);
4121

4122 4123
	BUG_ON(obj->pages);

4124 4125
	if (obj->base.import_attach)
		drm_prime_gem_destroy(&obj->base, NULL);
4126

4127 4128
	drm_gem_object_release(&obj->base);
	i915_gem_info_remove_obj(dev_priv, obj->base.size);
4129

4130
	kfree(obj->bit_17);
4131
	i915_gem_object_free(obj);
4132 4133
}

4134
struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
B
Ben Widawsky 已提交
4135
				     struct i915_address_space *vm)
4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146
{
	struct i915_vma *vma;
	list_for_each_entry(vma, &obj->vma_list, vma_link)
		if (vma->vm == vm)
			return vma;

	return NULL;
}

static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
					      struct i915_address_space *vm)
B
Ben Widawsky 已提交
4147 4148 4149 4150 4151 4152
{
	struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
	if (vma == NULL)
		return ERR_PTR(-ENOMEM);

	INIT_LIST_HEAD(&vma->vma_link);
B
Ben Widawsky 已提交
4153
	INIT_LIST_HEAD(&vma->mm_list);
4154
	INIT_LIST_HEAD(&vma->exec_list);
B
Ben Widawsky 已提交
4155 4156 4157
	vma->vm = vm;
	vma->obj = obj;

4158 4159 4160 4161 4162 4163
	/* Keep GGTT vmas first to make debug easier */
	if (i915_is_ggtt(vm))
		list_add(&vma->vma_link, &obj->vma_list);
	else
		list_add_tail(&vma->vma_link, &obj->vma_list);

B
Ben Widawsky 已提交
4164 4165 4166
	return vma;
}

4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179
struct i915_vma *
i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
				  struct i915_address_space *vm)
{
	struct i915_vma *vma;

	vma = i915_gem_obj_to_vma(obj, vm);
	if (!vma)
		vma = __i915_gem_vma_create(obj, vm);

	return vma;
}

B
Ben Widawsky 已提交
4180 4181 4182
void i915_gem_vma_destroy(struct i915_vma *vma)
{
	WARN_ON(vma->node.allocated);
4183 4184 4185 4186 4187

	/* Keep the vma as a placeholder in the execbuffer reservation lists */
	if (!list_empty(&vma->exec_list))
		return;

4188 4189
	list_del(&vma->vma_link);

B
Ben Widawsky 已提交
4190 4191 4192
	kfree(vma);
}

4193 4194 4195 4196 4197
int
i915_gem_idle(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret;
4198

4199
	if (dev_priv->ums.mm_suspended) {
4200 4201
		mutex_unlock(&dev->struct_mutex);
		return 0;
4202 4203
	}

4204
	ret = i915_gpu_idle(dev);
4205 4206
	if (ret) {
		mutex_unlock(&dev->struct_mutex);
4207
		return ret;
4208
	}
4209
	i915_gem_retire_requests(dev);
4210

4211
	/* Under UMS, be paranoid and evict. */
4212
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
C
Chris Wilson 已提交
4213
		i915_gem_evict_everything(dev);
4214

4215
	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
4216 4217

	i915_kernel_lost_context(dev);
4218
	i915_gem_cleanup_ringbuffer(dev);
4219 4220 4221 4222

	/* Cancel the retire work handler, which should be idle now. */
	cancel_delayed_work_sync(&dev_priv->mm.retire_work);

4223 4224 4225
	return 0;
}

B
Ben Widawsky 已提交
4226 4227 4228 4229 4230 4231
void i915_gem_l3_remap(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	u32 misccpctl;
	int i;

4232
	if (!HAS_L3_GPU_CACHE(dev))
B
Ben Widawsky 已提交
4233 4234
		return;

4235
	if (!dev_priv->l3_parity.remap_info)
B
Ben Widawsky 已提交
4236 4237 4238 4239 4240 4241 4242 4243
		return;

	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
4244
		if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
B
Ben Widawsky 已提交
4245 4246
			DRM_DEBUG("0x%x was already programmed to %x\n",
				  GEN7_L3LOG_BASE + i, remap);
4247
		if (remap && !dev_priv->l3_parity.remap_info[i/4])
B
Ben Widawsky 已提交
4248
			DRM_DEBUG_DRIVER("Clearing remapped register\n");
4249
		I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
B
Ben Widawsky 已提交
4250 4251 4252 4253 4254 4255 4256 4257
	}

	/* Make sure all the writes land before disabling dop clock gating */
	POSTING_READ(GEN7_L3LOG_BASE);

	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
}

4258 4259 4260 4261
void i915_gem_init_swizzling(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;

4262
	if (INTEL_INFO(dev)->gen < 5 ||
4263 4264 4265 4266 4267 4268
	    dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		return;

	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
				 DISP_TILE_SURFACE_SWIZZLING);

4269 4270 4271
	if (IS_GEN5(dev))
		return;

4272 4273
	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
	if (IS_GEN6(dev))
4274
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4275
	else if (IS_GEN7(dev))
4276
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4277 4278
	else
		BUG();
4279
}
D
Daniel Vetter 已提交
4280

4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296
static bool
intel_enable_blt(struct drm_device *dev)
{
	if (!HAS_BLT(dev))
		return false;

	/* The blitter was dysfunctional on early prototypes */
	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		DRM_INFO("BLT not supported on this pre-production hardware;"
			 " graphics performance will be degraded.\n");
		return false;
	}

	return true;
}

4297
static int i915_gem_init_rings(struct drm_device *dev)
4298
{
4299
	struct drm_i915_private *dev_priv = dev->dev_private;
4300
	int ret;
4301

4302
	ret = intel_init_render_ring_buffer(dev);
4303
	if (ret)
4304
		return ret;
4305 4306

	if (HAS_BSD(dev)) {
4307
		ret = intel_init_bsd_ring_buffer(dev);
4308 4309
		if (ret)
			goto cleanup_render_ring;
4310
	}
4311

4312
	if (intel_enable_blt(dev)) {
4313 4314 4315 4316 4317
		ret = intel_init_blt_ring_buffer(dev);
		if (ret)
			goto cleanup_bsd_ring;
	}

B
Ben Widawsky 已提交
4318 4319 4320 4321 4322 4323 4324
	if (HAS_VEBOX(dev)) {
		ret = intel_init_vebox_ring_buffer(dev);
		if (ret)
			goto cleanup_blt_ring;
	}


4325
	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
4326
	if (ret)
B
Ben Widawsky 已提交
4327
		goto cleanup_vebox_ring;
4328 4329 4330

	return 0;

B
Ben Widawsky 已提交
4331 4332
cleanup_vebox_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351
cleanup_blt_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
cleanup_bsd_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
cleanup_render_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);

	return ret;
}

int
i915_gem_init_hw(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret;

	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		return -EIO;

B
Ben Widawsky 已提交
4352
	if (dev_priv->ellc_size)
4353
		I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4354

4355 4356 4357 4358 4359
	if (IS_HSW_GT3(dev))
		I915_WRITE(MI_PREDICATE_RESULT_2, LOWER_SLICE_ENABLED);
	else
		I915_WRITE(MI_PREDICATE_RESULT_2, LOWER_SLICE_DISABLED);

4360 4361 4362 4363 4364 4365
	if (HAS_PCH_NOP(dev)) {
		u32 temp = I915_READ(GEN7_MSG_CTL);
		temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
		I915_WRITE(GEN7_MSG_CTL, temp);
	}

4366 4367 4368 4369 4370
	i915_gem_l3_remap(dev);

	i915_gem_init_swizzling(dev);

	ret = i915_gem_init_rings(dev);
4371 4372 4373
	if (ret)
		return ret;

4374 4375 4376 4377 4378
	/*
	 * XXX: There was some w/a described somewhere suggesting loading
	 * contexts before PPGTT.
	 */
	i915_gem_context_init(dev);
4379 4380 4381 4382 4383 4384 4385
	if (dev_priv->mm.aliasing_ppgtt) {
		ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
		if (ret) {
			i915_gem_cleanup_aliasing_ppgtt(dev);
			DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
		}
	}
D
Daniel Vetter 已提交
4386

4387
	return 0;
4388 4389
}

4390 4391 4392 4393 4394 4395
int i915_gem_init(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	mutex_lock(&dev->struct_mutex);
4396 4397 4398 4399 4400 4401 4402 4403

	if (IS_VALLEYVIEW(dev)) {
		/* VLVA0 (potential hack), BIOS isn't actually waking us */
		I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
		if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
	}

4404
	i915_gem_init_global_gtt(dev);
4405

4406 4407 4408 4409 4410 4411 4412
	ret = i915_gem_init_hw(dev);
	mutex_unlock(&dev->struct_mutex);
	if (ret) {
		i915_gem_cleanup_aliasing_ppgtt(dev);
		return ret;
	}

4413 4414 4415
	/* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		dev_priv->dri1.allow_batchbuffer = 1;
4416 4417 4418
	return 0;
}

4419 4420 4421 4422
void
i915_gem_cleanup_ringbuffer(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
4423
	struct intel_ring_buffer *ring;
4424
	int i;
4425

4426 4427
	for_each_ring(ring, dev_priv, i)
		intel_cleanup_ring_buffer(ring);
4428 4429
}

4430 4431 4432 4433
int
i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
4434
	struct drm_i915_private *dev_priv = dev->dev_private;
4435
	int ret;
4436

J
Jesse Barnes 已提交
4437 4438 4439
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return 0;

4440
	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
4441
		DRM_ERROR("Reenabling wedged hardware, good luck\n");
4442
		atomic_set(&dev_priv->gpu_error.reset_counter, 0);
4443 4444 4445
	}

	mutex_lock(&dev->struct_mutex);
4446
	dev_priv->ums.mm_suspended = 0;
4447

4448
	ret = i915_gem_init_hw(dev);
4449 4450
	if (ret != 0) {
		mutex_unlock(&dev->struct_mutex);
4451
		return ret;
4452
	}
4453

4454
	BUG_ON(!list_empty(&dev_priv->gtt.base.active_list));
4455
	mutex_unlock(&dev->struct_mutex);
4456

4457 4458 4459
	ret = drm_irq_install(dev);
	if (ret)
		goto cleanup_ringbuffer;
4460

4461
	return 0;
4462 4463 4464 4465

cleanup_ringbuffer:
	mutex_lock(&dev->struct_mutex);
	i915_gem_cleanup_ringbuffer(dev);
4466
	dev_priv->ums.mm_suspended = 1;
4467 4468 4469
	mutex_unlock(&dev->struct_mutex);

	return ret;
4470 4471 4472 4473 4474 4475
}

int
i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
4476 4477 4478
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

J
Jesse Barnes 已提交
4479 4480 4481
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return 0;

4482
	drm_irq_uninstall(dev);
4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495

	mutex_lock(&dev->struct_mutex);
	ret =  i915_gem_idle(dev);

	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
	 * We need to replace this with a semaphore, or something.
	 * And not confound ums.mm_suspended!
	 */
	if (ret != 0)
		dev_priv->ums.mm_suspended = 1;
	mutex_unlock(&dev->struct_mutex);

	return ret;
4496 4497 4498 4499 4500 4501 4502
}

void
i915_gem_lastclose(struct drm_device *dev)
{
	int ret;

4503 4504 4505
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return;

4506
	mutex_lock(&dev->struct_mutex);
4507 4508 4509
	ret = i915_gem_idle(dev);
	if (ret)
		DRM_ERROR("failed to idle hardware: %d\n", ret);
4510
	mutex_unlock(&dev->struct_mutex);
4511 4512
}

4513 4514 4515 4516 4517 4518 4519
static void
init_ring_lists(struct intel_ring_buffer *ring)
{
	INIT_LIST_HEAD(&ring->active_list);
	INIT_LIST_HEAD(&ring->request_list);
}

B
Ben Widawsky 已提交
4520 4521 4522 4523 4524 4525 4526 4527 4528 4529
static void i915_init_vm(struct drm_i915_private *dev_priv,
			 struct i915_address_space *vm)
{
	vm->dev = dev_priv->dev;
	INIT_LIST_HEAD(&vm->active_list);
	INIT_LIST_HEAD(&vm->inactive_list);
	INIT_LIST_HEAD(&vm->global_link);
	list_add(&vm->global_link, &dev_priv->vm_list);
}

4530 4531 4532 4533
void
i915_gem_load(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
4534 4535 4536 4537 4538 4539 4540
	int i;

	dev_priv->slab =
		kmem_cache_create("i915_gem_object",
				  sizeof(struct drm_i915_gem_object), 0,
				  SLAB_HWCACHE_ALIGN,
				  NULL);
4541

B
Ben Widawsky 已提交
4542 4543 4544
	INIT_LIST_HEAD(&dev_priv->vm_list);
	i915_init_vm(dev_priv, &dev_priv->gtt.base);

C
Chris Wilson 已提交
4545 4546
	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4547
	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4548 4549
	for (i = 0; i < I915_NUM_RINGS; i++)
		init_ring_lists(&dev_priv->ring[i]);
4550
	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4551
		INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4552 4553
	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
			  i915_gem_retire_work_handler);
4554
	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4555

4556 4557
	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
	if (IS_GEN3(dev)) {
4558 4559
		I915_WRITE(MI_ARB_STATE,
			   _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4560 4561
	}

4562 4563
	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;

4564
	/* Old X drivers will take 0-2 for front, back, depth buffers */
4565 4566
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		dev_priv->fence_reg_start = 3;
4567

4568 4569 4570
	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		dev_priv->num_fence_regs = 32;
	else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4571 4572 4573 4574
		dev_priv->num_fence_regs = 16;
	else
		dev_priv->num_fence_regs = 8;

4575
	/* Initialize fence registers to zero */
4576 4577
	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
	i915_gem_restore_fences(dev);
4578

4579
	i915_gem_detect_bit_6_swizzle(dev);
4580
	init_waitqueue_head(&dev_priv->pending_flip_queue);
4581

4582 4583
	dev_priv->mm.interruptible = true;

4584 4585 4586
	dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
	dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
	register_shrinker(&dev_priv->mm.inactive_shrinker);
4587
}
4588 4589 4590 4591 4592

/*
 * Create a physically contiguous memory object for this object
 * e.g. for cursor + overlay regs
 */
4593 4594
static int i915_gem_init_phys_object(struct drm_device *dev,
				     int id, int size, int align)
4595 4596 4597 4598 4599 4600 4601 4602
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_i915_gem_phys_object *phys_obj;
	int ret;

	if (dev_priv->mm.phys_objs[id - 1] || !size)
		return 0;

4603
	phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4604 4605 4606 4607 4608
	if (!phys_obj)
		return -ENOMEM;

	phys_obj->id = id;

4609
	phys_obj->handle = drm_pci_alloc(dev, size, align);
4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621
	if (!phys_obj->handle) {
		ret = -ENOMEM;
		goto kfree_obj;
	}
#ifdef CONFIG_X86
	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
#endif

	dev_priv->mm.phys_objs[id - 1] = phys_obj;

	return 0;
kfree_obj:
4622
	kfree(phys_obj);
4623 4624 4625
	return ret;
}

4626
static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_i915_gem_phys_object *phys_obj;

	if (!dev_priv->mm.phys_objs[id - 1])
		return;

	phys_obj = dev_priv->mm.phys_objs[id - 1];
	if (phys_obj->cur_obj) {
		i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
	}

#ifdef CONFIG_X86
	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
#endif
	drm_pci_free(dev, phys_obj->handle);
	kfree(phys_obj);
	dev_priv->mm.phys_objs[id - 1] = NULL;
}

void i915_gem_free_all_phys_object(struct drm_device *dev)
{
	int i;

4651
	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4652 4653 4654 4655
		i915_gem_free_phys_object(dev, i);
}

void i915_gem_detach_phys_object(struct drm_device *dev,
4656
				 struct drm_i915_gem_object *obj)
4657
{
A
Al Viro 已提交
4658
	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4659
	char *vaddr;
4660 4661 4662
	int i;
	int page_count;

4663
	if (!obj->phys_obj)
4664
		return;
4665
	vaddr = obj->phys_obj->handle->vaddr;
4666

4667
	page_count = obj->base.size / PAGE_SIZE;
4668
	for (i = 0; i < page_count; i++) {
4669
		struct page *page = shmem_read_mapping_page(mapping, i);
4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680
		if (!IS_ERR(page)) {
			char *dst = kmap_atomic(page);
			memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
			kunmap_atomic(dst);

			drm_clflush_pages(&page, 1);

			set_page_dirty(page);
			mark_page_accessed(page);
			page_cache_release(page);
		}
4681
	}
4682
	i915_gem_chipset_flush(dev);
4683

4684 4685
	obj->phys_obj->cur_obj = NULL;
	obj->phys_obj = NULL;
4686 4687 4688 4689
}

int
i915_gem_attach_phys_object(struct drm_device *dev,
4690
			    struct drm_i915_gem_object *obj,
4691 4692
			    int id,
			    int align)
4693
{
A
Al Viro 已提交
4694
	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4695 4696 4697 4698 4699 4700 4701 4702
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret = 0;
	int page_count;
	int i;

	if (id > I915_MAX_PHYS_OBJECT)
		return -EINVAL;

4703 4704
	if (obj->phys_obj) {
		if (obj->phys_obj->id == id)
4705 4706 4707 4708 4709 4710 4711
			return 0;
		i915_gem_detach_phys_object(dev, obj);
	}

	/* create a new object */
	if (!dev_priv->mm.phys_objs[id - 1]) {
		ret = i915_gem_init_phys_object(dev, id,
4712
						obj->base.size, align);
4713
		if (ret) {
4714 4715
			DRM_ERROR("failed to init phys object %d size: %zu\n",
				  id, obj->base.size);
4716
			return ret;
4717 4718 4719 4720
		}
	}

	/* bind to the object */
4721 4722
	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
	obj->phys_obj->cur_obj = obj;
4723

4724
	page_count = obj->base.size / PAGE_SIZE;
4725 4726

	for (i = 0; i < page_count; i++) {
4727 4728 4729
		struct page *page;
		char *dst, *src;

4730
		page = shmem_read_mapping_page(mapping, i);
4731 4732
		if (IS_ERR(page))
			return PTR_ERR(page);
4733

4734
		src = kmap_atomic(page);
4735
		dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4736
		memcpy(dst, src, PAGE_SIZE);
P
Peter Zijlstra 已提交
4737
		kunmap_atomic(src);
4738

4739 4740 4741
		mark_page_accessed(page);
		page_cache_release(page);
	}
4742

4743 4744 4745 4746
	return 0;
}

static int
4747 4748
i915_gem_phys_pwrite(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
4749 4750 4751
		     struct drm_i915_gem_pwrite *args,
		     struct drm_file *file_priv)
{
4752
	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
V
Ville Syrjälä 已提交
4753
	char __user *user_data = to_user_ptr(args->data_ptr);
4754

4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767
	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
		unsigned long unwritten;

		/* The physical object once assigned is fixed for the lifetime
		 * of the obj, so we can safely drop the lock and continue
		 * to access vaddr.
		 */
		mutex_unlock(&dev->struct_mutex);
		unwritten = copy_from_user(vaddr, user_data, args->size);
		mutex_lock(&dev->struct_mutex);
		if (unwritten)
			return -EFAULT;
	}
4768

4769
	i915_gem_chipset_flush(dev);
4770 4771
	return 0;
}
4772

4773
void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4774
{
4775
	struct drm_i915_file_private *file_priv = file->driver_priv;
4776 4777 4778 4779 4780

	/* Clean up our request list when the client is going away, so that
	 * later retire_requests won't dereference our soon-to-be-gone
	 * file_priv.
	 */
4781
	spin_lock(&file_priv->mm.lock);
4782 4783 4784 4785 4786 4787 4788 4789 4790
	while (!list_empty(&file_priv->mm.request_list)) {
		struct drm_i915_gem_request *request;

		request = list_first_entry(&file_priv->mm.request_list,
					   struct drm_i915_gem_request,
					   client_list);
		list_del(&request->client_list);
		request->file_priv = NULL;
	}
4791
	spin_unlock(&file_priv->mm.lock);
4792
}
4793

4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806
static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
{
	if (!mutex_is_locked(mutex))
		return false;

#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
	return mutex->owner == task;
#else
	/* Since UP may be pre-empted, we cannot assume that we own the lock */
	return false;
#endif
}

4807
static int
4808
i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
4809
{
4810 4811 4812 4813 4814
	struct drm_i915_private *dev_priv =
		container_of(shrinker,
			     struct drm_i915_private,
			     mm.inactive_shrinker);
	struct drm_device *dev = dev_priv->dev;
C
Chris Wilson 已提交
4815
	struct drm_i915_gem_object *obj;
4816
	int nr_to_scan = sc->nr_to_scan;
4817
	bool unlock = true;
4818 4819
	int cnt;

4820 4821 4822 4823
	if (!mutex_trylock(&dev->struct_mutex)) {
		if (!mutex_is_locked_by(&dev->struct_mutex, current))
			return 0;

4824 4825 4826
		if (dev_priv->mm.shrinker_no_lock_stealing)
			return 0;

4827 4828
		unlock = false;
	}
4829

C
Chris Wilson 已提交
4830 4831
	if (nr_to_scan) {
		nr_to_scan -= i915_gem_purge(dev_priv, nr_to_scan);
4832 4833 4834
		if (nr_to_scan > 0)
			nr_to_scan -= __i915_gem_shrink(dev_priv, nr_to_scan,
							false);
C
Chris Wilson 已提交
4835 4836
		if (nr_to_scan > 0)
			i915_gem_shrink_all(dev_priv);
4837 4838
	}

4839
	cnt = 0;
4840
	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
4841 4842
		if (obj->pages_pin_count == 0)
			cnt += obj->base.size >> PAGE_SHIFT;
4843 4844 4845 4846 4847

	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
		if (obj->active)
			continue;

4848
		if (obj->pin_count == 0 && obj->pages_pin_count == 0)
C
Chris Wilson 已提交
4849
			cnt += obj->base.size >> PAGE_SHIFT;
4850
	}
4851

4852 4853
	if (unlock)
		mutex_unlock(&dev->struct_mutex);
C
Chris Wilson 已提交
4854
	return cnt;
4855
}
4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881

/* All the new VM stuff */
unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
				  struct i915_address_space *vm)
{
	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
	struct i915_vma *vma;

	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		vm = &dev_priv->gtt.base;

	BUG_ON(list_empty(&o->vma_list));
	list_for_each_entry(vma, &o->vma_list, vma_link) {
		if (vma->vm == vm)
			return vma->node.start;

	}
	return -1;
}

bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
			struct i915_address_space *vm)
{
	struct i915_vma *vma;

	list_for_each_entry(vma, &o->vma_list, vma_link)
4882
		if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
4883 4884 4885 4886 4887 4888 4889
			return true;

	return false;
}

bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
{
4890
	struct i915_vma *vma;
4891

4892 4893
	list_for_each_entry(vma, &o->vma_list, vma_link)
		if (drm_mm_node_allocated(&vma->node))
4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915
			return true;

	return false;
}

unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
				struct i915_address_space *vm)
{
	struct drm_i915_private *dev_priv = o->base.dev->dev_private;
	struct i915_vma *vma;

	if (vm == &dev_priv->mm.aliasing_ppgtt->base)
		vm = &dev_priv->gtt.base;

	BUG_ON(list_empty(&o->vma_list));

	list_for_each_entry(vma, &o->vma_list, vma_link)
		if (vma->vm == vm)
			return vma->node.size;

	return 0;
}