i915_gem.c 115.0 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright © 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

28 29
#include <drm/drmP.h>
#include <drm/i915_drm.h>
30
#include "i915_drv.h"
C
Chris Wilson 已提交
31
#include "i915_trace.h"
32
#include "intel_drv.h"
33
#include <linux/shmem_fs.h>
34
#include <linux/slab.h>
35
#include <linux/swap.h>
J
Jesse Barnes 已提交
36
#include <linux/pci.h>
37
#include <linux/dma-buf.h>
38

39 40
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
41 42
static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
						    unsigned alignment,
43 44
						    bool map_and_fenceable,
						    bool nonblocking);
45 46
static int i915_gem_phys_pwrite(struct drm_device *dev,
				struct drm_i915_gem_object *obj,
47
				struct drm_i915_gem_pwrite *args,
48
				struct drm_file *file);
49

50 51 52 53 54 55
static void i915_gem_write_fence(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj);
static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
					 struct drm_i915_fence_reg *fence,
					 bool enable);

56
static int i915_gem_inactive_shrink(struct shrinker *shrinker,
57
				    struct shrink_control *sc);
C
Chris Wilson 已提交
58 59
static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
60
static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
61

62 63 64 65 66 67 68 69
static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
{
	if (obj->tiling_mode)
		i915_gem_release_mmap(obj);

	/* As we do not have an associated fence register, we will force
	 * a tiling change if we ever need to acquire one.
	 */
70
	obj->fence_dirty = false;
71 72 73
	obj->fence_reg = I915_FENCE_REG_NONE;
}

74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
/* some bookkeeping */
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
				  size_t size)
{
	dev_priv->mm.object_count++;
	dev_priv->mm.object_memory += size;
}

static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
				     size_t size)
{
	dev_priv->mm.object_count--;
	dev_priv->mm.object_memory -= size;
}

89
static int
90
i915_gem_wait_for_error(struct i915_gpu_error *error)
91 92 93
{
	int ret;

94 95
#define EXIT_COND (!i915_reset_in_progress(error) || \
		   i915_terminally_wedged(error))
96
	if (EXIT_COND)
97 98
		return 0;

99 100 101 102 103
	/*
	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
	 * userspace. If it takes that long something really bad is going on and
	 * we should simply try to bail out and fail as gracefully as possible.
	 */
104 105 106
	ret = wait_event_interruptible_timeout(error->reset_queue,
					       EXIT_COND,
					       10*HZ);
107 108 109 110
	if (ret == 0) {
		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		return -EIO;
	} else if (ret < 0) {
111
		return ret;
112
	}
113
#undef EXIT_COND
114

115
	return 0;
116 117
}

118
int i915_mutex_lock_interruptible(struct drm_device *dev)
119
{
120
	struct drm_i915_private *dev_priv = dev->dev_private;
121 122
	int ret;

123
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
124 125 126 127 128 129 130
	if (ret)
		return ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

131
	WARN_ON(i915_verify_lists(dev));
132 133
	return 0;
}
134

135
static inline bool
136
i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
137
{
C
Chris Wilson 已提交
138
	return obj->gtt_space && !obj->active;
139 140
}

J
Jesse Barnes 已提交
141 142
int
i915_gem_init_ioctl(struct drm_device *dev, void *data,
143
		    struct drm_file *file)
J
Jesse Barnes 已提交
144
{
145
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
146
	struct drm_i915_gem_init *args = data;
147

148 149 150
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return -ENODEV;

151 152 153
	if (args->gtt_start >= args->gtt_end ||
	    (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
		return -EINVAL;
J
Jesse Barnes 已提交
154

155 156 157 158
	/* GEM with user mode setting was never supported on ilk and later. */
	if (INTEL_INFO(dev)->gen >= 5)
		return -ENODEV;

J
Jesse Barnes 已提交
159
	mutex_lock(&dev->struct_mutex);
160 161
	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
				  args->gtt_end);
162
	dev_priv->gtt.mappable_end = args->gtt_end;
163 164
	mutex_unlock(&dev->struct_mutex);

165
	return 0;
166 167
}

168 169
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
170
			    struct drm_file *file)
171
{
172
	struct drm_i915_private *dev_priv = dev->dev_private;
173
	struct drm_i915_gem_get_aperture *args = data;
174 175
	struct drm_i915_gem_object *obj;
	size_t pinned;
176

177
	pinned = 0;
178
	mutex_lock(&dev->struct_mutex);
179
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
180 181
		if (obj->pin_count)
			pinned += obj->gtt_space->size;
182
	mutex_unlock(&dev->struct_mutex);
183

B
Ben Widawsky 已提交
184
	args->aper_size = dev_priv->gtt.total;
185
	args->aper_available_size = args->aper_size - pinned;
186

187 188 189
	return 0;
}

190 191 192 193 194 195 196 197 198 199 200 201
void *i915_gem_object_alloc(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	return kmem_cache_alloc(dev_priv->slab, GFP_KERNEL | __GFP_ZERO);
}

void i915_gem_object_free(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	kmem_cache_free(dev_priv->slab, obj);
}

202 203 204 205 206
static int
i915_gem_create(struct drm_file *file,
		struct drm_device *dev,
		uint64_t size,
		uint32_t *handle_p)
207
{
208
	struct drm_i915_gem_object *obj;
209 210
	int ret;
	u32 handle;
211

212
	size = roundup(size, PAGE_SIZE);
213 214
	if (size == 0)
		return -EINVAL;
215 216

	/* Allocate the new object */
217
	obj = i915_gem_alloc_object(dev, size);
218 219 220
	if (obj == NULL)
		return -ENOMEM;

221
	ret = drm_gem_handle_create(file, &obj->base, &handle);
222
	if (ret) {
223 224
		drm_gem_object_release(&obj->base);
		i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
225
		i915_gem_object_free(obj);
226
		return ret;
227
	}
228

229
	/* drop reference from allocate - handle holds it now */
230
	drm_gem_object_unreference(&obj->base);
231 232
	trace_i915_gem_object_create(obj);

233
	*handle_p = handle;
234 235 236
	return 0;
}

237 238 239 240 241 242
int
i915_gem_dumb_create(struct drm_file *file,
		     struct drm_device *dev,
		     struct drm_mode_create_dumb *args)
{
	/* have to work out size/pitch and return them */
243
	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
	args->size = args->pitch * args->height;
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

int i915_gem_dumb_destroy(struct drm_file *file,
			  struct drm_device *dev,
			  uint32_t handle)
{
	return drm_gem_handle_delete(file, handle);
}

/**
 * Creates a new mm object and returns a handle to it.
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file)
{
	struct drm_i915_gem_create *args = data;
264

265 266 267 268
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
static inline int
__copy_to_user_swizzled(char __user *cpu_vaddr,
			const char *gpu_vaddr, int gpu_offset,
			int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_to_user(cpu_vaddr + cpu_offset,
				     gpu_vaddr + swizzled_gpu_offset,
				     this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

295
static inline int
296 297
__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
			  const char __user *cpu_vaddr,
298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
			  int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
				       cpu_vaddr + cpu_offset,
				       this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

321 322 323
/* Per-page copy function for the shmem pread fastpath.
 * Flushes invalid cachelines before reading the target if
 * needs_clflush is set. */
324
static int
325 326 327 328 329 330 331
shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

332
	if (unlikely(page_do_bit17_swizzling))
333 334 335 336 337 338 339 340 341 342 343
		return -EINVAL;

	vaddr = kmap_atomic(page);
	if (needs_clflush)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_to_user_inatomic(user_data,
				      vaddr + shmem_page_offset,
				      page_length);
	kunmap_atomic(vaddr);

344
	return ret ? -EFAULT : 0;
345 346
}

347 348 349 350
static void
shmem_clflush_swizzled_range(char *addr, unsigned long length,
			     bool swizzled)
{
351
	if (unlikely(swizzled)) {
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
		unsigned long start = (unsigned long) addr;
		unsigned long end = (unsigned long) addr + length;

		/* For swizzling simply ensure that we always flush both
		 * channels. Lame, but simple and it works. Swizzled
		 * pwrite/pread is far from a hotpath - current userspace
		 * doesn't use it at all. */
		start = round_down(start, 128);
		end = round_up(end, 128);

		drm_clflush_virt_range((void *)start, end - start);
	} else {
		drm_clflush_virt_range(addr, length);
	}

}

369 370 371 372 373 374 375 376 377 378 379 380
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
static int
shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

	vaddr = kmap(page);
	if (needs_clflush)
381 382 383
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
384 385 386 387 388 389 390 391 392 393 394

	if (page_do_bit17_swizzling)
		ret = __copy_to_user_swizzled(user_data,
					      vaddr, shmem_page_offset,
					      page_length);
	else
		ret = __copy_to_user(user_data,
				     vaddr + shmem_page_offset,
				     page_length);
	kunmap(page);

395
	return ret ? - EFAULT : 0;
396 397
}

398
static int
399 400 401 402
i915_gem_shmem_pread(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pread *args,
		     struct drm_file *file)
403
{
404
	char __user *user_data;
405
	ssize_t remain;
406
	loff_t offset;
407
	int shmem_page_offset, page_length, ret = 0;
408
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
409
	int prefaulted = 0;
410
	int needs_clflush = 0;
411
	struct sg_page_iter sg_iter;
412

V
Ville Syrjälä 已提交
413
	user_data = to_user_ptr(args->data_ptr);
414 415
	remain = args->size;

416
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
417

418 419 420 421 422 423 424
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		/* If we're not in the cpu read domain, set ourself into the gtt
		 * read domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will dirty the data
		 * anyway again before the next pread happens. */
		if (obj->cache_level == I915_CACHE_NONE)
			needs_clflush = 1;
C
Chris Wilson 已提交
425 426 427 428 429
		if (obj->gtt_space) {
			ret = i915_gem_object_set_to_gtt_domain(obj, false);
			if (ret)
				return ret;
		}
430
	}
431

432 433 434 435 436 437
	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

	i915_gem_object_pin_pages(obj);

438
	offset = args->offset;
439

440 441
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
442
		struct page *page = sg_page_iter_page(&sg_iter);
443 444 445 446

		if (remain <= 0)
			break;

447 448 449 450 451
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
452
		shmem_page_offset = offset_in_page(offset);
453 454 455 456
		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

457 458 459
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

460 461 462 463 464
		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
		if (ret == 0)
			goto next_page;
465 466 467

		mutex_unlock(&dev->struct_mutex);

468
		if (!prefaulted) {
469
			ret = fault_in_multipages_writeable(user_data, remain);
470 471 472 473 474 475 476
			/* Userspace is tricking us, but we've already clobbered
			 * its pages with the prefault and promised to write the
			 * data up to the first fault. Hence ignore any errors
			 * and just continue. */
			(void)ret;
			prefaulted = 1;
		}
477

478 479 480
		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
481

482
		mutex_lock(&dev->struct_mutex);
483

484
next_page:
485 486
		mark_page_accessed(page);

487
		if (ret)
488 489
			goto out;

490
		remain -= page_length;
491
		user_data += page_length;
492 493 494
		offset += page_length;
	}

495
out:
496 497
	i915_gem_object_unpin_pages(obj);

498 499 500
	return ret;
}

501 502 503 504 505 506 507
/**
 * Reads data from the object referenced by handle.
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
508
		     struct drm_file *file)
509 510
{
	struct drm_i915_gem_pread *args = data;
511
	struct drm_i915_gem_object *obj;
512
	int ret = 0;
513

514 515 516 517
	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_WRITE,
V
Ville Syrjälä 已提交
518
		       to_user_ptr(args->data_ptr),
519 520 521
		       args->size))
		return -EFAULT;

522
	ret = i915_mutex_lock_interruptible(dev);
523
	if (ret)
524
		return ret;
525

526
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
527
	if (&obj->base == NULL) {
528 529
		ret = -ENOENT;
		goto unlock;
530
	}
531

532
	/* Bounds check source.  */
533 534
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
C
Chris Wilson 已提交
535
		ret = -EINVAL;
536
		goto out;
C
Chris Wilson 已提交
537 538
	}

539 540 541 542 543 544 545 546
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

C
Chris Wilson 已提交
547 548
	trace_i915_gem_object_pread(obj, args->offset, args->size);

549
	ret = i915_gem_shmem_pread(dev, obj, args, file);
550

551
out:
552
	drm_gem_object_unreference(&obj->base);
553
unlock:
554
	mutex_unlock(&dev->struct_mutex);
555
	return ret;
556 557
}

558 559
/* This is the fast write path which cannot handle
 * page faults in the source data
560
 */
561 562 563 564 565 566

static inline int
fast_user_write(struct io_mapping *mapping,
		loff_t page_base, int page_offset,
		char __user *user_data,
		int length)
567
{
568 569
	void __iomem *vaddr_atomic;
	void *vaddr;
570
	unsigned long unwritten;
571

P
Peter Zijlstra 已提交
572
	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
573 574 575
	/* We can use the cpu mem copy function because this is X86. */
	vaddr = (void __force*)vaddr_atomic + page_offset;
	unwritten = __copy_from_user_inatomic_nocache(vaddr,
576
						      user_data, length);
P
Peter Zijlstra 已提交
577
	io_mapping_unmap_atomic(vaddr_atomic);
578
	return unwritten;
579 580
}

581 582 583 584
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
 */
585
static int
586 587
i915_gem_gtt_pwrite_fast(struct drm_device *dev,
			 struct drm_i915_gem_object *obj,
588
			 struct drm_i915_gem_pwrite *args,
589
			 struct drm_file *file)
590
{
591
	drm_i915_private_t *dev_priv = dev->dev_private;
592
	ssize_t remain;
593
	loff_t offset, page_base;
594
	char __user *user_data;
D
Daniel Vetter 已提交
595 596
	int page_offset, page_length, ret;

597
	ret = i915_gem_object_pin(obj, 0, true, true);
D
Daniel Vetter 已提交
598 599 600 601 602 603 604 605 606 607
	if (ret)
		goto out;

	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		goto out_unpin;

	ret = i915_gem_object_put_fence(obj);
	if (ret)
		goto out_unpin;
608

V
Ville Syrjälä 已提交
609
	user_data = to_user_ptr(args->data_ptr);
610 611
	remain = args->size;

612
	offset = obj->gtt_offset + args->offset;
613 614 615 616

	while (remain > 0) {
		/* Operation in this page
		 *
617 618 619
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
620
		 */
621 622
		page_base = offset & PAGE_MASK;
		page_offset = offset_in_page(offset);
623 624 625 626 627
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		/* If we get a fault while copying data, then (presumably) our
628 629
		 * source page isn't available.  Return the error and we'll
		 * retry in the slow path.
630
		 */
B
Ben Widawsky 已提交
631
		if (fast_user_write(dev_priv->gtt.mappable, page_base,
D
Daniel Vetter 已提交
632 633 634 635
				    page_offset, user_data, page_length)) {
			ret = -EFAULT;
			goto out_unpin;
		}
636

637 638 639
		remain -= page_length;
		user_data += page_length;
		offset += page_length;
640 641
	}

D
Daniel Vetter 已提交
642 643 644
out_unpin:
	i915_gem_object_unpin(obj);
out:
645
	return ret;
646 647
}

648 649 650 651
/* Per-page copy function for the shmem pwrite fastpath.
 * Flushes invalid cachelines before writing to the target if
 * needs_clflush_before is set and flushes out any written cachelines after
 * writing if needs_clflush is set. */
652
static int
653 654 655 656 657
shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
658
{
659
	char *vaddr;
660
	int ret;
661

662
	if (unlikely(page_do_bit17_swizzling))
663
		return -EINVAL;
664

665 666 667 668 669 670 671 672 673 674 675
	vaddr = kmap_atomic(page);
	if (needs_clflush_before)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
						user_data,
						page_length);
	if (needs_clflush_after)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	kunmap_atomic(vaddr);
676

677
	return ret ? -EFAULT : 0;
678 679
}

680 681
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
682
static int
683 684 685 686 687
shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
688
{
689 690
	char *vaddr;
	int ret;
691

692
	vaddr = kmap(page);
693
	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
694 695 696
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
697 698
	if (page_do_bit17_swizzling)
		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
699 700
						user_data,
						page_length);
701 702 703 704 705
	else
		ret = __copy_from_user(vaddr + shmem_page_offset,
				       user_data,
				       page_length);
	if (needs_clflush_after)
706 707 708
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
709
	kunmap(page);
710

711
	return ret ? -EFAULT : 0;
712 713 714
}

static int
715 716 717 718
i915_gem_shmem_pwrite(struct drm_device *dev,
		      struct drm_i915_gem_object *obj,
		      struct drm_i915_gem_pwrite *args,
		      struct drm_file *file)
719 720
{
	ssize_t remain;
721 722
	loff_t offset;
	char __user *user_data;
723
	int shmem_page_offset, page_length, ret = 0;
724
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
725
	int hit_slowpath = 0;
726 727
	int needs_clflush_after = 0;
	int needs_clflush_before = 0;
728
	struct sg_page_iter sg_iter;
729

V
Ville Syrjälä 已提交
730
	user_data = to_user_ptr(args->data_ptr);
731 732
	remain = args->size;

733
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
734

735 736 737 738 739 740 741
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		/* If we're not in the cpu write domain, set ourself into the gtt
		 * write domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will use the data
		 * right away and we therefore have to clflush anyway. */
		if (obj->cache_level == I915_CACHE_NONE)
			needs_clflush_after = 1;
C
Chris Wilson 已提交
742 743 744 745 746
		if (obj->gtt_space) {
			ret = i915_gem_object_set_to_gtt_domain(obj, true);
			if (ret)
				return ret;
		}
747 748 749 750 751 752 753
	}
	/* Same trick applies for invalidate partially written cachelines before
	 * writing.  */
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
	    && obj->cache_level == I915_CACHE_NONE)
		needs_clflush_before = 1;

754 755 756 757 758 759
	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

	i915_gem_object_pin_pages(obj);

760
	offset = args->offset;
761
	obj->dirty = 1;
762

763 764
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
765
		struct page *page = sg_page_iter_page(&sg_iter);
766
		int partial_cacheline_write;
767

768 769 770
		if (remain <= 0)
			break;

771 772 773 774 775
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
776
		shmem_page_offset = offset_in_page(offset);
777 778 779 780 781

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

782 783 784 785 786 787 788
		/* If we don't overwrite a cacheline completely we need to be
		 * careful to have up-to-date data by first clflushing. Don't
		 * overcomplicate things and flush the entire patch. */
		partial_cacheline_write = needs_clflush_before &&
			((shmem_page_offset | page_length)
				& (boot_cpu_data.x86_clflush_size - 1));

789 790 791
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

792 793 794 795 796 797
		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
		if (ret == 0)
			goto next_page;
798 799 800

		hit_slowpath = 1;
		mutex_unlock(&dev->struct_mutex);
801 802 803 804
		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
805

806
		mutex_lock(&dev->struct_mutex);
807

808
next_page:
809 810 811
		set_page_dirty(page);
		mark_page_accessed(page);

812
		if (ret)
813 814
			goto out;

815
		remain -= page_length;
816
		user_data += page_length;
817
		offset += page_length;
818 819
	}

820
out:
821 822
	i915_gem_object_unpin_pages(obj);

823
	if (hit_slowpath) {
824 825 826 827 828 829 830
		/*
		 * Fixup: Flush cpu caches in case we didn't flush the dirty
		 * cachelines in-line while writing and the object moved
		 * out of the cpu write domain while we've dropped the lock.
		 */
		if (!needs_clflush_after &&
		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
831
			i915_gem_clflush_object(obj);
832
			i915_gem_chipset_flush(dev);
833
		}
834
	}
835

836
	if (needs_clflush_after)
837
		i915_gem_chipset_flush(dev);
838

839
	return ret;
840 841 842 843 844 845 846 847 848
}

/**
 * Writes data to the object referenced by handle.
 *
 * On error, the contents of the buffer that were to be modified are undefined.
 */
int
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
849
		      struct drm_file *file)
850 851
{
	struct drm_i915_gem_pwrite *args = data;
852
	struct drm_i915_gem_object *obj;
853 854 855 856 857 858
	int ret;

	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_READ,
V
Ville Syrjälä 已提交
859
		       to_user_ptr(args->data_ptr),
860 861 862
		       args->size))
		return -EFAULT;

V
Ville Syrjälä 已提交
863
	ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
864
					   args->size);
865 866
	if (ret)
		return -EFAULT;
867

868
	ret = i915_mutex_lock_interruptible(dev);
869
	if (ret)
870
		return ret;
871

872
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
873
	if (&obj->base == NULL) {
874 875
		ret = -ENOENT;
		goto unlock;
876
	}
877

878
	/* Bounds check destination. */
879 880
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
C
Chris Wilson 已提交
881
		ret = -EINVAL;
882
		goto out;
C
Chris Wilson 已提交
883 884
	}

885 886 887 888 889 890 891 892
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

C
Chris Wilson 已提交
893 894
	trace_i915_gem_object_pwrite(obj, args->offset, args->size);

D
Daniel Vetter 已提交
895
	ret = -EFAULT;
896 897 898 899 900 901
	/* We can only do the GTT pwrite on untiled buffers, as otherwise
	 * it would end up going through the fenced access, and we'll get
	 * different detiling behavior between reading and writing.
	 * pread/pwrite currently are reading and writing from the CPU
	 * perspective, requiring manual detiling by the client.
	 */
902
	if (obj->phys_obj) {
903
		ret = i915_gem_phys_pwrite(dev, obj, args, file);
904 905 906
		goto out;
	}

907
	if (obj->cache_level == I915_CACHE_NONE &&
908
	    obj->tiling_mode == I915_TILING_NONE &&
909
	    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
910
		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
D
Daniel Vetter 已提交
911 912 913
		/* Note that the gtt paths might fail with non-page-backed user
		 * pointers (e.g. gtt mappings when moving data between
		 * textures). Fallback to the shmem path in that case. */
914
	}
915

916
	if (ret == -EFAULT || ret == -ENOSPC)
D
Daniel Vetter 已提交
917
		ret = i915_gem_shmem_pwrite(dev, obj, args, file);
918

919
out:
920
	drm_gem_object_unreference(&obj->base);
921
unlock:
922
	mutex_unlock(&dev->struct_mutex);
923 924 925
	return ret;
}

926
int
927
i915_gem_check_wedge(struct i915_gpu_error *error,
928 929
		     bool interruptible)
{
930
	if (i915_reset_in_progress(error)) {
931 932 933 934 935
		/* Non-interruptible callers can't handle -EAGAIN, hence return
		 * -EIO unconditionally for these. */
		if (!interruptible)
			return -EIO;

936 937
		/* Recovery complete, but the reset failed ... */
		if (i915_terminally_wedged(error))
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958
			return -EIO;

		return -EAGAIN;
	}

	return 0;
}

/*
 * Compare seqno against outstanding lazy request. Emit a request if they are
 * equal.
 */
static int
i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
{
	int ret;

	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));

	ret = 0;
	if (seqno == ring->outstanding_lazy_request)
959
		ret = i915_add_request(ring, NULL);
960 961 962 963 964 965 966 967

	return ret;
}

/**
 * __wait_seqno - wait until execution of seqno has finished
 * @ring: the ring expected to report seqno
 * @seqno: duh!
968
 * @reset_counter: reset sequence associated with the given seqno
969 970 971
 * @interruptible: do an interruptible wait (normally yes)
 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
 *
972 973 974 975 976 977 978
 * Note: It is of utmost importance that the passed in seqno and reset_counter
 * values have been read by the caller in an smp safe manner. Where read-side
 * locks are involved, it is sufficient to read the reset_counter before
 * unlocking the lock that protects the seqno. For lockless tricks, the
 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
 * inserted.
 *
979 980 981 982
 * Returns 0 if the seqno was found within the alloted time. Else returns the
 * errno with remaining time filled in timeout argument.
 */
static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
983
			unsigned reset_counter,
984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
			bool interruptible, struct timespec *timeout)
{
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
	struct timespec before, now, wait_time={1,0};
	unsigned long timeout_jiffies;
	long end;
	bool wait_forever = true;
	int ret;

	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		return 0;

	trace_i915_gem_request_wait_begin(ring, seqno);

	if (timeout != NULL) {
		wait_time = *timeout;
		wait_forever = false;
	}

1003
	timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
1004 1005 1006 1007 1008 1009 1010 1011 1012

	if (WARN_ON(!ring->irq_get(ring)))
		return -ENODEV;

	/* Record current time in case interrupted by signal, or wedged * */
	getrawmonotonic(&before);

#define EXIT_COND \
	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
1013 1014
	 i915_reset_in_progress(&dev_priv->gpu_error) || \
	 reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
1015 1016 1017 1018 1019 1020 1021 1022 1023
	do {
		if (interruptible)
			end = wait_event_interruptible_timeout(ring->irq_queue,
							       EXIT_COND,
							       timeout_jiffies);
		else
			end = wait_event_timeout(ring->irq_queue, EXIT_COND,
						 timeout_jiffies);

1024 1025 1026 1027 1028 1029 1030
		/* We need to check whether any gpu reset happened in between
		 * the caller grabbing the seqno and now ... */
		if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
			end = -EAGAIN;

		/* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
		 * gone. */
1031
		ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
		if (ret)
			end = ret;
	} while (end == 0 && wait_forever);

	getrawmonotonic(&now);

	ring->irq_put(ring);
	trace_i915_gem_request_wait_end(ring, seqno);
#undef EXIT_COND

	if (timeout) {
		struct timespec sleep_time = timespec_sub(now, before);
		*timeout = timespec_sub(*timeout, sleep_time);
1045 1046
		if (!timespec_valid(timeout)) /* i.e. negative time remains */
			set_normalized_timespec(timeout, 0, 0);
1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
	}

	switch (end) {
	case -EIO:
	case -EAGAIN: /* Wedged */
	case -ERESTARTSYS: /* Signal */
		return (int)end;
	case 0: /* Timeout */
		return -ETIME;
	default: /* Completed */
		WARN_ON(end < 0); /* We're not aware of other errors */
		return 0;
	}
}

/**
 * Waits for a sequence number to be signaled, and cleans up the
 * request and object lists appropriately for that event.
 */
int
i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
{
	struct drm_device *dev = ring->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	bool interruptible = dev_priv->mm.interruptible;
	int ret;

	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
	BUG_ON(seqno == 0);

1077
	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1078 1079 1080 1081 1082 1083 1084
	if (ret)
		return ret;

	ret = i915_gem_check_olr(ring, seqno);
	if (ret)
		return ret;

1085 1086 1087
	return __wait_seqno(ring, seqno,
			    atomic_read(&dev_priv->gpu_error.reset_counter),
			    interruptible, NULL);
1088 1089
}

1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108
static int
i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
				     struct intel_ring_buffer *ring)
{
	i915_gem_retire_requests_ring(ring);

	/* Manually manage the write flush as we may have not yet
	 * retired the buffer.
	 *
	 * Note that the last_write_seqno is always the earlier of
	 * the two (read/write) seqno, so if we haved successfully waited,
	 * we know we have passed the last write.
	 */
	obj->last_write_seqno = 0;
	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;

	return 0;
}

1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
/**
 * Ensures that all rendering to the object has completed and the object is
 * safe to unbind from the GTT or access from the CPU.
 */
static __must_check int
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
			       bool readonly)
{
	struct intel_ring_buffer *ring = obj->ring;
	u32 seqno;
	int ret;

	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
	if (seqno == 0)
		return 0;

	ret = i915_wait_seqno(ring, seqno);
	if (ret)
		return ret;

1129
	return i915_gem_object_wait_rendering__tail(obj, ring);
1130 1131
}

1132 1133 1134 1135 1136 1137 1138 1139 1140 1141
/* A nonblocking variant of the above wait. This is a highly dangerous routine
 * as the object state may change during this call.
 */
static __must_check int
i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
					    bool readonly)
{
	struct drm_device *dev = obj->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_ring_buffer *ring = obj->ring;
1142
	unsigned reset_counter;
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
	u32 seqno;
	int ret;

	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
	BUG_ON(!dev_priv->mm.interruptible);

	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
	if (seqno == 0)
		return 0;

1153
	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1154 1155 1156 1157 1158 1159 1160
	if (ret)
		return ret;

	ret = i915_gem_check_olr(ring, seqno);
	if (ret)
		return ret;

1161
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1162
	mutex_unlock(&dev->struct_mutex);
1163
	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
1164
	mutex_lock(&dev->struct_mutex);
1165 1166
	if (ret)
		return ret;
1167

1168
	return i915_gem_object_wait_rendering__tail(obj, ring);
1169 1170
}

1171
/**
1172 1173
 * Called when user space prepares to use an object with the CPU, either
 * through the mmap ioctl's mapping or a GTT mapping.
1174 1175 1176
 */
int
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1177
			  struct drm_file *file)
1178 1179
{
	struct drm_i915_gem_set_domain *args = data;
1180
	struct drm_i915_gem_object *obj;
1181 1182
	uint32_t read_domains = args->read_domains;
	uint32_t write_domain = args->write_domain;
1183 1184
	int ret;

1185
	/* Only handle setting domains to types used by the CPU. */
1186
	if (write_domain & I915_GEM_GPU_DOMAINS)
1187 1188
		return -EINVAL;

1189
	if (read_domains & I915_GEM_GPU_DOMAINS)
1190 1191 1192 1193 1194 1195 1196 1197
		return -EINVAL;

	/* Having something in the write domain implies it's in the read
	 * domain, and only that read domain.  Enforce that in the request.
	 */
	if (write_domain != 0 && read_domains != write_domain)
		return -EINVAL;

1198
	ret = i915_mutex_lock_interruptible(dev);
1199
	if (ret)
1200
		return ret;
1201

1202
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1203
	if (&obj->base == NULL) {
1204 1205
		ret = -ENOENT;
		goto unlock;
1206
	}
1207

1208 1209 1210 1211 1212 1213 1214 1215
	/* Try to flush the object off the GPU without holding the lock.
	 * We will repeat the flush holding the lock in the normal manner
	 * to catch cases where we are gazumped.
	 */
	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
	if (ret)
		goto unref;

1216 1217
	if (read_domains & I915_GEM_DOMAIN_GTT) {
		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1218 1219 1220 1221 1222 1223 1224

		/* Silently promote "you're not bound, there was nothing to do"
		 * to success, since the client was just asking us to
		 * make sure everything was done.
		 */
		if (ret == -EINVAL)
			ret = 0;
1225
	} else {
1226
		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1227 1228
	}

1229
unref:
1230
	drm_gem_object_unreference(&obj->base);
1231
unlock:
1232 1233 1234 1235 1236 1237 1238 1239 1240
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Called when user space has done writes to this buffer
 */
int
i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1241
			 struct drm_file *file)
1242 1243
{
	struct drm_i915_gem_sw_finish *args = data;
1244
	struct drm_i915_gem_object *obj;
1245 1246
	int ret = 0;

1247
	ret = i915_mutex_lock_interruptible(dev);
1248
	if (ret)
1249
		return ret;
1250

1251
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1252
	if (&obj->base == NULL) {
1253 1254
		ret = -ENOENT;
		goto unlock;
1255 1256 1257
	}

	/* Pinned buffers may be scanout, so flush the cache */
1258
	if (obj->pin_count)
1259 1260
		i915_gem_object_flush_cpu_write_domain(obj);

1261
	drm_gem_object_unreference(&obj->base);
1262
unlock:
1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Maps the contents of an object, returning the address it is mapped
 * into.
 *
 * While the mapping holds a reference on the contents of the object, it doesn't
 * imply a ref on the object itself.
 */
int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1276
		    struct drm_file *file)
1277 1278 1279 1280 1281
{
	struct drm_i915_gem_mmap *args = data;
	struct drm_gem_object *obj;
	unsigned long addr;

1282
	obj = drm_gem_object_lookup(dev, file, args->handle);
1283
	if (obj == NULL)
1284
		return -ENOENT;
1285

1286 1287 1288 1289 1290 1291 1292 1293
	/* prime objects have no backing filp to GEM mmap
	 * pages from.
	 */
	if (!obj->filp) {
		drm_gem_object_unreference_unlocked(obj);
		return -EINVAL;
	}

1294
	addr = vm_mmap(obj->filp, 0, args->size,
1295 1296
		       PROT_READ | PROT_WRITE, MAP_SHARED,
		       args->offset);
1297
	drm_gem_object_unreference_unlocked(obj);
1298 1299 1300 1301 1302 1303 1304 1305
	if (IS_ERR((void *)addr))
		return addr;

	args->addr_ptr = (uint64_t) addr;

	return 0;
}

1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
/**
 * i915_gem_fault - fault a page into the GTT
 * vma: VMA in question
 * vmf: fault info
 *
 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
 * from userspace.  The fault handler takes care of binding the object to
 * the GTT (if needed), allocating and programming a fence register (again,
 * only if needed based on whether the old reg is still valid or the object
 * is tiled) and inserting a new PTE into the faulting process.
 *
 * Note that the faulting process may involve evicting existing objects
 * from the GTT and/or fence registers to make room.  So performance may
 * suffer if the GTT working set is large or there are few fence registers
 * left.
 */
int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
1324 1325
	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
	struct drm_device *dev = obj->base.dev;
1326
	drm_i915_private_t *dev_priv = dev->dev_private;
1327 1328 1329
	pgoff_t page_offset;
	unsigned long pfn;
	int ret = 0;
1330
	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1331 1332 1333 1334 1335

	/* We don't use vmf->pgoff since that has the fake offset */
	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
		PAGE_SHIFT;

1336 1337 1338
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto out;
1339

C
Chris Wilson 已提交
1340 1341
	trace_i915_gem_object_fault(obj, page_offset, true, write);

1342 1343 1344 1345 1346 1347
	/* Access to snoopable pages through the GTT is incoherent. */
	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
		ret = -EINVAL;
		goto unlock;
	}

1348
	/* Now bind it into the GTT if needed */
1349 1350 1351
	ret = i915_gem_object_pin(obj, 0, true, false);
	if (ret)
		goto unlock;
1352

1353 1354 1355
	ret = i915_gem_object_set_to_gtt_domain(obj, write);
	if (ret)
		goto unpin;
1356

1357
	ret = i915_gem_object_get_fence(obj);
1358
	if (ret)
1359
		goto unpin;
1360

1361 1362
	obj->fault_mappable = true;

B
Ben Widawsky 已提交
1363
	pfn = ((dev_priv->gtt.mappable_base + obj->gtt_offset) >> PAGE_SHIFT) +
1364 1365 1366 1367
		page_offset;

	/* Finally, remap it using the new GTT offset */
	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1368 1369
unpin:
	i915_gem_object_unpin(obj);
1370
unlock:
1371
	mutex_unlock(&dev->struct_mutex);
1372
out:
1373
	switch (ret) {
1374
	case -EIO:
1375 1376 1377
		/* If this -EIO is due to a gpu hang, give the reset code a
		 * chance to clean up the mess. Otherwise return the proper
		 * SIGBUS. */
1378
		if (i915_terminally_wedged(&dev_priv->gpu_error))
1379
			return VM_FAULT_SIGBUS;
1380
	case -EAGAIN:
1381 1382 1383 1384 1385 1386 1387
		/* Give the error handler a chance to run and move the
		 * objects off the GPU active list. Next time we service the
		 * fault, we should be able to transition the page into the
		 * GTT without touching the GPU (and so avoid further
		 * EIO/EGAIN). If the GPU is wedged, then there is no issue
		 * with coherency, just lost writes.
		 */
1388
		set_need_resched();
1389 1390
	case 0:
	case -ERESTARTSYS:
1391
	case -EINTR:
1392 1393 1394 1395 1396
	case -EBUSY:
		/*
		 * EBUSY is ok: this just means that another thread
		 * already did the job.
		 */
1397
		return VM_FAULT_NOPAGE;
1398 1399
	case -ENOMEM:
		return VM_FAULT_OOM;
1400 1401
	case -ENOSPC:
		return VM_FAULT_SIGBUS;
1402
	default:
1403
		WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1404
		return VM_FAULT_SIGBUS;
1405 1406 1407
	}
}

1408 1409 1410 1411
/**
 * i915_gem_release_mmap - remove physical page mappings
 * @obj: obj in question
 *
1412
 * Preserve the reservation of the mmapping with the DRM core code, but
1413 1414 1415 1416 1417 1418 1419 1420 1421
 * relinquish ownership of the pages back to the system.
 *
 * It is vital that we remove the page mapping if we have mapped a tiled
 * object through the GTT and then lose the fence register due to
 * resource pressure. Similarly if the object has been moved out of the
 * aperture, than pages mapped into userspace must be revoked. Removing the
 * mapping will then trigger a page fault on the next user access, allowing
 * fixup by i915_gem_fault().
 */
1422
void
1423
i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1424
{
1425 1426
	if (!obj->fault_mappable)
		return;
1427

1428 1429 1430 1431
	if (obj->base.dev->dev_mapping)
		unmap_mapping_range(obj->base.dev->dev_mapping,
				    (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
				    obj->base.size, 1);
1432

1433
	obj->fault_mappable = false;
1434 1435
}

1436
uint32_t
1437
i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1438
{
1439
	uint32_t gtt_size;
1440 1441

	if (INTEL_INFO(dev)->gen >= 4 ||
1442 1443
	    tiling_mode == I915_TILING_NONE)
		return size;
1444 1445 1446

	/* Previous chips need a power-of-two fence region when tiling */
	if (INTEL_INFO(dev)->gen == 3)
1447
		gtt_size = 1024*1024;
1448
	else
1449
		gtt_size = 512*1024;
1450

1451 1452
	while (gtt_size < size)
		gtt_size <<= 1;
1453

1454
	return gtt_size;
1455 1456
}

1457 1458 1459 1460 1461
/**
 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 * @obj: object to check
 *
 * Return the required GTT alignment for an object, taking into account
1462
 * potential fence register mapping.
1463
 */
1464 1465 1466
uint32_t
i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
			   int tiling_mode, bool fenced)
1467 1468 1469 1470 1471
{
	/*
	 * Minimum alignment is 4k (GTT page size), but might be greater
	 * if a fence register is needed for the object.
	 */
1472
	if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1473
	    tiling_mode == I915_TILING_NONE)
1474 1475
		return 4096;

1476 1477 1478 1479
	/*
	 * Previous chips need to be aligned to the size of the smallest
	 * fence register that can contain the object.
	 */
1480
	return i915_gem_get_gtt_size(dev, size, tiling_mode);
1481 1482
}

1483 1484 1485 1486 1487 1488 1489 1490
static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	int ret;

	if (obj->base.map_list.map)
		return 0;

1491 1492
	dev_priv->mm.shrinker_no_lock_stealing = true;

1493 1494
	ret = drm_gem_create_mmap_offset(&obj->base);
	if (ret != -ENOSPC)
1495
		goto out;
1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506

	/* Badly fragmented mmap space? The only way we can recover
	 * space is by destroying unwanted objects. We can't randomly release
	 * mmap_offsets as userspace expects them to be persistent for the
	 * lifetime of the objects. The closest we can is to release the
	 * offsets on purgeable objects by truncating it and marking it purged,
	 * which prevents userspace from ever using that object again.
	 */
	i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
	ret = drm_gem_create_mmap_offset(&obj->base);
	if (ret != -ENOSPC)
1507
		goto out;
1508 1509

	i915_gem_shrink_all(dev_priv);
1510 1511 1512 1513 1514
	ret = drm_gem_create_mmap_offset(&obj->base);
out:
	dev_priv->mm.shrinker_no_lock_stealing = false;

	return ret;
1515 1516 1517 1518 1519 1520 1521 1522 1523 1524
}

static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
{
	if (!obj->base.map_list.map)
		return;

	drm_gem_free_mmap_offset(&obj->base);
}

1525
int
1526 1527 1528 1529
i915_gem_mmap_gtt(struct drm_file *file,
		  struct drm_device *dev,
		  uint32_t handle,
		  uint64_t *offset)
1530
{
1531
	struct drm_i915_private *dev_priv = dev->dev_private;
1532
	struct drm_i915_gem_object *obj;
1533 1534
	int ret;

1535
	ret = i915_mutex_lock_interruptible(dev);
1536
	if (ret)
1537
		return ret;
1538

1539
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1540
	if (&obj->base == NULL) {
1541 1542 1543
		ret = -ENOENT;
		goto unlock;
	}
1544

B
Ben Widawsky 已提交
1545
	if (obj->base.size > dev_priv->gtt.mappable_end) {
1546
		ret = -E2BIG;
1547
		goto out;
1548 1549
	}

1550
	if (obj->madv != I915_MADV_WILLNEED) {
1551
		DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1552 1553
		ret = -EINVAL;
		goto out;
1554 1555
	}

1556 1557 1558
	ret = i915_gem_object_create_mmap_offset(obj);
	if (ret)
		goto out;
1559

1560
	*offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1561

1562
out:
1563
	drm_gem_object_unreference(&obj->base);
1564
unlock:
1565
	mutex_unlock(&dev->struct_mutex);
1566
	return ret;
1567 1568
}

1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592
/**
 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 * @dev: DRM device
 * @data: GTT mapping ioctl data
 * @file: GEM object info
 *
 * Simply returns the fake offset to userspace so it can mmap it.
 * The mmap call will end up in drm_gem_mmap(), which will set things
 * up so we can get faults in the handler above.
 *
 * The fault handler will take care of binding the object into the GTT
 * (since it may have been evicted to make room for something), allocating
 * a fence register, and mapping the appropriate aperture address into
 * userspace.
 */
int
i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file)
{
	struct drm_i915_gem_mmap_gtt *args = data;

	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
}

D
Daniel Vetter 已提交
1593 1594 1595
/* Immediately discard the backing storage */
static void
i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1596 1597 1598
{
	struct inode *inode;

1599
	i915_gem_object_free_mmap_offset(obj);
1600

1601 1602
	if (obj->base.filp == NULL)
		return;
1603

D
Daniel Vetter 已提交
1604 1605 1606 1607 1608
	/* Our goal here is to return as much of the memory as
	 * is possible back to the system as we are called from OOM.
	 * To do this we must instruct the shmfs to drop all of its
	 * backing pages, *now*.
	 */
A
Al Viro 已提交
1609
	inode = file_inode(obj->base.filp);
D
Daniel Vetter 已提交
1610
	shmem_truncate_range(inode, 0, (loff_t)-1);
1611

D
Daniel Vetter 已提交
1612 1613
	obj->madv = __I915_MADV_PURGED;
}
1614

D
Daniel Vetter 已提交
1615 1616 1617 1618
static inline int
i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
{
	return obj->madv == I915_MADV_DONTNEED;
1619 1620
}

1621
static void
1622
i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1623
{
1624 1625
	struct sg_page_iter sg_iter;
	int ret;
1626

1627
	BUG_ON(obj->madv == __I915_MADV_PURGED);
1628

C
Chris Wilson 已提交
1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
	ret = i915_gem_object_set_to_cpu_domain(obj, true);
	if (ret) {
		/* In the event of a disaster, abandon all caches and
		 * hope for the best.
		 */
		WARN_ON(ret != -EIO);
		i915_gem_clflush_object(obj);
		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	}

1639
	if (i915_gem_object_needs_bit17_swizzle(obj))
1640 1641
		i915_gem_object_save_bit_17_swizzle(obj);

1642 1643
	if (obj->madv == I915_MADV_DONTNEED)
		obj->dirty = 0;
1644

1645
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1646
		struct page *page = sg_page_iter_page(&sg_iter);
1647

1648
		if (obj->dirty)
1649
			set_page_dirty(page);
1650

1651
		if (obj->madv == I915_MADV_WILLNEED)
1652
			mark_page_accessed(page);
1653

1654
		page_cache_release(page);
1655
	}
1656
	obj->dirty = 0;
1657

1658 1659
	sg_free_table(obj->pages);
	kfree(obj->pages);
1660
}
C
Chris Wilson 已提交
1661

1662
int
1663 1664 1665 1666
i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
{
	const struct drm_i915_gem_object_ops *ops = obj->ops;

1667
	if (obj->pages == NULL)
1668 1669 1670
		return 0;

	BUG_ON(obj->gtt_space);
C
Chris Wilson 已提交
1671

1672 1673 1674
	if (obj->pages_pin_count)
		return -EBUSY;

1675 1676 1677
	/* ->put_pages might need to allocate memory for the bit17 swizzle
	 * array, hence protect them from being reaped by removing them from gtt
	 * lists early. */
1678
	list_del(&obj->global_list);
1679

1680
	ops->put_pages(obj);
1681
	obj->pages = NULL;
1682

C
Chris Wilson 已提交
1683 1684 1685 1686 1687 1688 1689
	if (i915_gem_object_is_purgeable(obj))
		i915_gem_object_truncate(obj);

	return 0;
}

static long
1690 1691
__i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
		  bool purgeable_only)
C
Chris Wilson 已提交
1692 1693 1694 1695 1696 1697
{
	struct drm_i915_gem_object *obj, *next;
	long count = 0;

	list_for_each_entry_safe(obj, next,
				 &dev_priv->mm.unbound_list,
1698
				 global_list) {
1699
		if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
1700
		    i915_gem_object_put_pages(obj) == 0) {
C
Chris Wilson 已提交
1701 1702 1703 1704 1705 1706 1707 1708 1709
			count += obj->base.size >> PAGE_SHIFT;
			if (count >= target)
				return count;
		}
	}

	list_for_each_entry_safe(obj, next,
				 &dev_priv->mm.inactive_list,
				 mm_list) {
1710
		if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
C
Chris Wilson 已提交
1711
		    i915_gem_object_unbind(obj) == 0 &&
1712
		    i915_gem_object_put_pages(obj) == 0) {
C
Chris Wilson 已提交
1713 1714 1715 1716 1717 1718 1719 1720 1721
			count += obj->base.size >> PAGE_SHIFT;
			if (count >= target)
				return count;
		}
	}

	return count;
}

1722 1723 1724 1725 1726 1727
static long
i915_gem_purge(struct drm_i915_private *dev_priv, long target)
{
	return __i915_gem_shrink(dev_priv, target, true);
}

C
Chris Wilson 已提交
1728 1729 1730 1731 1732 1733 1734
static void
i915_gem_shrink_all(struct drm_i915_private *dev_priv)
{
	struct drm_i915_gem_object *obj, *next;

	i915_gem_evict_everything(dev_priv->dev);

1735 1736
	list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
				 global_list)
1737
		i915_gem_object_put_pages(obj);
D
Daniel Vetter 已提交
1738 1739
}

1740
static int
C
Chris Wilson 已提交
1741
i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
1742
{
C
Chris Wilson 已提交
1743
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1744 1745
	int page_count, i;
	struct address_space *mapping;
1746 1747
	struct sg_table *st;
	struct scatterlist *sg;
1748
	struct sg_page_iter sg_iter;
1749
	struct page *page;
1750
	unsigned long last_pfn = 0;	/* suppress gcc warning */
C
Chris Wilson 已提交
1751
	gfp_t gfp;
1752

C
Chris Wilson 已提交
1753 1754 1755 1756 1757 1758 1759
	/* Assert that the object is not currently in any GPU domain. As it
	 * wasn't in the GTT, there shouldn't be any way it could have been in
	 * a GPU cache
	 */
	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);

1760 1761 1762 1763
	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (st == NULL)
		return -ENOMEM;

1764
	page_count = obj->base.size / PAGE_SIZE;
1765 1766 1767
	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		sg_free_table(st);
		kfree(st);
1768
		return -ENOMEM;
1769
	}
1770

1771 1772 1773 1774 1775
	/* Get the list of pages out of our struct file.  They'll be pinned
	 * at this point until we release them.
	 *
	 * Fail silently without starting the shrinker
	 */
A
Al Viro 已提交
1776
	mapping = file_inode(obj->base.filp)->i_mapping;
C
Chris Wilson 已提交
1777
	gfp = mapping_gfp_mask(mapping);
1778
	gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
C
Chris Wilson 已提交
1779
	gfp &= ~(__GFP_IO | __GFP_WAIT);
1780 1781 1782
	sg = st->sgl;
	st->nents = 0;
	for (i = 0; i < page_count; i++) {
C
Chris Wilson 已提交
1783 1784 1785 1786 1787 1788 1789 1790 1791 1792
		page = shmem_read_mapping_page_gfp(mapping, i, gfp);
		if (IS_ERR(page)) {
			i915_gem_purge(dev_priv, page_count);
			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
		}
		if (IS_ERR(page)) {
			/* We've tried hard to allocate the memory by reaping
			 * our own buffer, now let the real VM do its job and
			 * go down in flames if truly OOM.
			 */
1793
			gfp &= ~(__GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD);
C
Chris Wilson 已提交
1794 1795 1796 1797 1798 1799 1800
			gfp |= __GFP_IO | __GFP_WAIT;

			i915_gem_shrink_all(dev_priv);
			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
			if (IS_ERR(page))
				goto err_pages;

1801
			gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
C
Chris Wilson 已提交
1802 1803
			gfp &= ~(__GFP_IO | __GFP_WAIT);
		}
1804 1805 1806 1807 1808 1809 1810 1811
#ifdef CONFIG_SWIOTLB
		if (swiotlb_nr_tbl()) {
			st->nents++;
			sg_set_page(sg, page, PAGE_SIZE, 0);
			sg = sg_next(sg);
			continue;
		}
#endif
1812 1813 1814 1815 1816 1817 1818 1819 1820
		if (!i || page_to_pfn(page) != last_pfn + 1) {
			if (i)
				sg = sg_next(sg);
			st->nents++;
			sg_set_page(sg, page, PAGE_SIZE, 0);
		} else {
			sg->length += PAGE_SIZE;
		}
		last_pfn = page_to_pfn(page);
1821
	}
1822 1823 1824 1825
#ifdef CONFIG_SWIOTLB
	if (!swiotlb_nr_tbl())
#endif
		sg_mark_end(sg);
1826 1827
	obj->pages = st;

1828
	if (i915_gem_object_needs_bit17_swizzle(obj))
1829 1830 1831 1832 1833
		i915_gem_object_do_bit_17_swizzle(obj);

	return 0;

err_pages:
1834 1835
	sg_mark_end(sg);
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
1836
		page_cache_release(sg_page_iter_page(&sg_iter));
1837 1838
	sg_free_table(st);
	kfree(st);
1839
	return PTR_ERR(page);
1840 1841
}

1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855
/* Ensure that the associated pages are gathered from the backing storage
 * and pinned into our object. i915_gem_object_get_pages() may be called
 * multiple times before they are released by a single call to
 * i915_gem_object_put_pages() - once the pages are no longer referenced
 * either as a result of memory pressure (reaping pages under the shrinker)
 * or as the object is itself released.
 */
int
i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	const struct drm_i915_gem_object_ops *ops = obj->ops;
	int ret;

1856
	if (obj->pages)
1857 1858
		return 0;

1859 1860 1861 1862 1863
	if (obj->madv != I915_MADV_WILLNEED) {
		DRM_ERROR("Attempting to obtain a purgeable object\n");
		return -EINVAL;
	}

1864 1865
	BUG_ON(obj->pages_pin_count);

1866 1867 1868 1869
	ret = ops->get_pages(obj);
	if (ret)
		return ret;

1870
	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
1871
	return 0;
1872 1873
}

1874
void
1875
i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1876
			       struct intel_ring_buffer *ring)
1877
{
1878
	struct drm_device *dev = obj->base.dev;
1879
	struct drm_i915_private *dev_priv = dev->dev_private;
1880
	u32 seqno = intel_ring_get_seqno(ring);
1881

1882
	BUG_ON(ring == NULL);
1883 1884 1885 1886
	if (obj->ring != ring && obj->last_write_seqno) {
		/* Keep the seqno relative to the current ring */
		obj->last_write_seqno = seqno;
	}
1887
	obj->ring = ring;
1888 1889

	/* Add a reference if we're newly entering the active list. */
1890 1891 1892
	if (!obj->active) {
		drm_gem_object_reference(&obj->base);
		obj->active = 1;
1893
	}
1894

1895
	/* Move from whatever list we were on to the tail of execution. */
1896 1897
	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
	list_move_tail(&obj->ring_list, &ring->active_list);
1898

1899
	obj->last_read_seqno = seqno;
1900

1901
	if (obj->fenced_gpu_access) {
1902 1903
		obj->last_fenced_seqno = seqno;

1904 1905 1906 1907 1908 1909 1910 1911
		/* Bump MRU to take account of the delayed flush */
		if (obj->fence_reg != I915_FENCE_REG_NONE) {
			struct drm_i915_fence_reg *reg;

			reg = &dev_priv->fence_regs[obj->fence_reg];
			list_move_tail(&reg->lru_list,
				       &dev_priv->mm.fence_list);
		}
1912 1913 1914 1915 1916
	}
}

static void
i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1917
{
1918
	struct drm_device *dev = obj->base.dev;
1919
	struct drm_i915_private *dev_priv = dev->dev_private;
1920

1921
	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
1922
	BUG_ON(!obj->active);
1923

1924
	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1925

1926
	list_del_init(&obj->ring_list);
1927 1928
	obj->ring = NULL;

1929 1930 1931 1932 1933
	obj->last_read_seqno = 0;
	obj->last_write_seqno = 0;
	obj->base.write_domain = 0;

	obj->last_fenced_seqno = 0;
1934 1935 1936 1937 1938 1939
	obj->fenced_gpu_access = false;

	obj->active = 0;
	drm_gem_object_unreference(&obj->base);

	WARN_ON(i915_verify_lists(dev));
1940
}
1941

1942
static int
1943
i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
1944
{
1945 1946 1947
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_ring_buffer *ring;
	int ret, i, j;
1948

1949
	/* Carefully retire all requests without writing to the rings */
1950
	for_each_ring(ring, dev_priv, i) {
1951 1952 1953
		ret = intel_ring_idle(ring);
		if (ret)
			return ret;
1954 1955
	}
	i915_gem_retire_requests(dev);
1956 1957

	/* Finally reset hw state */
1958
	for_each_ring(ring, dev_priv, i) {
1959
		intel_ring_init_seqno(ring, seqno);
1960

1961 1962 1963
		for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
			ring->sync_seqno[j] = 0;
	}
1964

1965
	return 0;
1966 1967
}

1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993
int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	if (seqno == 0)
		return -EINVAL;

	/* HWS page needs to be set less than what we
	 * will inject to ring
	 */
	ret = i915_gem_init_seqno(dev, seqno - 1);
	if (ret)
		return ret;

	/* Carefully set the last_seqno value so that wrap
	 * detection still works
	 */
	dev_priv->next_seqno = seqno;
	dev_priv->last_seqno = seqno - 1;
	if (dev_priv->last_seqno == 0)
		dev_priv->last_seqno--;

	return 0;
}

1994 1995
int
i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
1996
{
1997 1998 1999 2000
	struct drm_i915_private *dev_priv = dev->dev_private;

	/* reserve 0 for non-seqno */
	if (dev_priv->next_seqno == 0) {
2001
		int ret = i915_gem_init_seqno(dev, 0);
2002 2003
		if (ret)
			return ret;
2004

2005 2006
		dev_priv->next_seqno = 1;
	}
2007

2008
	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2009
	return 0;
2010 2011
}

2012 2013
int __i915_add_request(struct intel_ring_buffer *ring,
		       struct drm_file *file,
2014
		       struct drm_i915_gem_object *obj,
2015
		       u32 *out_seqno)
2016
{
C
Chris Wilson 已提交
2017
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
2018
	struct drm_i915_gem_request *request;
2019
	u32 request_ring_position, request_start;
2020
	int was_empty;
2021 2022
	int ret;

2023
	request_start = intel_ring_get_tail(ring);
2024 2025 2026 2027 2028 2029 2030
	/*
	 * Emit any outstanding flushes - execbuf can fail to emit the flush
	 * after having emitted the batchbuffer command. Hence we need to fix
	 * things up similar to emitting the lazy request. The difference here
	 * is that the flush _must_ happen before the next request, no matter
	 * what.
	 */
2031 2032 2033
	ret = intel_ring_flush_all_caches(ring);
	if (ret)
		return ret;
2034

2035 2036 2037
	request = kmalloc(sizeof(*request), GFP_KERNEL);
	if (request == NULL)
		return -ENOMEM;
2038

2039

2040 2041 2042 2043 2044 2045 2046
	/* Record the position of the start of the request so that
	 * should we detect the updated seqno part-way through the
	 * GPU processing the request, we never over-estimate the
	 * position of the head.
	 */
	request_ring_position = intel_ring_get_tail(ring);

2047
	ret = ring->add_request(ring);
2048 2049 2050 2051
	if (ret) {
		kfree(request);
		return ret;
	}
2052

2053
	request->seqno = intel_ring_get_seqno(ring);
2054
	request->ring = ring;
2055
	request->head = request_start;
2056
	request->tail = request_ring_position;
2057
	request->ctx = ring->last_context;
2058 2059 2060 2061 2062 2063 2064 2065
	request->batch_obj = obj;

	/* Whilst this request exists, batch_obj will be on the
	 * active_list, and so will hold the active reference. Only when this
	 * request is retired will the the batch_obj be moved onto the
	 * inactive_list and lose its active reference. Hence we do not need
	 * to explicitly hold another reference here.
	 */
2066 2067 2068 2069

	if (request->ctx)
		i915_gem_context_reference(request->ctx);

2070
	request->emitted_jiffies = jiffies;
2071 2072
	was_empty = list_empty(&ring->request_list);
	list_add_tail(&request->list, &ring->request_list);
2073
	request->file_priv = NULL;
2074

C
Chris Wilson 已提交
2075 2076 2077
	if (file) {
		struct drm_i915_file_private *file_priv = file->driver_priv;

2078
		spin_lock(&file_priv->mm.lock);
2079
		request->file_priv = file_priv;
2080
		list_add_tail(&request->client_list,
2081
			      &file_priv->mm.request_list);
2082
		spin_unlock(&file_priv->mm.lock);
2083
	}
2084

2085
	trace_i915_gem_request_add(ring, request->seqno);
2086
	ring->outstanding_lazy_request = 0;
C
Chris Wilson 已提交
2087

B
Ben Gamari 已提交
2088
	if (!dev_priv->mm.suspended) {
2089
		if (i915_enable_hangcheck) {
2090
			mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2091
				  round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
2092
		}
2093
		if (was_empty) {
2094
			queue_delayed_work(dev_priv->wq,
2095 2096
					   &dev_priv->mm.retire_work,
					   round_jiffies_up_relative(HZ));
2097 2098
			intel_mark_busy(dev_priv->dev);
		}
B
Ben Gamari 已提交
2099
	}
2100

2101
	if (out_seqno)
2102
		*out_seqno = request->seqno;
2103
	return 0;
2104 2105
}

2106 2107
static inline void
i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2108
{
2109
	struct drm_i915_file_private *file_priv = request->file_priv;
2110

2111 2112
	if (!file_priv)
		return;
C
Chris Wilson 已提交
2113

2114
	spin_lock(&file_priv->mm.lock);
2115 2116 2117 2118
	if (request->file_priv) {
		list_del(&request->client_list);
		request->file_priv = NULL;
	}
2119
	spin_unlock(&file_priv->mm.lock);
2120 2121
}

2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209
static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj)
{
	if (acthd >= obj->gtt_offset &&
	    acthd < obj->gtt_offset + obj->base.size)
		return true;

	return false;
}

static bool i915_head_inside_request(const u32 acthd_unmasked,
				     const u32 request_start,
				     const u32 request_end)
{
	const u32 acthd = acthd_unmasked & HEAD_ADDR;

	if (request_start < request_end) {
		if (acthd >= request_start && acthd < request_end)
			return true;
	} else if (request_start > request_end) {
		if (acthd >= request_start || acthd < request_end)
			return true;
	}

	return false;
}

static bool i915_request_guilty(struct drm_i915_gem_request *request,
				const u32 acthd, bool *inside)
{
	/* There is a possibility that unmasked head address
	 * pointing inside the ring, matches the batch_obj address range.
	 * However this is extremely unlikely.
	 */

	if (request->batch_obj) {
		if (i915_head_inside_object(acthd, request->batch_obj)) {
			*inside = true;
			return true;
		}
	}

	if (i915_head_inside_request(acthd, request->head, request->tail)) {
		*inside = false;
		return true;
	}

	return false;
}

static void i915_set_reset_status(struct intel_ring_buffer *ring,
				  struct drm_i915_gem_request *request,
				  u32 acthd)
{
	struct i915_ctx_hang_stats *hs = NULL;
	bool inside, guilty;

	/* Innocent until proven guilty */
	guilty = false;

	if (ring->hangcheck.action != wait &&
	    i915_request_guilty(request, acthd, &inside)) {
		DRM_ERROR("%s hung %s bo (0x%x ctx %d) at 0x%x\n",
			  ring->name,
			  inside ? "inside" : "flushing",
			  request->batch_obj ?
			  request->batch_obj->gtt_offset : 0,
			  request->ctx ? request->ctx->id : 0,
			  acthd);

		guilty = true;
	}

	/* If contexts are disabled or this is the default context, use
	 * file_priv->reset_state
	 */
	if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
		hs = &request->ctx->hang_stats;
	else if (request->file_priv)
		hs = &request->file_priv->hang_stats;

	if (hs) {
		if (guilty)
			hs->batch_active++;
		else
			hs->batch_pending++;
	}
}

2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220
static void i915_gem_free_request(struct drm_i915_gem_request *request)
{
	list_del(&request->list);
	i915_gem_request_remove_from_client(request);

	if (request->ctx)
		i915_gem_context_unreference(request->ctx);

	kfree(request);
}

2221 2222
static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
				      struct intel_ring_buffer *ring)
2223
{
2224 2225 2226 2227 2228 2229
	u32 completed_seqno;
	u32 acthd;

	acthd = intel_ring_get_active_head(ring);
	completed_seqno = ring->get_seqno(ring, false);

2230 2231
	while (!list_empty(&ring->request_list)) {
		struct drm_i915_gem_request *request;
2232

2233 2234 2235
		request = list_first_entry(&ring->request_list,
					   struct drm_i915_gem_request,
					   list);
2236

2237 2238 2239
		if (request->seqno > completed_seqno)
			i915_set_reset_status(ring, request, acthd);

2240
		i915_gem_free_request(request);
2241
	}
2242

2243
	while (!list_empty(&ring->active_list)) {
2244
		struct drm_i915_gem_object *obj;
2245

2246 2247 2248
		obj = list_first_entry(&ring->active_list,
				       struct drm_i915_gem_object,
				       ring_list);
2249

2250
		i915_gem_object_move_to_inactive(obj);
2251 2252 2253
	}
}

2254 2255 2256 2257 2258
static void i915_gem_reset_fences(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int i;

2259
	for (i = 0; i < dev_priv->num_fence_regs; i++) {
2260
		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2261

2262 2263
		if (reg->obj)
			i915_gem_object_fence_lost(reg->obj);
2264

2265 2266
		i915_gem_write_fence(dev, i, NULL);

2267 2268 2269
		reg->pin_count = 0;
		reg->obj = NULL;
		INIT_LIST_HEAD(&reg->lru_list);
2270
	}
2271 2272

	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
2273 2274
}

2275
void i915_gem_reset(struct drm_device *dev)
2276
{
2277
	struct drm_i915_private *dev_priv = dev->dev_private;
2278
	struct drm_i915_gem_object *obj;
2279
	struct intel_ring_buffer *ring;
2280
	int i;
2281

2282 2283
	for_each_ring(ring, dev_priv, i)
		i915_gem_reset_ring_lists(dev_priv, ring);
2284 2285 2286 2287

	/* Move everything out of the GPU domains to ensure we do any
	 * necessary invalidation upon reuse.
	 */
2288
	list_for_each_entry(obj,
2289
			    &dev_priv->mm.inactive_list,
2290
			    mm_list)
2291
	{
2292
		obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2293
	}
2294 2295

	/* The fence registers are invalidated so clear them out */
2296
	i915_gem_reset_fences(dev);
2297 2298 2299 2300 2301
}

/**
 * This function clears the request list as sequence numbers are passed.
 */
2302
void
C
Chris Wilson 已提交
2303
i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2304 2305 2306
{
	uint32_t seqno;

C
Chris Wilson 已提交
2307
	if (list_empty(&ring->request_list))
2308 2309
		return;

C
Chris Wilson 已提交
2310
	WARN_ON(i915_verify_lists(ring->dev));
2311

2312
	seqno = ring->get_seqno(ring, true);
2313

2314
	while (!list_empty(&ring->request_list)) {
2315 2316
		struct drm_i915_gem_request *request;

2317
		request = list_first_entry(&ring->request_list,
2318 2319 2320
					   struct drm_i915_gem_request,
					   list);

2321
		if (!i915_seqno_passed(seqno, request->seqno))
2322 2323
			break;

C
Chris Wilson 已提交
2324
		trace_i915_gem_request_retire(ring, request->seqno);
2325 2326 2327 2328 2329 2330
		/* We know the GPU must have read the request to have
		 * sent us the seqno + interrupt, so use the position
		 * of tail of the request to update the last known position
		 * of the GPU head.
		 */
		ring->last_retired_head = request->tail;
2331

2332
		i915_gem_free_request(request);
2333
	}
2334

2335 2336 2337 2338
	/* Move any buffers on the active list that are no longer referenced
	 * by the ringbuffer to the flushing/inactive lists as appropriate.
	 */
	while (!list_empty(&ring->active_list)) {
2339
		struct drm_i915_gem_object *obj;
2340

2341
		obj = list_first_entry(&ring->active_list,
2342 2343
				      struct drm_i915_gem_object,
				      ring_list);
2344

2345
		if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2346
			break;
2347

2348
		i915_gem_object_move_to_inactive(obj);
2349
	}
2350

C
Chris Wilson 已提交
2351 2352
	if (unlikely(ring->trace_irq_seqno &&
		     i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2353
		ring->irq_put(ring);
C
Chris Wilson 已提交
2354
		ring->trace_irq_seqno = 0;
2355
	}
2356

C
Chris Wilson 已提交
2357
	WARN_ON(i915_verify_lists(ring->dev));
2358 2359
}

2360 2361 2362 2363
void
i915_gem_retire_requests(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2364
	struct intel_ring_buffer *ring;
2365
	int i;
2366

2367 2368
	for_each_ring(ring, dev_priv, i)
		i915_gem_retire_requests_ring(ring);
2369 2370
}

2371
static void
2372 2373 2374 2375
i915_gem_retire_work_handler(struct work_struct *work)
{
	drm_i915_private_t *dev_priv;
	struct drm_device *dev;
2376
	struct intel_ring_buffer *ring;
2377 2378
	bool idle;
	int i;
2379 2380 2381 2382 2383

	dev_priv = container_of(work, drm_i915_private_t,
				mm.retire_work.work);
	dev = dev_priv->dev;

2384 2385
	/* Come back later if the device is busy... */
	if (!mutex_trylock(&dev->struct_mutex)) {
2386 2387
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
				   round_jiffies_up_relative(HZ));
2388 2389
		return;
	}
2390

2391
	i915_gem_retire_requests(dev);
2392

2393 2394
	/* Send a periodic flush down the ring so we don't hold onto GEM
	 * objects indefinitely.
2395
	 */
2396
	idle = true;
2397
	for_each_ring(ring, dev_priv, i) {
2398
		if (ring->gpu_caches_dirty)
2399
			i915_add_request(ring, NULL);
2400 2401

		idle &= list_empty(&ring->request_list);
2402 2403
	}

2404
	if (!dev_priv->mm.suspended && !idle)
2405 2406
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
				   round_jiffies_up_relative(HZ));
2407 2408
	if (idle)
		intel_mark_idle(dev);
2409

2410 2411 2412
	mutex_unlock(&dev->struct_mutex);
}

2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423
/**
 * Ensures that an object will eventually get non-busy by flushing any required
 * write domains, emitting any outstanding lazy request and retiring and
 * completed requests.
 */
static int
i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
{
	int ret;

	if (obj->active) {
2424
		ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2425 2426 2427 2428 2429 2430 2431 2432 2433
		if (ret)
			return ret;

		i915_gem_retire_requests_ring(obj->ring);
	}

	return 0;
}

2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458
/**
 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
 * @DRM_IOCTL_ARGS: standard ioctl arguments
 *
 * Returns 0 if successful, else an error is returned with the remaining time in
 * the timeout parameter.
 *  -ETIME: object is still busy after timeout
 *  -ERESTARTSYS: signal interrupted the wait
 *  -ENONENT: object doesn't exist
 * Also possible, but rare:
 *  -EAGAIN: GPU wedged
 *  -ENOMEM: damn
 *  -ENODEV: Internal IRQ fail
 *  -E?: The add request failed
 *
 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
 * non-zero timeout parameter the wait ioctl will wait for the given number of
 * nanoseconds on an object becoming unbusy. Since the wait itself does so
 * without holding struct_mutex the object may become re-busied before this
 * function completes. A similar but shorter * race condition exists in the busy
 * ioctl
 */
int
i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
2459
	drm_i915_private_t *dev_priv = dev->dev_private;
2460 2461 2462
	struct drm_i915_gem_wait *args = data;
	struct drm_i915_gem_object *obj;
	struct intel_ring_buffer *ring = NULL;
2463
	struct timespec timeout_stack, *timeout = NULL;
2464
	unsigned reset_counter;
2465 2466 2467
	u32 seqno = 0;
	int ret = 0;

2468 2469 2470 2471
	if (args->timeout_ns >= 0) {
		timeout_stack = ns_to_timespec(args->timeout_ns);
		timeout = &timeout_stack;
	}
2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482

	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
	if (&obj->base == NULL) {
		mutex_unlock(&dev->struct_mutex);
		return -ENOENT;
	}

2483 2484
	/* Need to make sure the object gets inactive eventually. */
	ret = i915_gem_object_flush_active(obj);
2485 2486 2487 2488
	if (ret)
		goto out;

	if (obj->active) {
2489
		seqno = obj->last_read_seqno;
2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504
		ring = obj->ring;
	}

	if (seqno == 0)
		 goto out;

	/* Do this after OLR check to make sure we make forward progress polling
	 * on this IOCTL with a 0 timeout (like busy ioctl)
	 */
	if (!args->timeout_ns) {
		ret = -ETIME;
		goto out;
	}

	drm_gem_object_unreference(&obj->base);
2505
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2506 2507
	mutex_unlock(&dev->struct_mutex);

2508
	ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
2509
	if (timeout)
2510
		args->timeout_ns = timespec_to_ns(timeout);
2511 2512 2513 2514 2515 2516 2517 2518
	return ret;

out:
	drm_gem_object_unreference(&obj->base);
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
/**
 * i915_gem_object_sync - sync an object to a ring.
 *
 * @obj: object which may be in use on another ring.
 * @to: ring we wish to use the object on. May be NULL.
 *
 * This code is meant to abstract object synchronization with the GPU.
 * Calling with NULL implies synchronizing the object with the CPU
 * rather than a particular GPU ring.
 *
 * Returns 0 if successful, else propagates up the lower layer error.
 */
2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541
int
i915_gem_object_sync(struct drm_i915_gem_object *obj,
		     struct intel_ring_buffer *to)
{
	struct intel_ring_buffer *from = obj->ring;
	u32 seqno;
	int ret, idx;

	if (from == NULL || to == from)
		return 0;

2542
	if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2543
		return i915_gem_object_wait_rendering(obj, false);
2544 2545 2546

	idx = intel_ring_sync_index(from, to);

2547
	seqno = obj->last_read_seqno;
2548 2549 2550
	if (seqno <= from->sync_seqno[idx])
		return 0;

2551 2552 2553
	ret = i915_gem_check_olr(obj->ring, seqno);
	if (ret)
		return ret;
2554

2555
	ret = to->sync_to(to, from, seqno);
2556
	if (!ret)
2557 2558 2559 2560 2561
		/* We use last_read_seqno because sync_to()
		 * might have just caused seqno wrap under
		 * the radar.
		 */
		from->sync_seqno[idx] = obj->last_read_seqno;
2562

2563
	return ret;
2564 2565
}

2566 2567 2568 2569 2570 2571 2572
static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
{
	u32 old_write_domain, old_read_domains;

	/* Force a pagefault for domain tracking on next user access */
	i915_gem_release_mmap(obj);

2573 2574 2575
	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		return;

2576 2577 2578
	/* Wait for any direct GTT access to complete */
	mb();

2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589
	old_read_domains = obj->base.read_domains;
	old_write_domain = obj->base.write_domain;

	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;

	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);
}

2590 2591 2592
/**
 * Unbinds an object from the GTT aperture.
 */
2593
int
2594
i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2595
{
2596
	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2597
	int ret;
2598

2599
	if (obj->gtt_space == NULL)
2600 2601
		return 0;

2602 2603
	if (obj->pin_count)
		return -EBUSY;
2604

2605 2606
	BUG_ON(obj->pages == NULL);

2607
	ret = i915_gem_object_finish_gpu(obj);
2608
	if (ret)
2609 2610 2611 2612 2613 2614
		return ret;
	/* Continue on if we fail due to EIO, the GPU is hung so we
	 * should be safe and we need to cleanup or else we might
	 * cause memory corruption through use-after-free.
	 */

2615
	i915_gem_object_finish_gtt(obj);
2616

2617
	/* release the fence reg _after_ flushing */
2618
	ret = i915_gem_object_put_fence(obj);
2619
	if (ret)
2620
		return ret;
2621

C
Chris Wilson 已提交
2622 2623
	trace_i915_gem_object_unbind(obj);

2624 2625
	if (obj->has_global_gtt_mapping)
		i915_gem_gtt_unbind_object(obj);
2626 2627 2628 2629
	if (obj->has_aliasing_ppgtt_mapping) {
		i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		obj->has_aliasing_ppgtt_mapping = 0;
	}
2630
	i915_gem_gtt_finish_object(obj);
B
Ben Widawsky 已提交
2631
	i915_gem_object_unpin_pages(obj);
2632

C
Chris Wilson 已提交
2633
	list_del(&obj->mm_list);
2634
	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2635
	/* Avoid an unnecessary call to unbind on rebind. */
2636
	obj->map_and_fenceable = true;
2637

2638 2639 2640
	drm_mm_put_block(obj->gtt_space);
	obj->gtt_space = NULL;
	obj->gtt_offset = 0;
2641

2642
	return 0;
2643 2644
}

2645
int i915_gpu_idle(struct drm_device *dev)
2646 2647
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2648
	struct intel_ring_buffer *ring;
2649
	int ret, i;
2650 2651

	/* Flush everything onto the inactive list. */
2652
	for_each_ring(ring, dev_priv, i) {
2653 2654 2655 2656
		ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
		if (ret)
			return ret;

2657
		ret = intel_ring_idle(ring);
2658 2659 2660
		if (ret)
			return ret;
	}
2661

2662
	return 0;
2663 2664
}

2665 2666
static void i965_write_fence_reg(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
2667 2668
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2669 2670
	int fence_reg;
	int fence_pitch_shift;
2671

2672 2673 2674 2675 2676 2677 2678 2679
	if (INTEL_INFO(dev)->gen >= 6) {
		fence_reg = FENCE_REG_SANDYBRIDGE_0;
		fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
	} else {
		fence_reg = FENCE_REG_965_0;
		fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
	}

2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693
	fence_reg += reg * 8;

	/* To w/a incoherency with non-atomic 64-bit register updates,
	 * we split the 64-bit update into two 32-bit writes. In order
	 * for a partial fence not to be evaluated between writes, we
	 * precede the update with write to turn off the fence register,
	 * and only enable the fence as the last step.
	 *
	 * For extra levels of paranoia, we make sure each step lands
	 * before applying the next step.
	 */
	I915_WRITE(fence_reg, 0);
	POSTING_READ(fence_reg);

2694 2695
	if (obj) {
		u32 size = obj->gtt_space->size;
2696
		uint64_t val;
2697

2698 2699 2700
		val = (uint64_t)((obj->gtt_offset + size - 4096) &
				 0xfffff000) << 32;
		val |= obj->gtt_offset & 0xfffff000;
2701
		val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
2702 2703 2704
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I965_FENCE_TILING_Y_SHIFT;
		val |= I965_FENCE_REG_VALID;
2705

2706 2707 2708 2709 2710 2711 2712 2713 2714
		I915_WRITE(fence_reg + 4, val >> 32);
		POSTING_READ(fence_reg + 4);

		I915_WRITE(fence_reg + 0, val);
		POSTING_READ(fence_reg);
	} else {
		I915_WRITE(fence_reg + 4, 0);
		POSTING_READ(fence_reg + 4);
	}
2715 2716
}

2717 2718
static void i915_write_fence_reg(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
2719 2720
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2721
	u32 val;
2722

2723 2724 2725 2726
	if (obj) {
		u32 size = obj->gtt_space->size;
		int pitch_val;
		int tile_width;
2727

2728 2729 2730 2731 2732
		WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
		     (size & -size) != size ||
		     (obj->gtt_offset & (size - 1)),
		     "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		     obj->gtt_offset, obj->map_and_fenceable, size);
2733

2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758
		if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
			tile_width = 128;
		else
			tile_width = 512;

		/* Note: pitch better be a power of two tile widths */
		pitch_val = obj->stride / tile_width;
		pitch_val = ffs(pitch_val) - 1;

		val = obj->gtt_offset;
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
		val |= I915_FENCE_SIZE_BITS(size);
		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
		val |= I830_FENCE_REG_VALID;
	} else
		val = 0;

	if (reg < 8)
		reg = FENCE_REG_830_0 + reg * 4;
	else
		reg = FENCE_REG_945_8 + (reg - 8) * 4;

	I915_WRITE(reg, val);
	POSTING_READ(reg);
2759 2760
}

2761 2762
static void i830_write_fence_reg(struct drm_device *dev, int reg,
				struct drm_i915_gem_object *obj)
2763 2764 2765 2766
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	uint32_t val;

2767 2768 2769
	if (obj) {
		u32 size = obj->gtt_space->size;
		uint32_t pitch_val;
2770

2771 2772 2773 2774 2775
		WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
		     (size & -size) != size ||
		     (obj->gtt_offset & (size - 1)),
		     "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
		     obj->gtt_offset, size);
2776

2777 2778
		pitch_val = obj->stride / 128;
		pitch_val = ffs(pitch_val) - 1;
2779

2780 2781 2782 2783 2784 2785 2786 2787
		val = obj->gtt_offset;
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
		val |= I830_FENCE_SIZE_BITS(size);
		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
		val |= I830_FENCE_REG_VALID;
	} else
		val = 0;
2788

2789 2790 2791 2792
	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
	POSTING_READ(FENCE_REG_830_0 + reg * 4);
}

2793 2794 2795 2796 2797
inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
{
	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
}

2798 2799 2800
static void i915_gem_write_fence(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
{
2801 2802 2803 2804 2805 2806 2807 2808
	struct drm_i915_private *dev_priv = dev->dev_private;

	/* Ensure that all CPU reads are completed before installing a fence
	 * and all writes before removing the fence.
	 */
	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		mb();

2809 2810
	switch (INTEL_INFO(dev)->gen) {
	case 7:
2811
	case 6:
2812 2813 2814 2815
	case 5:
	case 4: i965_write_fence_reg(dev, reg, obj); break;
	case 3: i915_write_fence_reg(dev, reg, obj); break;
	case 2: i830_write_fence_reg(dev, reg, obj); break;
2816
	default: BUG();
2817
	}
2818 2819 2820 2821 2822 2823

	/* And similarly be paranoid that no direct access to this region
	 * is reordered to before the fence is installed.
	 */
	if (i915_gem_object_needs_mb(obj))
		mb();
2824 2825
}

2826 2827 2828 2829 2830 2831 2832 2833 2834 2835
static inline int fence_number(struct drm_i915_private *dev_priv,
			       struct drm_i915_fence_reg *fence)
{
	return fence - dev_priv->fence_regs;
}

static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
					 struct drm_i915_fence_reg *fence,
					 bool enable)
{
2836
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2837 2838 2839
	int reg = fence_number(dev_priv, fence);

	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2840 2841

	if (enable) {
2842
		obj->fence_reg = reg;
2843 2844 2845 2846 2847 2848 2849 2850 2851
		fence->obj = obj;
		list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
	} else {
		obj->fence_reg = I915_FENCE_REG_NONE;
		fence->obj = NULL;
		list_del_init(&fence->lru_list);
	}
}

2852
static int
2853
i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
2854
{
2855
	if (obj->last_fenced_seqno) {
2856
		int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2857 2858
		if (ret)
			return ret;
2859 2860 2861 2862

		obj->last_fenced_seqno = 0;
	}

2863
	obj->fenced_gpu_access = false;
2864 2865 2866 2867 2868 2869
	return 0;
}

int
i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
{
2870
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2871
	struct drm_i915_fence_reg *fence;
2872 2873
	int ret;

2874
	ret = i915_gem_object_wait_fence(obj);
2875 2876 2877
	if (ret)
		return ret;

2878 2879
	if (obj->fence_reg == I915_FENCE_REG_NONE)
		return 0;
2880

2881 2882
	fence = &dev_priv->fence_regs[obj->fence_reg];

2883
	i915_gem_object_fence_lost(obj);
2884
	i915_gem_object_update_fence(obj, fence, false);
2885 2886 2887 2888 2889

	return 0;
}

static struct drm_i915_fence_reg *
C
Chris Wilson 已提交
2890
i915_find_fence_reg(struct drm_device *dev)
2891 2892
{
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
2893
	struct drm_i915_fence_reg *reg, *avail;
2894
	int i;
2895 2896

	/* First try to find a free reg */
2897
	avail = NULL;
2898 2899 2900
	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		reg = &dev_priv->fence_regs[i];
		if (!reg->obj)
2901
			return reg;
2902

2903
		if (!reg->pin_count)
2904
			avail = reg;
2905 2906
	}

2907 2908
	if (avail == NULL)
		return NULL;
2909 2910

	/* None available, try to steal one or wait for a user to finish */
2911
	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2912
		if (reg->pin_count)
2913 2914
			continue;

C
Chris Wilson 已提交
2915
		return reg;
2916 2917
	}

C
Chris Wilson 已提交
2918
	return NULL;
2919 2920
}

2921
/**
2922
 * i915_gem_object_get_fence - set up fencing for an object
2923 2924 2925 2926 2927 2928 2929 2930 2931
 * @obj: object to map through a fence reg
 *
 * When mapping objects through the GTT, userspace wants to be able to write
 * to them without having to worry about swizzling if the object is tiled.
 * This function walks the fence regs looking for a free one for @obj,
 * stealing one if it can't find any.
 *
 * It then sets up the reg based on the object's properties: address, pitch
 * and tiling format.
2932 2933
 *
 * For an untiled surface, this removes any existing fence.
2934
 */
2935
int
2936
i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
2937
{
2938
	struct drm_device *dev = obj->base.dev;
J
Jesse Barnes 已提交
2939
	struct drm_i915_private *dev_priv = dev->dev_private;
2940
	bool enable = obj->tiling_mode != I915_TILING_NONE;
2941
	struct drm_i915_fence_reg *reg;
2942
	int ret;
2943

2944 2945 2946
	/* Have we updated the tiling parameters upon the object and so
	 * will need to serialise the write to the associated fence register?
	 */
2947
	if (obj->fence_dirty) {
2948
		ret = i915_gem_object_wait_fence(obj);
2949 2950 2951
		if (ret)
			return ret;
	}
2952

2953
	/* Just update our place in the LRU if our fence is getting reused. */
2954 2955
	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		reg = &dev_priv->fence_regs[obj->fence_reg];
2956
		if (!obj->fence_dirty) {
2957 2958 2959 2960 2961 2962 2963 2964
			list_move_tail(&reg->lru_list,
				       &dev_priv->mm.fence_list);
			return 0;
		}
	} else if (enable) {
		reg = i915_find_fence_reg(dev);
		if (reg == NULL)
			return -EDEADLK;
2965

2966 2967 2968
		if (reg->obj) {
			struct drm_i915_gem_object *old = reg->obj;

2969
			ret = i915_gem_object_wait_fence(old);
2970 2971 2972
			if (ret)
				return ret;

2973
			i915_gem_object_fence_lost(old);
2974
		}
2975
	} else
2976 2977
		return 0;

2978
	i915_gem_object_update_fence(obj, reg, enable);
2979
	obj->fence_dirty = false;
2980

2981
	return 0;
2982 2983
}

2984 2985 2986 2987 2988 2989 2990 2991
static bool i915_gem_valid_gtt_space(struct drm_device *dev,
				     struct drm_mm_node *gtt_space,
				     unsigned long cache_level)
{
	struct drm_mm_node *other;

	/* On non-LLC machines we have to be careful when putting differing
	 * types of snoopable memory together to avoid the prefetcher
2992
	 * crossing memory domains and dying.
2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020
	 */
	if (HAS_LLC(dev))
		return true;

	if (gtt_space == NULL)
		return true;

	if (list_empty(&gtt_space->node_list))
		return true;

	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
	if (other->allocated && !other->hole_follows && other->color != cache_level)
		return false;

	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		return false;

	return true;
}

static void i915_gem_verify_gtt(struct drm_device *dev)
{
#if WATCH_GTT
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_gem_object *obj;
	int err = 0;

3021
	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053
		if (obj->gtt_space == NULL) {
			printk(KERN_ERR "object found on GTT list with no space reserved\n");
			err++;
			continue;
		}

		if (obj->cache_level != obj->gtt_space->color) {
			printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
			       obj->gtt_space->start,
			       obj->gtt_space->start + obj->gtt_space->size,
			       obj->cache_level,
			       obj->gtt_space->color);
			err++;
			continue;
		}

		if (!i915_gem_valid_gtt_space(dev,
					      obj->gtt_space,
					      obj->cache_level)) {
			printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
			       obj->gtt_space->start,
			       obj->gtt_space->start + obj->gtt_space->size,
			       obj->cache_level);
			err++;
			continue;
		}
	}

	WARN_ON(err);
#endif
}

3054 3055 3056 3057
/**
 * Finds free space in the GTT aperture and binds the object there.
 */
static int
3058
i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
3059
			    unsigned alignment,
3060 3061
			    bool map_and_fenceable,
			    bool nonblocking)
3062
{
3063
	struct drm_device *dev = obj->base.dev;
3064
	drm_i915_private_t *dev_priv = dev->dev_private;
3065
	struct drm_mm_node *node;
3066
	u32 size, fence_size, fence_alignment, unfenced_alignment;
3067
	bool mappable, fenceable;
3068 3069
	size_t gtt_max = map_and_fenceable ?
		dev_priv->gtt.mappable_end : dev_priv->gtt.total;
3070
	int ret;
3071

3072 3073 3074 3075 3076
	fence_size = i915_gem_get_gtt_size(dev,
					   obj->base.size,
					   obj->tiling_mode);
	fence_alignment = i915_gem_get_gtt_alignment(dev,
						     obj->base.size,
3077
						     obj->tiling_mode, true);
3078
	unfenced_alignment =
3079
		i915_gem_get_gtt_alignment(dev,
3080
						    obj->base.size,
3081
						    obj->tiling_mode, false);
3082

3083
	if (alignment == 0)
3084 3085
		alignment = map_and_fenceable ? fence_alignment :
						unfenced_alignment;
3086
	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
3087 3088 3089 3090
		DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		return -EINVAL;
	}

3091
	size = map_and_fenceable ? fence_size : obj->base.size;
3092

3093 3094 3095
	/* If the object is bigger than the entire aperture, reject it early
	 * before evicting everything in a vain attempt to find space.
	 */
3096
	if (obj->base.size > gtt_max) {
3097
		DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
3098 3099
			  obj->base.size,
			  map_and_fenceable ? "mappable" : "total",
3100
			  gtt_max);
3101 3102 3103
		return -E2BIG;
	}

3104
	ret = i915_gem_object_get_pages(obj);
C
Chris Wilson 已提交
3105 3106 3107
	if (ret)
		return ret;

3108 3109
	i915_gem_object_pin_pages(obj);

3110 3111 3112 3113 3114 3115
	node = kzalloc(sizeof(*node), GFP_KERNEL);
	if (node == NULL) {
		i915_gem_object_unpin_pages(obj);
		return -ENOMEM;
	}

3116 3117 3118 3119
search_free:
	ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
						  size, alignment,
						  obj->cache_level, 0, gtt_max);
3120
	if (ret) {
3121
		ret = i915_gem_evict_something(dev, size, alignment,
3122
					       obj->cache_level,
3123 3124
					       map_and_fenceable,
					       nonblocking);
3125 3126
		if (ret == 0)
			goto search_free;
3127

3128 3129 3130
		i915_gem_object_unpin_pages(obj);
		kfree(node);
		return ret;
3131
	}
3132
	if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
3133
		i915_gem_object_unpin_pages(obj);
3134
		drm_mm_put_block(node);
3135
		return -EINVAL;
3136 3137
	}

3138
	ret = i915_gem_gtt_prepare_object(obj);
3139
	if (ret) {
3140
		i915_gem_object_unpin_pages(obj);
3141
		drm_mm_put_block(node);
C
Chris Wilson 已提交
3142
		return ret;
3143 3144
	}

3145
	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3146
	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3147

3148 3149
	obj->gtt_space = node;
	obj->gtt_offset = node->start;
C
Chris Wilson 已提交
3150

3151
	fenceable =
3152 3153
		node->size == fence_size &&
		(node->start & (fence_alignment - 1)) == 0;
3154

3155
	mappable =
B
Ben Widawsky 已提交
3156
		obj->gtt_offset + obj->base.size <= dev_priv->gtt.mappable_end;
3157

3158
	obj->map_and_fenceable = mappable && fenceable;
3159

C
Chris Wilson 已提交
3160
	trace_i915_gem_object_bind(obj, map_and_fenceable);
3161
	i915_gem_verify_gtt(dev);
3162 3163 3164 3165
	return 0;
}

void
3166
i915_gem_clflush_object(struct drm_i915_gem_object *obj)
3167 3168 3169 3170 3171
{
	/* If we don't have a page list set up, then we're not pinned
	 * to GPU, and we can ignore the cache flush because it'll happen
	 * again at bind time.
	 */
3172
	if (obj->pages == NULL)
3173 3174
		return;

3175 3176 3177 3178 3179 3180 3181
	/*
	 * Stolen memory is always coherent with the GPU as it is explicitly
	 * marked as wc by the system, or the system is cache-coherent.
	 */
	if (obj->stolen)
		return;

3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192
	/* If the GPU is snooping the contents of the CPU cache,
	 * we do not need to manually clear the CPU cache lines.  However,
	 * the caches are only snooped when the render cache is
	 * flushed/invalidated.  As we always have to emit invalidations
	 * and flushes when moving into and out of the RENDER domain, correct
	 * snooping behaviour occurs naturally as the result of our domain
	 * tracking.
	 */
	if (obj->cache_level != I915_CACHE_NONE)
		return;

C
Chris Wilson 已提交
3193
	trace_i915_gem_object_clflush(obj);
3194

3195
	drm_clflush_sg(obj->pages);
3196 3197 3198 3199
}

/** Flushes the GTT write domain for the object if it's dirty. */
static void
3200
i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3201
{
C
Chris Wilson 已提交
3202 3203
	uint32_t old_write_domain;

3204
	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3205 3206
		return;

3207
	/* No actual flushing is required for the GTT write domain.  Writes
3208 3209
	 * to it immediately go to main memory as far as we know, so there's
	 * no chipset flush.  It also doesn't land in render cache.
3210 3211 3212 3213
	 *
	 * However, we do have to enforce the order so that all writes through
	 * the GTT land before any writes to the device, such as updates to
	 * the GATT itself.
3214
	 */
3215 3216
	wmb();

3217 3218
	old_write_domain = obj->base.write_domain;
	obj->base.write_domain = 0;
C
Chris Wilson 已提交
3219 3220

	trace_i915_gem_object_change_domain(obj,
3221
					    obj->base.read_domains,
C
Chris Wilson 已提交
3222
					    old_write_domain);
3223 3224 3225 3226
}

/** Flushes the CPU write domain for the object if it's dirty. */
static void
3227
i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3228
{
C
Chris Wilson 已提交
3229
	uint32_t old_write_domain;
3230

3231
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3232 3233 3234
		return;

	i915_gem_clflush_object(obj);
3235
	i915_gem_chipset_flush(obj->base.dev);
3236 3237
	old_write_domain = obj->base.write_domain;
	obj->base.write_domain = 0;
C
Chris Wilson 已提交
3238 3239

	trace_i915_gem_object_change_domain(obj,
3240
					    obj->base.read_domains,
C
Chris Wilson 已提交
3241
					    old_write_domain);
3242 3243
}

3244 3245 3246 3247 3248 3249
/**
 * Moves a single object to the GTT read, and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
J
Jesse Barnes 已提交
3250
int
3251
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3252
{
3253
	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
C
Chris Wilson 已提交
3254
	uint32_t old_write_domain, old_read_domains;
3255
	int ret;
3256

3257
	/* Not valid to be called on unbound objects. */
3258
	if (obj->gtt_space == NULL)
3259 3260
		return -EINVAL;

3261 3262 3263
	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		return 0;

3264
	ret = i915_gem_object_wait_rendering(obj, !write);
3265 3266 3267
	if (ret)
		return ret;

3268
	i915_gem_object_flush_cpu_write_domain(obj);
C
Chris Wilson 已提交
3269

3270 3271 3272 3273 3274 3275 3276
	/* Serialise direct access to this object with the barriers for
	 * coherent writes from the GPU, by effectively invalidating the
	 * GTT domain upon first access.
	 */
	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		mb();

3277 3278
	old_write_domain = obj->base.write_domain;
	old_read_domains = obj->base.read_domains;
C
Chris Wilson 已提交
3279

3280 3281 3282
	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3283 3284
	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3285
	if (write) {
3286 3287 3288
		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		obj->dirty = 1;
3289 3290
	}

C
Chris Wilson 已提交
3291 3292 3293 3294
	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);

3295 3296 3297 3298
	/* And bump the LRU for this access */
	if (i915_gem_object_is_inactive(obj))
		list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);

3299 3300 3301
	return 0;
}

3302 3303 3304
int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
				    enum i915_cache_level cache_level)
{
3305 3306
	struct drm_device *dev = obj->base.dev;
	drm_i915_private_t *dev_priv = dev->dev_private;
3307 3308 3309 3310 3311 3312 3313 3314 3315 3316
	int ret;

	if (obj->cache_level == cache_level)
		return 0;

	if (obj->pin_count) {
		DRM_DEBUG("can not change the cache level of pinned objects\n");
		return -EBUSY;
	}

3317 3318 3319 3320 3321 3322
	if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
		ret = i915_gem_object_unbind(obj);
		if (ret)
			return ret;
	}

3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333
	if (obj->gtt_space) {
		ret = i915_gem_object_finish_gpu(obj);
		if (ret)
			return ret;

		i915_gem_object_finish_gtt(obj);

		/* Before SandyBridge, you could not use tiling or fence
		 * registers with snooped memory, so relinquish any fences
		 * currently pointing to our region in the aperture.
		 */
3334
		if (INTEL_INFO(dev)->gen < 6) {
3335 3336 3337 3338 3339
			ret = i915_gem_object_put_fence(obj);
			if (ret)
				return ret;
		}

3340 3341
		if (obj->has_global_gtt_mapping)
			i915_gem_gtt_bind_object(obj, cache_level);
3342 3343 3344
		if (obj->has_aliasing_ppgtt_mapping)
			i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
					       obj, cache_level);
3345 3346

		obj->gtt_space->color = cache_level;
3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372
	}

	if (cache_level == I915_CACHE_NONE) {
		u32 old_read_domains, old_write_domain;

		/* If we're coming from LLC cached, then we haven't
		 * actually been tracking whether the data is in the
		 * CPU cache or not, since we only allow one bit set
		 * in obj->write_domain and have been skipping the clflushes.
		 * Just set it to the CPU cache for now.
		 */
		WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);

		old_read_domains = obj->base.read_domains;
		old_write_domain = obj->base.write_domain;

		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		obj->base.write_domain = I915_GEM_DOMAIN_CPU;

		trace_i915_gem_object_change_domain(obj,
						    old_read_domains,
						    old_write_domain);
	}

	obj->cache_level = cache_level;
3373
	i915_gem_verify_gtt(dev);
3374 3375 3376
	return 0;
}

B
Ben Widawsky 已提交
3377 3378
int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3379
{
B
Ben Widawsky 已提交
3380
	struct drm_i915_gem_caching *args = data;
3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393
	struct drm_i915_gem_object *obj;
	int ret;

	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
	if (&obj->base == NULL) {
		ret = -ENOENT;
		goto unlock;
	}

B
Ben Widawsky 已提交
3394
	args->caching = obj->cache_level != I915_CACHE_NONE;
3395 3396 3397 3398 3399 3400 3401

	drm_gem_object_unreference(&obj->base);
unlock:
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

B
Ben Widawsky 已提交
3402 3403
int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3404
{
B
Ben Widawsky 已提交
3405
	struct drm_i915_gem_caching *args = data;
3406 3407 3408 3409
	struct drm_i915_gem_object *obj;
	enum i915_cache_level level;
	int ret;

B
Ben Widawsky 已提交
3410 3411
	switch (args->caching) {
	case I915_CACHING_NONE:
3412 3413
		level = I915_CACHE_NONE;
		break;
B
Ben Widawsky 已提交
3414
	case I915_CACHING_CACHED:
3415 3416 3417 3418 3419 3420
		level = I915_CACHE_LLC;
		break;
	default:
		return -EINVAL;
	}

B
Ben Widawsky 已提交
3421 3422 3423 3424
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
	if (&obj->base == NULL) {
		ret = -ENOENT;
		goto unlock;
	}

	ret = i915_gem_object_set_cache_level(obj, level);

	drm_gem_object_unreference(&obj->base);
unlock:
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

3439
/*
3440 3441 3442
 * Prepare buffer for display plane (scanout, cursors, etc).
 * Can be called from an uninterruptible phase (modesetting) and allows
 * any flushes to be pipelined (for pageflips).
3443 3444
 */
int
3445 3446
i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
				     u32 alignment,
3447
				     struct intel_ring_buffer *pipelined)
3448
{
3449
	u32 old_read_domains, old_write_domain;
3450 3451
	int ret;

3452
	if (pipelined != obj->ring) {
3453 3454
		ret = i915_gem_object_sync(obj, pipelined);
		if (ret)
3455 3456 3457
			return ret;
	}

3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470
	/* The display engine is not coherent with the LLC cache on gen6.  As
	 * a result, we make sure that the pinning that is about to occur is
	 * done with uncached PTEs. This is lowest common denominator for all
	 * chipsets.
	 *
	 * However for gen6+, we could do better by using the GFDT bit instead
	 * of uncaching, which would allow us to flush all the LLC-cached data
	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
	 */
	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
	if (ret)
		return ret;

3471 3472 3473 3474
	/* As the user may map the buffer once pinned in the display plane
	 * (e.g. libkms for the bootup splash), we have to ensure that we
	 * always use map_and_fenceable for all scanout buffers.
	 */
3475
	ret = i915_gem_object_pin(obj, alignment, true, false);
3476 3477 3478
	if (ret)
		return ret;

3479 3480
	i915_gem_object_flush_cpu_write_domain(obj);

3481
	old_write_domain = obj->base.write_domain;
3482
	old_read_domains = obj->base.read_domains;
3483 3484 3485 3486

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3487
	obj->base.write_domain = 0;
3488
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3489 3490 3491

	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
3492
					    old_write_domain);
3493 3494 3495 3496

	return 0;
}

3497
int
3498
i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3499
{
3500 3501
	int ret;

3502
	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3503 3504
		return 0;

3505
	ret = i915_gem_object_wait_rendering(obj, false);
3506 3507 3508
	if (ret)
		return ret;

3509 3510
	/* Ensure that we invalidate the GPU's caches and TLBs. */
	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3511
	return 0;
3512 3513
}

3514 3515 3516 3517 3518 3519
/**
 * Moves a single object to the CPU read, and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
3520
int
3521
i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3522
{
C
Chris Wilson 已提交
3523
	uint32_t old_write_domain, old_read_domains;
3524 3525
	int ret;

3526 3527 3528
	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		return 0;

3529
	ret = i915_gem_object_wait_rendering(obj, !write);
3530 3531 3532
	if (ret)
		return ret;

3533
	i915_gem_object_flush_gtt_write_domain(obj);
3534

3535 3536
	old_write_domain = obj->base.write_domain;
	old_read_domains = obj->base.read_domains;
C
Chris Wilson 已提交
3537

3538
	/* Flush the CPU cache if it's still invalid. */
3539
	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3540 3541
		i915_gem_clflush_object(obj);

3542
		obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3543 3544 3545 3546 3547
	}

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3548
	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3549 3550 3551 3552 3553

	/* If we're writing through the CPU, then the GPU read domains will
	 * need to be invalidated at next use.
	 */
	if (write) {
3554 3555
		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3556
	}
3557

C
Chris Wilson 已提交
3558 3559 3560 3561
	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);

3562 3563 3564
	return 0;
}

3565 3566 3567
/* Throttle our rendering by waiting until the ring has completed our requests
 * emitted over 20 msec ago.
 *
3568 3569 3570 3571
 * Note that if we were to use the current jiffies each time around the loop,
 * we wouldn't escape the function with any frames outstanding if the time to
 * render a frame was over 20ms.
 *
3572 3573 3574
 * This should get us reasonable parallelism between CPU and GPU but also
 * relatively low latency when blocking on a particular request to finish.
 */
3575
static int
3576
i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3577
{
3578 3579
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_file_private *file_priv = file->driver_priv;
3580
	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3581 3582
	struct drm_i915_gem_request *request;
	struct intel_ring_buffer *ring = NULL;
3583
	unsigned reset_counter;
3584 3585
	u32 seqno = 0;
	int ret;
3586

3587 3588 3589 3590 3591 3592 3593
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
	if (ret)
		return ret;

	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
	if (ret)
		return ret;
3594

3595
	spin_lock(&file_priv->mm.lock);
3596
	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3597 3598
		if (time_after_eq(request->emitted_jiffies, recent_enough))
			break;
3599

3600 3601
		ring = request->ring;
		seqno = request->seqno;
3602
	}
3603
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3604
	spin_unlock(&file_priv->mm.lock);
3605

3606 3607
	if (seqno == 0)
		return 0;
3608

3609
	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3610 3611
	if (ret == 0)
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3612 3613 3614 3615

	return ret;
}

3616
int
3617 3618
i915_gem_object_pin(struct drm_i915_gem_object *obj,
		    uint32_t alignment,
3619 3620
		    bool map_and_fenceable,
		    bool nonblocking)
3621 3622 3623
{
	int ret;

3624 3625
	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		return -EBUSY;
3626

3627 3628 3629 3630
	if (obj->gtt_space != NULL) {
		if ((alignment && obj->gtt_offset & (alignment - 1)) ||
		    (map_and_fenceable && !obj->map_and_fenceable)) {
			WARN(obj->pin_count,
3631
			     "bo is already pinned with incorrect alignment:"
3632 3633
			     " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
			     " obj->map_and_fenceable=%d\n",
3634
			     obj->gtt_offset, alignment,
3635
			     map_and_fenceable,
3636
			     obj->map_and_fenceable);
3637 3638 3639 3640 3641 3642
			ret = i915_gem_object_unbind(obj);
			if (ret)
				return ret;
		}
	}

3643
	if (obj->gtt_space == NULL) {
3644 3645
		struct drm_i915_private *dev_priv = obj->base.dev->dev_private;

3646
		ret = i915_gem_object_bind_to_gtt(obj, alignment,
3647 3648
						  map_and_fenceable,
						  nonblocking);
3649
		if (ret)
3650
			return ret;
3651 3652 3653

		if (!dev_priv->mm.aliasing_ppgtt)
			i915_gem_gtt_bind_object(obj, obj->cache_level);
3654
	}
J
Jesse Barnes 已提交
3655

3656 3657 3658
	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		i915_gem_gtt_bind_object(obj, obj->cache_level);

3659
	obj->pin_count++;
3660
	obj->pin_mappable |= map_and_fenceable;
3661 3662 3663 3664 3665

	return 0;
}

void
3666
i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3667
{
3668 3669
	BUG_ON(obj->pin_count == 0);
	BUG_ON(obj->gtt_space == NULL);
3670

3671
	if (--obj->pin_count == 0)
3672
		obj->pin_mappable = false;
3673 3674 3675 3676
}

int
i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3677
		   struct drm_file *file)
3678 3679
{
	struct drm_i915_gem_pin *args = data;
3680
	struct drm_i915_gem_object *obj;
3681 3682
	int ret;

3683 3684 3685
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;
3686

3687
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3688
	if (&obj->base == NULL) {
3689 3690
		ret = -ENOENT;
		goto unlock;
3691 3692
	}

3693
	if (obj->madv != I915_MADV_WILLNEED) {
C
Chris Wilson 已提交
3694
		DRM_ERROR("Attempting to pin a purgeable buffer\n");
3695 3696
		ret = -EINVAL;
		goto out;
3697 3698
	}

3699
	if (obj->pin_filp != NULL && obj->pin_filp != file) {
J
Jesse Barnes 已提交
3700 3701
		DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
			  args->handle);
3702 3703
		ret = -EINVAL;
		goto out;
J
Jesse Barnes 已提交
3704 3705
	}

3706
	if (obj->user_pin_count == 0) {
3707
		ret = i915_gem_object_pin(obj, args->alignment, true, false);
3708 3709
		if (ret)
			goto out;
3710 3711
	}

3712 3713 3714
	obj->user_pin_count++;
	obj->pin_filp = file;

3715 3716 3717
	/* XXX - flush the CPU caches for pinned objects
	 * as the X server doesn't manage domains yet
	 */
3718
	i915_gem_object_flush_cpu_write_domain(obj);
3719
	args->offset = obj->gtt_offset;
3720
out:
3721
	drm_gem_object_unreference(&obj->base);
3722
unlock:
3723
	mutex_unlock(&dev->struct_mutex);
3724
	return ret;
3725 3726 3727 3728
}

int
i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3729
		     struct drm_file *file)
3730 3731
{
	struct drm_i915_gem_pin *args = data;
3732
	struct drm_i915_gem_object *obj;
3733
	int ret;
3734

3735 3736 3737
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;
3738

3739
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3740
	if (&obj->base == NULL) {
3741 3742
		ret = -ENOENT;
		goto unlock;
3743
	}
3744

3745
	if (obj->pin_filp != file) {
J
Jesse Barnes 已提交
3746 3747
		DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
			  args->handle);
3748 3749
		ret = -EINVAL;
		goto out;
J
Jesse Barnes 已提交
3750
	}
3751 3752 3753
	obj->user_pin_count--;
	if (obj->user_pin_count == 0) {
		obj->pin_filp = NULL;
J
Jesse Barnes 已提交
3754 3755
		i915_gem_object_unpin(obj);
	}
3756

3757
out:
3758
	drm_gem_object_unreference(&obj->base);
3759
unlock:
3760
	mutex_unlock(&dev->struct_mutex);
3761
	return ret;
3762 3763 3764 3765
}

int
i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3766
		    struct drm_file *file)
3767 3768
{
	struct drm_i915_gem_busy *args = data;
3769
	struct drm_i915_gem_object *obj;
3770 3771
	int ret;

3772
	ret = i915_mutex_lock_interruptible(dev);
3773
	if (ret)
3774
		return ret;
3775

3776
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3777
	if (&obj->base == NULL) {
3778 3779
		ret = -ENOENT;
		goto unlock;
3780
	}
3781

3782 3783 3784 3785
	/* Count all active objects as busy, even if they are currently not used
	 * by the gpu. Users of this interface expect objects to eventually
	 * become non-busy without any further actions, therefore emit any
	 * necessary flushes here.
3786
	 */
3787
	ret = i915_gem_object_flush_active(obj);
3788

3789
	args->busy = obj->active;
3790 3791 3792 3793
	if (obj->ring) {
		BUILD_BUG_ON(I915_NUM_RINGS > 16);
		args->busy |= intel_ring_flag(obj->ring) << 16;
	}
3794

3795
	drm_gem_object_unreference(&obj->base);
3796
unlock:
3797
	mutex_unlock(&dev->struct_mutex);
3798
	return ret;
3799 3800 3801 3802 3803 3804
}

int
i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file_priv)
{
3805
	return i915_gem_ring_throttle(dev, file_priv);
3806 3807
}

3808 3809 3810 3811 3812
int
i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
	struct drm_i915_gem_madvise *args = data;
3813
	struct drm_i915_gem_object *obj;
3814
	int ret;
3815 3816 3817 3818 3819 3820 3821 3822 3823

	switch (args->madv) {
	case I915_MADV_DONTNEED:
	case I915_MADV_WILLNEED:
	    break;
	default:
	    return -EINVAL;
	}

3824 3825 3826 3827
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

3828
	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3829
	if (&obj->base == NULL) {
3830 3831
		ret = -ENOENT;
		goto unlock;
3832 3833
	}

3834
	if (obj->pin_count) {
3835 3836
		ret = -EINVAL;
		goto out;
3837 3838
	}

3839 3840
	if (obj->madv != __I915_MADV_PURGED)
		obj->madv = args->madv;
3841

C
Chris Wilson 已提交
3842 3843
	/* if the object is no longer attached, discard its backing storage */
	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
3844 3845
		i915_gem_object_truncate(obj);

3846
	args->retained = obj->madv != __I915_MADV_PURGED;
C
Chris Wilson 已提交
3847

3848
out:
3849
	drm_gem_object_unreference(&obj->base);
3850
unlock:
3851
	mutex_unlock(&dev->struct_mutex);
3852
	return ret;
3853 3854
}

3855 3856
void i915_gem_object_init(struct drm_i915_gem_object *obj,
			  const struct drm_i915_gem_object_ops *ops)
3857 3858
{
	INIT_LIST_HEAD(&obj->mm_list);
3859
	INIT_LIST_HEAD(&obj->global_list);
3860 3861 3862
	INIT_LIST_HEAD(&obj->ring_list);
	INIT_LIST_HEAD(&obj->exec_list);

3863 3864
	obj->ops = ops;

3865 3866 3867 3868 3869 3870 3871 3872
	obj->fence_reg = I915_FENCE_REG_NONE;
	obj->madv = I915_MADV_WILLNEED;
	/* Avoid an unnecessary call to unbind on the first bind. */
	obj->map_and_fenceable = true;

	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
}

3873 3874 3875 3876 3877
static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
	.get_pages = i915_gem_object_get_pages_gtt,
	.put_pages = i915_gem_object_put_pages_gtt,
};

3878 3879
struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
						  size_t size)
3880
{
3881
	struct drm_i915_gem_object *obj;
3882
	struct address_space *mapping;
D
Daniel Vetter 已提交
3883
	gfp_t mask;
3884

3885
	obj = i915_gem_object_alloc(dev);
3886 3887
	if (obj == NULL)
		return NULL;
3888

3889
	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3890
		i915_gem_object_free(obj);
3891 3892
		return NULL;
	}
3893

3894 3895 3896 3897 3898 3899 3900
	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
	if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
		/* 965gm cannot relocate objects above 4GiB. */
		mask &= ~__GFP_HIGHMEM;
		mask |= __GFP_DMA32;
	}

A
Al Viro 已提交
3901
	mapping = file_inode(obj->base.filp)->i_mapping;
3902
	mapping_set_gfp_mask(mapping, mask);
3903

3904
	i915_gem_object_init(obj, &i915_gem_object_ops);
3905

3906 3907
	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3908

3909 3910
	if (HAS_LLC(dev)) {
		/* On some devices, we can have the GPU use the LLC (the CPU
3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925
		 * cache) for about a 10% performance improvement
		 * compared to uncached.  Graphics requests other than
		 * display scanout are coherent with the CPU in
		 * accessing this cache.  This means in this mode we
		 * don't need to clflush on the CPU side, and on the
		 * GPU side we only need to flush internal caches to
		 * get data visible to the CPU.
		 *
		 * However, we maintain the display planes as UC, and so
		 * need to rebind when first used as such.
		 */
		obj->cache_level = I915_CACHE_LLC;
	} else
		obj->cache_level = I915_CACHE_NONE;

3926
	return obj;
3927 3928 3929 3930 3931
}

int i915_gem_init_object(struct drm_gem_object *obj)
{
	BUG();
3932

3933 3934 3935
	return 0;
}

3936
void i915_gem_free_object(struct drm_gem_object *gem_obj)
3937
{
3938
	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3939
	struct drm_device *dev = obj->base.dev;
3940
	drm_i915_private_t *dev_priv = dev->dev_private;
3941

3942 3943
	trace_i915_gem_object_destroy(obj);

3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958
	if (obj->phys_obj)
		i915_gem_detach_phys_object(dev, obj);

	obj->pin_count = 0;
	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		bool was_interruptible;

		was_interruptible = dev_priv->mm.interruptible;
		dev_priv->mm.interruptible = false;

		WARN_ON(i915_gem_object_unbind(obj));

		dev_priv->mm.interruptible = was_interruptible;
	}

B
Ben Widawsky 已提交
3959 3960 3961 3962 3963
	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
	 * before progressing. */
	if (obj->stolen)
		i915_gem_object_unpin_pages(obj);

B
Ben Widawsky 已提交
3964 3965
	if (WARN_ON(obj->pages_pin_count))
		obj->pages_pin_count = 0;
3966
	i915_gem_object_put_pages(obj);
3967
	i915_gem_object_free_mmap_offset(obj);
3968
	i915_gem_object_release_stolen(obj);
3969

3970 3971
	BUG_ON(obj->pages);

3972 3973
	if (obj->base.import_attach)
		drm_prime_gem_destroy(&obj->base, NULL);
3974

3975 3976
	drm_gem_object_release(&obj->base);
	i915_gem_info_remove_obj(dev_priv, obj->base.size);
3977

3978
	kfree(obj->bit_17);
3979
	i915_gem_object_free(obj);
3980 3981
}

3982 3983 3984 3985 3986
int
i915_gem_idle(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret;
3987

3988
	mutex_lock(&dev->struct_mutex);
C
Chris Wilson 已提交
3989

3990
	if (dev_priv->mm.suspended) {
3991 3992
		mutex_unlock(&dev->struct_mutex);
		return 0;
3993 3994
	}

3995
	ret = i915_gpu_idle(dev);
3996 3997
	if (ret) {
		mutex_unlock(&dev->struct_mutex);
3998
		return ret;
3999
	}
4000
	i915_gem_retire_requests(dev);
4001

4002
	/* Under UMS, be paranoid and evict. */
4003
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
C
Chris Wilson 已提交
4004
		i915_gem_evict_everything(dev);
4005

4006 4007
	i915_gem_reset_fences(dev);

4008 4009 4010 4011 4012
	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
	 * We need to replace this with a semaphore, or something.
	 * And not confound mm.suspended!
	 */
	dev_priv->mm.suspended = 1;
4013
	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
4014 4015

	i915_kernel_lost_context(dev);
4016
	i915_gem_cleanup_ringbuffer(dev);
4017

4018 4019
	mutex_unlock(&dev->struct_mutex);

4020 4021 4022
	/* Cancel the retire work handler, which should be idle now. */
	cancel_delayed_work_sync(&dev_priv->mm.retire_work);

4023 4024 4025
	return 0;
}

B
Ben Widawsky 已提交
4026 4027 4028 4029 4030 4031
void i915_gem_l3_remap(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	u32 misccpctl;
	int i;

4032
	if (!HAS_L3_GPU_CACHE(dev))
B
Ben Widawsky 已提交
4033 4034
		return;

4035
	if (!dev_priv->l3_parity.remap_info)
B
Ben Widawsky 已提交
4036 4037 4038 4039 4040 4041 4042 4043
		return;

	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
4044
		if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
B
Ben Widawsky 已提交
4045 4046
			DRM_DEBUG("0x%x was already programmed to %x\n",
				  GEN7_L3LOG_BASE + i, remap);
4047
		if (remap && !dev_priv->l3_parity.remap_info[i/4])
B
Ben Widawsky 已提交
4048
			DRM_DEBUG_DRIVER("Clearing remapped register\n");
4049
		I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
B
Ben Widawsky 已提交
4050 4051 4052 4053 4054 4055 4056 4057
	}

	/* Make sure all the writes land before disabling dop clock gating */
	POSTING_READ(GEN7_L3LOG_BASE);

	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
}

4058 4059 4060 4061
void i915_gem_init_swizzling(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;

4062
	if (INTEL_INFO(dev)->gen < 5 ||
4063 4064 4065 4066 4067 4068
	    dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		return;

	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
				 DISP_TILE_SURFACE_SWIZZLING);

4069 4070 4071
	if (IS_GEN5(dev))
		return;

4072 4073
	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
	if (IS_GEN6(dev))
4074
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4075
	else if (IS_GEN7(dev))
4076
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4077 4078
	else
		BUG();
4079
}
D
Daniel Vetter 已提交
4080

4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096
static bool
intel_enable_blt(struct drm_device *dev)
{
	if (!HAS_BLT(dev))
		return false;

	/* The blitter was dysfunctional on early prototypes */
	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		DRM_INFO("BLT not supported on this pre-production hardware;"
			 " graphics performance will be degraded.\n");
		return false;
	}

	return true;
}

4097
static int i915_gem_init_rings(struct drm_device *dev)
4098
{
4099
	struct drm_i915_private *dev_priv = dev->dev_private;
4100
	int ret;
4101

4102
	ret = intel_init_render_ring_buffer(dev);
4103
	if (ret)
4104
		return ret;
4105 4106

	if (HAS_BSD(dev)) {
4107
		ret = intel_init_bsd_ring_buffer(dev);
4108 4109
		if (ret)
			goto cleanup_render_ring;
4110
	}
4111

4112
	if (intel_enable_blt(dev)) {
4113 4114 4115 4116 4117
		ret = intel_init_blt_ring_buffer(dev);
		if (ret)
			goto cleanup_bsd_ring;
	}

B
Ben Widawsky 已提交
4118 4119 4120 4121 4122 4123 4124
	if (HAS_VEBOX(dev)) {
		ret = intel_init_vebox_ring_buffer(dev);
		if (ret)
			goto cleanup_blt_ring;
	}


4125
	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
4126
	if (ret)
B
Ben Widawsky 已提交
4127
		goto cleanup_vebox_ring;
4128 4129 4130

	return 0;

B
Ben Widawsky 已提交
4131 4132
cleanup_vebox_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154
cleanup_blt_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
cleanup_bsd_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
cleanup_render_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);

	return ret;
}

int
i915_gem_init_hw(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret;

	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		return -EIO;

	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);

4155 4156 4157 4158 4159 4160
	if (HAS_PCH_NOP(dev)) {
		u32 temp = I915_READ(GEN7_MSG_CTL);
		temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
		I915_WRITE(GEN7_MSG_CTL, temp);
	}

4161 4162 4163 4164 4165
	i915_gem_l3_remap(dev);

	i915_gem_init_swizzling(dev);

	ret = i915_gem_init_rings(dev);
4166 4167 4168
	if (ret)
		return ret;

4169 4170 4171 4172 4173
	/*
	 * XXX: There was some w/a described somewhere suggesting loading
	 * contexts before PPGTT.
	 */
	i915_gem_context_init(dev);
4174 4175 4176 4177 4178 4179 4180
	if (dev_priv->mm.aliasing_ppgtt) {
		ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
		if (ret) {
			i915_gem_cleanup_aliasing_ppgtt(dev);
			DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
		}
	}
D
Daniel Vetter 已提交
4181

4182
	return 0;
4183 4184
}

4185 4186 4187 4188 4189 4190
int i915_gem_init(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	mutex_lock(&dev->struct_mutex);
4191 4192 4193 4194 4195 4196 4197 4198

	if (IS_VALLEYVIEW(dev)) {
		/* VLVA0 (potential hack), BIOS isn't actually waking us */
		I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
		if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
	}

4199
	i915_gem_init_global_gtt(dev);
4200

4201 4202 4203 4204 4205 4206 4207
	ret = i915_gem_init_hw(dev);
	mutex_unlock(&dev->struct_mutex);
	if (ret) {
		i915_gem_cleanup_aliasing_ppgtt(dev);
		return ret;
	}

4208 4209 4210
	/* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		dev_priv->dri1.allow_batchbuffer = 1;
4211 4212 4213
	return 0;
}

4214 4215 4216 4217
void
i915_gem_cleanup_ringbuffer(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
4218
	struct intel_ring_buffer *ring;
4219
	int i;
4220

4221 4222
	for_each_ring(ring, dev_priv, i)
		intel_cleanup_ring_buffer(ring);
4223 4224
}

4225 4226 4227 4228 4229
int
i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
4230
	int ret;
4231

J
Jesse Barnes 已提交
4232 4233 4234
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return 0;

4235
	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
4236
		DRM_ERROR("Reenabling wedged hardware, good luck\n");
4237
		atomic_set(&dev_priv->gpu_error.reset_counter, 0);
4238 4239 4240
	}

	mutex_lock(&dev->struct_mutex);
4241 4242
	dev_priv->mm.suspended = 0;

4243
	ret = i915_gem_init_hw(dev);
4244 4245
	if (ret != 0) {
		mutex_unlock(&dev->struct_mutex);
4246
		return ret;
4247
	}
4248

4249
	BUG_ON(!list_empty(&dev_priv->mm.active_list));
4250
	mutex_unlock(&dev->struct_mutex);
4251

4252 4253 4254
	ret = drm_irq_install(dev);
	if (ret)
		goto cleanup_ringbuffer;
4255

4256
	return 0;
4257 4258 4259 4260 4261 4262 4263 4264

cleanup_ringbuffer:
	mutex_lock(&dev->struct_mutex);
	i915_gem_cleanup_ringbuffer(dev);
	dev_priv->mm.suspended = 1;
	mutex_unlock(&dev->struct_mutex);

	return ret;
4265 4266 4267 4268 4269 4270
}

int
i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
J
Jesse Barnes 已提交
4271 4272 4273
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return 0;

4274
	drm_irq_uninstall(dev);
4275
	return i915_gem_idle(dev);
4276 4277 4278 4279 4280 4281 4282
}

void
i915_gem_lastclose(struct drm_device *dev)
{
	int ret;

4283 4284 4285
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return;

4286 4287 4288
	ret = i915_gem_idle(dev);
	if (ret)
		DRM_ERROR("failed to idle hardware: %d\n", ret);
4289 4290
}

4291 4292 4293 4294 4295 4296 4297
static void
init_ring_lists(struct intel_ring_buffer *ring)
{
	INIT_LIST_HEAD(&ring->active_list);
	INIT_LIST_HEAD(&ring->request_list);
}

4298 4299 4300 4301
void
i915_gem_load(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
4302 4303 4304 4305 4306 4307 4308
	int i;

	dev_priv->slab =
		kmem_cache_create("i915_gem_object",
				  sizeof(struct drm_i915_gem_object), 0,
				  SLAB_HWCACHE_ALIGN,
				  NULL);
4309

4310
	INIT_LIST_HEAD(&dev_priv->mm.active_list);
4311
	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
C
Chris Wilson 已提交
4312 4313
	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4314
	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4315 4316
	for (i = 0; i < I915_NUM_RINGS; i++)
		init_ring_lists(&dev_priv->ring[i]);
4317
	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4318
		INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4319 4320
	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
			  i915_gem_retire_work_handler);
4321
	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4322

4323 4324
	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
	if (IS_GEN3(dev)) {
4325 4326
		I915_WRITE(MI_ARB_STATE,
			   _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4327 4328
	}

4329 4330
	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;

4331
	/* Old X drivers will take 0-2 for front, back, depth buffers */
4332 4333
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		dev_priv->fence_reg_start = 3;
4334

4335 4336 4337
	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		dev_priv->num_fence_regs = 32;
	else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4338 4339 4340 4341
		dev_priv->num_fence_regs = 16;
	else
		dev_priv->num_fence_regs = 8;

4342
	/* Initialize fence registers to zero */
4343
	i915_gem_reset_fences(dev);
4344

4345
	i915_gem_detect_bit_6_swizzle(dev);
4346
	init_waitqueue_head(&dev_priv->pending_flip_queue);
4347

4348 4349
	dev_priv->mm.interruptible = true;

4350 4351 4352
	dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
	dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
	register_shrinker(&dev_priv->mm.inactive_shrinker);
4353
}
4354 4355 4356 4357 4358

/*
 * Create a physically contiguous memory object for this object
 * e.g. for cursor + overlay regs
 */
4359 4360
static int i915_gem_init_phys_object(struct drm_device *dev,
				     int id, int size, int align)
4361 4362 4363 4364 4365 4366 4367 4368
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_i915_gem_phys_object *phys_obj;
	int ret;

	if (dev_priv->mm.phys_objs[id - 1] || !size)
		return 0;

4369
	phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4370 4371 4372 4373 4374
	if (!phys_obj)
		return -ENOMEM;

	phys_obj->id = id;

4375
	phys_obj->handle = drm_pci_alloc(dev, size, align);
4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387
	if (!phys_obj->handle) {
		ret = -ENOMEM;
		goto kfree_obj;
	}
#ifdef CONFIG_X86
	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
#endif

	dev_priv->mm.phys_objs[id - 1] = phys_obj;

	return 0;
kfree_obj:
4388
	kfree(phys_obj);
4389 4390 4391
	return ret;
}

4392
static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_i915_gem_phys_object *phys_obj;

	if (!dev_priv->mm.phys_objs[id - 1])
		return;

	phys_obj = dev_priv->mm.phys_objs[id - 1];
	if (phys_obj->cur_obj) {
		i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
	}

#ifdef CONFIG_X86
	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
#endif
	drm_pci_free(dev, phys_obj->handle);
	kfree(phys_obj);
	dev_priv->mm.phys_objs[id - 1] = NULL;
}

void i915_gem_free_all_phys_object(struct drm_device *dev)
{
	int i;

4417
	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4418 4419 4420 4421
		i915_gem_free_phys_object(dev, i);
}

void i915_gem_detach_phys_object(struct drm_device *dev,
4422
				 struct drm_i915_gem_object *obj)
4423
{
A
Al Viro 已提交
4424
	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4425
	char *vaddr;
4426 4427 4428
	int i;
	int page_count;

4429
	if (!obj->phys_obj)
4430
		return;
4431
	vaddr = obj->phys_obj->handle->vaddr;
4432

4433
	page_count = obj->base.size / PAGE_SIZE;
4434
	for (i = 0; i < page_count; i++) {
4435
		struct page *page = shmem_read_mapping_page(mapping, i);
4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446
		if (!IS_ERR(page)) {
			char *dst = kmap_atomic(page);
			memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
			kunmap_atomic(dst);

			drm_clflush_pages(&page, 1);

			set_page_dirty(page);
			mark_page_accessed(page);
			page_cache_release(page);
		}
4447
	}
4448
	i915_gem_chipset_flush(dev);
4449

4450 4451
	obj->phys_obj->cur_obj = NULL;
	obj->phys_obj = NULL;
4452 4453 4454 4455
}

int
i915_gem_attach_phys_object(struct drm_device *dev,
4456
			    struct drm_i915_gem_object *obj,
4457 4458
			    int id,
			    int align)
4459
{
A
Al Viro 已提交
4460
	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4461 4462 4463 4464 4465 4466 4467 4468
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret = 0;
	int page_count;
	int i;

	if (id > I915_MAX_PHYS_OBJECT)
		return -EINVAL;

4469 4470
	if (obj->phys_obj) {
		if (obj->phys_obj->id == id)
4471 4472 4473 4474 4475 4476 4477
			return 0;
		i915_gem_detach_phys_object(dev, obj);
	}

	/* create a new object */
	if (!dev_priv->mm.phys_objs[id - 1]) {
		ret = i915_gem_init_phys_object(dev, id,
4478
						obj->base.size, align);
4479
		if (ret) {
4480 4481
			DRM_ERROR("failed to init phys object %d size: %zu\n",
				  id, obj->base.size);
4482
			return ret;
4483 4484 4485 4486
		}
	}

	/* bind to the object */
4487 4488
	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
	obj->phys_obj->cur_obj = obj;
4489

4490
	page_count = obj->base.size / PAGE_SIZE;
4491 4492

	for (i = 0; i < page_count; i++) {
4493 4494 4495
		struct page *page;
		char *dst, *src;

4496
		page = shmem_read_mapping_page(mapping, i);
4497 4498
		if (IS_ERR(page))
			return PTR_ERR(page);
4499

4500
		src = kmap_atomic(page);
4501
		dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4502
		memcpy(dst, src, PAGE_SIZE);
P
Peter Zijlstra 已提交
4503
		kunmap_atomic(src);
4504

4505 4506 4507
		mark_page_accessed(page);
		page_cache_release(page);
	}
4508

4509 4510 4511 4512
	return 0;
}

static int
4513 4514
i915_gem_phys_pwrite(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
4515 4516 4517
		     struct drm_i915_gem_pwrite *args,
		     struct drm_file *file_priv)
{
4518
	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
V
Ville Syrjälä 已提交
4519
	char __user *user_data = to_user_ptr(args->data_ptr);
4520

4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533
	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
		unsigned long unwritten;

		/* The physical object once assigned is fixed for the lifetime
		 * of the obj, so we can safely drop the lock and continue
		 * to access vaddr.
		 */
		mutex_unlock(&dev->struct_mutex);
		unwritten = copy_from_user(vaddr, user_data, args->size);
		mutex_lock(&dev->struct_mutex);
		if (unwritten)
			return -EFAULT;
	}
4534

4535
	i915_gem_chipset_flush(dev);
4536 4537
	return 0;
}
4538

4539
void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4540
{
4541
	struct drm_i915_file_private *file_priv = file->driver_priv;
4542 4543 4544 4545 4546

	/* Clean up our request list when the client is going away, so that
	 * later retire_requests won't dereference our soon-to-be-gone
	 * file_priv.
	 */
4547
	spin_lock(&file_priv->mm.lock);
4548 4549 4550 4551 4552 4553 4554 4555 4556
	while (!list_empty(&file_priv->mm.request_list)) {
		struct drm_i915_gem_request *request;

		request = list_first_entry(&file_priv->mm.request_list,
					   struct drm_i915_gem_request,
					   client_list);
		list_del(&request->client_list);
		request->file_priv = NULL;
	}
4557
	spin_unlock(&file_priv->mm.lock);
4558
}
4559

4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572
static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
{
	if (!mutex_is_locked(mutex))
		return false;

#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
	return mutex->owner == task;
#else
	/* Since UP may be pre-empted, we cannot assume that we own the lock */
	return false;
#endif
}

4573
static int
4574
i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
4575
{
4576 4577 4578 4579 4580
	struct drm_i915_private *dev_priv =
		container_of(shrinker,
			     struct drm_i915_private,
			     mm.inactive_shrinker);
	struct drm_device *dev = dev_priv->dev;
C
Chris Wilson 已提交
4581
	struct drm_i915_gem_object *obj;
4582
	int nr_to_scan = sc->nr_to_scan;
4583
	bool unlock = true;
4584 4585
	int cnt;

4586 4587 4588 4589
	if (!mutex_trylock(&dev->struct_mutex)) {
		if (!mutex_is_locked_by(&dev->struct_mutex, current))
			return 0;

4590 4591 4592
		if (dev_priv->mm.shrinker_no_lock_stealing)
			return 0;

4593 4594
		unlock = false;
	}
4595

C
Chris Wilson 已提交
4596 4597
	if (nr_to_scan) {
		nr_to_scan -= i915_gem_purge(dev_priv, nr_to_scan);
4598 4599 4600
		if (nr_to_scan > 0)
			nr_to_scan -= __i915_gem_shrink(dev_priv, nr_to_scan,
							false);
C
Chris Wilson 已提交
4601 4602
		if (nr_to_scan > 0)
			i915_gem_shrink_all(dev_priv);
4603 4604
	}

4605
	cnt = 0;
4606
	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
4607 4608
		if (obj->pages_pin_count == 0)
			cnt += obj->base.size >> PAGE_SHIFT;
4609
	list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list)
4610
		if (obj->pin_count == 0 && obj->pages_pin_count == 0)
C
Chris Wilson 已提交
4611
			cnt += obj->base.size >> PAGE_SHIFT;
4612

4613 4614
	if (unlock)
		mutex_unlock(&dev->struct_mutex);
C
Chris Wilson 已提交
4615
	return cnt;
4616
}