i915_gem.c 116.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright © 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

28 29
#include <drm/drmP.h>
#include <drm/i915_drm.h>
30
#include "i915_drv.h"
C
Chris Wilson 已提交
31
#include "i915_trace.h"
32
#include "intel_drv.h"
33
#include <linux/shmem_fs.h>
34
#include <linux/slab.h>
35
#include <linux/swap.h>
J
Jesse Barnes 已提交
36
#include <linux/pci.h>
37
#include <linux/dma-buf.h>
38

39 40
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
41 42
static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
						    unsigned alignment,
43 44
						    bool map_and_fenceable,
						    bool nonblocking);
45 46
static int i915_gem_phys_pwrite(struct drm_device *dev,
				struct drm_i915_gem_object *obj,
47
				struct drm_i915_gem_pwrite *args,
48
				struct drm_file *file);
49

50 51 52 53 54 55
static void i915_gem_write_fence(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj);
static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
					 struct drm_i915_fence_reg *fence,
					 bool enable);

56
static int i915_gem_inactive_shrink(struct shrinker *shrinker,
57
				    struct shrink_control *sc);
C
Chris Wilson 已提交
58 59
static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
60
static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
61

62 63 64 65 66 67 68 69
static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
{
	if (obj->tiling_mode)
		i915_gem_release_mmap(obj);

	/* As we do not have an associated fence register, we will force
	 * a tiling change if we ever need to acquire one.
	 */
70
	obj->fence_dirty = false;
71 72 73
	obj->fence_reg = I915_FENCE_REG_NONE;
}

74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
/* some bookkeeping */
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
				  size_t size)
{
	dev_priv->mm.object_count++;
	dev_priv->mm.object_memory += size;
}

static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
				     size_t size)
{
	dev_priv->mm.object_count--;
	dev_priv->mm.object_memory -= size;
}

89
static int
90
i915_gem_wait_for_error(struct i915_gpu_error *error)
91 92 93
{
	int ret;

94 95
#define EXIT_COND (!i915_reset_in_progress(error) || \
		   i915_terminally_wedged(error))
96
	if (EXIT_COND)
97 98
		return 0;

99 100 101 102 103
	/*
	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
	 * userspace. If it takes that long something really bad is going on and
	 * we should simply try to bail out and fail as gracefully as possible.
	 */
104 105 106
	ret = wait_event_interruptible_timeout(error->reset_queue,
					       EXIT_COND,
					       10*HZ);
107 108 109 110
	if (ret == 0) {
		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		return -EIO;
	} else if (ret < 0) {
111
		return ret;
112
	}
113
#undef EXIT_COND
114

115
	return 0;
116 117
}

118
int i915_mutex_lock_interruptible(struct drm_device *dev)
119
{
120
	struct drm_i915_private *dev_priv = dev->dev_private;
121 122
	int ret;

123
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
124 125 126 127 128 129 130
	if (ret)
		return ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

131
	WARN_ON(i915_verify_lists(dev));
132 133
	return 0;
}
134

135
static inline bool
136
i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
137
{
138
	return i915_gem_obj_ggtt_bound(obj) && !obj->active;
139 140
}

J
Jesse Barnes 已提交
141 142
int
i915_gem_init_ioctl(struct drm_device *dev, void *data,
143
		    struct drm_file *file)
J
Jesse Barnes 已提交
144
{
145
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
146
	struct drm_i915_gem_init *args = data;
147

148 149 150
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return -ENODEV;

151 152 153
	if (args->gtt_start >= args->gtt_end ||
	    (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
		return -EINVAL;
J
Jesse Barnes 已提交
154

155 156 157 158
	/* GEM with user mode setting was never supported on ilk and later. */
	if (INTEL_INFO(dev)->gen >= 5)
		return -ENODEV;

J
Jesse Barnes 已提交
159
	mutex_lock(&dev->struct_mutex);
160 161
	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
				  args->gtt_end);
162
	dev_priv->gtt.mappable_end = args->gtt_end;
163 164
	mutex_unlock(&dev->struct_mutex);

165
	return 0;
166 167
}

168 169
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
170
			    struct drm_file *file)
171
{
172
	struct drm_i915_private *dev_priv = dev->dev_private;
173
	struct drm_i915_gem_get_aperture *args = data;
174 175
	struct drm_i915_gem_object *obj;
	size_t pinned;
176

177
	pinned = 0;
178
	mutex_lock(&dev->struct_mutex);
179
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
180
		if (obj->pin_count)
181
			pinned += i915_gem_obj_ggtt_size(obj);
182
	mutex_unlock(&dev->struct_mutex);
183

B
Ben Widawsky 已提交
184
	args->aper_size = dev_priv->gtt.total;
185
	args->aper_available_size = args->aper_size - pinned;
186

187 188 189
	return 0;
}

190 191 192 193 194 195 196 197 198 199 200 201
void *i915_gem_object_alloc(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	return kmem_cache_alloc(dev_priv->slab, GFP_KERNEL | __GFP_ZERO);
}

void i915_gem_object_free(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	kmem_cache_free(dev_priv->slab, obj);
}

202 203 204 205 206
static int
i915_gem_create(struct drm_file *file,
		struct drm_device *dev,
		uint64_t size,
		uint32_t *handle_p)
207
{
208
	struct drm_i915_gem_object *obj;
209 210
	int ret;
	u32 handle;
211

212
	size = roundup(size, PAGE_SIZE);
213 214
	if (size == 0)
		return -EINVAL;
215 216

	/* Allocate the new object */
217
	obj = i915_gem_alloc_object(dev, size);
218 219 220
	if (obj == NULL)
		return -ENOMEM;

221
	ret = drm_gem_handle_create(file, &obj->base, &handle);
222
	if (ret) {
223 224
		drm_gem_object_release(&obj->base);
		i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
225
		i915_gem_object_free(obj);
226
		return ret;
227
	}
228

229
	/* drop reference from allocate - handle holds it now */
230
	drm_gem_object_unreference(&obj->base);
231 232
	trace_i915_gem_object_create(obj);

233
	*handle_p = handle;
234 235 236
	return 0;
}

237 238 239 240 241 242
int
i915_gem_dumb_create(struct drm_file *file,
		     struct drm_device *dev,
		     struct drm_mode_create_dumb *args)
{
	/* have to work out size/pitch and return them */
243
	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
	args->size = args->pitch * args->height;
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

int i915_gem_dumb_destroy(struct drm_file *file,
			  struct drm_device *dev,
			  uint32_t handle)
{
	return drm_gem_handle_delete(file, handle);
}

/**
 * Creates a new mm object and returns a handle to it.
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file)
{
	struct drm_i915_gem_create *args = data;
264

265 266 267 268
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
static inline int
__copy_to_user_swizzled(char __user *cpu_vaddr,
			const char *gpu_vaddr, int gpu_offset,
			int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_to_user(cpu_vaddr + cpu_offset,
				     gpu_vaddr + swizzled_gpu_offset,
				     this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

295
static inline int
296 297
__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
			  const char __user *cpu_vaddr,
298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
			  int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
				       cpu_vaddr + cpu_offset,
				       this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

321 322 323
/* Per-page copy function for the shmem pread fastpath.
 * Flushes invalid cachelines before reading the target if
 * needs_clflush is set. */
324
static int
325 326 327 328 329 330 331
shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

332
	if (unlikely(page_do_bit17_swizzling))
333 334 335 336 337 338 339 340 341 342 343
		return -EINVAL;

	vaddr = kmap_atomic(page);
	if (needs_clflush)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_to_user_inatomic(user_data,
				      vaddr + shmem_page_offset,
				      page_length);
	kunmap_atomic(vaddr);

344
	return ret ? -EFAULT : 0;
345 346
}

347 348 349 350
static void
shmem_clflush_swizzled_range(char *addr, unsigned long length,
			     bool swizzled)
{
351
	if (unlikely(swizzled)) {
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
		unsigned long start = (unsigned long) addr;
		unsigned long end = (unsigned long) addr + length;

		/* For swizzling simply ensure that we always flush both
		 * channels. Lame, but simple and it works. Swizzled
		 * pwrite/pread is far from a hotpath - current userspace
		 * doesn't use it at all. */
		start = round_down(start, 128);
		end = round_up(end, 128);

		drm_clflush_virt_range((void *)start, end - start);
	} else {
		drm_clflush_virt_range(addr, length);
	}

}

369 370 371 372 373 374 375 376 377 378 379 380
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
static int
shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

	vaddr = kmap(page);
	if (needs_clflush)
381 382 383
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
384 385 386 387 388 389 390 391 392 393 394

	if (page_do_bit17_swizzling)
		ret = __copy_to_user_swizzled(user_data,
					      vaddr, shmem_page_offset,
					      page_length);
	else
		ret = __copy_to_user(user_data,
				     vaddr + shmem_page_offset,
				     page_length);
	kunmap(page);

395
	return ret ? - EFAULT : 0;
396 397
}

398
static int
399 400 401 402
i915_gem_shmem_pread(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pread *args,
		     struct drm_file *file)
403
{
404
	char __user *user_data;
405
	ssize_t remain;
406
	loff_t offset;
407
	int shmem_page_offset, page_length, ret = 0;
408
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
409
	int prefaulted = 0;
410
	int needs_clflush = 0;
411
	struct sg_page_iter sg_iter;
412

V
Ville Syrjälä 已提交
413
	user_data = to_user_ptr(args->data_ptr);
414 415
	remain = args->size;

416
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
417

418 419 420 421 422 423 424
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		/* If we're not in the cpu read domain, set ourself into the gtt
		 * read domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will dirty the data
		 * anyway again before the next pread happens. */
		if (obj->cache_level == I915_CACHE_NONE)
			needs_clflush = 1;
425
		if (i915_gem_obj_ggtt_bound(obj)) {
C
Chris Wilson 已提交
426 427 428 429
			ret = i915_gem_object_set_to_gtt_domain(obj, false);
			if (ret)
				return ret;
		}
430
	}
431

432 433 434 435 436 437
	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

	i915_gem_object_pin_pages(obj);

438
	offset = args->offset;
439

440 441
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
442
		struct page *page = sg_page_iter_page(&sg_iter);
443 444 445 446

		if (remain <= 0)
			break;

447 448 449 450 451
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
452
		shmem_page_offset = offset_in_page(offset);
453 454 455 456
		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

457 458 459
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

460 461 462 463 464
		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
		if (ret == 0)
			goto next_page;
465 466 467

		mutex_unlock(&dev->struct_mutex);

468
		if (!prefaulted) {
469
			ret = fault_in_multipages_writeable(user_data, remain);
470 471 472 473 474 475 476
			/* Userspace is tricking us, but we've already clobbered
			 * its pages with the prefault and promised to write the
			 * data up to the first fault. Hence ignore any errors
			 * and just continue. */
			(void)ret;
			prefaulted = 1;
		}
477

478 479 480
		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
481

482
		mutex_lock(&dev->struct_mutex);
483

484
next_page:
485 486
		mark_page_accessed(page);

487
		if (ret)
488 489
			goto out;

490
		remain -= page_length;
491
		user_data += page_length;
492 493 494
		offset += page_length;
	}

495
out:
496 497
	i915_gem_object_unpin_pages(obj);

498 499 500
	return ret;
}

501 502 503 504 505 506 507
/**
 * Reads data from the object referenced by handle.
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
508
		     struct drm_file *file)
509 510
{
	struct drm_i915_gem_pread *args = data;
511
	struct drm_i915_gem_object *obj;
512
	int ret = 0;
513

514 515 516 517
	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_WRITE,
V
Ville Syrjälä 已提交
518
		       to_user_ptr(args->data_ptr),
519 520 521
		       args->size))
		return -EFAULT;

522
	ret = i915_mutex_lock_interruptible(dev);
523
	if (ret)
524
		return ret;
525

526
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
527
	if (&obj->base == NULL) {
528 529
		ret = -ENOENT;
		goto unlock;
530
	}
531

532
	/* Bounds check source.  */
533 534
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
C
Chris Wilson 已提交
535
		ret = -EINVAL;
536
		goto out;
C
Chris Wilson 已提交
537 538
	}

539 540 541 542 543 544 545 546
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

C
Chris Wilson 已提交
547 548
	trace_i915_gem_object_pread(obj, args->offset, args->size);

549
	ret = i915_gem_shmem_pread(dev, obj, args, file);
550

551
out:
552
	drm_gem_object_unreference(&obj->base);
553
unlock:
554
	mutex_unlock(&dev->struct_mutex);
555
	return ret;
556 557
}

558 559
/* This is the fast write path which cannot handle
 * page faults in the source data
560
 */
561 562 563 564 565 566

static inline int
fast_user_write(struct io_mapping *mapping,
		loff_t page_base, int page_offset,
		char __user *user_data,
		int length)
567
{
568 569
	void __iomem *vaddr_atomic;
	void *vaddr;
570
	unsigned long unwritten;
571

P
Peter Zijlstra 已提交
572
	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
573 574 575
	/* We can use the cpu mem copy function because this is X86. */
	vaddr = (void __force*)vaddr_atomic + page_offset;
	unwritten = __copy_from_user_inatomic_nocache(vaddr,
576
						      user_data, length);
P
Peter Zijlstra 已提交
577
	io_mapping_unmap_atomic(vaddr_atomic);
578
	return unwritten;
579 580
}

581 582 583 584
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
 */
585
static int
586 587
i915_gem_gtt_pwrite_fast(struct drm_device *dev,
			 struct drm_i915_gem_object *obj,
588
			 struct drm_i915_gem_pwrite *args,
589
			 struct drm_file *file)
590
{
591
	drm_i915_private_t *dev_priv = dev->dev_private;
592
	ssize_t remain;
593
	loff_t offset, page_base;
594
	char __user *user_data;
D
Daniel Vetter 已提交
595 596
	int page_offset, page_length, ret;

597
	ret = i915_gem_object_pin(obj, 0, true, true);
D
Daniel Vetter 已提交
598 599 600 601 602 603 604 605 606 607
	if (ret)
		goto out;

	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		goto out_unpin;

	ret = i915_gem_object_put_fence(obj);
	if (ret)
		goto out_unpin;
608

V
Ville Syrjälä 已提交
609
	user_data = to_user_ptr(args->data_ptr);
610 611
	remain = args->size;

612
	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
613 614 615 616

	while (remain > 0) {
		/* Operation in this page
		 *
617 618 619
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
620
		 */
621 622
		page_base = offset & PAGE_MASK;
		page_offset = offset_in_page(offset);
623 624 625 626 627
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		/* If we get a fault while copying data, then (presumably) our
628 629
		 * source page isn't available.  Return the error and we'll
		 * retry in the slow path.
630
		 */
B
Ben Widawsky 已提交
631
		if (fast_user_write(dev_priv->gtt.mappable, page_base,
D
Daniel Vetter 已提交
632 633 634 635
				    page_offset, user_data, page_length)) {
			ret = -EFAULT;
			goto out_unpin;
		}
636

637 638 639
		remain -= page_length;
		user_data += page_length;
		offset += page_length;
640 641
	}

D
Daniel Vetter 已提交
642 643 644
out_unpin:
	i915_gem_object_unpin(obj);
out:
645
	return ret;
646 647
}

648 649 650 651
/* Per-page copy function for the shmem pwrite fastpath.
 * Flushes invalid cachelines before writing to the target if
 * needs_clflush_before is set and flushes out any written cachelines after
 * writing if needs_clflush is set. */
652
static int
653 654 655 656 657
shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
658
{
659
	char *vaddr;
660
	int ret;
661

662
	if (unlikely(page_do_bit17_swizzling))
663
		return -EINVAL;
664

665 666 667 668 669 670 671 672 673 674 675
	vaddr = kmap_atomic(page);
	if (needs_clflush_before)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
						user_data,
						page_length);
	if (needs_clflush_after)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	kunmap_atomic(vaddr);
676

677
	return ret ? -EFAULT : 0;
678 679
}

680 681
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
682
static int
683 684 685 686 687
shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
688
{
689 690
	char *vaddr;
	int ret;
691

692
	vaddr = kmap(page);
693
	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
694 695 696
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
697 698
	if (page_do_bit17_swizzling)
		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
699 700
						user_data,
						page_length);
701 702 703 704 705
	else
		ret = __copy_from_user(vaddr + shmem_page_offset,
				       user_data,
				       page_length);
	if (needs_clflush_after)
706 707 708
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
709
	kunmap(page);
710

711
	return ret ? -EFAULT : 0;
712 713 714
}

static int
715 716 717 718
i915_gem_shmem_pwrite(struct drm_device *dev,
		      struct drm_i915_gem_object *obj,
		      struct drm_i915_gem_pwrite *args,
		      struct drm_file *file)
719 720
{
	ssize_t remain;
721 722
	loff_t offset;
	char __user *user_data;
723
	int shmem_page_offset, page_length, ret = 0;
724
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
725
	int hit_slowpath = 0;
726 727
	int needs_clflush_after = 0;
	int needs_clflush_before = 0;
728
	struct sg_page_iter sg_iter;
729

V
Ville Syrjälä 已提交
730
	user_data = to_user_ptr(args->data_ptr);
731 732
	remain = args->size;

733
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
734

735 736 737 738 739 740 741
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		/* If we're not in the cpu write domain, set ourself into the gtt
		 * write domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will use the data
		 * right away and we therefore have to clflush anyway. */
		if (obj->cache_level == I915_CACHE_NONE)
			needs_clflush_after = 1;
742
		if (i915_gem_obj_ggtt_bound(obj)) {
C
Chris Wilson 已提交
743 744 745 746
			ret = i915_gem_object_set_to_gtt_domain(obj, true);
			if (ret)
				return ret;
		}
747 748 749 750 751 752 753
	}
	/* Same trick applies for invalidate partially written cachelines before
	 * writing.  */
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
	    && obj->cache_level == I915_CACHE_NONE)
		needs_clflush_before = 1;

754 755 756 757 758 759
	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

	i915_gem_object_pin_pages(obj);

760
	offset = args->offset;
761
	obj->dirty = 1;
762

763 764
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
765
		struct page *page = sg_page_iter_page(&sg_iter);
766
		int partial_cacheline_write;
767

768 769 770
		if (remain <= 0)
			break;

771 772 773 774 775
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
776
		shmem_page_offset = offset_in_page(offset);
777 778 779 780 781

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

782 783 784 785 786 787 788
		/* If we don't overwrite a cacheline completely we need to be
		 * careful to have up-to-date data by first clflushing. Don't
		 * overcomplicate things and flush the entire patch. */
		partial_cacheline_write = needs_clflush_before &&
			((shmem_page_offset | page_length)
				& (boot_cpu_data.x86_clflush_size - 1));

789 790 791
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

792 793 794 795 796 797
		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
		if (ret == 0)
			goto next_page;
798 799 800

		hit_slowpath = 1;
		mutex_unlock(&dev->struct_mutex);
801 802 803 804
		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
805

806
		mutex_lock(&dev->struct_mutex);
807

808
next_page:
809 810 811
		set_page_dirty(page);
		mark_page_accessed(page);

812
		if (ret)
813 814
			goto out;

815
		remain -= page_length;
816
		user_data += page_length;
817
		offset += page_length;
818 819
	}

820
out:
821 822
	i915_gem_object_unpin_pages(obj);

823
	if (hit_slowpath) {
824 825 826 827 828 829 830
		/*
		 * Fixup: Flush cpu caches in case we didn't flush the dirty
		 * cachelines in-line while writing and the object moved
		 * out of the cpu write domain while we've dropped the lock.
		 */
		if (!needs_clflush_after &&
		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
831
			i915_gem_clflush_object(obj);
832
			i915_gem_chipset_flush(dev);
833
		}
834
	}
835

836
	if (needs_clflush_after)
837
		i915_gem_chipset_flush(dev);
838

839
	return ret;
840 841 842 843 844 845 846 847 848
}

/**
 * Writes data to the object referenced by handle.
 *
 * On error, the contents of the buffer that were to be modified are undefined.
 */
int
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
849
		      struct drm_file *file)
850 851
{
	struct drm_i915_gem_pwrite *args = data;
852
	struct drm_i915_gem_object *obj;
853 854 855 856 857 858
	int ret;

	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_READ,
V
Ville Syrjälä 已提交
859
		       to_user_ptr(args->data_ptr),
860 861 862
		       args->size))
		return -EFAULT;

V
Ville Syrjälä 已提交
863
	ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
864
					   args->size);
865 866
	if (ret)
		return -EFAULT;
867

868
	ret = i915_mutex_lock_interruptible(dev);
869
	if (ret)
870
		return ret;
871

872
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
873
	if (&obj->base == NULL) {
874 875
		ret = -ENOENT;
		goto unlock;
876
	}
877

878
	/* Bounds check destination. */
879 880
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
C
Chris Wilson 已提交
881
		ret = -EINVAL;
882
		goto out;
C
Chris Wilson 已提交
883 884
	}

885 886 887 888 889 890 891 892
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

C
Chris Wilson 已提交
893 894
	trace_i915_gem_object_pwrite(obj, args->offset, args->size);

D
Daniel Vetter 已提交
895
	ret = -EFAULT;
896 897 898 899 900 901
	/* We can only do the GTT pwrite on untiled buffers, as otherwise
	 * it would end up going through the fenced access, and we'll get
	 * different detiling behavior between reading and writing.
	 * pread/pwrite currently are reading and writing from the CPU
	 * perspective, requiring manual detiling by the client.
	 */
902
	if (obj->phys_obj) {
903
		ret = i915_gem_phys_pwrite(dev, obj, args, file);
904 905 906
		goto out;
	}

907
	if (obj->cache_level == I915_CACHE_NONE &&
908
	    obj->tiling_mode == I915_TILING_NONE &&
909
	    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
910
		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
D
Daniel Vetter 已提交
911 912 913
		/* Note that the gtt paths might fail with non-page-backed user
		 * pointers (e.g. gtt mappings when moving data between
		 * textures). Fallback to the shmem path in that case. */
914
	}
915

916
	if (ret == -EFAULT || ret == -ENOSPC)
D
Daniel Vetter 已提交
917
		ret = i915_gem_shmem_pwrite(dev, obj, args, file);
918

919
out:
920
	drm_gem_object_unreference(&obj->base);
921
unlock:
922
	mutex_unlock(&dev->struct_mutex);
923 924 925
	return ret;
}

926
int
927
i915_gem_check_wedge(struct i915_gpu_error *error,
928 929
		     bool interruptible)
{
930
	if (i915_reset_in_progress(error)) {
931 932 933 934 935
		/* Non-interruptible callers can't handle -EAGAIN, hence return
		 * -EIO unconditionally for these. */
		if (!interruptible)
			return -EIO;

936 937
		/* Recovery complete, but the reset failed ... */
		if (i915_terminally_wedged(error))
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958
			return -EIO;

		return -EAGAIN;
	}

	return 0;
}

/*
 * Compare seqno against outstanding lazy request. Emit a request if they are
 * equal.
 */
static int
i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
{
	int ret;

	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));

	ret = 0;
	if (seqno == ring->outstanding_lazy_request)
959
		ret = i915_add_request(ring, NULL);
960 961 962 963 964 965 966 967

	return ret;
}

/**
 * __wait_seqno - wait until execution of seqno has finished
 * @ring: the ring expected to report seqno
 * @seqno: duh!
968
 * @reset_counter: reset sequence associated with the given seqno
969 970 971
 * @interruptible: do an interruptible wait (normally yes)
 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
 *
972 973 974 975 976 977 978
 * Note: It is of utmost importance that the passed in seqno and reset_counter
 * values have been read by the caller in an smp safe manner. Where read-side
 * locks are involved, it is sufficient to read the reset_counter before
 * unlocking the lock that protects the seqno. For lockless tricks, the
 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
 * inserted.
 *
979 980 981 982
 * Returns 0 if the seqno was found within the alloted time. Else returns the
 * errno with remaining time filled in timeout argument.
 */
static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
983
			unsigned reset_counter,
984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
			bool interruptible, struct timespec *timeout)
{
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
	struct timespec before, now, wait_time={1,0};
	unsigned long timeout_jiffies;
	long end;
	bool wait_forever = true;
	int ret;

	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		return 0;

	trace_i915_gem_request_wait_begin(ring, seqno);

	if (timeout != NULL) {
		wait_time = *timeout;
		wait_forever = false;
	}

1003
	timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
1004 1005 1006 1007 1008 1009 1010 1011 1012

	if (WARN_ON(!ring->irq_get(ring)))
		return -ENODEV;

	/* Record current time in case interrupted by signal, or wedged * */
	getrawmonotonic(&before);

#define EXIT_COND \
	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
1013 1014
	 i915_reset_in_progress(&dev_priv->gpu_error) || \
	 reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
1015 1016 1017 1018 1019 1020 1021 1022 1023
	do {
		if (interruptible)
			end = wait_event_interruptible_timeout(ring->irq_queue,
							       EXIT_COND,
							       timeout_jiffies);
		else
			end = wait_event_timeout(ring->irq_queue, EXIT_COND,
						 timeout_jiffies);

1024 1025 1026 1027 1028 1029 1030
		/* We need to check whether any gpu reset happened in between
		 * the caller grabbing the seqno and now ... */
		if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
			end = -EAGAIN;

		/* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
		 * gone. */
1031
		ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044
		if (ret)
			end = ret;
	} while (end == 0 && wait_forever);

	getrawmonotonic(&now);

	ring->irq_put(ring);
	trace_i915_gem_request_wait_end(ring, seqno);
#undef EXIT_COND

	if (timeout) {
		struct timespec sleep_time = timespec_sub(now, before);
		*timeout = timespec_sub(*timeout, sleep_time);
1045 1046
		if (!timespec_valid(timeout)) /* i.e. negative time remains */
			set_normalized_timespec(timeout, 0, 0);
1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
	}

	switch (end) {
	case -EIO:
	case -EAGAIN: /* Wedged */
	case -ERESTARTSYS: /* Signal */
		return (int)end;
	case 0: /* Timeout */
		return -ETIME;
	default: /* Completed */
		WARN_ON(end < 0); /* We're not aware of other errors */
		return 0;
	}
}

/**
 * Waits for a sequence number to be signaled, and cleans up the
 * request and object lists appropriately for that event.
 */
int
i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
{
	struct drm_device *dev = ring->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	bool interruptible = dev_priv->mm.interruptible;
	int ret;

	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
	BUG_ON(seqno == 0);

1077
	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1078 1079 1080 1081 1082 1083 1084
	if (ret)
		return ret;

	ret = i915_gem_check_olr(ring, seqno);
	if (ret)
		return ret;

1085 1086 1087
	return __wait_seqno(ring, seqno,
			    atomic_read(&dev_priv->gpu_error.reset_counter),
			    interruptible, NULL);
1088 1089
}

1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108
static int
i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
				     struct intel_ring_buffer *ring)
{
	i915_gem_retire_requests_ring(ring);

	/* Manually manage the write flush as we may have not yet
	 * retired the buffer.
	 *
	 * Note that the last_write_seqno is always the earlier of
	 * the two (read/write) seqno, so if we haved successfully waited,
	 * we know we have passed the last write.
	 */
	obj->last_write_seqno = 0;
	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;

	return 0;
}

1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
/**
 * Ensures that all rendering to the object has completed and the object is
 * safe to unbind from the GTT or access from the CPU.
 */
static __must_check int
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
			       bool readonly)
{
	struct intel_ring_buffer *ring = obj->ring;
	u32 seqno;
	int ret;

	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
	if (seqno == 0)
		return 0;

	ret = i915_wait_seqno(ring, seqno);
	if (ret)
		return ret;

1129
	return i915_gem_object_wait_rendering__tail(obj, ring);
1130 1131
}

1132 1133 1134 1135 1136 1137 1138 1139 1140 1141
/* A nonblocking variant of the above wait. This is a highly dangerous routine
 * as the object state may change during this call.
 */
static __must_check int
i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
					    bool readonly)
{
	struct drm_device *dev = obj->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_ring_buffer *ring = obj->ring;
1142
	unsigned reset_counter;
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
	u32 seqno;
	int ret;

	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
	BUG_ON(!dev_priv->mm.interruptible);

	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
	if (seqno == 0)
		return 0;

1153
	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1154 1155 1156 1157 1158 1159 1160
	if (ret)
		return ret;

	ret = i915_gem_check_olr(ring, seqno);
	if (ret)
		return ret;

1161
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1162
	mutex_unlock(&dev->struct_mutex);
1163
	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
1164
	mutex_lock(&dev->struct_mutex);
1165 1166
	if (ret)
		return ret;
1167

1168
	return i915_gem_object_wait_rendering__tail(obj, ring);
1169 1170
}

1171
/**
1172 1173
 * Called when user space prepares to use an object with the CPU, either
 * through the mmap ioctl's mapping or a GTT mapping.
1174 1175 1176
 */
int
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1177
			  struct drm_file *file)
1178 1179
{
	struct drm_i915_gem_set_domain *args = data;
1180
	struct drm_i915_gem_object *obj;
1181 1182
	uint32_t read_domains = args->read_domains;
	uint32_t write_domain = args->write_domain;
1183 1184
	int ret;

1185
	/* Only handle setting domains to types used by the CPU. */
1186
	if (write_domain & I915_GEM_GPU_DOMAINS)
1187 1188
		return -EINVAL;

1189
	if (read_domains & I915_GEM_GPU_DOMAINS)
1190 1191 1192 1193 1194 1195 1196 1197
		return -EINVAL;

	/* Having something in the write domain implies it's in the read
	 * domain, and only that read domain.  Enforce that in the request.
	 */
	if (write_domain != 0 && read_domains != write_domain)
		return -EINVAL;

1198
	ret = i915_mutex_lock_interruptible(dev);
1199
	if (ret)
1200
		return ret;
1201

1202
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1203
	if (&obj->base == NULL) {
1204 1205
		ret = -ENOENT;
		goto unlock;
1206
	}
1207

1208 1209 1210 1211 1212 1213 1214 1215
	/* Try to flush the object off the GPU without holding the lock.
	 * We will repeat the flush holding the lock in the normal manner
	 * to catch cases where we are gazumped.
	 */
	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
	if (ret)
		goto unref;

1216 1217
	if (read_domains & I915_GEM_DOMAIN_GTT) {
		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1218 1219 1220 1221 1222 1223 1224

		/* Silently promote "you're not bound, there was nothing to do"
		 * to success, since the client was just asking us to
		 * make sure everything was done.
		 */
		if (ret == -EINVAL)
			ret = 0;
1225
	} else {
1226
		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1227 1228
	}

1229
unref:
1230
	drm_gem_object_unreference(&obj->base);
1231
unlock:
1232 1233 1234 1235 1236 1237 1238 1239 1240
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Called when user space has done writes to this buffer
 */
int
i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1241
			 struct drm_file *file)
1242 1243
{
	struct drm_i915_gem_sw_finish *args = data;
1244
	struct drm_i915_gem_object *obj;
1245 1246
	int ret = 0;

1247
	ret = i915_mutex_lock_interruptible(dev);
1248
	if (ret)
1249
		return ret;
1250

1251
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1252
	if (&obj->base == NULL) {
1253 1254
		ret = -ENOENT;
		goto unlock;
1255 1256 1257
	}

	/* Pinned buffers may be scanout, so flush the cache */
1258
	if (obj->pin_count)
1259 1260
		i915_gem_object_flush_cpu_write_domain(obj);

1261
	drm_gem_object_unreference(&obj->base);
1262
unlock:
1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Maps the contents of an object, returning the address it is mapped
 * into.
 *
 * While the mapping holds a reference on the contents of the object, it doesn't
 * imply a ref on the object itself.
 */
int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1276
		    struct drm_file *file)
1277 1278 1279 1280 1281
{
	struct drm_i915_gem_mmap *args = data;
	struct drm_gem_object *obj;
	unsigned long addr;

1282
	obj = drm_gem_object_lookup(dev, file, args->handle);
1283
	if (obj == NULL)
1284
		return -ENOENT;
1285

1286 1287 1288 1289 1290 1291 1292 1293
	/* prime objects have no backing filp to GEM mmap
	 * pages from.
	 */
	if (!obj->filp) {
		drm_gem_object_unreference_unlocked(obj);
		return -EINVAL;
	}

1294
	addr = vm_mmap(obj->filp, 0, args->size,
1295 1296
		       PROT_READ | PROT_WRITE, MAP_SHARED,
		       args->offset);
1297
	drm_gem_object_unreference_unlocked(obj);
1298 1299 1300 1301 1302 1303 1304 1305
	if (IS_ERR((void *)addr))
		return addr;

	args->addr_ptr = (uint64_t) addr;

	return 0;
}

1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
/**
 * i915_gem_fault - fault a page into the GTT
 * vma: VMA in question
 * vmf: fault info
 *
 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
 * from userspace.  The fault handler takes care of binding the object to
 * the GTT (if needed), allocating and programming a fence register (again,
 * only if needed based on whether the old reg is still valid or the object
 * is tiled) and inserting a new PTE into the faulting process.
 *
 * Note that the faulting process may involve evicting existing objects
 * from the GTT and/or fence registers to make room.  So performance may
 * suffer if the GTT working set is large or there are few fence registers
 * left.
 */
int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
1324 1325
	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
	struct drm_device *dev = obj->base.dev;
1326
	drm_i915_private_t *dev_priv = dev->dev_private;
1327 1328 1329
	pgoff_t page_offset;
	unsigned long pfn;
	int ret = 0;
1330
	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1331 1332 1333 1334 1335

	/* We don't use vmf->pgoff since that has the fake offset */
	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
		PAGE_SHIFT;

1336 1337 1338
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto out;
1339

C
Chris Wilson 已提交
1340 1341
	trace_i915_gem_object_fault(obj, page_offset, true, write);

1342 1343 1344 1345 1346 1347
	/* Access to snoopable pages through the GTT is incoherent. */
	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
		ret = -EINVAL;
		goto unlock;
	}

1348
	/* Now bind it into the GTT if needed */
1349 1350 1351
	ret = i915_gem_object_pin(obj, 0, true, false);
	if (ret)
		goto unlock;
1352

1353 1354 1355
	ret = i915_gem_object_set_to_gtt_domain(obj, write);
	if (ret)
		goto unpin;
1356

1357
	ret = i915_gem_object_get_fence(obj);
1358
	if (ret)
1359
		goto unpin;
1360

1361 1362
	obj->fault_mappable = true;

1363 1364 1365
	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
	pfn >>= PAGE_SHIFT;
	pfn += page_offset;
1366 1367 1368

	/* Finally, remap it using the new GTT offset */
	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1369 1370
unpin:
	i915_gem_object_unpin(obj);
1371
unlock:
1372
	mutex_unlock(&dev->struct_mutex);
1373
out:
1374
	switch (ret) {
1375
	case -EIO:
1376 1377 1378
		/* If this -EIO is due to a gpu hang, give the reset code a
		 * chance to clean up the mess. Otherwise return the proper
		 * SIGBUS. */
1379
		if (i915_terminally_wedged(&dev_priv->gpu_error))
1380
			return VM_FAULT_SIGBUS;
1381
	case -EAGAIN:
1382 1383 1384 1385 1386 1387 1388
		/* Give the error handler a chance to run and move the
		 * objects off the GPU active list. Next time we service the
		 * fault, we should be able to transition the page into the
		 * GTT without touching the GPU (and so avoid further
		 * EIO/EGAIN). If the GPU is wedged, then there is no issue
		 * with coherency, just lost writes.
		 */
1389
		set_need_resched();
1390 1391
	case 0:
	case -ERESTARTSYS:
1392
	case -EINTR:
1393 1394 1395 1396 1397
	case -EBUSY:
		/*
		 * EBUSY is ok: this just means that another thread
		 * already did the job.
		 */
1398
		return VM_FAULT_NOPAGE;
1399 1400
	case -ENOMEM:
		return VM_FAULT_OOM;
1401 1402
	case -ENOSPC:
		return VM_FAULT_SIGBUS;
1403
	default:
1404
		WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1405
		return VM_FAULT_SIGBUS;
1406 1407 1408
	}
}

1409 1410 1411 1412
/**
 * i915_gem_release_mmap - remove physical page mappings
 * @obj: obj in question
 *
1413
 * Preserve the reservation of the mmapping with the DRM core code, but
1414 1415 1416 1417 1418 1419 1420 1421 1422
 * relinquish ownership of the pages back to the system.
 *
 * It is vital that we remove the page mapping if we have mapped a tiled
 * object through the GTT and then lose the fence register due to
 * resource pressure. Similarly if the object has been moved out of the
 * aperture, than pages mapped into userspace must be revoked. Removing the
 * mapping will then trigger a page fault on the next user access, allowing
 * fixup by i915_gem_fault().
 */
1423
void
1424
i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1425
{
1426 1427
	if (!obj->fault_mappable)
		return;
1428

1429 1430 1431 1432
	if (obj->base.dev->dev_mapping)
		unmap_mapping_range(obj->base.dev->dev_mapping,
				    (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
				    obj->base.size, 1);
1433

1434
	obj->fault_mappable = false;
1435 1436
}

1437
uint32_t
1438
i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1439
{
1440
	uint32_t gtt_size;
1441 1442

	if (INTEL_INFO(dev)->gen >= 4 ||
1443 1444
	    tiling_mode == I915_TILING_NONE)
		return size;
1445 1446 1447

	/* Previous chips need a power-of-two fence region when tiling */
	if (INTEL_INFO(dev)->gen == 3)
1448
		gtt_size = 1024*1024;
1449
	else
1450
		gtt_size = 512*1024;
1451

1452 1453
	while (gtt_size < size)
		gtt_size <<= 1;
1454

1455
	return gtt_size;
1456 1457
}

1458 1459 1460 1461 1462
/**
 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 * @obj: object to check
 *
 * Return the required GTT alignment for an object, taking into account
1463
 * potential fence register mapping.
1464
 */
1465 1466 1467
uint32_t
i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
			   int tiling_mode, bool fenced)
1468 1469 1470 1471 1472
{
	/*
	 * Minimum alignment is 4k (GTT page size), but might be greater
	 * if a fence register is needed for the object.
	 */
1473
	if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1474
	    tiling_mode == I915_TILING_NONE)
1475 1476
		return 4096;

1477 1478 1479 1480
	/*
	 * Previous chips need to be aligned to the size of the smallest
	 * fence register that can contain the object.
	 */
1481
	return i915_gem_get_gtt_size(dev, size, tiling_mode);
1482 1483
}

1484 1485 1486 1487 1488 1489 1490 1491
static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	int ret;

	if (obj->base.map_list.map)
		return 0;

1492 1493
	dev_priv->mm.shrinker_no_lock_stealing = true;

1494 1495
	ret = drm_gem_create_mmap_offset(&obj->base);
	if (ret != -ENOSPC)
1496
		goto out;
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507

	/* Badly fragmented mmap space? The only way we can recover
	 * space is by destroying unwanted objects. We can't randomly release
	 * mmap_offsets as userspace expects them to be persistent for the
	 * lifetime of the objects. The closest we can is to release the
	 * offsets on purgeable objects by truncating it and marking it purged,
	 * which prevents userspace from ever using that object again.
	 */
	i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
	ret = drm_gem_create_mmap_offset(&obj->base);
	if (ret != -ENOSPC)
1508
		goto out;
1509 1510

	i915_gem_shrink_all(dev_priv);
1511 1512 1513 1514 1515
	ret = drm_gem_create_mmap_offset(&obj->base);
out:
	dev_priv->mm.shrinker_no_lock_stealing = false;

	return ret;
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525
}

static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
{
	if (!obj->base.map_list.map)
		return;

	drm_gem_free_mmap_offset(&obj->base);
}

1526
int
1527 1528 1529 1530
i915_gem_mmap_gtt(struct drm_file *file,
		  struct drm_device *dev,
		  uint32_t handle,
		  uint64_t *offset)
1531
{
1532
	struct drm_i915_private *dev_priv = dev->dev_private;
1533
	struct drm_i915_gem_object *obj;
1534 1535
	int ret;

1536
	ret = i915_mutex_lock_interruptible(dev);
1537
	if (ret)
1538
		return ret;
1539

1540
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1541
	if (&obj->base == NULL) {
1542 1543 1544
		ret = -ENOENT;
		goto unlock;
	}
1545

B
Ben Widawsky 已提交
1546
	if (obj->base.size > dev_priv->gtt.mappable_end) {
1547
		ret = -E2BIG;
1548
		goto out;
1549 1550
	}

1551
	if (obj->madv != I915_MADV_WILLNEED) {
1552
		DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1553 1554
		ret = -EINVAL;
		goto out;
1555 1556
	}

1557 1558 1559
	ret = i915_gem_object_create_mmap_offset(obj);
	if (ret)
		goto out;
1560

1561
	*offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1562

1563
out:
1564
	drm_gem_object_unreference(&obj->base);
1565
unlock:
1566
	mutex_unlock(&dev->struct_mutex);
1567
	return ret;
1568 1569
}

1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593
/**
 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 * @dev: DRM device
 * @data: GTT mapping ioctl data
 * @file: GEM object info
 *
 * Simply returns the fake offset to userspace so it can mmap it.
 * The mmap call will end up in drm_gem_mmap(), which will set things
 * up so we can get faults in the handler above.
 *
 * The fault handler will take care of binding the object into the GTT
 * (since it may have been evicted to make room for something), allocating
 * a fence register, and mapping the appropriate aperture address into
 * userspace.
 */
int
i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file)
{
	struct drm_i915_gem_mmap_gtt *args = data;

	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
}

D
Daniel Vetter 已提交
1594 1595 1596
/* Immediately discard the backing storage */
static void
i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1597 1598 1599
{
	struct inode *inode;

1600
	i915_gem_object_free_mmap_offset(obj);
1601

1602 1603
	if (obj->base.filp == NULL)
		return;
1604

D
Daniel Vetter 已提交
1605 1606 1607 1608 1609
	/* Our goal here is to return as much of the memory as
	 * is possible back to the system as we are called from OOM.
	 * To do this we must instruct the shmfs to drop all of its
	 * backing pages, *now*.
	 */
A
Al Viro 已提交
1610
	inode = file_inode(obj->base.filp);
D
Daniel Vetter 已提交
1611
	shmem_truncate_range(inode, 0, (loff_t)-1);
1612

D
Daniel Vetter 已提交
1613 1614
	obj->madv = __I915_MADV_PURGED;
}
1615

D
Daniel Vetter 已提交
1616 1617 1618 1619
static inline int
i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
{
	return obj->madv == I915_MADV_DONTNEED;
1620 1621
}

1622
static void
1623
i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1624
{
1625 1626
	struct sg_page_iter sg_iter;
	int ret;
1627

1628
	BUG_ON(obj->madv == __I915_MADV_PURGED);
1629

C
Chris Wilson 已提交
1630 1631 1632 1633 1634 1635 1636 1637 1638 1639
	ret = i915_gem_object_set_to_cpu_domain(obj, true);
	if (ret) {
		/* In the event of a disaster, abandon all caches and
		 * hope for the best.
		 */
		WARN_ON(ret != -EIO);
		i915_gem_clflush_object(obj);
		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	}

1640
	if (i915_gem_object_needs_bit17_swizzle(obj))
1641 1642
		i915_gem_object_save_bit_17_swizzle(obj);

1643 1644
	if (obj->madv == I915_MADV_DONTNEED)
		obj->dirty = 0;
1645

1646
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1647
		struct page *page = sg_page_iter_page(&sg_iter);
1648

1649
		if (obj->dirty)
1650
			set_page_dirty(page);
1651

1652
		if (obj->madv == I915_MADV_WILLNEED)
1653
			mark_page_accessed(page);
1654

1655
		page_cache_release(page);
1656
	}
1657
	obj->dirty = 0;
1658

1659 1660
	sg_free_table(obj->pages);
	kfree(obj->pages);
1661
}
C
Chris Wilson 已提交
1662

1663
int
1664 1665 1666 1667
i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
{
	const struct drm_i915_gem_object_ops *ops = obj->ops;

1668
	if (obj->pages == NULL)
1669 1670
		return 0;

1671
	BUG_ON(i915_gem_obj_ggtt_bound(obj));
C
Chris Wilson 已提交
1672

1673 1674 1675
	if (obj->pages_pin_count)
		return -EBUSY;

1676 1677 1678
	/* ->put_pages might need to allocate memory for the bit17 swizzle
	 * array, hence protect them from being reaped by removing them from gtt
	 * lists early. */
1679
	list_del(&obj->global_list);
1680

1681
	ops->put_pages(obj);
1682
	obj->pages = NULL;
1683

C
Chris Wilson 已提交
1684 1685 1686 1687 1688 1689 1690
	if (i915_gem_object_is_purgeable(obj))
		i915_gem_object_truncate(obj);

	return 0;
}

static long
1691 1692
__i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
		  bool purgeable_only)
C
Chris Wilson 已提交
1693 1694 1695 1696 1697 1698
{
	struct drm_i915_gem_object *obj, *next;
	long count = 0;

	list_for_each_entry_safe(obj, next,
				 &dev_priv->mm.unbound_list,
1699
				 global_list) {
1700
		if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
1701
		    i915_gem_object_put_pages(obj) == 0) {
C
Chris Wilson 已提交
1702 1703 1704 1705 1706 1707 1708 1709 1710
			count += obj->base.size >> PAGE_SHIFT;
			if (count >= target)
				return count;
		}
	}

	list_for_each_entry_safe(obj, next,
				 &dev_priv->mm.inactive_list,
				 mm_list) {
1711
		if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
C
Chris Wilson 已提交
1712
		    i915_gem_object_unbind(obj) == 0 &&
1713
		    i915_gem_object_put_pages(obj) == 0) {
C
Chris Wilson 已提交
1714 1715 1716 1717 1718 1719 1720 1721 1722
			count += obj->base.size >> PAGE_SHIFT;
			if (count >= target)
				return count;
		}
	}

	return count;
}

1723 1724 1725 1726 1727 1728
static long
i915_gem_purge(struct drm_i915_private *dev_priv, long target)
{
	return __i915_gem_shrink(dev_priv, target, true);
}

C
Chris Wilson 已提交
1729 1730 1731 1732 1733 1734 1735
static void
i915_gem_shrink_all(struct drm_i915_private *dev_priv)
{
	struct drm_i915_gem_object *obj, *next;

	i915_gem_evict_everything(dev_priv->dev);

1736 1737
	list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
				 global_list)
1738
		i915_gem_object_put_pages(obj);
D
Daniel Vetter 已提交
1739 1740
}

1741
static int
C
Chris Wilson 已提交
1742
i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
1743
{
C
Chris Wilson 已提交
1744
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1745 1746
	int page_count, i;
	struct address_space *mapping;
1747 1748
	struct sg_table *st;
	struct scatterlist *sg;
1749
	struct sg_page_iter sg_iter;
1750
	struct page *page;
1751
	unsigned long last_pfn = 0;	/* suppress gcc warning */
C
Chris Wilson 已提交
1752
	gfp_t gfp;
1753

C
Chris Wilson 已提交
1754 1755 1756 1757 1758 1759 1760
	/* Assert that the object is not currently in any GPU domain. As it
	 * wasn't in the GTT, there shouldn't be any way it could have been in
	 * a GPU cache
	 */
	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);

1761 1762 1763 1764
	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (st == NULL)
		return -ENOMEM;

1765
	page_count = obj->base.size / PAGE_SIZE;
1766 1767 1768
	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		sg_free_table(st);
		kfree(st);
1769
		return -ENOMEM;
1770
	}
1771

1772 1773 1774 1775 1776
	/* Get the list of pages out of our struct file.  They'll be pinned
	 * at this point until we release them.
	 *
	 * Fail silently without starting the shrinker
	 */
A
Al Viro 已提交
1777
	mapping = file_inode(obj->base.filp)->i_mapping;
C
Chris Wilson 已提交
1778
	gfp = mapping_gfp_mask(mapping);
1779
	gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
C
Chris Wilson 已提交
1780
	gfp &= ~(__GFP_IO | __GFP_WAIT);
1781 1782 1783
	sg = st->sgl;
	st->nents = 0;
	for (i = 0; i < page_count; i++) {
C
Chris Wilson 已提交
1784 1785 1786 1787 1788 1789 1790 1791 1792 1793
		page = shmem_read_mapping_page_gfp(mapping, i, gfp);
		if (IS_ERR(page)) {
			i915_gem_purge(dev_priv, page_count);
			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
		}
		if (IS_ERR(page)) {
			/* We've tried hard to allocate the memory by reaping
			 * our own buffer, now let the real VM do its job and
			 * go down in flames if truly OOM.
			 */
1794
			gfp &= ~(__GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD);
C
Chris Wilson 已提交
1795 1796 1797 1798 1799 1800 1801
			gfp |= __GFP_IO | __GFP_WAIT;

			i915_gem_shrink_all(dev_priv);
			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
			if (IS_ERR(page))
				goto err_pages;

1802
			gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
C
Chris Wilson 已提交
1803 1804
			gfp &= ~(__GFP_IO | __GFP_WAIT);
		}
1805 1806 1807 1808 1809 1810 1811 1812
#ifdef CONFIG_SWIOTLB
		if (swiotlb_nr_tbl()) {
			st->nents++;
			sg_set_page(sg, page, PAGE_SIZE, 0);
			sg = sg_next(sg);
			continue;
		}
#endif
1813 1814 1815 1816 1817 1818 1819 1820 1821
		if (!i || page_to_pfn(page) != last_pfn + 1) {
			if (i)
				sg = sg_next(sg);
			st->nents++;
			sg_set_page(sg, page, PAGE_SIZE, 0);
		} else {
			sg->length += PAGE_SIZE;
		}
		last_pfn = page_to_pfn(page);
1822
	}
1823 1824 1825 1826
#ifdef CONFIG_SWIOTLB
	if (!swiotlb_nr_tbl())
#endif
		sg_mark_end(sg);
1827 1828
	obj->pages = st;

1829
	if (i915_gem_object_needs_bit17_swizzle(obj))
1830 1831 1832 1833 1834
		i915_gem_object_do_bit_17_swizzle(obj);

	return 0;

err_pages:
1835 1836
	sg_mark_end(sg);
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
1837
		page_cache_release(sg_page_iter_page(&sg_iter));
1838 1839
	sg_free_table(st);
	kfree(st);
1840
	return PTR_ERR(page);
1841 1842
}

1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856
/* Ensure that the associated pages are gathered from the backing storage
 * and pinned into our object. i915_gem_object_get_pages() may be called
 * multiple times before they are released by a single call to
 * i915_gem_object_put_pages() - once the pages are no longer referenced
 * either as a result of memory pressure (reaping pages under the shrinker)
 * or as the object is itself released.
 */
int
i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	const struct drm_i915_gem_object_ops *ops = obj->ops;
	int ret;

1857
	if (obj->pages)
1858 1859
		return 0;

1860 1861 1862 1863 1864
	if (obj->madv != I915_MADV_WILLNEED) {
		DRM_ERROR("Attempting to obtain a purgeable object\n");
		return -EINVAL;
	}

1865 1866
	BUG_ON(obj->pages_pin_count);

1867 1868 1869 1870
	ret = ops->get_pages(obj);
	if (ret)
		return ret;

1871
	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
1872
	return 0;
1873 1874
}

1875
void
1876
i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1877
			       struct intel_ring_buffer *ring)
1878
{
1879
	struct drm_device *dev = obj->base.dev;
1880
	struct drm_i915_private *dev_priv = dev->dev_private;
1881
	u32 seqno = intel_ring_get_seqno(ring);
1882

1883
	BUG_ON(ring == NULL);
1884
	obj->ring = ring;
1885 1886

	/* Add a reference if we're newly entering the active list. */
1887 1888 1889
	if (!obj->active) {
		drm_gem_object_reference(&obj->base);
		obj->active = 1;
1890
	}
1891

1892
	/* Move from whatever list we were on to the tail of execution. */
1893 1894
	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
	list_move_tail(&obj->ring_list, &ring->active_list);
1895

1896
	obj->last_read_seqno = seqno;
1897

1898
	if (obj->fenced_gpu_access) {
1899 1900
		obj->last_fenced_seqno = seqno;

1901 1902 1903 1904 1905 1906 1907 1908
		/* Bump MRU to take account of the delayed flush */
		if (obj->fence_reg != I915_FENCE_REG_NONE) {
			struct drm_i915_fence_reg *reg;

			reg = &dev_priv->fence_regs[obj->fence_reg];
			list_move_tail(&reg->lru_list,
				       &dev_priv->mm.fence_list);
		}
1909 1910 1911 1912 1913
	}
}

static void
i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1914
{
1915
	struct drm_device *dev = obj->base.dev;
1916
	struct drm_i915_private *dev_priv = dev->dev_private;
1917

1918
	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
1919
	BUG_ON(!obj->active);
1920

1921
	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1922

1923
	list_del_init(&obj->ring_list);
1924 1925
	obj->ring = NULL;

1926 1927 1928 1929 1930
	obj->last_read_seqno = 0;
	obj->last_write_seqno = 0;
	obj->base.write_domain = 0;

	obj->last_fenced_seqno = 0;
1931 1932 1933 1934 1935 1936
	obj->fenced_gpu_access = false;

	obj->active = 0;
	drm_gem_object_unreference(&obj->base);

	WARN_ON(i915_verify_lists(dev));
1937
}
1938

1939
static int
1940
i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
1941
{
1942 1943 1944
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_ring_buffer *ring;
	int ret, i, j;
1945

1946
	/* Carefully retire all requests without writing to the rings */
1947
	for_each_ring(ring, dev_priv, i) {
1948 1949 1950
		ret = intel_ring_idle(ring);
		if (ret)
			return ret;
1951 1952
	}
	i915_gem_retire_requests(dev);
1953 1954

	/* Finally reset hw state */
1955
	for_each_ring(ring, dev_priv, i) {
1956
		intel_ring_init_seqno(ring, seqno);
1957

1958 1959 1960
		for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
			ring->sync_seqno[j] = 0;
	}
1961

1962
	return 0;
1963 1964
}

1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990
int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	if (seqno == 0)
		return -EINVAL;

	/* HWS page needs to be set less than what we
	 * will inject to ring
	 */
	ret = i915_gem_init_seqno(dev, seqno - 1);
	if (ret)
		return ret;

	/* Carefully set the last_seqno value so that wrap
	 * detection still works
	 */
	dev_priv->next_seqno = seqno;
	dev_priv->last_seqno = seqno - 1;
	if (dev_priv->last_seqno == 0)
		dev_priv->last_seqno--;

	return 0;
}

1991 1992
int
i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
1993
{
1994 1995 1996 1997
	struct drm_i915_private *dev_priv = dev->dev_private;

	/* reserve 0 for non-seqno */
	if (dev_priv->next_seqno == 0) {
1998
		int ret = i915_gem_init_seqno(dev, 0);
1999 2000
		if (ret)
			return ret;
2001

2002 2003
		dev_priv->next_seqno = 1;
	}
2004

2005
	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2006
	return 0;
2007 2008
}

2009 2010
int __i915_add_request(struct intel_ring_buffer *ring,
		       struct drm_file *file,
2011
		       struct drm_i915_gem_object *obj,
2012
		       u32 *out_seqno)
2013
{
C
Chris Wilson 已提交
2014
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
2015
	struct drm_i915_gem_request *request;
2016
	u32 request_ring_position, request_start;
2017
	int was_empty;
2018 2019
	int ret;

2020
	request_start = intel_ring_get_tail(ring);
2021 2022 2023 2024 2025 2026 2027
	/*
	 * Emit any outstanding flushes - execbuf can fail to emit the flush
	 * after having emitted the batchbuffer command. Hence we need to fix
	 * things up similar to emitting the lazy request. The difference here
	 * is that the flush _must_ happen before the next request, no matter
	 * what.
	 */
2028 2029 2030
	ret = intel_ring_flush_all_caches(ring);
	if (ret)
		return ret;
2031

2032 2033 2034
	request = kmalloc(sizeof(*request), GFP_KERNEL);
	if (request == NULL)
		return -ENOMEM;
2035

2036

2037 2038 2039 2040 2041 2042 2043
	/* Record the position of the start of the request so that
	 * should we detect the updated seqno part-way through the
	 * GPU processing the request, we never over-estimate the
	 * position of the head.
	 */
	request_ring_position = intel_ring_get_tail(ring);

2044
	ret = ring->add_request(ring);
2045 2046 2047 2048
	if (ret) {
		kfree(request);
		return ret;
	}
2049

2050
	request->seqno = intel_ring_get_seqno(ring);
2051
	request->ring = ring;
2052
	request->head = request_start;
2053
	request->tail = request_ring_position;
2054
	request->ctx = ring->last_context;
2055 2056 2057 2058 2059 2060 2061 2062
	request->batch_obj = obj;

	/* Whilst this request exists, batch_obj will be on the
	 * active_list, and so will hold the active reference. Only when this
	 * request is retired will the the batch_obj be moved onto the
	 * inactive_list and lose its active reference. Hence we do not need
	 * to explicitly hold another reference here.
	 */
2063 2064 2065 2066

	if (request->ctx)
		i915_gem_context_reference(request->ctx);

2067
	request->emitted_jiffies = jiffies;
2068 2069
	was_empty = list_empty(&ring->request_list);
	list_add_tail(&request->list, &ring->request_list);
2070
	request->file_priv = NULL;
2071

C
Chris Wilson 已提交
2072 2073 2074
	if (file) {
		struct drm_i915_file_private *file_priv = file->driver_priv;

2075
		spin_lock(&file_priv->mm.lock);
2076
		request->file_priv = file_priv;
2077
		list_add_tail(&request->client_list,
2078
			      &file_priv->mm.request_list);
2079
		spin_unlock(&file_priv->mm.lock);
2080
	}
2081

2082
	trace_i915_gem_request_add(ring, request->seqno);
2083
	ring->outstanding_lazy_request = 0;
C
Chris Wilson 已提交
2084

2085
	if (!dev_priv->ums.mm_suspended) {
2086
		if (i915_enable_hangcheck) {
2087
			mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2088
				  round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
2089
		}
2090
		if (was_empty) {
2091
			queue_delayed_work(dev_priv->wq,
2092 2093
					   &dev_priv->mm.retire_work,
					   round_jiffies_up_relative(HZ));
2094 2095
			intel_mark_busy(dev_priv->dev);
		}
B
Ben Gamari 已提交
2096
	}
2097

2098
	if (out_seqno)
2099
		*out_seqno = request->seqno;
2100
	return 0;
2101 2102
}

2103 2104
static inline void
i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2105
{
2106
	struct drm_i915_file_private *file_priv = request->file_priv;
2107

2108 2109
	if (!file_priv)
		return;
C
Chris Wilson 已提交
2110

2111
	spin_lock(&file_priv->mm.lock);
2112 2113 2114 2115
	if (request->file_priv) {
		list_del(&request->client_list);
		request->file_priv = NULL;
	}
2116
	spin_unlock(&file_priv->mm.lock);
2117 2118
}

2119 2120
static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj)
{
2121 2122
	if (acthd >= i915_gem_obj_ggtt_offset(obj) &&
	    acthd < i915_gem_obj_ggtt_offset(obj) + obj->base.size)
2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179
		return true;

	return false;
}

static bool i915_head_inside_request(const u32 acthd_unmasked,
				     const u32 request_start,
				     const u32 request_end)
{
	const u32 acthd = acthd_unmasked & HEAD_ADDR;

	if (request_start < request_end) {
		if (acthd >= request_start && acthd < request_end)
			return true;
	} else if (request_start > request_end) {
		if (acthd >= request_start || acthd < request_end)
			return true;
	}

	return false;
}

static bool i915_request_guilty(struct drm_i915_gem_request *request,
				const u32 acthd, bool *inside)
{
	/* There is a possibility that unmasked head address
	 * pointing inside the ring, matches the batch_obj address range.
	 * However this is extremely unlikely.
	 */

	if (request->batch_obj) {
		if (i915_head_inside_object(acthd, request->batch_obj)) {
			*inside = true;
			return true;
		}
	}

	if (i915_head_inside_request(acthd, request->head, request->tail)) {
		*inside = false;
		return true;
	}

	return false;
}

static void i915_set_reset_status(struct intel_ring_buffer *ring,
				  struct drm_i915_gem_request *request,
				  u32 acthd)
{
	struct i915_ctx_hang_stats *hs = NULL;
	bool inside, guilty;

	/* Innocent until proven guilty */
	guilty = false;

	if (ring->hangcheck.action != wait &&
	    i915_request_guilty(request, acthd, &inside)) {
2180
		DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
2181 2182 2183
			  ring->name,
			  inside ? "inside" : "flushing",
			  request->batch_obj ?
2184
			  i915_gem_obj_ggtt_offset(request->batch_obj) : 0,
2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206
			  request->ctx ? request->ctx->id : 0,
			  acthd);

		guilty = true;
	}

	/* If contexts are disabled or this is the default context, use
	 * file_priv->reset_state
	 */
	if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
		hs = &request->ctx->hang_stats;
	else if (request->file_priv)
		hs = &request->file_priv->hang_stats;

	if (hs) {
		if (guilty)
			hs->batch_active++;
		else
			hs->batch_pending++;
	}
}

2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217
static void i915_gem_free_request(struct drm_i915_gem_request *request)
{
	list_del(&request->list);
	i915_gem_request_remove_from_client(request);

	if (request->ctx)
		i915_gem_context_unreference(request->ctx);

	kfree(request);
}

2218 2219
static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
				      struct intel_ring_buffer *ring)
2220
{
2221 2222 2223 2224 2225 2226
	u32 completed_seqno;
	u32 acthd;

	acthd = intel_ring_get_active_head(ring);
	completed_seqno = ring->get_seqno(ring, false);

2227 2228
	while (!list_empty(&ring->request_list)) {
		struct drm_i915_gem_request *request;
2229

2230 2231 2232
		request = list_first_entry(&ring->request_list,
					   struct drm_i915_gem_request,
					   list);
2233

2234 2235 2236
		if (request->seqno > completed_seqno)
			i915_set_reset_status(ring, request, acthd);

2237
		i915_gem_free_request(request);
2238
	}
2239

2240
	while (!list_empty(&ring->active_list)) {
2241
		struct drm_i915_gem_object *obj;
2242

2243 2244 2245
		obj = list_first_entry(&ring->active_list,
				       struct drm_i915_gem_object,
				       ring_list);
2246

2247
		i915_gem_object_move_to_inactive(obj);
2248 2249 2250
	}
}

2251 2252 2253 2254 2255
static void i915_gem_reset_fences(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int i;

2256
	for (i = 0; i < dev_priv->num_fence_regs; i++) {
2257
		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2258

2259 2260
		if (reg->obj)
			i915_gem_object_fence_lost(reg->obj);
2261

2262 2263
		i915_gem_write_fence(dev, i, NULL);

2264 2265 2266
		reg->pin_count = 0;
		reg->obj = NULL;
		INIT_LIST_HEAD(&reg->lru_list);
2267
	}
2268 2269

	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
2270 2271
}

2272
void i915_gem_reset(struct drm_device *dev)
2273
{
2274
	struct drm_i915_private *dev_priv = dev->dev_private;
2275
	struct drm_i915_gem_object *obj;
2276
	struct intel_ring_buffer *ring;
2277
	int i;
2278

2279 2280
	for_each_ring(ring, dev_priv, i)
		i915_gem_reset_ring_lists(dev_priv, ring);
2281 2282 2283 2284

	/* Move everything out of the GPU domains to ensure we do any
	 * necessary invalidation upon reuse.
	 */
2285
	list_for_each_entry(obj,
2286
			    &dev_priv->mm.inactive_list,
2287
			    mm_list)
2288
	{
2289
		obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2290
	}
2291 2292

	/* The fence registers are invalidated so clear them out */
2293
	i915_gem_reset_fences(dev);
2294 2295 2296 2297 2298
}

/**
 * This function clears the request list as sequence numbers are passed.
 */
2299
void
C
Chris Wilson 已提交
2300
i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2301 2302 2303
{
	uint32_t seqno;

C
Chris Wilson 已提交
2304
	if (list_empty(&ring->request_list))
2305 2306
		return;

C
Chris Wilson 已提交
2307
	WARN_ON(i915_verify_lists(ring->dev));
2308

2309
	seqno = ring->get_seqno(ring, true);
2310

2311
	while (!list_empty(&ring->request_list)) {
2312 2313
		struct drm_i915_gem_request *request;

2314
		request = list_first_entry(&ring->request_list,
2315 2316 2317
					   struct drm_i915_gem_request,
					   list);

2318
		if (!i915_seqno_passed(seqno, request->seqno))
2319 2320
			break;

C
Chris Wilson 已提交
2321
		trace_i915_gem_request_retire(ring, request->seqno);
2322 2323 2324 2325 2326 2327
		/* We know the GPU must have read the request to have
		 * sent us the seqno + interrupt, so use the position
		 * of tail of the request to update the last known position
		 * of the GPU head.
		 */
		ring->last_retired_head = request->tail;
2328

2329
		i915_gem_free_request(request);
2330
	}
2331

2332 2333 2334 2335
	/* Move any buffers on the active list that are no longer referenced
	 * by the ringbuffer to the flushing/inactive lists as appropriate.
	 */
	while (!list_empty(&ring->active_list)) {
2336
		struct drm_i915_gem_object *obj;
2337

2338
		obj = list_first_entry(&ring->active_list,
2339 2340
				      struct drm_i915_gem_object,
				      ring_list);
2341

2342
		if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2343
			break;
2344

2345
		i915_gem_object_move_to_inactive(obj);
2346
	}
2347

C
Chris Wilson 已提交
2348 2349
	if (unlikely(ring->trace_irq_seqno &&
		     i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2350
		ring->irq_put(ring);
C
Chris Wilson 已提交
2351
		ring->trace_irq_seqno = 0;
2352
	}
2353

C
Chris Wilson 已提交
2354
	WARN_ON(i915_verify_lists(ring->dev));
2355 2356
}

2357 2358 2359 2360
void
i915_gem_retire_requests(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2361
	struct intel_ring_buffer *ring;
2362
	int i;
2363

2364 2365
	for_each_ring(ring, dev_priv, i)
		i915_gem_retire_requests_ring(ring);
2366 2367
}

2368
static void
2369 2370 2371 2372
i915_gem_retire_work_handler(struct work_struct *work)
{
	drm_i915_private_t *dev_priv;
	struct drm_device *dev;
2373
	struct intel_ring_buffer *ring;
2374 2375
	bool idle;
	int i;
2376 2377 2378 2379 2380

	dev_priv = container_of(work, drm_i915_private_t,
				mm.retire_work.work);
	dev = dev_priv->dev;

2381 2382
	/* Come back later if the device is busy... */
	if (!mutex_trylock(&dev->struct_mutex)) {
2383 2384
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
				   round_jiffies_up_relative(HZ));
2385 2386
		return;
	}
2387

2388
	i915_gem_retire_requests(dev);
2389

2390 2391
	/* Send a periodic flush down the ring so we don't hold onto GEM
	 * objects indefinitely.
2392
	 */
2393
	idle = true;
2394
	for_each_ring(ring, dev_priv, i) {
2395
		if (ring->gpu_caches_dirty)
2396
			i915_add_request(ring, NULL);
2397 2398

		idle &= list_empty(&ring->request_list);
2399 2400
	}

2401
	if (!dev_priv->ums.mm_suspended && !idle)
2402 2403
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
				   round_jiffies_up_relative(HZ));
2404 2405
	if (idle)
		intel_mark_idle(dev);
2406

2407 2408 2409
	mutex_unlock(&dev->struct_mutex);
}

2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420
/**
 * Ensures that an object will eventually get non-busy by flushing any required
 * write domains, emitting any outstanding lazy request and retiring and
 * completed requests.
 */
static int
i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
{
	int ret;

	if (obj->active) {
2421
		ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2422 2423 2424 2425 2426 2427 2428 2429 2430
		if (ret)
			return ret;

		i915_gem_retire_requests_ring(obj->ring);
	}

	return 0;
}

2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455
/**
 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
 * @DRM_IOCTL_ARGS: standard ioctl arguments
 *
 * Returns 0 if successful, else an error is returned with the remaining time in
 * the timeout parameter.
 *  -ETIME: object is still busy after timeout
 *  -ERESTARTSYS: signal interrupted the wait
 *  -ENONENT: object doesn't exist
 * Also possible, but rare:
 *  -EAGAIN: GPU wedged
 *  -ENOMEM: damn
 *  -ENODEV: Internal IRQ fail
 *  -E?: The add request failed
 *
 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
 * non-zero timeout parameter the wait ioctl will wait for the given number of
 * nanoseconds on an object becoming unbusy. Since the wait itself does so
 * without holding struct_mutex the object may become re-busied before this
 * function completes. A similar but shorter * race condition exists in the busy
 * ioctl
 */
int
i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
2456
	drm_i915_private_t *dev_priv = dev->dev_private;
2457 2458 2459
	struct drm_i915_gem_wait *args = data;
	struct drm_i915_gem_object *obj;
	struct intel_ring_buffer *ring = NULL;
2460
	struct timespec timeout_stack, *timeout = NULL;
2461
	unsigned reset_counter;
2462 2463 2464
	u32 seqno = 0;
	int ret = 0;

2465 2466 2467 2468
	if (args->timeout_ns >= 0) {
		timeout_stack = ns_to_timespec(args->timeout_ns);
		timeout = &timeout_stack;
	}
2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479

	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
	if (&obj->base == NULL) {
		mutex_unlock(&dev->struct_mutex);
		return -ENOENT;
	}

2480 2481
	/* Need to make sure the object gets inactive eventually. */
	ret = i915_gem_object_flush_active(obj);
2482 2483 2484 2485
	if (ret)
		goto out;

	if (obj->active) {
2486
		seqno = obj->last_read_seqno;
2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501
		ring = obj->ring;
	}

	if (seqno == 0)
		 goto out;

	/* Do this after OLR check to make sure we make forward progress polling
	 * on this IOCTL with a 0 timeout (like busy ioctl)
	 */
	if (!args->timeout_ns) {
		ret = -ETIME;
		goto out;
	}

	drm_gem_object_unreference(&obj->base);
2502
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2503 2504
	mutex_unlock(&dev->struct_mutex);

2505
	ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
2506
	if (timeout)
2507
		args->timeout_ns = timespec_to_ns(timeout);
2508 2509 2510 2511 2512 2513 2514 2515
	return ret;

out:
	drm_gem_object_unreference(&obj->base);
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527
/**
 * i915_gem_object_sync - sync an object to a ring.
 *
 * @obj: object which may be in use on another ring.
 * @to: ring we wish to use the object on. May be NULL.
 *
 * This code is meant to abstract object synchronization with the GPU.
 * Calling with NULL implies synchronizing the object with the CPU
 * rather than a particular GPU ring.
 *
 * Returns 0 if successful, else propagates up the lower layer error.
 */
2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538
int
i915_gem_object_sync(struct drm_i915_gem_object *obj,
		     struct intel_ring_buffer *to)
{
	struct intel_ring_buffer *from = obj->ring;
	u32 seqno;
	int ret, idx;

	if (from == NULL || to == from)
		return 0;

2539
	if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2540
		return i915_gem_object_wait_rendering(obj, false);
2541 2542 2543

	idx = intel_ring_sync_index(from, to);

2544
	seqno = obj->last_read_seqno;
2545 2546 2547
	if (seqno <= from->sync_seqno[idx])
		return 0;

2548 2549 2550
	ret = i915_gem_check_olr(obj->ring, seqno);
	if (ret)
		return ret;
2551

2552
	ret = to->sync_to(to, from, seqno);
2553
	if (!ret)
2554 2555 2556 2557 2558
		/* We use last_read_seqno because sync_to()
		 * might have just caused seqno wrap under
		 * the radar.
		 */
		from->sync_seqno[idx] = obj->last_read_seqno;
2559

2560
	return ret;
2561 2562
}

2563 2564 2565 2566 2567 2568 2569
static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
{
	u32 old_write_domain, old_read_domains;

	/* Force a pagefault for domain tracking on next user access */
	i915_gem_release_mmap(obj);

2570 2571 2572
	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		return;

2573 2574 2575
	/* Wait for any direct GTT access to complete */
	mb();

2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586
	old_read_domains = obj->base.read_domains;
	old_write_domain = obj->base.write_domain;

	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;

	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);
}

2587 2588 2589
/**
 * Unbinds an object from the GTT aperture.
 */
2590
int
2591
i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2592
{
2593
	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2594
	int ret;
2595

2596
	if (!i915_gem_obj_ggtt_bound(obj))
2597 2598
		return 0;

2599 2600
	if (obj->pin_count)
		return -EBUSY;
2601

2602 2603
	BUG_ON(obj->pages == NULL);

2604
	ret = i915_gem_object_finish_gpu(obj);
2605
	if (ret)
2606 2607 2608 2609 2610 2611
		return ret;
	/* Continue on if we fail due to EIO, the GPU is hung so we
	 * should be safe and we need to cleanup or else we might
	 * cause memory corruption through use-after-free.
	 */

2612
	i915_gem_object_finish_gtt(obj);
2613

2614
	/* release the fence reg _after_ flushing */
2615
	ret = i915_gem_object_put_fence(obj);
2616
	if (ret)
2617
		return ret;
2618

C
Chris Wilson 已提交
2619 2620
	trace_i915_gem_object_unbind(obj);

2621 2622
	if (obj->has_global_gtt_mapping)
		i915_gem_gtt_unbind_object(obj);
2623 2624 2625 2626
	if (obj->has_aliasing_ppgtt_mapping) {
		i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		obj->has_aliasing_ppgtt_mapping = 0;
	}
2627
	i915_gem_gtt_finish_object(obj);
B
Ben Widawsky 已提交
2628
	i915_gem_object_unpin_pages(obj);
2629

C
Chris Wilson 已提交
2630
	list_del(&obj->mm_list);
2631
	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2632
	/* Avoid an unnecessary call to unbind on rebind. */
2633
	obj->map_and_fenceable = true;
2634

2635
	drm_mm_remove_node(&obj->gtt_space);
2636

2637
	return 0;
2638 2639
}

2640
int i915_gpu_idle(struct drm_device *dev)
2641 2642
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2643
	struct intel_ring_buffer *ring;
2644
	int ret, i;
2645 2646

	/* Flush everything onto the inactive list. */
2647
	for_each_ring(ring, dev_priv, i) {
2648 2649 2650 2651
		ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
		if (ret)
			return ret;

2652
		ret = intel_ring_idle(ring);
2653 2654 2655
		if (ret)
			return ret;
	}
2656

2657
	return 0;
2658 2659
}

2660 2661
static void i965_write_fence_reg(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
2662 2663
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2664 2665
	int fence_reg;
	int fence_pitch_shift;
2666 2667
	uint64_t val;

2668 2669 2670 2671 2672 2673 2674 2675
	if (INTEL_INFO(dev)->gen >= 6) {
		fence_reg = FENCE_REG_SANDYBRIDGE_0;
		fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
	} else {
		fence_reg = FENCE_REG_965_0;
		fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
	}

2676
	if (obj) {
2677
		u32 size = i915_gem_obj_ggtt_size(obj);
2678

2679
		val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
2680
				 0xfffff000) << 32;
2681
		val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
2682
		val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
2683 2684 2685 2686 2687
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I965_FENCE_TILING_Y_SHIFT;
		val |= I965_FENCE_REG_VALID;
	} else
		val = 0;
2688

2689 2690 2691
	fence_reg += reg * 8;
	I915_WRITE64(fence_reg, val);
	POSTING_READ(fence_reg);
2692 2693
}

2694 2695
static void i915_write_fence_reg(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
2696 2697
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2698
	u32 val;
2699

2700
	if (obj) {
2701
		u32 size = i915_gem_obj_ggtt_size(obj);
2702 2703
		int pitch_val;
		int tile_width;
2704

2705
		WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
2706
		     (size & -size) != size ||
2707 2708 2709
		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		     "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		     i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2710

2711 2712 2713 2714 2715 2716 2717 2718 2719
		if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
			tile_width = 128;
		else
			tile_width = 512;

		/* Note: pitch better be a power of two tile widths */
		pitch_val = obj->stride / tile_width;
		pitch_val = ffs(pitch_val) - 1;

2720
		val = i915_gem_obj_ggtt_offset(obj);
2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
		val |= I915_FENCE_SIZE_BITS(size);
		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
		val |= I830_FENCE_REG_VALID;
	} else
		val = 0;

	if (reg < 8)
		reg = FENCE_REG_830_0 + reg * 4;
	else
		reg = FENCE_REG_945_8 + (reg - 8) * 4;

	I915_WRITE(reg, val);
	POSTING_READ(reg);
2736 2737
}

2738 2739
static void i830_write_fence_reg(struct drm_device *dev, int reg,
				struct drm_i915_gem_object *obj)
2740 2741 2742 2743
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	uint32_t val;

2744
	if (obj) {
2745
		u32 size = i915_gem_obj_ggtt_size(obj);
2746
		uint32_t pitch_val;
2747

2748
		WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
2749
		     (size & -size) != size ||
2750 2751 2752
		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		     "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
		     i915_gem_obj_ggtt_offset(obj), size);
2753

2754 2755
		pitch_val = obj->stride / 128;
		pitch_val = ffs(pitch_val) - 1;
2756

2757
		val = i915_gem_obj_ggtt_offset(obj);
2758 2759 2760 2761 2762 2763 2764
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
		val |= I830_FENCE_SIZE_BITS(size);
		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
		val |= I830_FENCE_REG_VALID;
	} else
		val = 0;
2765

2766 2767 2768 2769
	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
	POSTING_READ(FENCE_REG_830_0 + reg * 4);
}

2770 2771 2772 2773 2774
inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
{
	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
}

2775 2776 2777
static void i915_gem_write_fence(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
{
2778 2779 2780 2781 2782 2783 2784 2785
	struct drm_i915_private *dev_priv = dev->dev_private;

	/* Ensure that all CPU reads are completed before installing a fence
	 * and all writes before removing the fence.
	 */
	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		mb();

2786 2787
	switch (INTEL_INFO(dev)->gen) {
	case 7:
2788
	case 6:
2789 2790 2791 2792
	case 5:
	case 4: i965_write_fence_reg(dev, reg, obj); break;
	case 3: i915_write_fence_reg(dev, reg, obj); break;
	case 2: i830_write_fence_reg(dev, reg, obj); break;
2793
	default: BUG();
2794
	}
2795 2796 2797 2798 2799 2800

	/* And similarly be paranoid that no direct access to this region
	 * is reordered to before the fence is installed.
	 */
	if (i915_gem_object_needs_mb(obj))
		mb();
2801 2802
}

2803 2804 2805 2806 2807 2808
static inline int fence_number(struct drm_i915_private *dev_priv,
			       struct drm_i915_fence_reg *fence)
{
	return fence - dev_priv->fence_regs;
}

2809 2810 2811 2812 2813 2814
struct write_fence {
	struct drm_device *dev;
	struct drm_i915_gem_object *obj;
	int fence;
};

2815 2816
static void i915_gem_write_fence__ipi(void *data)
{
2817 2818 2819
	struct write_fence *args = data;

	/* Required for SNB+ with LLC */
2820
	wbinvd();
2821 2822 2823

	/* Required for VLV */
	i915_gem_write_fence(args->dev, args->fence, args->obj);
2824 2825
}

2826 2827 2828 2829
static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
					 struct drm_i915_fence_reg *fence,
					 bool enable)
{
2830 2831 2832 2833 2834 2835
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	struct write_fence args = {
		.dev = obj->base.dev,
		.fence = fence_number(dev_priv, fence),
		.obj = enable ? obj : NULL,
	};
2836 2837 2838 2839 2840 2841 2842 2843 2844 2845

	/* In order to fully serialize access to the fenced region and
	 * the update to the fence register we need to take extreme
	 * measures on SNB+. In theory, the write to the fence register
	 * flushes all memory transactions before, and coupled with the
	 * mb() placed around the register write we serialise all memory
	 * operations with respect to the changes in the tiler. Yet, on
	 * SNB+ we need to take a step further and emit an explicit wbinvd()
	 * on each processor in order to manually flush all memory
	 * transactions before updating the fence register.
2846 2847 2848 2849 2850
	 *
	 * However, Valleyview complicates matter. There the wbinvd is
	 * insufficient and unlike SNB/IVB requires the serialising
	 * register write. (Note that that register write by itself is
	 * conversely not sufficient for SNB+.) To compromise, we do both.
2851
	 */
2852 2853 2854 2855
	if (INTEL_INFO(args.dev)->gen >= 6)
		on_each_cpu(i915_gem_write_fence__ipi, &args, 1);
	else
		i915_gem_write_fence(args.dev, args.fence, args.obj);
2856 2857

	if (enable) {
2858
		obj->fence_reg = args.fence;
2859 2860 2861 2862 2863 2864 2865 2866 2867
		fence->obj = obj;
		list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
	} else {
		obj->fence_reg = I915_FENCE_REG_NONE;
		fence->obj = NULL;
		list_del_init(&fence->lru_list);
	}
}

2868
static int
2869
i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
2870
{
2871
	if (obj->last_fenced_seqno) {
2872
		int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2873 2874
		if (ret)
			return ret;
2875 2876 2877 2878

		obj->last_fenced_seqno = 0;
	}

2879
	obj->fenced_gpu_access = false;
2880 2881 2882 2883 2884 2885
	return 0;
}

int
i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
{
2886
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2887
	struct drm_i915_fence_reg *fence;
2888 2889
	int ret;

2890
	ret = i915_gem_object_wait_fence(obj);
2891 2892 2893
	if (ret)
		return ret;

2894 2895
	if (obj->fence_reg == I915_FENCE_REG_NONE)
		return 0;
2896

2897 2898
	fence = &dev_priv->fence_regs[obj->fence_reg];

2899
	i915_gem_object_fence_lost(obj);
2900
	i915_gem_object_update_fence(obj, fence, false);
2901 2902 2903 2904 2905

	return 0;
}

static struct drm_i915_fence_reg *
C
Chris Wilson 已提交
2906
i915_find_fence_reg(struct drm_device *dev)
2907 2908
{
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
2909
	struct drm_i915_fence_reg *reg, *avail;
2910
	int i;
2911 2912

	/* First try to find a free reg */
2913
	avail = NULL;
2914 2915 2916
	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		reg = &dev_priv->fence_regs[i];
		if (!reg->obj)
2917
			return reg;
2918

2919
		if (!reg->pin_count)
2920
			avail = reg;
2921 2922
	}

2923 2924
	if (avail == NULL)
		return NULL;
2925 2926

	/* None available, try to steal one or wait for a user to finish */
2927
	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2928
		if (reg->pin_count)
2929 2930
			continue;

C
Chris Wilson 已提交
2931
		return reg;
2932 2933
	}

C
Chris Wilson 已提交
2934
	return NULL;
2935 2936
}

2937
/**
2938
 * i915_gem_object_get_fence - set up fencing for an object
2939 2940 2941 2942 2943 2944 2945 2946 2947
 * @obj: object to map through a fence reg
 *
 * When mapping objects through the GTT, userspace wants to be able to write
 * to them without having to worry about swizzling if the object is tiled.
 * This function walks the fence regs looking for a free one for @obj,
 * stealing one if it can't find any.
 *
 * It then sets up the reg based on the object's properties: address, pitch
 * and tiling format.
2948 2949
 *
 * For an untiled surface, this removes any existing fence.
2950
 */
2951
int
2952
i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
2953
{
2954
	struct drm_device *dev = obj->base.dev;
J
Jesse Barnes 已提交
2955
	struct drm_i915_private *dev_priv = dev->dev_private;
2956
	bool enable = obj->tiling_mode != I915_TILING_NONE;
2957
	struct drm_i915_fence_reg *reg;
2958
	int ret;
2959

2960 2961 2962
	/* Have we updated the tiling parameters upon the object and so
	 * will need to serialise the write to the associated fence register?
	 */
2963
	if (obj->fence_dirty) {
2964
		ret = i915_gem_object_wait_fence(obj);
2965 2966 2967
		if (ret)
			return ret;
	}
2968

2969
	/* Just update our place in the LRU if our fence is getting reused. */
2970 2971
	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		reg = &dev_priv->fence_regs[obj->fence_reg];
2972
		if (!obj->fence_dirty) {
2973 2974 2975 2976 2977 2978 2979 2980
			list_move_tail(&reg->lru_list,
				       &dev_priv->mm.fence_list);
			return 0;
		}
	} else if (enable) {
		reg = i915_find_fence_reg(dev);
		if (reg == NULL)
			return -EDEADLK;
2981

2982 2983 2984
		if (reg->obj) {
			struct drm_i915_gem_object *old = reg->obj;

2985
			ret = i915_gem_object_wait_fence(old);
2986 2987 2988
			if (ret)
				return ret;

2989
			i915_gem_object_fence_lost(old);
2990
		}
2991
	} else
2992 2993
		return 0;

2994
	i915_gem_object_update_fence(obj, reg, enable);
2995
	obj->fence_dirty = false;
2996

2997
	return 0;
2998 2999
}

3000 3001 3002 3003 3004 3005 3006 3007
static bool i915_gem_valid_gtt_space(struct drm_device *dev,
				     struct drm_mm_node *gtt_space,
				     unsigned long cache_level)
{
	struct drm_mm_node *other;

	/* On non-LLC machines we have to be careful when putting differing
	 * types of snoopable memory together to avoid the prefetcher
3008
	 * crossing memory domains and dying.
3009 3010 3011 3012
	 */
	if (HAS_LLC(dev))
		return true;

3013
	if (!drm_mm_node_allocated(gtt_space))
3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036
		return true;

	if (list_empty(&gtt_space->node_list))
		return true;

	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
	if (other->allocated && !other->hole_follows && other->color != cache_level)
		return false;

	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		return false;

	return true;
}

static void i915_gem_verify_gtt(struct drm_device *dev)
{
#if WATCH_GTT
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_gem_object *obj;
	int err = 0;

3037
	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3038 3039 3040 3041 3042 3043 3044 3045
		if (obj->gtt_space == NULL) {
			printk(KERN_ERR "object found on GTT list with no space reserved\n");
			err++;
			continue;
		}

		if (obj->cache_level != obj->gtt_space->color) {
			printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
3046 3047
			       i915_gem_obj_ggtt_offset(obj),
			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3048 3049 3050 3051 3052 3053 3054 3055 3056 3057
			       obj->cache_level,
			       obj->gtt_space->color);
			err++;
			continue;
		}

		if (!i915_gem_valid_gtt_space(dev,
					      obj->gtt_space,
					      obj->cache_level)) {
			printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
3058 3059
			       i915_gem_obj_ggtt_offset(obj),
			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3060 3061 3062 3063 3064 3065 3066 3067 3068 3069
			       obj->cache_level);
			err++;
			continue;
		}
	}

	WARN_ON(err);
#endif
}

3070 3071 3072 3073
/**
 * Finds free space in the GTT aperture and binds the object there.
 */
static int
3074
i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
3075
			    unsigned alignment,
3076 3077
			    bool map_and_fenceable,
			    bool nonblocking)
3078
{
3079
	struct drm_device *dev = obj->base.dev;
3080
	drm_i915_private_t *dev_priv = dev->dev_private;
3081
	u32 size, fence_size, fence_alignment, unfenced_alignment;
3082
	bool mappable, fenceable;
3083 3084
	size_t gtt_max = map_and_fenceable ?
		dev_priv->gtt.mappable_end : dev_priv->gtt.total;
3085
	int ret;
3086

3087 3088 3089 3090 3091
	fence_size = i915_gem_get_gtt_size(dev,
					   obj->base.size,
					   obj->tiling_mode);
	fence_alignment = i915_gem_get_gtt_alignment(dev,
						     obj->base.size,
3092
						     obj->tiling_mode, true);
3093
	unfenced_alignment =
3094
		i915_gem_get_gtt_alignment(dev,
3095
						    obj->base.size,
3096
						    obj->tiling_mode, false);
3097

3098
	if (alignment == 0)
3099 3100
		alignment = map_and_fenceable ? fence_alignment :
						unfenced_alignment;
3101
	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
3102 3103 3104 3105
		DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		return -EINVAL;
	}

3106
	size = map_and_fenceable ? fence_size : obj->base.size;
3107

3108 3109 3110
	/* If the object is bigger than the entire aperture, reject it early
	 * before evicting everything in a vain attempt to find space.
	 */
3111
	if (obj->base.size > gtt_max) {
3112
		DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
3113 3114
			  obj->base.size,
			  map_and_fenceable ? "mappable" : "total",
3115
			  gtt_max);
3116 3117 3118
		return -E2BIG;
	}

3119
	ret = i915_gem_object_get_pages(obj);
C
Chris Wilson 已提交
3120 3121 3122
	if (ret)
		return ret;

3123 3124
	i915_gem_object_pin_pages(obj);

3125
search_free:
3126 3127
	ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space,
						  &obj->gtt_space,
3128 3129
						  size, alignment,
						  obj->cache_level, 0, gtt_max);
3130
	if (ret) {
3131
		ret = i915_gem_evict_something(dev, size, alignment,
3132
					       obj->cache_level,
3133 3134
					       map_and_fenceable,
					       nonblocking);
3135 3136
		if (ret == 0)
			goto search_free;
3137

3138 3139
		i915_gem_object_unpin_pages(obj);
		return ret;
3140
	}
3141 3142
	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &obj->gtt_space,
					      obj->cache_level))) {
3143
		i915_gem_object_unpin_pages(obj);
3144
		drm_mm_remove_node(&obj->gtt_space);
3145
		return -EINVAL;
3146 3147
	}

3148
	ret = i915_gem_gtt_prepare_object(obj);
3149
	if (ret) {
3150
		i915_gem_object_unpin_pages(obj);
3151
		drm_mm_remove_node(&obj->gtt_space);
C
Chris Wilson 已提交
3152
		return ret;
3153 3154
	}

3155
	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3156
	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3157

3158
	fenceable =
3159 3160
		i915_gem_obj_ggtt_size(obj) == fence_size &&
		(i915_gem_obj_ggtt_offset(obj) & (fence_alignment - 1)) == 0;
3161

3162 3163
	mappable = i915_gem_obj_ggtt_offset(obj) + obj->base.size <=
		dev_priv->gtt.mappable_end;
3164

3165
	obj->map_and_fenceable = mappable && fenceable;
3166

C
Chris Wilson 已提交
3167
	trace_i915_gem_object_bind(obj, map_and_fenceable);
3168
	i915_gem_verify_gtt(dev);
3169 3170 3171 3172
	return 0;
}

void
3173
i915_gem_clflush_object(struct drm_i915_gem_object *obj)
3174 3175 3176 3177 3178
{
	/* If we don't have a page list set up, then we're not pinned
	 * to GPU, and we can ignore the cache flush because it'll happen
	 * again at bind time.
	 */
3179
	if (obj->pages == NULL)
3180 3181
		return;

3182 3183 3184 3185 3186 3187 3188
	/*
	 * Stolen memory is always coherent with the GPU as it is explicitly
	 * marked as wc by the system, or the system is cache-coherent.
	 */
	if (obj->stolen)
		return;

3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199
	/* If the GPU is snooping the contents of the CPU cache,
	 * we do not need to manually clear the CPU cache lines.  However,
	 * the caches are only snooped when the render cache is
	 * flushed/invalidated.  As we always have to emit invalidations
	 * and flushes when moving into and out of the RENDER domain, correct
	 * snooping behaviour occurs naturally as the result of our domain
	 * tracking.
	 */
	if (obj->cache_level != I915_CACHE_NONE)
		return;

C
Chris Wilson 已提交
3200
	trace_i915_gem_object_clflush(obj);
3201

3202
	drm_clflush_sg(obj->pages);
3203 3204 3205 3206
}

/** Flushes the GTT write domain for the object if it's dirty. */
static void
3207
i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3208
{
C
Chris Wilson 已提交
3209 3210
	uint32_t old_write_domain;

3211
	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3212 3213
		return;

3214
	/* No actual flushing is required for the GTT write domain.  Writes
3215 3216
	 * to it immediately go to main memory as far as we know, so there's
	 * no chipset flush.  It also doesn't land in render cache.
3217 3218 3219 3220
	 *
	 * However, we do have to enforce the order so that all writes through
	 * the GTT land before any writes to the device, such as updates to
	 * the GATT itself.
3221
	 */
3222 3223
	wmb();

3224 3225
	old_write_domain = obj->base.write_domain;
	obj->base.write_domain = 0;
C
Chris Wilson 已提交
3226 3227

	trace_i915_gem_object_change_domain(obj,
3228
					    obj->base.read_domains,
C
Chris Wilson 已提交
3229
					    old_write_domain);
3230 3231 3232 3233
}

/** Flushes the CPU write domain for the object if it's dirty. */
static void
3234
i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3235
{
C
Chris Wilson 已提交
3236
	uint32_t old_write_domain;
3237

3238
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3239 3240 3241
		return;

	i915_gem_clflush_object(obj);
3242
	i915_gem_chipset_flush(obj->base.dev);
3243 3244
	old_write_domain = obj->base.write_domain;
	obj->base.write_domain = 0;
C
Chris Wilson 已提交
3245 3246

	trace_i915_gem_object_change_domain(obj,
3247
					    obj->base.read_domains,
C
Chris Wilson 已提交
3248
					    old_write_domain);
3249 3250
}

3251 3252 3253 3254 3255 3256
/**
 * Moves a single object to the GTT read, and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
J
Jesse Barnes 已提交
3257
int
3258
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3259
{
3260
	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
C
Chris Wilson 已提交
3261
	uint32_t old_write_domain, old_read_domains;
3262
	int ret;
3263

3264
	/* Not valid to be called on unbound objects. */
3265
	if (!i915_gem_obj_ggtt_bound(obj))
3266 3267
		return -EINVAL;

3268 3269 3270
	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		return 0;

3271
	ret = i915_gem_object_wait_rendering(obj, !write);
3272 3273 3274
	if (ret)
		return ret;

3275
	i915_gem_object_flush_cpu_write_domain(obj);
C
Chris Wilson 已提交
3276

3277 3278 3279 3280 3281 3282 3283
	/* Serialise direct access to this object with the barriers for
	 * coherent writes from the GPU, by effectively invalidating the
	 * GTT domain upon first access.
	 */
	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		mb();

3284 3285
	old_write_domain = obj->base.write_domain;
	old_read_domains = obj->base.read_domains;
C
Chris Wilson 已提交
3286

3287 3288 3289
	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3290 3291
	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3292
	if (write) {
3293 3294 3295
		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		obj->dirty = 1;
3296 3297
	}

C
Chris Wilson 已提交
3298 3299 3300 3301
	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);

3302 3303 3304 3305
	/* And bump the LRU for this access */
	if (i915_gem_object_is_inactive(obj))
		list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);

3306 3307 3308
	return 0;
}

3309 3310 3311
int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
				    enum i915_cache_level cache_level)
{
3312 3313
	struct drm_device *dev = obj->base.dev;
	drm_i915_private_t *dev_priv = dev->dev_private;
3314 3315 3316 3317 3318 3319 3320 3321 3322 3323
	int ret;

	if (obj->cache_level == cache_level)
		return 0;

	if (obj->pin_count) {
		DRM_DEBUG("can not change the cache level of pinned objects\n");
		return -EBUSY;
	}

3324
	if (!i915_gem_valid_gtt_space(dev, &obj->gtt_space, cache_level)) {
3325 3326 3327 3328 3329
		ret = i915_gem_object_unbind(obj);
		if (ret)
			return ret;
	}

3330
	if (i915_gem_obj_ggtt_bound(obj)) {
3331 3332 3333 3334 3335 3336 3337 3338 3339 3340
		ret = i915_gem_object_finish_gpu(obj);
		if (ret)
			return ret;

		i915_gem_object_finish_gtt(obj);

		/* Before SandyBridge, you could not use tiling or fence
		 * registers with snooped memory, so relinquish any fences
		 * currently pointing to our region in the aperture.
		 */
3341
		if (INTEL_INFO(dev)->gen < 6) {
3342 3343 3344 3345 3346
			ret = i915_gem_object_put_fence(obj);
			if (ret)
				return ret;
		}

3347 3348
		if (obj->has_global_gtt_mapping)
			i915_gem_gtt_bind_object(obj, cache_level);
3349 3350 3351
		if (obj->has_aliasing_ppgtt_mapping)
			i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
					       obj, cache_level);
3352

3353
		i915_gem_obj_ggtt_set_color(obj, cache_level);
3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379
	}

	if (cache_level == I915_CACHE_NONE) {
		u32 old_read_domains, old_write_domain;

		/* If we're coming from LLC cached, then we haven't
		 * actually been tracking whether the data is in the
		 * CPU cache or not, since we only allow one bit set
		 * in obj->write_domain and have been skipping the clflushes.
		 * Just set it to the CPU cache for now.
		 */
		WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);

		old_read_domains = obj->base.read_domains;
		old_write_domain = obj->base.write_domain;

		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		obj->base.write_domain = I915_GEM_DOMAIN_CPU;

		trace_i915_gem_object_change_domain(obj,
						    old_read_domains,
						    old_write_domain);
	}

	obj->cache_level = cache_level;
3380
	i915_gem_verify_gtt(dev);
3381 3382 3383
	return 0;
}

B
Ben Widawsky 已提交
3384 3385
int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3386
{
B
Ben Widawsky 已提交
3387
	struct drm_i915_gem_caching *args = data;
3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400
	struct drm_i915_gem_object *obj;
	int ret;

	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
	if (&obj->base == NULL) {
		ret = -ENOENT;
		goto unlock;
	}

B
Ben Widawsky 已提交
3401
	args->caching = obj->cache_level != I915_CACHE_NONE;
3402 3403 3404 3405 3406 3407 3408

	drm_gem_object_unreference(&obj->base);
unlock:
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

B
Ben Widawsky 已提交
3409 3410
int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3411
{
B
Ben Widawsky 已提交
3412
	struct drm_i915_gem_caching *args = data;
3413 3414 3415 3416
	struct drm_i915_gem_object *obj;
	enum i915_cache_level level;
	int ret;

B
Ben Widawsky 已提交
3417 3418
	switch (args->caching) {
	case I915_CACHING_NONE:
3419 3420
		level = I915_CACHE_NONE;
		break;
B
Ben Widawsky 已提交
3421
	case I915_CACHING_CACHED:
3422 3423 3424 3425 3426 3427
		level = I915_CACHE_LLC;
		break;
	default:
		return -EINVAL;
	}

B
Ben Widawsky 已提交
3428 3429 3430 3431
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
	if (&obj->base == NULL) {
		ret = -ENOENT;
		goto unlock;
	}

	ret = i915_gem_object_set_cache_level(obj, level);

	drm_gem_object_unreference(&obj->base);
unlock:
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

3446
/*
3447 3448 3449
 * Prepare buffer for display plane (scanout, cursors, etc).
 * Can be called from an uninterruptible phase (modesetting) and allows
 * any flushes to be pipelined (for pageflips).
3450 3451
 */
int
3452 3453
i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
				     u32 alignment,
3454
				     struct intel_ring_buffer *pipelined)
3455
{
3456
	u32 old_read_domains, old_write_domain;
3457 3458
	int ret;

3459
	if (pipelined != obj->ring) {
3460 3461
		ret = i915_gem_object_sync(obj, pipelined);
		if (ret)
3462 3463 3464
			return ret;
	}

3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477
	/* The display engine is not coherent with the LLC cache on gen6.  As
	 * a result, we make sure that the pinning that is about to occur is
	 * done with uncached PTEs. This is lowest common denominator for all
	 * chipsets.
	 *
	 * However for gen6+, we could do better by using the GFDT bit instead
	 * of uncaching, which would allow us to flush all the LLC-cached data
	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
	 */
	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
	if (ret)
		return ret;

3478 3479 3480 3481
	/* As the user may map the buffer once pinned in the display plane
	 * (e.g. libkms for the bootup splash), we have to ensure that we
	 * always use map_and_fenceable for all scanout buffers.
	 */
3482
	ret = i915_gem_object_pin(obj, alignment, true, false);
3483 3484 3485
	if (ret)
		return ret;

3486 3487
	i915_gem_object_flush_cpu_write_domain(obj);

3488
	old_write_domain = obj->base.write_domain;
3489
	old_read_domains = obj->base.read_domains;
3490 3491 3492 3493

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3494
	obj->base.write_domain = 0;
3495
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3496 3497 3498

	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
3499
					    old_write_domain);
3500 3501 3502 3503

	return 0;
}

3504
int
3505
i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3506
{
3507 3508
	int ret;

3509
	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3510 3511
		return 0;

3512
	ret = i915_gem_object_wait_rendering(obj, false);
3513 3514 3515
	if (ret)
		return ret;

3516 3517
	/* Ensure that we invalidate the GPU's caches and TLBs. */
	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3518
	return 0;
3519 3520
}

3521 3522 3523 3524 3525 3526
/**
 * Moves a single object to the CPU read, and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
3527
int
3528
i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3529
{
C
Chris Wilson 已提交
3530
	uint32_t old_write_domain, old_read_domains;
3531 3532
	int ret;

3533 3534 3535
	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		return 0;

3536
	ret = i915_gem_object_wait_rendering(obj, !write);
3537 3538 3539
	if (ret)
		return ret;

3540
	i915_gem_object_flush_gtt_write_domain(obj);
3541

3542 3543
	old_write_domain = obj->base.write_domain;
	old_read_domains = obj->base.read_domains;
C
Chris Wilson 已提交
3544

3545
	/* Flush the CPU cache if it's still invalid. */
3546
	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3547 3548
		i915_gem_clflush_object(obj);

3549
		obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3550 3551 3552 3553 3554
	}

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3555
	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3556 3557 3558 3559 3560

	/* If we're writing through the CPU, then the GPU read domains will
	 * need to be invalidated at next use.
	 */
	if (write) {
3561 3562
		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3563
	}
3564

C
Chris Wilson 已提交
3565 3566 3567 3568
	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);

3569 3570 3571
	return 0;
}

3572 3573 3574
/* Throttle our rendering by waiting until the ring has completed our requests
 * emitted over 20 msec ago.
 *
3575 3576 3577 3578
 * Note that if we were to use the current jiffies each time around the loop,
 * we wouldn't escape the function with any frames outstanding if the time to
 * render a frame was over 20ms.
 *
3579 3580 3581
 * This should get us reasonable parallelism between CPU and GPU but also
 * relatively low latency when blocking on a particular request to finish.
 */
3582
static int
3583
i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3584
{
3585 3586
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_file_private *file_priv = file->driver_priv;
3587
	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3588 3589
	struct drm_i915_gem_request *request;
	struct intel_ring_buffer *ring = NULL;
3590
	unsigned reset_counter;
3591 3592
	u32 seqno = 0;
	int ret;
3593

3594 3595 3596 3597 3598 3599 3600
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
	if (ret)
		return ret;

	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
	if (ret)
		return ret;
3601

3602
	spin_lock(&file_priv->mm.lock);
3603
	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3604 3605
		if (time_after_eq(request->emitted_jiffies, recent_enough))
			break;
3606

3607 3608
		ring = request->ring;
		seqno = request->seqno;
3609
	}
3610
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3611
	spin_unlock(&file_priv->mm.lock);
3612

3613 3614
	if (seqno == 0)
		return 0;
3615

3616
	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3617 3618
	if (ret == 0)
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3619 3620 3621 3622

	return ret;
}

3623
int
3624 3625
i915_gem_object_pin(struct drm_i915_gem_object *obj,
		    uint32_t alignment,
3626 3627
		    bool map_and_fenceable,
		    bool nonblocking)
3628 3629 3630
{
	int ret;

3631 3632
	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		return -EBUSY;
3633

3634 3635
	if (i915_gem_obj_ggtt_bound(obj)) {
		if ((alignment && i915_gem_obj_ggtt_offset(obj) & (alignment - 1)) ||
3636 3637
		    (map_and_fenceable && !obj->map_and_fenceable)) {
			WARN(obj->pin_count,
3638
			     "bo is already pinned with incorrect alignment:"
3639
			     " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
3640
			     " obj->map_and_fenceable=%d\n",
3641
			     i915_gem_obj_ggtt_offset(obj), alignment,
3642
			     map_and_fenceable,
3643
			     obj->map_and_fenceable);
3644 3645 3646 3647 3648 3649
			ret = i915_gem_object_unbind(obj);
			if (ret)
				return ret;
		}
	}

3650
	if (!i915_gem_obj_ggtt_bound(obj)) {
3651 3652
		struct drm_i915_private *dev_priv = obj->base.dev->dev_private;

3653
		ret = i915_gem_object_bind_to_gtt(obj, alignment,
3654 3655
						  map_and_fenceable,
						  nonblocking);
3656
		if (ret)
3657
			return ret;
3658 3659 3660

		if (!dev_priv->mm.aliasing_ppgtt)
			i915_gem_gtt_bind_object(obj, obj->cache_level);
3661
	}
J
Jesse Barnes 已提交
3662

3663 3664 3665
	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		i915_gem_gtt_bind_object(obj, obj->cache_level);

3666
	obj->pin_count++;
3667
	obj->pin_mappable |= map_and_fenceable;
3668 3669 3670 3671 3672

	return 0;
}

void
3673
i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3674
{
3675
	BUG_ON(obj->pin_count == 0);
3676
	BUG_ON(!i915_gem_obj_ggtt_bound(obj));
3677

3678
	if (--obj->pin_count == 0)
3679
		obj->pin_mappable = false;
3680 3681 3682 3683
}

int
i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3684
		   struct drm_file *file)
3685 3686
{
	struct drm_i915_gem_pin *args = data;
3687
	struct drm_i915_gem_object *obj;
3688 3689
	int ret;

3690 3691 3692
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;
3693

3694
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3695
	if (&obj->base == NULL) {
3696 3697
		ret = -ENOENT;
		goto unlock;
3698 3699
	}

3700
	if (obj->madv != I915_MADV_WILLNEED) {
C
Chris Wilson 已提交
3701
		DRM_ERROR("Attempting to pin a purgeable buffer\n");
3702 3703
		ret = -EINVAL;
		goto out;
3704 3705
	}

3706
	if (obj->pin_filp != NULL && obj->pin_filp != file) {
J
Jesse Barnes 已提交
3707 3708
		DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
			  args->handle);
3709 3710
		ret = -EINVAL;
		goto out;
J
Jesse Barnes 已提交
3711 3712
	}

3713
	if (obj->user_pin_count == 0) {
3714
		ret = i915_gem_object_pin(obj, args->alignment, true, false);
3715 3716
		if (ret)
			goto out;
3717 3718
	}

3719 3720 3721
	obj->user_pin_count++;
	obj->pin_filp = file;

3722 3723 3724
	/* XXX - flush the CPU caches for pinned objects
	 * as the X server doesn't manage domains yet
	 */
3725
	i915_gem_object_flush_cpu_write_domain(obj);
3726
	args->offset = i915_gem_obj_ggtt_offset(obj);
3727
out:
3728
	drm_gem_object_unreference(&obj->base);
3729
unlock:
3730
	mutex_unlock(&dev->struct_mutex);
3731
	return ret;
3732 3733 3734 3735
}

int
i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3736
		     struct drm_file *file)
3737 3738
{
	struct drm_i915_gem_pin *args = data;
3739
	struct drm_i915_gem_object *obj;
3740
	int ret;
3741

3742 3743 3744
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;
3745

3746
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3747
	if (&obj->base == NULL) {
3748 3749
		ret = -ENOENT;
		goto unlock;
3750
	}
3751

3752
	if (obj->pin_filp != file) {
J
Jesse Barnes 已提交
3753 3754
		DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
			  args->handle);
3755 3756
		ret = -EINVAL;
		goto out;
J
Jesse Barnes 已提交
3757
	}
3758 3759 3760
	obj->user_pin_count--;
	if (obj->user_pin_count == 0) {
		obj->pin_filp = NULL;
J
Jesse Barnes 已提交
3761 3762
		i915_gem_object_unpin(obj);
	}
3763

3764
out:
3765
	drm_gem_object_unreference(&obj->base);
3766
unlock:
3767
	mutex_unlock(&dev->struct_mutex);
3768
	return ret;
3769 3770 3771 3772
}

int
i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3773
		    struct drm_file *file)
3774 3775
{
	struct drm_i915_gem_busy *args = data;
3776
	struct drm_i915_gem_object *obj;
3777 3778
	int ret;

3779
	ret = i915_mutex_lock_interruptible(dev);
3780
	if (ret)
3781
		return ret;
3782

3783
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3784
	if (&obj->base == NULL) {
3785 3786
		ret = -ENOENT;
		goto unlock;
3787
	}
3788

3789 3790 3791 3792
	/* Count all active objects as busy, even if they are currently not used
	 * by the gpu. Users of this interface expect objects to eventually
	 * become non-busy without any further actions, therefore emit any
	 * necessary flushes here.
3793
	 */
3794
	ret = i915_gem_object_flush_active(obj);
3795

3796
	args->busy = obj->active;
3797 3798 3799 3800
	if (obj->ring) {
		BUILD_BUG_ON(I915_NUM_RINGS > 16);
		args->busy |= intel_ring_flag(obj->ring) << 16;
	}
3801

3802
	drm_gem_object_unreference(&obj->base);
3803
unlock:
3804
	mutex_unlock(&dev->struct_mutex);
3805
	return ret;
3806 3807 3808 3809 3810 3811
}

int
i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file_priv)
{
3812
	return i915_gem_ring_throttle(dev, file_priv);
3813 3814
}

3815 3816 3817 3818 3819
int
i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
	struct drm_i915_gem_madvise *args = data;
3820
	struct drm_i915_gem_object *obj;
3821
	int ret;
3822 3823 3824 3825 3826 3827 3828 3829 3830

	switch (args->madv) {
	case I915_MADV_DONTNEED:
	case I915_MADV_WILLNEED:
	    break;
	default:
	    return -EINVAL;
	}

3831 3832 3833 3834
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

3835
	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3836
	if (&obj->base == NULL) {
3837 3838
		ret = -ENOENT;
		goto unlock;
3839 3840
	}

3841
	if (obj->pin_count) {
3842 3843
		ret = -EINVAL;
		goto out;
3844 3845
	}

3846 3847
	if (obj->madv != __I915_MADV_PURGED)
		obj->madv = args->madv;
3848

C
Chris Wilson 已提交
3849 3850
	/* if the object is no longer attached, discard its backing storage */
	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
3851 3852
		i915_gem_object_truncate(obj);

3853
	args->retained = obj->madv != __I915_MADV_PURGED;
C
Chris Wilson 已提交
3854

3855
out:
3856
	drm_gem_object_unreference(&obj->base);
3857
unlock:
3858
	mutex_unlock(&dev->struct_mutex);
3859
	return ret;
3860 3861
}

3862 3863
void i915_gem_object_init(struct drm_i915_gem_object *obj,
			  const struct drm_i915_gem_object_ops *ops)
3864 3865
{
	INIT_LIST_HEAD(&obj->mm_list);
3866
	INIT_LIST_HEAD(&obj->global_list);
3867 3868 3869
	INIT_LIST_HEAD(&obj->ring_list);
	INIT_LIST_HEAD(&obj->exec_list);

3870 3871
	obj->ops = ops;

3872 3873 3874 3875 3876 3877 3878 3879
	obj->fence_reg = I915_FENCE_REG_NONE;
	obj->madv = I915_MADV_WILLNEED;
	/* Avoid an unnecessary call to unbind on the first bind. */
	obj->map_and_fenceable = true;

	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
}

3880 3881 3882 3883 3884
static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
	.get_pages = i915_gem_object_get_pages_gtt,
	.put_pages = i915_gem_object_put_pages_gtt,
};

3885 3886
struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
						  size_t size)
3887
{
3888
	struct drm_i915_gem_object *obj;
3889
	struct address_space *mapping;
D
Daniel Vetter 已提交
3890
	gfp_t mask;
3891

3892
	obj = i915_gem_object_alloc(dev);
3893 3894
	if (obj == NULL)
		return NULL;
3895

3896
	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3897
		i915_gem_object_free(obj);
3898 3899
		return NULL;
	}
3900

3901 3902 3903 3904 3905 3906 3907
	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
	if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
		/* 965gm cannot relocate objects above 4GiB. */
		mask &= ~__GFP_HIGHMEM;
		mask |= __GFP_DMA32;
	}

A
Al Viro 已提交
3908
	mapping = file_inode(obj->base.filp)->i_mapping;
3909
	mapping_set_gfp_mask(mapping, mask);
3910

3911
	i915_gem_object_init(obj, &i915_gem_object_ops);
3912

3913 3914
	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3915

3916 3917
	if (HAS_LLC(dev)) {
		/* On some devices, we can have the GPU use the LLC (the CPU
3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932
		 * cache) for about a 10% performance improvement
		 * compared to uncached.  Graphics requests other than
		 * display scanout are coherent with the CPU in
		 * accessing this cache.  This means in this mode we
		 * don't need to clflush on the CPU side, and on the
		 * GPU side we only need to flush internal caches to
		 * get data visible to the CPU.
		 *
		 * However, we maintain the display planes as UC, and so
		 * need to rebind when first used as such.
		 */
		obj->cache_level = I915_CACHE_LLC;
	} else
		obj->cache_level = I915_CACHE_NONE;

3933
	return obj;
3934 3935 3936 3937 3938
}

int i915_gem_init_object(struct drm_gem_object *obj)
{
	BUG();
3939

3940 3941 3942
	return 0;
}

3943
void i915_gem_free_object(struct drm_gem_object *gem_obj)
3944
{
3945
	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3946
	struct drm_device *dev = obj->base.dev;
3947
	drm_i915_private_t *dev_priv = dev->dev_private;
3948

3949 3950
	trace_i915_gem_object_destroy(obj);

3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965
	if (obj->phys_obj)
		i915_gem_detach_phys_object(dev, obj);

	obj->pin_count = 0;
	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		bool was_interruptible;

		was_interruptible = dev_priv->mm.interruptible;
		dev_priv->mm.interruptible = false;

		WARN_ON(i915_gem_object_unbind(obj));

		dev_priv->mm.interruptible = was_interruptible;
	}

B
Ben Widawsky 已提交
3966 3967 3968 3969 3970
	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
	 * before progressing. */
	if (obj->stolen)
		i915_gem_object_unpin_pages(obj);

B
Ben Widawsky 已提交
3971 3972
	if (WARN_ON(obj->pages_pin_count))
		obj->pages_pin_count = 0;
3973
	i915_gem_object_put_pages(obj);
3974
	i915_gem_object_free_mmap_offset(obj);
3975
	i915_gem_object_release_stolen(obj);
3976

3977 3978
	BUG_ON(obj->pages);

3979 3980
	if (obj->base.import_attach)
		drm_prime_gem_destroy(&obj->base, NULL);
3981

3982 3983
	drm_gem_object_release(&obj->base);
	i915_gem_info_remove_obj(dev_priv, obj->base.size);
3984

3985
	kfree(obj->bit_17);
3986
	i915_gem_object_free(obj);
3987 3988
}

3989 3990 3991 3992 3993
int
i915_gem_idle(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret;
3994

3995
	if (dev_priv->ums.mm_suspended) {
3996 3997
		mutex_unlock(&dev->struct_mutex);
		return 0;
3998 3999
	}

4000
	ret = i915_gpu_idle(dev);
4001 4002
	if (ret) {
		mutex_unlock(&dev->struct_mutex);
4003
		return ret;
4004
	}
4005
	i915_gem_retire_requests(dev);
4006

4007
	/* Under UMS, be paranoid and evict. */
4008
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
C
Chris Wilson 已提交
4009
		i915_gem_evict_everything(dev);
4010

4011 4012
	i915_gem_reset_fences(dev);

4013
	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
4014 4015

	i915_kernel_lost_context(dev);
4016
	i915_gem_cleanup_ringbuffer(dev);
4017 4018 4019 4020

	/* Cancel the retire work handler, which should be idle now. */
	cancel_delayed_work_sync(&dev_priv->mm.retire_work);

4021 4022 4023
	return 0;
}

B
Ben Widawsky 已提交
4024 4025 4026 4027 4028 4029
void i915_gem_l3_remap(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	u32 misccpctl;
	int i;

4030
	if (!HAS_L3_GPU_CACHE(dev))
B
Ben Widawsky 已提交
4031 4032
		return;

4033
	if (!dev_priv->l3_parity.remap_info)
B
Ben Widawsky 已提交
4034 4035 4036 4037 4038 4039 4040 4041
		return;

	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
4042
		if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
B
Ben Widawsky 已提交
4043 4044
			DRM_DEBUG("0x%x was already programmed to %x\n",
				  GEN7_L3LOG_BASE + i, remap);
4045
		if (remap && !dev_priv->l3_parity.remap_info[i/4])
B
Ben Widawsky 已提交
4046
			DRM_DEBUG_DRIVER("Clearing remapped register\n");
4047
		I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
B
Ben Widawsky 已提交
4048 4049 4050 4051 4052 4053 4054 4055
	}

	/* Make sure all the writes land before disabling dop clock gating */
	POSTING_READ(GEN7_L3LOG_BASE);

	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
}

4056 4057 4058 4059
void i915_gem_init_swizzling(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;

4060
	if (INTEL_INFO(dev)->gen < 5 ||
4061 4062 4063 4064 4065 4066
	    dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		return;

	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
				 DISP_TILE_SURFACE_SWIZZLING);

4067 4068 4069
	if (IS_GEN5(dev))
		return;

4070 4071
	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
	if (IS_GEN6(dev))
4072
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4073
	else if (IS_GEN7(dev))
4074
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4075 4076
	else
		BUG();
4077
}
D
Daniel Vetter 已提交
4078

4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094
static bool
intel_enable_blt(struct drm_device *dev)
{
	if (!HAS_BLT(dev))
		return false;

	/* The blitter was dysfunctional on early prototypes */
	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		DRM_INFO("BLT not supported on this pre-production hardware;"
			 " graphics performance will be degraded.\n");
		return false;
	}

	return true;
}

4095
static int i915_gem_init_rings(struct drm_device *dev)
4096
{
4097
	struct drm_i915_private *dev_priv = dev->dev_private;
4098
	int ret;
4099

4100
	ret = intel_init_render_ring_buffer(dev);
4101
	if (ret)
4102
		return ret;
4103 4104

	if (HAS_BSD(dev)) {
4105
		ret = intel_init_bsd_ring_buffer(dev);
4106 4107
		if (ret)
			goto cleanup_render_ring;
4108
	}
4109

4110
	if (intel_enable_blt(dev)) {
4111 4112 4113 4114 4115
		ret = intel_init_blt_ring_buffer(dev);
		if (ret)
			goto cleanup_bsd_ring;
	}

B
Ben Widawsky 已提交
4116 4117 4118 4119 4120 4121 4122
	if (HAS_VEBOX(dev)) {
		ret = intel_init_vebox_ring_buffer(dev);
		if (ret)
			goto cleanup_blt_ring;
	}


4123
	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
4124
	if (ret)
B
Ben Widawsky 已提交
4125
		goto cleanup_vebox_ring;
4126 4127 4128

	return 0;

B
Ben Widawsky 已提交
4129 4130
cleanup_vebox_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152
cleanup_blt_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
cleanup_bsd_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
cleanup_render_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);

	return ret;
}

int
i915_gem_init_hw(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret;

	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		return -EIO;

	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);

4153 4154 4155 4156 4157 4158
	if (HAS_PCH_NOP(dev)) {
		u32 temp = I915_READ(GEN7_MSG_CTL);
		temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
		I915_WRITE(GEN7_MSG_CTL, temp);
	}

4159 4160 4161 4162 4163
	i915_gem_l3_remap(dev);

	i915_gem_init_swizzling(dev);

	ret = i915_gem_init_rings(dev);
4164 4165 4166
	if (ret)
		return ret;

4167 4168 4169 4170 4171
	/*
	 * XXX: There was some w/a described somewhere suggesting loading
	 * contexts before PPGTT.
	 */
	i915_gem_context_init(dev);
4172 4173 4174 4175 4176 4177 4178
	if (dev_priv->mm.aliasing_ppgtt) {
		ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
		if (ret) {
			i915_gem_cleanup_aliasing_ppgtt(dev);
			DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
		}
	}
D
Daniel Vetter 已提交
4179

4180
	return 0;
4181 4182
}

4183 4184 4185 4186 4187 4188
int i915_gem_init(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	mutex_lock(&dev->struct_mutex);
4189 4190 4191 4192 4193 4194 4195 4196

	if (IS_VALLEYVIEW(dev)) {
		/* VLVA0 (potential hack), BIOS isn't actually waking us */
		I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
		if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
	}

4197
	i915_gem_init_global_gtt(dev);
4198

4199 4200 4201 4202 4203 4204 4205
	ret = i915_gem_init_hw(dev);
	mutex_unlock(&dev->struct_mutex);
	if (ret) {
		i915_gem_cleanup_aliasing_ppgtt(dev);
		return ret;
	}

4206 4207 4208
	/* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		dev_priv->dri1.allow_batchbuffer = 1;
4209 4210 4211
	return 0;
}

4212 4213 4214 4215
void
i915_gem_cleanup_ringbuffer(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
4216
	struct intel_ring_buffer *ring;
4217
	int i;
4218

4219 4220
	for_each_ring(ring, dev_priv, i)
		intel_cleanup_ring_buffer(ring);
4221 4222
}

4223 4224 4225 4226
int
i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
4227
	struct drm_i915_private *dev_priv = dev->dev_private;
4228
	int ret;
4229

J
Jesse Barnes 已提交
4230 4231 4232
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return 0;

4233
	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
4234
		DRM_ERROR("Reenabling wedged hardware, good luck\n");
4235
		atomic_set(&dev_priv->gpu_error.reset_counter, 0);
4236 4237 4238
	}

	mutex_lock(&dev->struct_mutex);
4239
	dev_priv->ums.mm_suspended = 0;
4240

4241
	ret = i915_gem_init_hw(dev);
4242 4243
	if (ret != 0) {
		mutex_unlock(&dev->struct_mutex);
4244
		return ret;
4245
	}
4246

4247
	BUG_ON(!list_empty(&dev_priv->mm.active_list));
4248
	mutex_unlock(&dev->struct_mutex);
4249

4250 4251 4252
	ret = drm_irq_install(dev);
	if (ret)
		goto cleanup_ringbuffer;
4253

4254
	return 0;
4255 4256 4257 4258

cleanup_ringbuffer:
	mutex_lock(&dev->struct_mutex);
	i915_gem_cleanup_ringbuffer(dev);
4259
	dev_priv->ums.mm_suspended = 1;
4260 4261 4262
	mutex_unlock(&dev->struct_mutex);

	return ret;
4263 4264 4265 4266 4267 4268
}

int
i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
4269 4270 4271
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

J
Jesse Barnes 已提交
4272 4273 4274
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return 0;

4275
	drm_irq_uninstall(dev);
4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288

	mutex_lock(&dev->struct_mutex);
	ret =  i915_gem_idle(dev);

	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
	 * We need to replace this with a semaphore, or something.
	 * And not confound ums.mm_suspended!
	 */
	if (ret != 0)
		dev_priv->ums.mm_suspended = 1;
	mutex_unlock(&dev->struct_mutex);

	return ret;
4289 4290 4291 4292 4293 4294 4295
}

void
i915_gem_lastclose(struct drm_device *dev)
{
	int ret;

4296 4297 4298
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return;

4299
	mutex_lock(&dev->struct_mutex);
4300 4301 4302
	ret = i915_gem_idle(dev);
	if (ret)
		DRM_ERROR("failed to idle hardware: %d\n", ret);
4303
	mutex_unlock(&dev->struct_mutex);
4304 4305
}

4306 4307 4308 4309 4310 4311 4312
static void
init_ring_lists(struct intel_ring_buffer *ring)
{
	INIT_LIST_HEAD(&ring->active_list);
	INIT_LIST_HEAD(&ring->request_list);
}

4313 4314 4315 4316
void
i915_gem_load(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
4317 4318 4319 4320 4321 4322 4323
	int i;

	dev_priv->slab =
		kmem_cache_create("i915_gem_object",
				  sizeof(struct drm_i915_gem_object), 0,
				  SLAB_HWCACHE_ALIGN,
				  NULL);
4324

4325
	INIT_LIST_HEAD(&dev_priv->mm.active_list);
4326
	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
C
Chris Wilson 已提交
4327 4328
	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4329
	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4330 4331
	for (i = 0; i < I915_NUM_RINGS; i++)
		init_ring_lists(&dev_priv->ring[i]);
4332
	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4333
		INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4334 4335
	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
			  i915_gem_retire_work_handler);
4336
	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4337

4338 4339
	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
	if (IS_GEN3(dev)) {
4340 4341
		I915_WRITE(MI_ARB_STATE,
			   _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4342 4343
	}

4344 4345
	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;

4346
	/* Old X drivers will take 0-2 for front, back, depth buffers */
4347 4348
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		dev_priv->fence_reg_start = 3;
4349

4350 4351 4352
	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		dev_priv->num_fence_regs = 32;
	else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4353 4354 4355 4356
		dev_priv->num_fence_regs = 16;
	else
		dev_priv->num_fence_regs = 8;

4357
	/* Initialize fence registers to zero */
4358
	i915_gem_reset_fences(dev);
4359

4360
	i915_gem_detect_bit_6_swizzle(dev);
4361
	init_waitqueue_head(&dev_priv->pending_flip_queue);
4362

4363 4364
	dev_priv->mm.interruptible = true;

4365 4366 4367
	dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
	dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
	register_shrinker(&dev_priv->mm.inactive_shrinker);
4368
}
4369 4370 4371 4372 4373

/*
 * Create a physically contiguous memory object for this object
 * e.g. for cursor + overlay regs
 */
4374 4375
static int i915_gem_init_phys_object(struct drm_device *dev,
				     int id, int size, int align)
4376 4377 4378 4379 4380 4381 4382 4383
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_i915_gem_phys_object *phys_obj;
	int ret;

	if (dev_priv->mm.phys_objs[id - 1] || !size)
		return 0;

4384
	phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4385 4386 4387 4388 4389
	if (!phys_obj)
		return -ENOMEM;

	phys_obj->id = id;

4390
	phys_obj->handle = drm_pci_alloc(dev, size, align);
4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402
	if (!phys_obj->handle) {
		ret = -ENOMEM;
		goto kfree_obj;
	}
#ifdef CONFIG_X86
	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
#endif

	dev_priv->mm.phys_objs[id - 1] = phys_obj;

	return 0;
kfree_obj:
4403
	kfree(phys_obj);
4404 4405 4406
	return ret;
}

4407
static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_i915_gem_phys_object *phys_obj;

	if (!dev_priv->mm.phys_objs[id - 1])
		return;

	phys_obj = dev_priv->mm.phys_objs[id - 1];
	if (phys_obj->cur_obj) {
		i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
	}

#ifdef CONFIG_X86
	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
#endif
	drm_pci_free(dev, phys_obj->handle);
	kfree(phys_obj);
	dev_priv->mm.phys_objs[id - 1] = NULL;
}

void i915_gem_free_all_phys_object(struct drm_device *dev)
{
	int i;

4432
	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4433 4434 4435 4436
		i915_gem_free_phys_object(dev, i);
}

void i915_gem_detach_phys_object(struct drm_device *dev,
4437
				 struct drm_i915_gem_object *obj)
4438
{
A
Al Viro 已提交
4439
	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4440
	char *vaddr;
4441 4442 4443
	int i;
	int page_count;

4444
	if (!obj->phys_obj)
4445
		return;
4446
	vaddr = obj->phys_obj->handle->vaddr;
4447

4448
	page_count = obj->base.size / PAGE_SIZE;
4449
	for (i = 0; i < page_count; i++) {
4450
		struct page *page = shmem_read_mapping_page(mapping, i);
4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461
		if (!IS_ERR(page)) {
			char *dst = kmap_atomic(page);
			memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
			kunmap_atomic(dst);

			drm_clflush_pages(&page, 1);

			set_page_dirty(page);
			mark_page_accessed(page);
			page_cache_release(page);
		}
4462
	}
4463
	i915_gem_chipset_flush(dev);
4464

4465 4466
	obj->phys_obj->cur_obj = NULL;
	obj->phys_obj = NULL;
4467 4468 4469 4470
}

int
i915_gem_attach_phys_object(struct drm_device *dev,
4471
			    struct drm_i915_gem_object *obj,
4472 4473
			    int id,
			    int align)
4474
{
A
Al Viro 已提交
4475
	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4476 4477 4478 4479 4480 4481 4482 4483
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret = 0;
	int page_count;
	int i;

	if (id > I915_MAX_PHYS_OBJECT)
		return -EINVAL;

4484 4485
	if (obj->phys_obj) {
		if (obj->phys_obj->id == id)
4486 4487 4488 4489 4490 4491 4492
			return 0;
		i915_gem_detach_phys_object(dev, obj);
	}

	/* create a new object */
	if (!dev_priv->mm.phys_objs[id - 1]) {
		ret = i915_gem_init_phys_object(dev, id,
4493
						obj->base.size, align);
4494
		if (ret) {
4495 4496
			DRM_ERROR("failed to init phys object %d size: %zu\n",
				  id, obj->base.size);
4497
			return ret;
4498 4499 4500 4501
		}
	}

	/* bind to the object */
4502 4503
	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
	obj->phys_obj->cur_obj = obj;
4504

4505
	page_count = obj->base.size / PAGE_SIZE;
4506 4507

	for (i = 0; i < page_count; i++) {
4508 4509 4510
		struct page *page;
		char *dst, *src;

4511
		page = shmem_read_mapping_page(mapping, i);
4512 4513
		if (IS_ERR(page))
			return PTR_ERR(page);
4514

4515
		src = kmap_atomic(page);
4516
		dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4517
		memcpy(dst, src, PAGE_SIZE);
P
Peter Zijlstra 已提交
4518
		kunmap_atomic(src);
4519

4520 4521 4522
		mark_page_accessed(page);
		page_cache_release(page);
	}
4523

4524 4525 4526 4527
	return 0;
}

static int
4528 4529
i915_gem_phys_pwrite(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
4530 4531 4532
		     struct drm_i915_gem_pwrite *args,
		     struct drm_file *file_priv)
{
4533
	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
V
Ville Syrjälä 已提交
4534
	char __user *user_data = to_user_ptr(args->data_ptr);
4535

4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548
	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
		unsigned long unwritten;

		/* The physical object once assigned is fixed for the lifetime
		 * of the obj, so we can safely drop the lock and continue
		 * to access vaddr.
		 */
		mutex_unlock(&dev->struct_mutex);
		unwritten = copy_from_user(vaddr, user_data, args->size);
		mutex_lock(&dev->struct_mutex);
		if (unwritten)
			return -EFAULT;
	}
4549

4550
	i915_gem_chipset_flush(dev);
4551 4552
	return 0;
}
4553

4554
void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4555
{
4556
	struct drm_i915_file_private *file_priv = file->driver_priv;
4557 4558 4559 4560 4561

	/* Clean up our request list when the client is going away, so that
	 * later retire_requests won't dereference our soon-to-be-gone
	 * file_priv.
	 */
4562
	spin_lock(&file_priv->mm.lock);
4563 4564 4565 4566 4567 4568 4569 4570 4571
	while (!list_empty(&file_priv->mm.request_list)) {
		struct drm_i915_gem_request *request;

		request = list_first_entry(&file_priv->mm.request_list,
					   struct drm_i915_gem_request,
					   client_list);
		list_del(&request->client_list);
		request->file_priv = NULL;
	}
4572
	spin_unlock(&file_priv->mm.lock);
4573
}
4574

4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587
static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
{
	if (!mutex_is_locked(mutex))
		return false;

#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
	return mutex->owner == task;
#else
	/* Since UP may be pre-empted, we cannot assume that we own the lock */
	return false;
#endif
}

4588
static int
4589
i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
4590
{
4591 4592 4593 4594 4595
	struct drm_i915_private *dev_priv =
		container_of(shrinker,
			     struct drm_i915_private,
			     mm.inactive_shrinker);
	struct drm_device *dev = dev_priv->dev;
C
Chris Wilson 已提交
4596
	struct drm_i915_gem_object *obj;
4597
	int nr_to_scan = sc->nr_to_scan;
4598
	bool unlock = true;
4599 4600
	int cnt;

4601 4602 4603 4604
	if (!mutex_trylock(&dev->struct_mutex)) {
		if (!mutex_is_locked_by(&dev->struct_mutex, current))
			return 0;

4605 4606 4607
		if (dev_priv->mm.shrinker_no_lock_stealing)
			return 0;

4608 4609
		unlock = false;
	}
4610

C
Chris Wilson 已提交
4611 4612
	if (nr_to_scan) {
		nr_to_scan -= i915_gem_purge(dev_priv, nr_to_scan);
4613 4614 4615
		if (nr_to_scan > 0)
			nr_to_scan -= __i915_gem_shrink(dev_priv, nr_to_scan,
							false);
C
Chris Wilson 已提交
4616 4617
		if (nr_to_scan > 0)
			i915_gem_shrink_all(dev_priv);
4618 4619
	}

4620
	cnt = 0;
4621
	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
4622 4623
		if (obj->pages_pin_count == 0)
			cnt += obj->base.size >> PAGE_SHIFT;
4624
	list_for_each_entry(obj, &dev_priv->mm.inactive_list, global_list)
4625
		if (obj->pin_count == 0 && obj->pages_pin_count == 0)
C
Chris Wilson 已提交
4626
			cnt += obj->base.size >> PAGE_SHIFT;
4627

4628 4629
	if (unlock)
		mutex_unlock(&dev->struct_mutex);
C
Chris Wilson 已提交
4630
	return cnt;
4631
}