i915_gem.c 115.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
/*
 * Copyright © 2008 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

28
#include <drm/drmP.h>
29
#include <drm/drm_vma_manager.h>
30
#include <drm/i915_drm.h>
31
#include "i915_drv.h"
C
Chris Wilson 已提交
32
#include "i915_trace.h"
33
#include "intel_drv.h"
34
#include <linux/shmem_fs.h>
35
#include <linux/slab.h>
36
#include <linux/swap.h>
J
Jesse Barnes 已提交
37
#include <linux/pci.h>
38
#include <linux/dma-buf.h>
39

40 41
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
42 43
static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
						    unsigned alignment,
44 45
						    bool map_and_fenceable,
						    bool nonblocking);
46 47
static int i915_gem_phys_pwrite(struct drm_device *dev,
				struct drm_i915_gem_object *obj,
48
				struct drm_i915_gem_pwrite *args,
49
				struct drm_file *file);
50

51 52 53 54 55 56
static void i915_gem_write_fence(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj);
static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
					 struct drm_i915_fence_reg *fence,
					 bool enable);

57
static int i915_gem_inactive_shrink(struct shrinker *shrinker,
58
				    struct shrink_control *sc);
C
Chris Wilson 已提交
59 60
static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
61
static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
62

63 64 65 66 67 68 69 70
static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
{
	if (obj->tiling_mode)
		i915_gem_release_mmap(obj);

	/* As we do not have an associated fence register, we will force
	 * a tiling change if we ever need to acquire one.
	 */
71
	obj->fence_dirty = false;
72 73 74
	obj->fence_reg = I915_FENCE_REG_NONE;
}

75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
/* some bookkeeping */
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
				  size_t size)
{
	dev_priv->mm.object_count++;
	dev_priv->mm.object_memory += size;
}

static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
				     size_t size)
{
	dev_priv->mm.object_count--;
	dev_priv->mm.object_memory -= size;
}

90
static int
91
i915_gem_wait_for_error(struct i915_gpu_error *error)
92 93 94
{
	int ret;

95 96
#define EXIT_COND (!i915_reset_in_progress(error) || \
		   i915_terminally_wedged(error))
97
	if (EXIT_COND)
98 99
		return 0;

100 101 102 103 104
	/*
	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
	 * userspace. If it takes that long something really bad is going on and
	 * we should simply try to bail out and fail as gracefully as possible.
	 */
105 106 107
	ret = wait_event_interruptible_timeout(error->reset_queue,
					       EXIT_COND,
					       10*HZ);
108 109 110 111
	if (ret == 0) {
		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		return -EIO;
	} else if (ret < 0) {
112
		return ret;
113
	}
114
#undef EXIT_COND
115

116
	return 0;
117 118
}

119
int i915_mutex_lock_interruptible(struct drm_device *dev)
120
{
121
	struct drm_i915_private *dev_priv = dev->dev_private;
122 123
	int ret;

124
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
125 126 127 128 129 130 131
	if (ret)
		return ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

132
	WARN_ON(i915_verify_lists(dev));
133 134
	return 0;
}
135

136
static inline bool
137
i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
138
{
139
	return i915_gem_obj_ggtt_bound(obj) && !obj->active;
140 141
}

J
Jesse Barnes 已提交
142 143
int
i915_gem_init_ioctl(struct drm_device *dev, void *data,
144
		    struct drm_file *file)
J
Jesse Barnes 已提交
145
{
146
	struct drm_i915_private *dev_priv = dev->dev_private;
J
Jesse Barnes 已提交
147
	struct drm_i915_gem_init *args = data;
148

149 150 151
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return -ENODEV;

152 153 154
	if (args->gtt_start >= args->gtt_end ||
	    (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
		return -EINVAL;
J
Jesse Barnes 已提交
155

156 157 158 159
	/* GEM with user mode setting was never supported on ilk and later. */
	if (INTEL_INFO(dev)->gen >= 5)
		return -ENODEV;

J
Jesse Barnes 已提交
160
	mutex_lock(&dev->struct_mutex);
161 162
	i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
				  args->gtt_end);
163
	dev_priv->gtt.mappable_end = args->gtt_end;
164 165
	mutex_unlock(&dev->struct_mutex);

166
	return 0;
167 168
}

169 170
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
171
			    struct drm_file *file)
172
{
173
	struct drm_i915_private *dev_priv = dev->dev_private;
174
	struct drm_i915_gem_get_aperture *args = data;
175 176
	struct drm_i915_gem_object *obj;
	size_t pinned;
177

178
	pinned = 0;
179
	mutex_lock(&dev->struct_mutex);
180
	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
181
		if (obj->pin_count)
182
			pinned += i915_gem_obj_ggtt_size(obj);
183
	mutex_unlock(&dev->struct_mutex);
184

B
Ben Widawsky 已提交
185
	args->aper_size = dev_priv->gtt.total;
186
	args->aper_available_size = args->aper_size - pinned;
187

188 189 190
	return 0;
}

191 192 193 194 195 196 197 198 199 200 201 202
void *i915_gem_object_alloc(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	return kmem_cache_alloc(dev_priv->slab, GFP_KERNEL | __GFP_ZERO);
}

void i915_gem_object_free(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	kmem_cache_free(dev_priv->slab, obj);
}

203 204 205 206 207
static int
i915_gem_create(struct drm_file *file,
		struct drm_device *dev,
		uint64_t size,
		uint32_t *handle_p)
208
{
209
	struct drm_i915_gem_object *obj;
210 211
	int ret;
	u32 handle;
212

213
	size = roundup(size, PAGE_SIZE);
214 215
	if (size == 0)
		return -EINVAL;
216 217

	/* Allocate the new object */
218
	obj = i915_gem_alloc_object(dev, size);
219 220 221
	if (obj == NULL)
		return -ENOMEM;

222
	ret = drm_gem_handle_create(file, &obj->base, &handle);
223
	if (ret) {
224 225
		drm_gem_object_release(&obj->base);
		i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
226
		i915_gem_object_free(obj);
227
		return ret;
228
	}
229

230
	/* drop reference from allocate - handle holds it now */
231
	drm_gem_object_unreference(&obj->base);
232 233
	trace_i915_gem_object_create(obj);

234
	*handle_p = handle;
235 236 237
	return 0;
}

238 239 240 241 242 243
int
i915_gem_dumb_create(struct drm_file *file,
		     struct drm_device *dev,
		     struct drm_mode_create_dumb *args)
{
	/* have to work out size/pitch and return them */
244
	args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
	args->size = args->pitch * args->height;
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

int i915_gem_dumb_destroy(struct drm_file *file,
			  struct drm_device *dev,
			  uint32_t handle)
{
	return drm_gem_handle_delete(file, handle);
}

/**
 * Creates a new mm object and returns a handle to it.
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file)
{
	struct drm_i915_gem_create *args = data;
265

266 267 268 269
	return i915_gem_create(file, dev,
			       args->size, &args->handle);
}

270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
static inline int
__copy_to_user_swizzled(char __user *cpu_vaddr,
			const char *gpu_vaddr, int gpu_offset,
			int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_to_user(cpu_vaddr + cpu_offset,
				     gpu_vaddr + swizzled_gpu_offset,
				     this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

296
static inline int
297 298
__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
			  const char __user *cpu_vaddr,
299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321
			  int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
				       cpu_vaddr + cpu_offset,
				       this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

322 323 324
/* Per-page copy function for the shmem pread fastpath.
 * Flushes invalid cachelines before reading the target if
 * needs_clflush is set. */
325
static int
326 327 328 329 330 331 332
shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

333
	if (unlikely(page_do_bit17_swizzling))
334 335 336 337 338 339 340 341 342 343 344
		return -EINVAL;

	vaddr = kmap_atomic(page);
	if (needs_clflush)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_to_user_inatomic(user_data,
				      vaddr + shmem_page_offset,
				      page_length);
	kunmap_atomic(vaddr);

345
	return ret ? -EFAULT : 0;
346 347
}

348 349 350 351
static void
shmem_clflush_swizzled_range(char *addr, unsigned long length,
			     bool swizzled)
{
352
	if (unlikely(swizzled)) {
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
		unsigned long start = (unsigned long) addr;
		unsigned long end = (unsigned long) addr + length;

		/* For swizzling simply ensure that we always flush both
		 * channels. Lame, but simple and it works. Swizzled
		 * pwrite/pread is far from a hotpath - current userspace
		 * doesn't use it at all. */
		start = round_down(start, 128);
		end = round_up(end, 128);

		drm_clflush_virt_range((void *)start, end - start);
	} else {
		drm_clflush_virt_range(addr, length);
	}

}

370 371 372 373 374 375 376 377 378 379 380 381
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
static int
shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

	vaddr = kmap(page);
	if (needs_clflush)
382 383 384
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
385 386 387 388 389 390 391 392 393 394 395

	if (page_do_bit17_swizzling)
		ret = __copy_to_user_swizzled(user_data,
					      vaddr, shmem_page_offset,
					      page_length);
	else
		ret = __copy_to_user(user_data,
				     vaddr + shmem_page_offset,
				     page_length);
	kunmap(page);

396
	return ret ? - EFAULT : 0;
397 398
}

399
static int
400 401 402 403
i915_gem_shmem_pread(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pread *args,
		     struct drm_file *file)
404
{
405
	char __user *user_data;
406
	ssize_t remain;
407
	loff_t offset;
408
	int shmem_page_offset, page_length, ret = 0;
409
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
410
	int prefaulted = 0;
411
	int needs_clflush = 0;
412
	struct sg_page_iter sg_iter;
413

V
Ville Syrjälä 已提交
414
	user_data = to_user_ptr(args->data_ptr);
415 416
	remain = args->size;

417
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
418

419 420 421 422 423 424 425
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
		/* If we're not in the cpu read domain, set ourself into the gtt
		 * read domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will dirty the data
		 * anyway again before the next pread happens. */
		if (obj->cache_level == I915_CACHE_NONE)
			needs_clflush = 1;
426
		if (i915_gem_obj_ggtt_bound(obj)) {
C
Chris Wilson 已提交
427 428 429 430
			ret = i915_gem_object_set_to_gtt_domain(obj, false);
			if (ret)
				return ret;
		}
431
	}
432

433 434 435 436 437 438
	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

	i915_gem_object_pin_pages(obj);

439
	offset = args->offset;
440

441 442
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
443
		struct page *page = sg_page_iter_page(&sg_iter);
444 445 446 447

		if (remain <= 0)
			break;

448 449 450 451 452
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
453
		shmem_page_offset = offset_in_page(offset);
454 455 456 457
		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

458 459 460
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

461 462 463 464 465
		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
		if (ret == 0)
			goto next_page;
466 467 468

		mutex_unlock(&dev->struct_mutex);

469
		if (!prefaulted) {
470
			ret = fault_in_multipages_writeable(user_data, remain);
471 472 473 474 475 476 477
			/* Userspace is tricking us, but we've already clobbered
			 * its pages with the prefault and promised to write the
			 * data up to the first fault. Hence ignore any errors
			 * and just continue. */
			(void)ret;
			prefaulted = 1;
		}
478

479 480 481
		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
				       user_data, page_do_bit17_swizzling,
				       needs_clflush);
482

483
		mutex_lock(&dev->struct_mutex);
484

485
next_page:
486 487
		mark_page_accessed(page);

488
		if (ret)
489 490
			goto out;

491
		remain -= page_length;
492
		user_data += page_length;
493 494 495
		offset += page_length;
	}

496
out:
497 498
	i915_gem_object_unpin_pages(obj);

499 500 501
	return ret;
}

502 503 504 505 506 507 508
/**
 * Reads data from the object referenced by handle.
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
509
		     struct drm_file *file)
510 511
{
	struct drm_i915_gem_pread *args = data;
512
	struct drm_i915_gem_object *obj;
513
	int ret = 0;
514

515 516 517 518
	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_WRITE,
V
Ville Syrjälä 已提交
519
		       to_user_ptr(args->data_ptr),
520 521 522
		       args->size))
		return -EFAULT;

523
	ret = i915_mutex_lock_interruptible(dev);
524
	if (ret)
525
		return ret;
526

527
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
528
	if (&obj->base == NULL) {
529 530
		ret = -ENOENT;
		goto unlock;
531
	}
532

533
	/* Bounds check source.  */
534 535
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
C
Chris Wilson 已提交
536
		ret = -EINVAL;
537
		goto out;
C
Chris Wilson 已提交
538 539
	}

540 541 542 543 544 545 546 547
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

C
Chris Wilson 已提交
548 549
	trace_i915_gem_object_pread(obj, args->offset, args->size);

550
	ret = i915_gem_shmem_pread(dev, obj, args, file);
551

552
out:
553
	drm_gem_object_unreference(&obj->base);
554
unlock:
555
	mutex_unlock(&dev->struct_mutex);
556
	return ret;
557 558
}

559 560
/* This is the fast write path which cannot handle
 * page faults in the source data
561
 */
562 563 564 565 566 567

static inline int
fast_user_write(struct io_mapping *mapping,
		loff_t page_base, int page_offset,
		char __user *user_data,
		int length)
568
{
569 570
	void __iomem *vaddr_atomic;
	void *vaddr;
571
	unsigned long unwritten;
572

P
Peter Zijlstra 已提交
573
	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
574 575 576
	/* We can use the cpu mem copy function because this is X86. */
	vaddr = (void __force*)vaddr_atomic + page_offset;
	unwritten = __copy_from_user_inatomic_nocache(vaddr,
577
						      user_data, length);
P
Peter Zijlstra 已提交
578
	io_mapping_unmap_atomic(vaddr_atomic);
579
	return unwritten;
580 581
}

582 583 584 585
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
 */
586
static int
587 588
i915_gem_gtt_pwrite_fast(struct drm_device *dev,
			 struct drm_i915_gem_object *obj,
589
			 struct drm_i915_gem_pwrite *args,
590
			 struct drm_file *file)
591
{
592
	drm_i915_private_t *dev_priv = dev->dev_private;
593
	ssize_t remain;
594
	loff_t offset, page_base;
595
	char __user *user_data;
D
Daniel Vetter 已提交
596 597
	int page_offset, page_length, ret;

598
	ret = i915_gem_object_pin(obj, 0, true, true);
D
Daniel Vetter 已提交
599 600 601 602 603 604 605 606 607 608
	if (ret)
		goto out;

	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		goto out_unpin;

	ret = i915_gem_object_put_fence(obj);
	if (ret)
		goto out_unpin;
609

V
Ville Syrjälä 已提交
610
	user_data = to_user_ptr(args->data_ptr);
611 612
	remain = args->size;

613
	offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
614 615 616 617

	while (remain > 0) {
		/* Operation in this page
		 *
618 619 620
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
621
		 */
622 623
		page_base = offset & PAGE_MASK;
		page_offset = offset_in_page(offset);
624 625 626 627 628
		page_length = remain;
		if ((page_offset + remain) > PAGE_SIZE)
			page_length = PAGE_SIZE - page_offset;

		/* If we get a fault while copying data, then (presumably) our
629 630
		 * source page isn't available.  Return the error and we'll
		 * retry in the slow path.
631
		 */
B
Ben Widawsky 已提交
632
		if (fast_user_write(dev_priv->gtt.mappable, page_base,
D
Daniel Vetter 已提交
633 634 635 636
				    page_offset, user_data, page_length)) {
			ret = -EFAULT;
			goto out_unpin;
		}
637

638 639 640
		remain -= page_length;
		user_data += page_length;
		offset += page_length;
641 642
	}

D
Daniel Vetter 已提交
643 644 645
out_unpin:
	i915_gem_object_unpin(obj);
out:
646
	return ret;
647 648
}

649 650 651 652
/* Per-page copy function for the shmem pwrite fastpath.
 * Flushes invalid cachelines before writing to the target if
 * needs_clflush_before is set and flushes out any written cachelines after
 * writing if needs_clflush is set. */
653
static int
654 655 656 657 658
shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
659
{
660
	char *vaddr;
661
	int ret;
662

663
	if (unlikely(page_do_bit17_swizzling))
664
		return -EINVAL;
665

666 667 668 669 670 671 672 673 674 675 676
	vaddr = kmap_atomic(page);
	if (needs_clflush_before)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
						user_data,
						page_length);
	if (needs_clflush_after)
		drm_clflush_virt_range(vaddr + shmem_page_offset,
				       page_length);
	kunmap_atomic(vaddr);
677

678
	return ret ? -EFAULT : 0;
679 680
}

681 682
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
683
static int
684 685 686 687 688
shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
689
{
690 691
	char *vaddr;
	int ret;
692

693
	vaddr = kmap(page);
694
	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
695 696 697
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
698 699
	if (page_do_bit17_swizzling)
		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
700 701
						user_data,
						page_length);
702 703 704 705 706
	else
		ret = __copy_from_user(vaddr + shmem_page_offset,
				       user_data,
				       page_length);
	if (needs_clflush_after)
707 708 709
		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
					     page_length,
					     page_do_bit17_swizzling);
710
	kunmap(page);
711

712
	return ret ? -EFAULT : 0;
713 714 715
}

static int
716 717 718 719
i915_gem_shmem_pwrite(struct drm_device *dev,
		      struct drm_i915_gem_object *obj,
		      struct drm_i915_gem_pwrite *args,
		      struct drm_file *file)
720 721
{
	ssize_t remain;
722 723
	loff_t offset;
	char __user *user_data;
724
	int shmem_page_offset, page_length, ret = 0;
725
	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
726
	int hit_slowpath = 0;
727 728
	int needs_clflush_after = 0;
	int needs_clflush_before = 0;
729
	struct sg_page_iter sg_iter;
730

V
Ville Syrjälä 已提交
731
	user_data = to_user_ptr(args->data_ptr);
732 733
	remain = args->size;

734
	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
735

736 737 738 739 740 741 742
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
		/* If we're not in the cpu write domain, set ourself into the gtt
		 * write domain and manually flush cachelines (if required). This
		 * optimizes for the case when the gpu will use the data
		 * right away and we therefore have to clflush anyway. */
		if (obj->cache_level == I915_CACHE_NONE)
			needs_clflush_after = 1;
743
		if (i915_gem_obj_ggtt_bound(obj)) {
C
Chris Wilson 已提交
744 745 746 747
			ret = i915_gem_object_set_to_gtt_domain(obj, true);
			if (ret)
				return ret;
		}
748 749 750 751 752 753 754
	}
	/* Same trick applies for invalidate partially written cachelines before
	 * writing.  */
	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
	    && obj->cache_level == I915_CACHE_NONE)
		needs_clflush_before = 1;

755 756 757 758 759 760
	ret = i915_gem_object_get_pages(obj);
	if (ret)
		return ret;

	i915_gem_object_pin_pages(obj);

761
	offset = args->offset;
762
	obj->dirty = 1;
763

764 765
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
			 offset >> PAGE_SHIFT) {
766
		struct page *page = sg_page_iter_page(&sg_iter);
767
		int partial_cacheline_write;
768

769 770 771
		if (remain <= 0)
			break;

772 773 774 775 776
		/* Operation in this page
		 *
		 * shmem_page_offset = offset within page in shmem file
		 * page_length = bytes to copy for this page
		 */
777
		shmem_page_offset = offset_in_page(offset);
778 779 780 781 782

		page_length = remain;
		if ((shmem_page_offset + page_length) > PAGE_SIZE)
			page_length = PAGE_SIZE - shmem_page_offset;

783 784 785 786 787 788 789
		/* If we don't overwrite a cacheline completely we need to be
		 * careful to have up-to-date data by first clflushing. Don't
		 * overcomplicate things and flush the entire patch. */
		partial_cacheline_write = needs_clflush_before &&
			((shmem_page_offset | page_length)
				& (boot_cpu_data.x86_clflush_size - 1));

790 791 792
		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
			(page_to_phys(page) & (1 << 17)) != 0;

793 794 795 796 797 798
		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
		if (ret == 0)
			goto next_page;
799 800 801

		hit_slowpath = 1;
		mutex_unlock(&dev->struct_mutex);
802 803 804 805
		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
					user_data, page_do_bit17_swizzling,
					partial_cacheline_write,
					needs_clflush_after);
806

807
		mutex_lock(&dev->struct_mutex);
808

809
next_page:
810 811 812
		set_page_dirty(page);
		mark_page_accessed(page);

813
		if (ret)
814 815
			goto out;

816
		remain -= page_length;
817
		user_data += page_length;
818
		offset += page_length;
819 820
	}

821
out:
822 823
	i915_gem_object_unpin_pages(obj);

824
	if (hit_slowpath) {
825 826 827 828 829 830 831
		/*
		 * Fixup: Flush cpu caches in case we didn't flush the dirty
		 * cachelines in-line while writing and the object moved
		 * out of the cpu write domain while we've dropped the lock.
		 */
		if (!needs_clflush_after &&
		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
832
			i915_gem_clflush_object(obj);
833
			i915_gem_chipset_flush(dev);
834
		}
835
	}
836

837
	if (needs_clflush_after)
838
		i915_gem_chipset_flush(dev);
839

840
	return ret;
841 842 843 844 845 846 847 848 849
}

/**
 * Writes data to the object referenced by handle.
 *
 * On error, the contents of the buffer that were to be modified are undefined.
 */
int
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
850
		      struct drm_file *file)
851 852
{
	struct drm_i915_gem_pwrite *args = data;
853
	struct drm_i915_gem_object *obj;
854 855 856 857 858 859
	int ret;

	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_READ,
V
Ville Syrjälä 已提交
860
		       to_user_ptr(args->data_ptr),
861 862 863
		       args->size))
		return -EFAULT;

V
Ville Syrjälä 已提交
864
	ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
865
					   args->size);
866 867
	if (ret)
		return -EFAULT;
868

869
	ret = i915_mutex_lock_interruptible(dev);
870
	if (ret)
871
		return ret;
872

873
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
874
	if (&obj->base == NULL) {
875 876
		ret = -ENOENT;
		goto unlock;
877
	}
878

879
	/* Bounds check destination. */
880 881
	if (args->offset > obj->base.size ||
	    args->size > obj->base.size - args->offset) {
C
Chris Wilson 已提交
882
		ret = -EINVAL;
883
		goto out;
C
Chris Wilson 已提交
884 885
	}

886 887 888 889 890 891 892 893
	/* prime objects have no backing filp to GEM pread/pwrite
	 * pages from.
	 */
	if (!obj->base.filp) {
		ret = -EINVAL;
		goto out;
	}

C
Chris Wilson 已提交
894 895
	trace_i915_gem_object_pwrite(obj, args->offset, args->size);

D
Daniel Vetter 已提交
896
	ret = -EFAULT;
897 898 899 900 901 902
	/* We can only do the GTT pwrite on untiled buffers, as otherwise
	 * it would end up going through the fenced access, and we'll get
	 * different detiling behavior between reading and writing.
	 * pread/pwrite currently are reading and writing from the CPU
	 * perspective, requiring manual detiling by the client.
	 */
903
	if (obj->phys_obj) {
904
		ret = i915_gem_phys_pwrite(dev, obj, args, file);
905 906 907
		goto out;
	}

908
	if (obj->cache_level == I915_CACHE_NONE &&
909
	    obj->tiling_mode == I915_TILING_NONE &&
910
	    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
911
		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
D
Daniel Vetter 已提交
912 913 914
		/* Note that the gtt paths might fail with non-page-backed user
		 * pointers (e.g. gtt mappings when moving data between
		 * textures). Fallback to the shmem path in that case. */
915
	}
916

917
	if (ret == -EFAULT || ret == -ENOSPC)
D
Daniel Vetter 已提交
918
		ret = i915_gem_shmem_pwrite(dev, obj, args, file);
919

920
out:
921
	drm_gem_object_unreference(&obj->base);
922
unlock:
923
	mutex_unlock(&dev->struct_mutex);
924 925 926
	return ret;
}

927
int
928
i915_gem_check_wedge(struct i915_gpu_error *error,
929 930
		     bool interruptible)
{
931
	if (i915_reset_in_progress(error)) {
932 933 934 935 936
		/* Non-interruptible callers can't handle -EAGAIN, hence return
		 * -EIO unconditionally for these. */
		if (!interruptible)
			return -EIO;

937 938
		/* Recovery complete, but the reset failed ... */
		if (i915_terminally_wedged(error))
939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959
			return -EIO;

		return -EAGAIN;
	}

	return 0;
}

/*
 * Compare seqno against outstanding lazy request. Emit a request if they are
 * equal.
 */
static int
i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
{
	int ret;

	BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));

	ret = 0;
	if (seqno == ring->outstanding_lazy_request)
960
		ret = i915_add_request(ring, NULL);
961 962 963 964 965 966 967 968

	return ret;
}

/**
 * __wait_seqno - wait until execution of seqno has finished
 * @ring: the ring expected to report seqno
 * @seqno: duh!
969
 * @reset_counter: reset sequence associated with the given seqno
970 971 972
 * @interruptible: do an interruptible wait (normally yes)
 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
 *
973 974 975 976 977 978 979
 * Note: It is of utmost importance that the passed in seqno and reset_counter
 * values have been read by the caller in an smp safe manner. Where read-side
 * locks are involved, it is sufficient to read the reset_counter before
 * unlocking the lock that protects the seqno. For lockless tricks, the
 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
 * inserted.
 *
980 981 982 983
 * Returns 0 if the seqno was found within the alloted time. Else returns the
 * errno with remaining time filled in timeout argument.
 */
static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
984
			unsigned reset_counter,
985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003
			bool interruptible, struct timespec *timeout)
{
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
	struct timespec before, now, wait_time={1,0};
	unsigned long timeout_jiffies;
	long end;
	bool wait_forever = true;
	int ret;

	if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
		return 0;

	trace_i915_gem_request_wait_begin(ring, seqno);

	if (timeout != NULL) {
		wait_time = *timeout;
		wait_forever = false;
	}

1004
	timeout_jiffies = timespec_to_jiffies_timeout(&wait_time);
1005 1006 1007 1008 1009 1010 1011 1012 1013

	if (WARN_ON(!ring->irq_get(ring)))
		return -ENODEV;

	/* Record current time in case interrupted by signal, or wedged * */
	getrawmonotonic(&before);

#define EXIT_COND \
	(i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
1014 1015
	 i915_reset_in_progress(&dev_priv->gpu_error) || \
	 reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
1016 1017 1018 1019 1020 1021 1022 1023 1024
	do {
		if (interruptible)
			end = wait_event_interruptible_timeout(ring->irq_queue,
							       EXIT_COND,
							       timeout_jiffies);
		else
			end = wait_event_timeout(ring->irq_queue, EXIT_COND,
						 timeout_jiffies);

1025 1026 1027 1028 1029 1030 1031
		/* We need to check whether any gpu reset happened in between
		 * the caller grabbing the seqno and now ... */
		if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
			end = -EAGAIN;

		/* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
		 * gone. */
1032
		ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
		if (ret)
			end = ret;
	} while (end == 0 && wait_forever);

	getrawmonotonic(&now);

	ring->irq_put(ring);
	trace_i915_gem_request_wait_end(ring, seqno);
#undef EXIT_COND

	if (timeout) {
		struct timespec sleep_time = timespec_sub(now, before);
		*timeout = timespec_sub(*timeout, sleep_time);
1046 1047
		if (!timespec_valid(timeout)) /* i.e. negative time remains */
			set_normalized_timespec(timeout, 0, 0);
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
	}

	switch (end) {
	case -EIO:
	case -EAGAIN: /* Wedged */
	case -ERESTARTSYS: /* Signal */
		return (int)end;
	case 0: /* Timeout */
		return -ETIME;
	default: /* Completed */
		WARN_ON(end < 0); /* We're not aware of other errors */
		return 0;
	}
}

/**
 * Waits for a sequence number to be signaled, and cleans up the
 * request and object lists appropriately for that event.
 */
int
i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
{
	struct drm_device *dev = ring->dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	bool interruptible = dev_priv->mm.interruptible;
	int ret;

	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
	BUG_ON(seqno == 0);

1078
	ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1079 1080 1081 1082 1083 1084 1085
	if (ret)
		return ret;

	ret = i915_gem_check_olr(ring, seqno);
	if (ret)
		return ret;

1086 1087 1088
	return __wait_seqno(ring, seqno,
			    atomic_read(&dev_priv->gpu_error.reset_counter),
			    interruptible, NULL);
1089 1090
}

1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
static int
i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj,
				     struct intel_ring_buffer *ring)
{
	i915_gem_retire_requests_ring(ring);

	/* Manually manage the write flush as we may have not yet
	 * retired the buffer.
	 *
	 * Note that the last_write_seqno is always the earlier of
	 * the two (read/write) seqno, so if we haved successfully waited,
	 * we know we have passed the last write.
	 */
	obj->last_write_seqno = 0;
	obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;

	return 0;
}

1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
/**
 * Ensures that all rendering to the object has completed and the object is
 * safe to unbind from the GTT or access from the CPU.
 */
static __must_check int
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
			       bool readonly)
{
	struct intel_ring_buffer *ring = obj->ring;
	u32 seqno;
	int ret;

	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
	if (seqno == 0)
		return 0;

	ret = i915_wait_seqno(ring, seqno);
	if (ret)
		return ret;

1130
	return i915_gem_object_wait_rendering__tail(obj, ring);
1131 1132
}

1133 1134 1135 1136 1137 1138 1139 1140 1141 1142
/* A nonblocking variant of the above wait. This is a highly dangerous routine
 * as the object state may change during this call.
 */
static __must_check int
i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
					    bool readonly)
{
	struct drm_device *dev = obj->base.dev;
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_ring_buffer *ring = obj->ring;
1143
	unsigned reset_counter;
1144 1145 1146 1147 1148 1149 1150 1151 1152 1153
	u32 seqno;
	int ret;

	BUG_ON(!mutex_is_locked(&dev->struct_mutex));
	BUG_ON(!dev_priv->mm.interruptible);

	seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
	if (seqno == 0)
		return 0;

1154
	ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1155 1156 1157 1158 1159 1160 1161
	if (ret)
		return ret;

	ret = i915_gem_check_olr(ring, seqno);
	if (ret)
		return ret;

1162
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1163
	mutex_unlock(&dev->struct_mutex);
1164
	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
1165
	mutex_lock(&dev->struct_mutex);
1166 1167
	if (ret)
		return ret;
1168

1169
	return i915_gem_object_wait_rendering__tail(obj, ring);
1170 1171
}

1172
/**
1173 1174
 * Called when user space prepares to use an object with the CPU, either
 * through the mmap ioctl's mapping or a GTT mapping.
1175 1176 1177
 */
int
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1178
			  struct drm_file *file)
1179 1180
{
	struct drm_i915_gem_set_domain *args = data;
1181
	struct drm_i915_gem_object *obj;
1182 1183
	uint32_t read_domains = args->read_domains;
	uint32_t write_domain = args->write_domain;
1184 1185
	int ret;

1186
	/* Only handle setting domains to types used by the CPU. */
1187
	if (write_domain & I915_GEM_GPU_DOMAINS)
1188 1189
		return -EINVAL;

1190
	if (read_domains & I915_GEM_GPU_DOMAINS)
1191 1192 1193 1194 1195 1196 1197 1198
		return -EINVAL;

	/* Having something in the write domain implies it's in the read
	 * domain, and only that read domain.  Enforce that in the request.
	 */
	if (write_domain != 0 && read_domains != write_domain)
		return -EINVAL;

1199
	ret = i915_mutex_lock_interruptible(dev);
1200
	if (ret)
1201
		return ret;
1202

1203
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1204
	if (&obj->base == NULL) {
1205 1206
		ret = -ENOENT;
		goto unlock;
1207
	}
1208

1209 1210 1211 1212 1213 1214 1215 1216
	/* Try to flush the object off the GPU without holding the lock.
	 * We will repeat the flush holding the lock in the normal manner
	 * to catch cases where we are gazumped.
	 */
	ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
	if (ret)
		goto unref;

1217 1218
	if (read_domains & I915_GEM_DOMAIN_GTT) {
		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1219 1220 1221 1222 1223 1224 1225

		/* Silently promote "you're not bound, there was nothing to do"
		 * to success, since the client was just asking us to
		 * make sure everything was done.
		 */
		if (ret == -EINVAL)
			ret = 0;
1226
	} else {
1227
		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1228 1229
	}

1230
unref:
1231
	drm_gem_object_unreference(&obj->base);
1232
unlock:
1233 1234 1235 1236 1237 1238 1239 1240 1241
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Called when user space has done writes to this buffer
 */
int
i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1242
			 struct drm_file *file)
1243 1244
{
	struct drm_i915_gem_sw_finish *args = data;
1245
	struct drm_i915_gem_object *obj;
1246 1247
	int ret = 0;

1248
	ret = i915_mutex_lock_interruptible(dev);
1249
	if (ret)
1250
		return ret;
1251

1252
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1253
	if (&obj->base == NULL) {
1254 1255
		ret = -ENOENT;
		goto unlock;
1256 1257 1258
	}

	/* Pinned buffers may be scanout, so flush the cache */
1259
	if (obj->pin_count)
1260 1261
		i915_gem_object_flush_cpu_write_domain(obj);

1262
	drm_gem_object_unreference(&obj->base);
1263
unlock:
1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

/**
 * Maps the contents of an object, returning the address it is mapped
 * into.
 *
 * While the mapping holds a reference on the contents of the object, it doesn't
 * imply a ref on the object itself.
 */
int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1277
		    struct drm_file *file)
1278 1279 1280 1281 1282
{
	struct drm_i915_gem_mmap *args = data;
	struct drm_gem_object *obj;
	unsigned long addr;

1283
	obj = drm_gem_object_lookup(dev, file, args->handle);
1284
	if (obj == NULL)
1285
		return -ENOENT;
1286

1287 1288 1289 1290 1291 1292 1293 1294
	/* prime objects have no backing filp to GEM mmap
	 * pages from.
	 */
	if (!obj->filp) {
		drm_gem_object_unreference_unlocked(obj);
		return -EINVAL;
	}

1295
	addr = vm_mmap(obj->filp, 0, args->size,
1296 1297
		       PROT_READ | PROT_WRITE, MAP_SHARED,
		       args->offset);
1298
	drm_gem_object_unreference_unlocked(obj);
1299 1300 1301 1302 1303 1304 1305 1306
	if (IS_ERR((void *)addr))
		return addr;

	args->addr_ptr = (uint64_t) addr;

	return 0;
}

1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324
/**
 * i915_gem_fault - fault a page into the GTT
 * vma: VMA in question
 * vmf: fault info
 *
 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
 * from userspace.  The fault handler takes care of binding the object to
 * the GTT (if needed), allocating and programming a fence register (again,
 * only if needed based on whether the old reg is still valid or the object
 * is tiled) and inserting a new PTE into the faulting process.
 *
 * Note that the faulting process may involve evicting existing objects
 * from the GTT and/or fence registers to make room.  So performance may
 * suffer if the GTT working set is large or there are few fence registers
 * left.
 */
int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
1325 1326
	struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
	struct drm_device *dev = obj->base.dev;
1327
	drm_i915_private_t *dev_priv = dev->dev_private;
1328 1329 1330
	pgoff_t page_offset;
	unsigned long pfn;
	int ret = 0;
1331
	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1332 1333 1334 1335 1336

	/* We don't use vmf->pgoff since that has the fake offset */
	page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
		PAGE_SHIFT;

1337 1338 1339
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto out;
1340

C
Chris Wilson 已提交
1341 1342
	trace_i915_gem_object_fault(obj, page_offset, true, write);

1343 1344 1345 1346 1347 1348
	/* Access to snoopable pages through the GTT is incoherent. */
	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
		ret = -EINVAL;
		goto unlock;
	}

1349
	/* Now bind it into the GTT if needed */
1350 1351 1352
	ret = i915_gem_object_pin(obj, 0, true, false);
	if (ret)
		goto unlock;
1353

1354 1355 1356
	ret = i915_gem_object_set_to_gtt_domain(obj, write);
	if (ret)
		goto unpin;
1357

1358
	ret = i915_gem_object_get_fence(obj);
1359
	if (ret)
1360
		goto unpin;
1361

1362 1363
	obj->fault_mappable = true;

1364 1365 1366
	pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
	pfn >>= PAGE_SHIFT;
	pfn += page_offset;
1367 1368 1369

	/* Finally, remap it using the new GTT offset */
	ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1370 1371
unpin:
	i915_gem_object_unpin(obj);
1372
unlock:
1373
	mutex_unlock(&dev->struct_mutex);
1374
out:
1375
	switch (ret) {
1376
	case -EIO:
1377 1378 1379
		/* If this -EIO is due to a gpu hang, give the reset code a
		 * chance to clean up the mess. Otherwise return the proper
		 * SIGBUS. */
1380
		if (i915_terminally_wedged(&dev_priv->gpu_error))
1381
			return VM_FAULT_SIGBUS;
1382
	case -EAGAIN:
1383 1384 1385 1386 1387 1388 1389
		/* Give the error handler a chance to run and move the
		 * objects off the GPU active list. Next time we service the
		 * fault, we should be able to transition the page into the
		 * GTT without touching the GPU (and so avoid further
		 * EIO/EGAIN). If the GPU is wedged, then there is no issue
		 * with coherency, just lost writes.
		 */
1390
		set_need_resched();
1391 1392
	case 0:
	case -ERESTARTSYS:
1393
	case -EINTR:
1394 1395 1396 1397 1398
	case -EBUSY:
		/*
		 * EBUSY is ok: this just means that another thread
		 * already did the job.
		 */
1399
		return VM_FAULT_NOPAGE;
1400 1401
	case -ENOMEM:
		return VM_FAULT_OOM;
1402 1403
	case -ENOSPC:
		return VM_FAULT_SIGBUS;
1404
	default:
1405
		WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1406
		return VM_FAULT_SIGBUS;
1407 1408 1409
	}
}

1410 1411 1412 1413
/**
 * i915_gem_release_mmap - remove physical page mappings
 * @obj: obj in question
 *
1414
 * Preserve the reservation of the mmapping with the DRM core code, but
1415 1416 1417 1418 1419 1420 1421 1422 1423
 * relinquish ownership of the pages back to the system.
 *
 * It is vital that we remove the page mapping if we have mapped a tiled
 * object through the GTT and then lose the fence register due to
 * resource pressure. Similarly if the object has been moved out of the
 * aperture, than pages mapped into userspace must be revoked. Removing the
 * mapping will then trigger a page fault on the next user access, allowing
 * fixup by i915_gem_fault().
 */
1424
void
1425
i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1426
{
1427 1428
	if (!obj->fault_mappable)
		return;
1429

1430 1431
	if (obj->base.dev->dev_mapping)
		unmap_mapping_range(obj->base.dev->dev_mapping,
1432
				    (loff_t)drm_vma_node_offset_addr(&obj->base.vma_node),
1433
				    obj->base.size, 1);
1434

1435
	obj->fault_mappable = false;
1436 1437
}

1438
uint32_t
1439
i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1440
{
1441
	uint32_t gtt_size;
1442 1443

	if (INTEL_INFO(dev)->gen >= 4 ||
1444 1445
	    tiling_mode == I915_TILING_NONE)
		return size;
1446 1447 1448

	/* Previous chips need a power-of-two fence region when tiling */
	if (INTEL_INFO(dev)->gen == 3)
1449
		gtt_size = 1024*1024;
1450
	else
1451
		gtt_size = 512*1024;
1452

1453 1454
	while (gtt_size < size)
		gtt_size <<= 1;
1455

1456
	return gtt_size;
1457 1458
}

1459 1460 1461 1462 1463
/**
 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
 * @obj: object to check
 *
 * Return the required GTT alignment for an object, taking into account
1464
 * potential fence register mapping.
1465
 */
1466 1467 1468
uint32_t
i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
			   int tiling_mode, bool fenced)
1469 1470 1471 1472 1473
{
	/*
	 * Minimum alignment is 4k (GTT page size), but might be greater
	 * if a fence register is needed for the object.
	 */
1474
	if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
1475
	    tiling_mode == I915_TILING_NONE)
1476 1477
		return 4096;

1478 1479 1480 1481
	/*
	 * Previous chips need to be aligned to the size of the smallest
	 * fence register that can contain the object.
	 */
1482
	return i915_gem_get_gtt_size(dev, size, tiling_mode);
1483 1484
}

1485 1486 1487 1488 1489
static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	int ret;

1490
	if (drm_vma_node_has_offset(&obj->base.vma_node))
1491 1492
		return 0;

1493 1494
	dev_priv->mm.shrinker_no_lock_stealing = true;

1495 1496
	ret = drm_gem_create_mmap_offset(&obj->base);
	if (ret != -ENOSPC)
1497
		goto out;
1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508

	/* Badly fragmented mmap space? The only way we can recover
	 * space is by destroying unwanted objects. We can't randomly release
	 * mmap_offsets as userspace expects them to be persistent for the
	 * lifetime of the objects. The closest we can is to release the
	 * offsets on purgeable objects by truncating it and marking it purged,
	 * which prevents userspace from ever using that object again.
	 */
	i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
	ret = drm_gem_create_mmap_offset(&obj->base);
	if (ret != -ENOSPC)
1509
		goto out;
1510 1511

	i915_gem_shrink_all(dev_priv);
1512 1513 1514 1515 1516
	ret = drm_gem_create_mmap_offset(&obj->base);
out:
	dev_priv->mm.shrinker_no_lock_stealing = false;

	return ret;
1517 1518 1519 1520 1521 1522 1523
}

static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
{
	drm_gem_free_mmap_offset(&obj->base);
}

1524
int
1525 1526 1527 1528
i915_gem_mmap_gtt(struct drm_file *file,
		  struct drm_device *dev,
		  uint32_t handle,
		  uint64_t *offset)
1529
{
1530
	struct drm_i915_private *dev_priv = dev->dev_private;
1531
	struct drm_i915_gem_object *obj;
1532 1533
	int ret;

1534
	ret = i915_mutex_lock_interruptible(dev);
1535
	if (ret)
1536
		return ret;
1537

1538
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1539
	if (&obj->base == NULL) {
1540 1541 1542
		ret = -ENOENT;
		goto unlock;
	}
1543

B
Ben Widawsky 已提交
1544
	if (obj->base.size > dev_priv->gtt.mappable_end) {
1545
		ret = -E2BIG;
1546
		goto out;
1547 1548
	}

1549
	if (obj->madv != I915_MADV_WILLNEED) {
1550
		DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1551 1552
		ret = -EINVAL;
		goto out;
1553 1554
	}

1555 1556 1557
	ret = i915_gem_object_create_mmap_offset(obj);
	if (ret)
		goto out;
1558

1559
	*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
1560

1561
out:
1562
	drm_gem_object_unreference(&obj->base);
1563
unlock:
1564
	mutex_unlock(&dev->struct_mutex);
1565
	return ret;
1566 1567
}

1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591
/**
 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 * @dev: DRM device
 * @data: GTT mapping ioctl data
 * @file: GEM object info
 *
 * Simply returns the fake offset to userspace so it can mmap it.
 * The mmap call will end up in drm_gem_mmap(), which will set things
 * up so we can get faults in the handler above.
 *
 * The fault handler will take care of binding the object into the GTT
 * (since it may have been evicted to make room for something), allocating
 * a fence register, and mapping the appropriate aperture address into
 * userspace.
 */
int
i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file)
{
	struct drm_i915_gem_mmap_gtt *args = data;

	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
}

D
Daniel Vetter 已提交
1592 1593 1594
/* Immediately discard the backing storage */
static void
i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1595 1596 1597
{
	struct inode *inode;

1598
	i915_gem_object_free_mmap_offset(obj);
1599

1600 1601
	if (obj->base.filp == NULL)
		return;
1602

D
Daniel Vetter 已提交
1603 1604 1605 1606 1607
	/* Our goal here is to return as much of the memory as
	 * is possible back to the system as we are called from OOM.
	 * To do this we must instruct the shmfs to drop all of its
	 * backing pages, *now*.
	 */
A
Al Viro 已提交
1608
	inode = file_inode(obj->base.filp);
D
Daniel Vetter 已提交
1609
	shmem_truncate_range(inode, 0, (loff_t)-1);
1610

D
Daniel Vetter 已提交
1611 1612
	obj->madv = __I915_MADV_PURGED;
}
1613

D
Daniel Vetter 已提交
1614 1615 1616 1617
static inline int
i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
{
	return obj->madv == I915_MADV_DONTNEED;
1618 1619
}

1620
static void
1621
i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1622
{
1623 1624
	struct sg_page_iter sg_iter;
	int ret;
1625

1626
	BUG_ON(obj->madv == __I915_MADV_PURGED);
1627

C
Chris Wilson 已提交
1628 1629 1630 1631 1632 1633 1634 1635 1636 1637
	ret = i915_gem_object_set_to_cpu_domain(obj, true);
	if (ret) {
		/* In the event of a disaster, abandon all caches and
		 * hope for the best.
		 */
		WARN_ON(ret != -EIO);
		i915_gem_clflush_object(obj);
		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	}

1638
	if (i915_gem_object_needs_bit17_swizzle(obj))
1639 1640
		i915_gem_object_save_bit_17_swizzle(obj);

1641 1642
	if (obj->madv == I915_MADV_DONTNEED)
		obj->dirty = 0;
1643

1644
	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
1645
		struct page *page = sg_page_iter_page(&sg_iter);
1646

1647
		if (obj->dirty)
1648
			set_page_dirty(page);
1649

1650
		if (obj->madv == I915_MADV_WILLNEED)
1651
			mark_page_accessed(page);
1652

1653
		page_cache_release(page);
1654
	}
1655
	obj->dirty = 0;
1656

1657 1658
	sg_free_table(obj->pages);
	kfree(obj->pages);
1659
}
C
Chris Wilson 已提交
1660

1661
int
1662 1663 1664 1665
i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
{
	const struct drm_i915_gem_object_ops *ops = obj->ops;

1666
	if (obj->pages == NULL)
1667 1668
		return 0;

1669
	BUG_ON(i915_gem_obj_ggtt_bound(obj));
C
Chris Wilson 已提交
1670

1671 1672 1673
	if (obj->pages_pin_count)
		return -EBUSY;

1674 1675 1676
	/* ->put_pages might need to allocate memory for the bit17 swizzle
	 * array, hence protect them from being reaped by removing them from gtt
	 * lists early. */
1677
	list_del(&obj->global_list);
1678

1679
	ops->put_pages(obj);
1680
	obj->pages = NULL;
1681

C
Chris Wilson 已提交
1682 1683 1684 1685 1686 1687 1688
	if (i915_gem_object_is_purgeable(obj))
		i915_gem_object_truncate(obj);

	return 0;
}

static long
1689 1690
__i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
		  bool purgeable_only)
C
Chris Wilson 已提交
1691 1692 1693 1694 1695 1696
{
	struct drm_i915_gem_object *obj, *next;
	long count = 0;

	list_for_each_entry_safe(obj, next,
				 &dev_priv->mm.unbound_list,
1697
				 global_list) {
1698
		if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
1699
		    i915_gem_object_put_pages(obj) == 0) {
C
Chris Wilson 已提交
1700 1701 1702 1703 1704 1705 1706 1707 1708
			count += obj->base.size >> PAGE_SHIFT;
			if (count >= target)
				return count;
		}
	}

	list_for_each_entry_safe(obj, next,
				 &dev_priv->mm.inactive_list,
				 mm_list) {
1709
		if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
C
Chris Wilson 已提交
1710
		    i915_gem_object_unbind(obj) == 0 &&
1711
		    i915_gem_object_put_pages(obj) == 0) {
C
Chris Wilson 已提交
1712 1713 1714 1715 1716 1717 1718 1719 1720
			count += obj->base.size >> PAGE_SHIFT;
			if (count >= target)
				return count;
		}
	}

	return count;
}

1721 1722 1723 1724 1725 1726
static long
i915_gem_purge(struct drm_i915_private *dev_priv, long target)
{
	return __i915_gem_shrink(dev_priv, target, true);
}

C
Chris Wilson 已提交
1727 1728 1729 1730 1731 1732 1733
static void
i915_gem_shrink_all(struct drm_i915_private *dev_priv)
{
	struct drm_i915_gem_object *obj, *next;

	i915_gem_evict_everything(dev_priv->dev);

1734 1735
	list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
				 global_list)
1736
		i915_gem_object_put_pages(obj);
D
Daniel Vetter 已提交
1737 1738
}

1739
static int
C
Chris Wilson 已提交
1740
i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
1741
{
C
Chris Wilson 已提交
1742
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1743 1744
	int page_count, i;
	struct address_space *mapping;
1745 1746
	struct sg_table *st;
	struct scatterlist *sg;
1747
	struct sg_page_iter sg_iter;
1748
	struct page *page;
1749
	unsigned long last_pfn = 0;	/* suppress gcc warning */
C
Chris Wilson 已提交
1750
	gfp_t gfp;
1751

C
Chris Wilson 已提交
1752 1753 1754 1755 1756 1757 1758
	/* Assert that the object is not currently in any GPU domain. As it
	 * wasn't in the GTT, there shouldn't be any way it could have been in
	 * a GPU cache
	 */
	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);

1759 1760 1761 1762
	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (st == NULL)
		return -ENOMEM;

1763
	page_count = obj->base.size / PAGE_SIZE;
1764 1765 1766
	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		sg_free_table(st);
		kfree(st);
1767
		return -ENOMEM;
1768
	}
1769

1770 1771 1772 1773 1774
	/* Get the list of pages out of our struct file.  They'll be pinned
	 * at this point until we release them.
	 *
	 * Fail silently without starting the shrinker
	 */
A
Al Viro 已提交
1775
	mapping = file_inode(obj->base.filp)->i_mapping;
C
Chris Wilson 已提交
1776
	gfp = mapping_gfp_mask(mapping);
1777
	gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
C
Chris Wilson 已提交
1778
	gfp &= ~(__GFP_IO | __GFP_WAIT);
1779 1780 1781
	sg = st->sgl;
	st->nents = 0;
	for (i = 0; i < page_count; i++) {
C
Chris Wilson 已提交
1782 1783 1784 1785 1786 1787 1788 1789 1790 1791
		page = shmem_read_mapping_page_gfp(mapping, i, gfp);
		if (IS_ERR(page)) {
			i915_gem_purge(dev_priv, page_count);
			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
		}
		if (IS_ERR(page)) {
			/* We've tried hard to allocate the memory by reaping
			 * our own buffer, now let the real VM do its job and
			 * go down in flames if truly OOM.
			 */
1792
			gfp &= ~(__GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD);
C
Chris Wilson 已提交
1793 1794 1795 1796 1797 1798 1799
			gfp |= __GFP_IO | __GFP_WAIT;

			i915_gem_shrink_all(dev_priv);
			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
			if (IS_ERR(page))
				goto err_pages;

1800
			gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
C
Chris Wilson 已提交
1801 1802
			gfp &= ~(__GFP_IO | __GFP_WAIT);
		}
1803 1804 1805 1806 1807 1808 1809 1810
#ifdef CONFIG_SWIOTLB
		if (swiotlb_nr_tbl()) {
			st->nents++;
			sg_set_page(sg, page, PAGE_SIZE, 0);
			sg = sg_next(sg);
			continue;
		}
#endif
1811 1812 1813 1814 1815 1816 1817 1818 1819
		if (!i || page_to_pfn(page) != last_pfn + 1) {
			if (i)
				sg = sg_next(sg);
			st->nents++;
			sg_set_page(sg, page, PAGE_SIZE, 0);
		} else {
			sg->length += PAGE_SIZE;
		}
		last_pfn = page_to_pfn(page);
1820
	}
1821 1822 1823 1824
#ifdef CONFIG_SWIOTLB
	if (!swiotlb_nr_tbl())
#endif
		sg_mark_end(sg);
1825 1826
	obj->pages = st;

1827
	if (i915_gem_object_needs_bit17_swizzle(obj))
1828 1829 1830 1831 1832
		i915_gem_object_do_bit_17_swizzle(obj);

	return 0;

err_pages:
1833 1834
	sg_mark_end(sg);
	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
1835
		page_cache_release(sg_page_iter_page(&sg_iter));
1836 1837
	sg_free_table(st);
	kfree(st);
1838
	return PTR_ERR(page);
1839 1840
}

1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854
/* Ensure that the associated pages are gathered from the backing storage
 * and pinned into our object. i915_gem_object_get_pages() may be called
 * multiple times before they are released by a single call to
 * i915_gem_object_put_pages() - once the pages are no longer referenced
 * either as a result of memory pressure (reaping pages under the shrinker)
 * or as the object is itself released.
 */
int
i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
	const struct drm_i915_gem_object_ops *ops = obj->ops;
	int ret;

1855
	if (obj->pages)
1856 1857
		return 0;

1858 1859 1860 1861 1862
	if (obj->madv != I915_MADV_WILLNEED) {
		DRM_ERROR("Attempting to obtain a purgeable object\n");
		return -EINVAL;
	}

1863 1864
	BUG_ON(obj->pages_pin_count);

1865 1866 1867 1868
	ret = ops->get_pages(obj);
	if (ret)
		return ret;

1869
	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
1870
	return 0;
1871 1872
}

1873
void
1874
i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1875
			       struct intel_ring_buffer *ring)
1876
{
1877
	struct drm_device *dev = obj->base.dev;
1878
	struct drm_i915_private *dev_priv = dev->dev_private;
1879
	u32 seqno = intel_ring_get_seqno(ring);
1880

1881
	BUG_ON(ring == NULL);
1882 1883 1884 1885
	if (obj->ring != ring && obj->last_write_seqno) {
		/* Keep the seqno relative to the current ring */
		obj->last_write_seqno = seqno;
	}
1886
	obj->ring = ring;
1887 1888

	/* Add a reference if we're newly entering the active list. */
1889 1890 1891
	if (!obj->active) {
		drm_gem_object_reference(&obj->base);
		obj->active = 1;
1892
	}
1893

1894
	/* Move from whatever list we were on to the tail of execution. */
1895 1896
	list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
	list_move_tail(&obj->ring_list, &ring->active_list);
1897

1898
	obj->last_read_seqno = seqno;
1899

1900
	if (obj->fenced_gpu_access) {
1901 1902
		obj->last_fenced_seqno = seqno;

1903 1904 1905 1906 1907 1908 1909 1910
		/* Bump MRU to take account of the delayed flush */
		if (obj->fence_reg != I915_FENCE_REG_NONE) {
			struct drm_i915_fence_reg *reg;

			reg = &dev_priv->fence_regs[obj->fence_reg];
			list_move_tail(&reg->lru_list,
				       &dev_priv->mm.fence_list);
		}
1911 1912 1913 1914 1915
	}
}

static void
i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1916
{
1917
	struct drm_device *dev = obj->base.dev;
1918
	struct drm_i915_private *dev_priv = dev->dev_private;
1919

1920
	BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
1921
	BUG_ON(!obj->active);
1922

1923
	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1924

1925
	list_del_init(&obj->ring_list);
1926 1927
	obj->ring = NULL;

1928 1929 1930 1931 1932
	obj->last_read_seqno = 0;
	obj->last_write_seqno = 0;
	obj->base.write_domain = 0;

	obj->last_fenced_seqno = 0;
1933 1934 1935 1936 1937 1938
	obj->fenced_gpu_access = false;

	obj->active = 0;
	drm_gem_object_unreference(&obj->base);

	WARN_ON(i915_verify_lists(dev));
1939
}
1940

1941
static int
1942
i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
1943
{
1944 1945 1946
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct intel_ring_buffer *ring;
	int ret, i, j;
1947

1948
	/* Carefully retire all requests without writing to the rings */
1949
	for_each_ring(ring, dev_priv, i) {
1950 1951 1952
		ret = intel_ring_idle(ring);
		if (ret)
			return ret;
1953 1954
	}
	i915_gem_retire_requests(dev);
1955 1956

	/* Finally reset hw state */
1957
	for_each_ring(ring, dev_priv, i) {
1958
		intel_ring_init_seqno(ring, seqno);
1959

1960 1961 1962
		for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
			ring->sync_seqno[j] = 0;
	}
1963

1964
	return 0;
1965 1966
}

1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992
int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	if (seqno == 0)
		return -EINVAL;

	/* HWS page needs to be set less than what we
	 * will inject to ring
	 */
	ret = i915_gem_init_seqno(dev, seqno - 1);
	if (ret)
		return ret;

	/* Carefully set the last_seqno value so that wrap
	 * detection still works
	 */
	dev_priv->next_seqno = seqno;
	dev_priv->last_seqno = seqno - 1;
	if (dev_priv->last_seqno == 0)
		dev_priv->last_seqno--;

	return 0;
}

1993 1994
int
i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
1995
{
1996 1997 1998 1999
	struct drm_i915_private *dev_priv = dev->dev_private;

	/* reserve 0 for non-seqno */
	if (dev_priv->next_seqno == 0) {
2000
		int ret = i915_gem_init_seqno(dev, 0);
2001 2002
		if (ret)
			return ret;
2003

2004 2005
		dev_priv->next_seqno = 1;
	}
2006

2007
	*seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2008
	return 0;
2009 2010
}

2011 2012
int __i915_add_request(struct intel_ring_buffer *ring,
		       struct drm_file *file,
2013
		       struct drm_i915_gem_object *obj,
2014
		       u32 *out_seqno)
2015
{
C
Chris Wilson 已提交
2016
	drm_i915_private_t *dev_priv = ring->dev->dev_private;
2017
	struct drm_i915_gem_request *request;
2018
	u32 request_ring_position, request_start;
2019
	int was_empty;
2020 2021
	int ret;

2022
	request_start = intel_ring_get_tail(ring);
2023 2024 2025 2026 2027 2028 2029
	/*
	 * Emit any outstanding flushes - execbuf can fail to emit the flush
	 * after having emitted the batchbuffer command. Hence we need to fix
	 * things up similar to emitting the lazy request. The difference here
	 * is that the flush _must_ happen before the next request, no matter
	 * what.
	 */
2030 2031 2032
	ret = intel_ring_flush_all_caches(ring);
	if (ret)
		return ret;
2033

2034 2035 2036
	request = kmalloc(sizeof(*request), GFP_KERNEL);
	if (request == NULL)
		return -ENOMEM;
2037

2038

2039 2040 2041 2042 2043 2044 2045
	/* Record the position of the start of the request so that
	 * should we detect the updated seqno part-way through the
	 * GPU processing the request, we never over-estimate the
	 * position of the head.
	 */
	request_ring_position = intel_ring_get_tail(ring);

2046
	ret = ring->add_request(ring);
2047 2048 2049 2050
	if (ret) {
		kfree(request);
		return ret;
	}
2051

2052
	request->seqno = intel_ring_get_seqno(ring);
2053
	request->ring = ring;
2054
	request->head = request_start;
2055
	request->tail = request_ring_position;
2056
	request->ctx = ring->last_context;
2057 2058 2059 2060 2061 2062 2063 2064
	request->batch_obj = obj;

	/* Whilst this request exists, batch_obj will be on the
	 * active_list, and so will hold the active reference. Only when this
	 * request is retired will the the batch_obj be moved onto the
	 * inactive_list and lose its active reference. Hence we do not need
	 * to explicitly hold another reference here.
	 */
2065 2066 2067 2068

	if (request->ctx)
		i915_gem_context_reference(request->ctx);

2069
	request->emitted_jiffies = jiffies;
2070 2071
	was_empty = list_empty(&ring->request_list);
	list_add_tail(&request->list, &ring->request_list);
2072
	request->file_priv = NULL;
2073

C
Chris Wilson 已提交
2074 2075 2076
	if (file) {
		struct drm_i915_file_private *file_priv = file->driver_priv;

2077
		spin_lock(&file_priv->mm.lock);
2078
		request->file_priv = file_priv;
2079
		list_add_tail(&request->client_list,
2080
			      &file_priv->mm.request_list);
2081
		spin_unlock(&file_priv->mm.lock);
2082
	}
2083

2084
	trace_i915_gem_request_add(ring, request->seqno);
2085
	ring->outstanding_lazy_request = 0;
C
Chris Wilson 已提交
2086

2087
	if (!dev_priv->ums.mm_suspended) {
2088
		if (i915_enable_hangcheck) {
2089
			mod_timer(&dev_priv->gpu_error.hangcheck_timer,
2090
				  round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
2091
		}
2092
		if (was_empty) {
2093
			queue_delayed_work(dev_priv->wq,
2094 2095
					   &dev_priv->mm.retire_work,
					   round_jiffies_up_relative(HZ));
2096 2097
			intel_mark_busy(dev_priv->dev);
		}
B
Ben Gamari 已提交
2098
	}
2099

2100
	if (out_seqno)
2101
		*out_seqno = request->seqno;
2102
	return 0;
2103 2104
}

2105 2106
static inline void
i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
2107
{
2108
	struct drm_i915_file_private *file_priv = request->file_priv;
2109

2110 2111
	if (!file_priv)
		return;
C
Chris Wilson 已提交
2112

2113
	spin_lock(&file_priv->mm.lock);
2114 2115 2116 2117
	if (request->file_priv) {
		list_del(&request->client_list);
		request->file_priv = NULL;
	}
2118
	spin_unlock(&file_priv->mm.lock);
2119 2120
}

2121 2122
static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj)
{
2123 2124
	if (acthd >= i915_gem_obj_ggtt_offset(obj) &&
	    acthd < i915_gem_obj_ggtt_offset(obj) + obj->base.size)
2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181
		return true;

	return false;
}

static bool i915_head_inside_request(const u32 acthd_unmasked,
				     const u32 request_start,
				     const u32 request_end)
{
	const u32 acthd = acthd_unmasked & HEAD_ADDR;

	if (request_start < request_end) {
		if (acthd >= request_start && acthd < request_end)
			return true;
	} else if (request_start > request_end) {
		if (acthd >= request_start || acthd < request_end)
			return true;
	}

	return false;
}

static bool i915_request_guilty(struct drm_i915_gem_request *request,
				const u32 acthd, bool *inside)
{
	/* There is a possibility that unmasked head address
	 * pointing inside the ring, matches the batch_obj address range.
	 * However this is extremely unlikely.
	 */

	if (request->batch_obj) {
		if (i915_head_inside_object(acthd, request->batch_obj)) {
			*inside = true;
			return true;
		}
	}

	if (i915_head_inside_request(acthd, request->head, request->tail)) {
		*inside = false;
		return true;
	}

	return false;
}

static void i915_set_reset_status(struct intel_ring_buffer *ring,
				  struct drm_i915_gem_request *request,
				  u32 acthd)
{
	struct i915_ctx_hang_stats *hs = NULL;
	bool inside, guilty;

	/* Innocent until proven guilty */
	guilty = false;

	if (ring->hangcheck.action != wait &&
	    i915_request_guilty(request, acthd, &inside)) {
2182
		DRM_ERROR("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
2183 2184 2185
			  ring->name,
			  inside ? "inside" : "flushing",
			  request->batch_obj ?
2186
			  i915_gem_obj_ggtt_offset(request->batch_obj) : 0,
2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208
			  request->ctx ? request->ctx->id : 0,
			  acthd);

		guilty = true;
	}

	/* If contexts are disabled or this is the default context, use
	 * file_priv->reset_state
	 */
	if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
		hs = &request->ctx->hang_stats;
	else if (request->file_priv)
		hs = &request->file_priv->hang_stats;

	if (hs) {
		if (guilty)
			hs->batch_active++;
		else
			hs->batch_pending++;
	}
}

2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219
static void i915_gem_free_request(struct drm_i915_gem_request *request)
{
	list_del(&request->list);
	i915_gem_request_remove_from_client(request);

	if (request->ctx)
		i915_gem_context_unreference(request->ctx);

	kfree(request);
}

2220 2221
static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
				      struct intel_ring_buffer *ring)
2222
{
2223 2224 2225 2226 2227 2228
	u32 completed_seqno;
	u32 acthd;

	acthd = intel_ring_get_active_head(ring);
	completed_seqno = ring->get_seqno(ring, false);

2229 2230
	while (!list_empty(&ring->request_list)) {
		struct drm_i915_gem_request *request;
2231

2232 2233 2234
		request = list_first_entry(&ring->request_list,
					   struct drm_i915_gem_request,
					   list);
2235

2236 2237 2238
		if (request->seqno > completed_seqno)
			i915_set_reset_status(ring, request, acthd);

2239
		i915_gem_free_request(request);
2240
	}
2241

2242
	while (!list_empty(&ring->active_list)) {
2243
		struct drm_i915_gem_object *obj;
2244

2245 2246 2247
		obj = list_first_entry(&ring->active_list,
				       struct drm_i915_gem_object,
				       ring_list);
2248

2249
		i915_gem_object_move_to_inactive(obj);
2250 2251 2252
	}
}

2253
void i915_gem_restore_fences(struct drm_device *dev)
2254 2255 2256 2257
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int i;

2258
	for (i = 0; i < dev_priv->num_fence_regs; i++) {
2259
		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2260
		i915_gem_write_fence(dev, i, reg->obj);
2261 2262 2263
	}
}

2264
void i915_gem_reset(struct drm_device *dev)
2265
{
2266
	struct drm_i915_private *dev_priv = dev->dev_private;
2267
	struct drm_i915_gem_object *obj;
2268
	struct intel_ring_buffer *ring;
2269
	int i;
2270

2271 2272
	for_each_ring(ring, dev_priv, i)
		i915_gem_reset_ring_lists(dev_priv, ring);
2273 2274 2275 2276

	/* Move everything out of the GPU domains to ensure we do any
	 * necessary invalidation upon reuse.
	 */
2277
	list_for_each_entry(obj,
2278
			    &dev_priv->mm.inactive_list,
2279
			    mm_list)
2280
	{
2281
		obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2282
	}
2283

2284
	i915_gem_restore_fences(dev);
2285 2286 2287 2288 2289
}

/**
 * This function clears the request list as sequence numbers are passed.
 */
2290
void
C
Chris Wilson 已提交
2291
i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
2292 2293 2294
{
	uint32_t seqno;

C
Chris Wilson 已提交
2295
	if (list_empty(&ring->request_list))
2296 2297
		return;

C
Chris Wilson 已提交
2298
	WARN_ON(i915_verify_lists(ring->dev));
2299

2300
	seqno = ring->get_seqno(ring, true);
2301

2302
	while (!list_empty(&ring->request_list)) {
2303 2304
		struct drm_i915_gem_request *request;

2305
		request = list_first_entry(&ring->request_list,
2306 2307 2308
					   struct drm_i915_gem_request,
					   list);

2309
		if (!i915_seqno_passed(seqno, request->seqno))
2310 2311
			break;

C
Chris Wilson 已提交
2312
		trace_i915_gem_request_retire(ring, request->seqno);
2313 2314 2315 2316 2317 2318
		/* We know the GPU must have read the request to have
		 * sent us the seqno + interrupt, so use the position
		 * of tail of the request to update the last known position
		 * of the GPU head.
		 */
		ring->last_retired_head = request->tail;
2319

2320
		i915_gem_free_request(request);
2321
	}
2322

2323 2324 2325 2326
	/* Move any buffers on the active list that are no longer referenced
	 * by the ringbuffer to the flushing/inactive lists as appropriate.
	 */
	while (!list_empty(&ring->active_list)) {
2327
		struct drm_i915_gem_object *obj;
2328

2329
		obj = list_first_entry(&ring->active_list,
2330 2331
				      struct drm_i915_gem_object,
				      ring_list);
2332

2333
		if (!i915_seqno_passed(seqno, obj->last_read_seqno))
2334
			break;
2335

2336
		i915_gem_object_move_to_inactive(obj);
2337
	}
2338

C
Chris Wilson 已提交
2339 2340
	if (unlikely(ring->trace_irq_seqno &&
		     i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
2341
		ring->irq_put(ring);
C
Chris Wilson 已提交
2342
		ring->trace_irq_seqno = 0;
2343
	}
2344

C
Chris Wilson 已提交
2345
	WARN_ON(i915_verify_lists(ring->dev));
2346 2347
}

2348 2349 2350 2351
void
i915_gem_retire_requests(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2352
	struct intel_ring_buffer *ring;
2353
	int i;
2354

2355 2356
	for_each_ring(ring, dev_priv, i)
		i915_gem_retire_requests_ring(ring);
2357 2358
}

2359
static void
2360 2361 2362 2363
i915_gem_retire_work_handler(struct work_struct *work)
{
	drm_i915_private_t *dev_priv;
	struct drm_device *dev;
2364
	struct intel_ring_buffer *ring;
2365 2366
	bool idle;
	int i;
2367 2368 2369 2370 2371

	dev_priv = container_of(work, drm_i915_private_t,
				mm.retire_work.work);
	dev = dev_priv->dev;

2372 2373
	/* Come back later if the device is busy... */
	if (!mutex_trylock(&dev->struct_mutex)) {
2374 2375
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
				   round_jiffies_up_relative(HZ));
2376 2377
		return;
	}
2378

2379
	i915_gem_retire_requests(dev);
2380

2381 2382
	/* Send a periodic flush down the ring so we don't hold onto GEM
	 * objects indefinitely.
2383
	 */
2384
	idle = true;
2385
	for_each_ring(ring, dev_priv, i) {
2386
		if (ring->gpu_caches_dirty)
2387
			i915_add_request(ring, NULL);
2388 2389

		idle &= list_empty(&ring->request_list);
2390 2391
	}

2392
	if (!dev_priv->ums.mm_suspended && !idle)
2393 2394
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
				   round_jiffies_up_relative(HZ));
2395 2396
	if (idle)
		intel_mark_idle(dev);
2397

2398 2399 2400
	mutex_unlock(&dev->struct_mutex);
}

2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411
/**
 * Ensures that an object will eventually get non-busy by flushing any required
 * write domains, emitting any outstanding lazy request and retiring and
 * completed requests.
 */
static int
i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
{
	int ret;

	if (obj->active) {
2412
		ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2413 2414 2415 2416 2417 2418 2419 2420 2421
		if (ret)
			return ret;

		i915_gem_retire_requests_ring(obj->ring);
	}

	return 0;
}

2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446
/**
 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
 * @DRM_IOCTL_ARGS: standard ioctl arguments
 *
 * Returns 0 if successful, else an error is returned with the remaining time in
 * the timeout parameter.
 *  -ETIME: object is still busy after timeout
 *  -ERESTARTSYS: signal interrupted the wait
 *  -ENONENT: object doesn't exist
 * Also possible, but rare:
 *  -EAGAIN: GPU wedged
 *  -ENOMEM: damn
 *  -ENODEV: Internal IRQ fail
 *  -E?: The add request failed
 *
 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
 * non-zero timeout parameter the wait ioctl will wait for the given number of
 * nanoseconds on an object becoming unbusy. Since the wait itself does so
 * without holding struct_mutex the object may become re-busied before this
 * function completes. A similar but shorter * race condition exists in the busy
 * ioctl
 */
int
i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
2447
	drm_i915_private_t *dev_priv = dev->dev_private;
2448 2449 2450
	struct drm_i915_gem_wait *args = data;
	struct drm_i915_gem_object *obj;
	struct intel_ring_buffer *ring = NULL;
2451
	struct timespec timeout_stack, *timeout = NULL;
2452
	unsigned reset_counter;
2453 2454 2455
	u32 seqno = 0;
	int ret = 0;

2456 2457 2458 2459
	if (args->timeout_ns >= 0) {
		timeout_stack = ns_to_timespec(args->timeout_ns);
		timeout = &timeout_stack;
	}
2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470

	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
	if (&obj->base == NULL) {
		mutex_unlock(&dev->struct_mutex);
		return -ENOENT;
	}

2471 2472
	/* Need to make sure the object gets inactive eventually. */
	ret = i915_gem_object_flush_active(obj);
2473 2474 2475 2476
	if (ret)
		goto out;

	if (obj->active) {
2477
		seqno = obj->last_read_seqno;
2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492
		ring = obj->ring;
	}

	if (seqno == 0)
		 goto out;

	/* Do this after OLR check to make sure we make forward progress polling
	 * on this IOCTL with a 0 timeout (like busy ioctl)
	 */
	if (!args->timeout_ns) {
		ret = -ETIME;
		goto out;
	}

	drm_gem_object_unreference(&obj->base);
2493
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
2494 2495
	mutex_unlock(&dev->struct_mutex);

2496
	ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
2497
	if (timeout)
2498
		args->timeout_ns = timespec_to_ns(timeout);
2499 2500 2501 2502 2503 2504 2505 2506
	return ret;

out:
	drm_gem_object_unreference(&obj->base);
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518
/**
 * i915_gem_object_sync - sync an object to a ring.
 *
 * @obj: object which may be in use on another ring.
 * @to: ring we wish to use the object on. May be NULL.
 *
 * This code is meant to abstract object synchronization with the GPU.
 * Calling with NULL implies synchronizing the object with the CPU
 * rather than a particular GPU ring.
 *
 * Returns 0 if successful, else propagates up the lower layer error.
 */
2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529
int
i915_gem_object_sync(struct drm_i915_gem_object *obj,
		     struct intel_ring_buffer *to)
{
	struct intel_ring_buffer *from = obj->ring;
	u32 seqno;
	int ret, idx;

	if (from == NULL || to == from)
		return 0;

2530
	if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2531
		return i915_gem_object_wait_rendering(obj, false);
2532 2533 2534

	idx = intel_ring_sync_index(from, to);

2535
	seqno = obj->last_read_seqno;
2536 2537 2538
	if (seqno <= from->sync_seqno[idx])
		return 0;

2539 2540 2541
	ret = i915_gem_check_olr(obj->ring, seqno);
	if (ret)
		return ret;
2542

2543
	ret = to->sync_to(to, from, seqno);
2544
	if (!ret)
2545 2546 2547 2548 2549
		/* We use last_read_seqno because sync_to()
		 * might have just caused seqno wrap under
		 * the radar.
		 */
		from->sync_seqno[idx] = obj->last_read_seqno;
2550

2551
	return ret;
2552 2553
}

2554 2555 2556 2557 2558 2559 2560
static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
{
	u32 old_write_domain, old_read_domains;

	/* Force a pagefault for domain tracking on next user access */
	i915_gem_release_mmap(obj);

2561 2562 2563
	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		return;

2564 2565 2566
	/* Wait for any direct GTT access to complete */
	mb();

2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577
	old_read_domains = obj->base.read_domains;
	old_write_domain = obj->base.write_domain;

	obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
	obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;

	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);
}

2578 2579 2580
/**
 * Unbinds an object from the GTT aperture.
 */
2581
int
2582
i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2583
{
2584
	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2585
	int ret;
2586

2587
	if (!i915_gem_obj_ggtt_bound(obj))
2588 2589
		return 0;

2590 2591
	if (obj->pin_count)
		return -EBUSY;
2592

2593 2594
	BUG_ON(obj->pages == NULL);

2595
	ret = i915_gem_object_finish_gpu(obj);
2596
	if (ret)
2597 2598 2599 2600 2601 2602
		return ret;
	/* Continue on if we fail due to EIO, the GPU is hung so we
	 * should be safe and we need to cleanup or else we might
	 * cause memory corruption through use-after-free.
	 */

2603
	i915_gem_object_finish_gtt(obj);
2604

2605
	/* release the fence reg _after_ flushing */
2606
	ret = i915_gem_object_put_fence(obj);
2607
	if (ret)
2608
		return ret;
2609

C
Chris Wilson 已提交
2610 2611
	trace_i915_gem_object_unbind(obj);

2612 2613
	if (obj->has_global_gtt_mapping)
		i915_gem_gtt_unbind_object(obj);
2614 2615 2616 2617
	if (obj->has_aliasing_ppgtt_mapping) {
		i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
		obj->has_aliasing_ppgtt_mapping = 0;
	}
2618
	i915_gem_gtt_finish_object(obj);
B
Ben Widawsky 已提交
2619
	i915_gem_object_unpin_pages(obj);
2620

C
Chris Wilson 已提交
2621
	list_del(&obj->mm_list);
2622
	list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2623
	/* Avoid an unnecessary call to unbind on rebind. */
2624
	obj->map_and_fenceable = true;
2625

2626
	drm_mm_remove_node(&obj->gtt_space);
2627

2628
	return 0;
2629 2630
}

2631
int i915_gpu_idle(struct drm_device *dev)
2632 2633
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2634
	struct intel_ring_buffer *ring;
2635
	int ret, i;
2636 2637

	/* Flush everything onto the inactive list. */
2638
	for_each_ring(ring, dev_priv, i) {
2639 2640 2641 2642
		ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
		if (ret)
			return ret;

2643
		ret = intel_ring_idle(ring);
2644 2645 2646
		if (ret)
			return ret;
	}
2647

2648
	return 0;
2649 2650
}

2651 2652
static void i965_write_fence_reg(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
2653 2654
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2655 2656
	int fence_reg;
	int fence_pitch_shift;
2657

2658 2659 2660 2661 2662 2663 2664 2665
	if (INTEL_INFO(dev)->gen >= 6) {
		fence_reg = FENCE_REG_SANDYBRIDGE_0;
		fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
	} else {
		fence_reg = FENCE_REG_965_0;
		fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
	}

2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679
	fence_reg += reg * 8;

	/* To w/a incoherency with non-atomic 64-bit register updates,
	 * we split the 64-bit update into two 32-bit writes. In order
	 * for a partial fence not to be evaluated between writes, we
	 * precede the update with write to turn off the fence register,
	 * and only enable the fence as the last step.
	 *
	 * For extra levels of paranoia, we make sure each step lands
	 * before applying the next step.
	 */
	I915_WRITE(fence_reg, 0);
	POSTING_READ(fence_reg);

2680
	if (obj) {
2681
		u32 size = i915_gem_obj_ggtt_size(obj);
2682
		uint64_t val;
2683

2684
		val = (uint64_t)((i915_gem_obj_ggtt_offset(obj) + size - 4096) &
2685
				 0xfffff000) << 32;
2686
		val |= i915_gem_obj_ggtt_offset(obj) & 0xfffff000;
2687
		val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
2688 2689 2690
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I965_FENCE_TILING_Y_SHIFT;
		val |= I965_FENCE_REG_VALID;
2691

2692 2693 2694 2695 2696 2697 2698 2699 2700
		I915_WRITE(fence_reg + 4, val >> 32);
		POSTING_READ(fence_reg + 4);

		I915_WRITE(fence_reg + 0, val);
		POSTING_READ(fence_reg);
	} else {
		I915_WRITE(fence_reg + 4, 0);
		POSTING_READ(fence_reg + 4);
	}
2701 2702
}

2703 2704
static void i915_write_fence_reg(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
2705 2706
{
	drm_i915_private_t *dev_priv = dev->dev_private;
2707
	u32 val;
2708

2709
	if (obj) {
2710
		u32 size = i915_gem_obj_ggtt_size(obj);
2711 2712
		int pitch_val;
		int tile_width;
2713

2714
		WARN((i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK) ||
2715
		     (size & -size) != size ||
2716 2717 2718
		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		     "object 0x%08lx [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
		     i915_gem_obj_ggtt_offset(obj), obj->map_and_fenceable, size);
2719

2720 2721 2722 2723 2724 2725 2726 2727 2728
		if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
			tile_width = 128;
		else
			tile_width = 512;

		/* Note: pitch better be a power of two tile widths */
		pitch_val = obj->stride / tile_width;
		pitch_val = ffs(pitch_val) - 1;

2729
		val = i915_gem_obj_ggtt_offset(obj);
2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
		val |= I915_FENCE_SIZE_BITS(size);
		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
		val |= I830_FENCE_REG_VALID;
	} else
		val = 0;

	if (reg < 8)
		reg = FENCE_REG_830_0 + reg * 4;
	else
		reg = FENCE_REG_945_8 + (reg - 8) * 4;

	I915_WRITE(reg, val);
	POSTING_READ(reg);
2745 2746
}

2747 2748
static void i830_write_fence_reg(struct drm_device *dev, int reg,
				struct drm_i915_gem_object *obj)
2749 2750 2751 2752
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	uint32_t val;

2753
	if (obj) {
2754
		u32 size = i915_gem_obj_ggtt_size(obj);
2755
		uint32_t pitch_val;
2756

2757
		WARN((i915_gem_obj_ggtt_offset(obj) & ~I830_FENCE_START_MASK) ||
2758
		     (size & -size) != size ||
2759 2760 2761
		     (i915_gem_obj_ggtt_offset(obj) & (size - 1)),
		     "object 0x%08lx not 512K or pot-size 0x%08x aligned\n",
		     i915_gem_obj_ggtt_offset(obj), size);
2762

2763 2764
		pitch_val = obj->stride / 128;
		pitch_val = ffs(pitch_val) - 1;
2765

2766
		val = i915_gem_obj_ggtt_offset(obj);
2767 2768 2769 2770 2771 2772 2773
		if (obj->tiling_mode == I915_TILING_Y)
			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
		val |= I830_FENCE_SIZE_BITS(size);
		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
		val |= I830_FENCE_REG_VALID;
	} else
		val = 0;
2774

2775 2776 2777 2778
	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
	POSTING_READ(FENCE_REG_830_0 + reg * 4);
}

2779 2780 2781 2782 2783
inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
{
	return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
}

2784 2785 2786
static void i915_gem_write_fence(struct drm_device *dev, int reg,
				 struct drm_i915_gem_object *obj)
{
2787 2788 2789 2790 2791 2792 2793 2794
	struct drm_i915_private *dev_priv = dev->dev_private;

	/* Ensure that all CPU reads are completed before installing a fence
	 * and all writes before removing the fence.
	 */
	if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
		mb();

2795 2796
	switch (INTEL_INFO(dev)->gen) {
	case 7:
2797
	case 6:
2798 2799 2800 2801
	case 5:
	case 4: i965_write_fence_reg(dev, reg, obj); break;
	case 3: i915_write_fence_reg(dev, reg, obj); break;
	case 2: i830_write_fence_reg(dev, reg, obj); break;
2802
	default: BUG();
2803
	}
2804 2805 2806 2807 2808 2809

	/* And similarly be paranoid that no direct access to this region
	 * is reordered to before the fence is installed.
	 */
	if (i915_gem_object_needs_mb(obj))
		mb();
2810 2811
}

2812 2813 2814 2815 2816 2817 2818 2819 2820 2821
static inline int fence_number(struct drm_i915_private *dev_priv,
			       struct drm_i915_fence_reg *fence)
{
	return fence - dev_priv->fence_regs;
}

static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
					 struct drm_i915_fence_reg *fence,
					 bool enable)
{
2822
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2823 2824 2825
	int reg = fence_number(dev_priv, fence);

	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2826 2827

	if (enable) {
2828
		obj->fence_reg = reg;
2829 2830 2831 2832 2833 2834 2835 2836 2837
		fence->obj = obj;
		list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
	} else {
		obj->fence_reg = I915_FENCE_REG_NONE;
		fence->obj = NULL;
		list_del_init(&fence->lru_list);
	}
}

2838
static int
2839
i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
2840
{
2841
	if (obj->last_fenced_seqno) {
2842
		int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2843 2844
		if (ret)
			return ret;
2845 2846 2847 2848

		obj->last_fenced_seqno = 0;
	}

2849
	obj->fenced_gpu_access = false;
2850 2851 2852 2853 2854 2855
	return 0;
}

int
i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
{
2856
	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2857
	struct drm_i915_fence_reg *fence;
2858 2859
	int ret;

2860
	ret = i915_gem_object_wait_fence(obj);
2861 2862 2863
	if (ret)
		return ret;

2864 2865
	if (obj->fence_reg == I915_FENCE_REG_NONE)
		return 0;
2866

2867 2868
	fence = &dev_priv->fence_regs[obj->fence_reg];

2869
	i915_gem_object_fence_lost(obj);
2870
	i915_gem_object_update_fence(obj, fence, false);
2871 2872 2873 2874 2875

	return 0;
}

static struct drm_i915_fence_reg *
C
Chris Wilson 已提交
2876
i915_find_fence_reg(struct drm_device *dev)
2877 2878
{
	struct drm_i915_private *dev_priv = dev->dev_private;
C
Chris Wilson 已提交
2879
	struct drm_i915_fence_reg *reg, *avail;
2880
	int i;
2881 2882

	/* First try to find a free reg */
2883
	avail = NULL;
2884 2885 2886
	for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
		reg = &dev_priv->fence_regs[i];
		if (!reg->obj)
2887
			return reg;
2888

2889
		if (!reg->pin_count)
2890
			avail = reg;
2891 2892
	}

2893 2894
	if (avail == NULL)
		return NULL;
2895 2896

	/* None available, try to steal one or wait for a user to finish */
2897
	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2898
		if (reg->pin_count)
2899 2900
			continue;

C
Chris Wilson 已提交
2901
		return reg;
2902 2903
	}

C
Chris Wilson 已提交
2904
	return NULL;
2905 2906
}

2907
/**
2908
 * i915_gem_object_get_fence - set up fencing for an object
2909 2910 2911 2912 2913 2914 2915 2916 2917
 * @obj: object to map through a fence reg
 *
 * When mapping objects through the GTT, userspace wants to be able to write
 * to them without having to worry about swizzling if the object is tiled.
 * This function walks the fence regs looking for a free one for @obj,
 * stealing one if it can't find any.
 *
 * It then sets up the reg based on the object's properties: address, pitch
 * and tiling format.
2918 2919
 *
 * For an untiled surface, this removes any existing fence.
2920
 */
2921
int
2922
i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
2923
{
2924
	struct drm_device *dev = obj->base.dev;
J
Jesse Barnes 已提交
2925
	struct drm_i915_private *dev_priv = dev->dev_private;
2926
	bool enable = obj->tiling_mode != I915_TILING_NONE;
2927
	struct drm_i915_fence_reg *reg;
2928
	int ret;
2929

2930 2931 2932
	/* Have we updated the tiling parameters upon the object and so
	 * will need to serialise the write to the associated fence register?
	 */
2933
	if (obj->fence_dirty) {
2934
		ret = i915_gem_object_wait_fence(obj);
2935 2936 2937
		if (ret)
			return ret;
	}
2938

2939
	/* Just update our place in the LRU if our fence is getting reused. */
2940 2941
	if (obj->fence_reg != I915_FENCE_REG_NONE) {
		reg = &dev_priv->fence_regs[obj->fence_reg];
2942
		if (!obj->fence_dirty) {
2943 2944 2945 2946 2947 2948 2949 2950
			list_move_tail(&reg->lru_list,
				       &dev_priv->mm.fence_list);
			return 0;
		}
	} else if (enable) {
		reg = i915_find_fence_reg(dev);
		if (reg == NULL)
			return -EDEADLK;
2951

2952 2953 2954
		if (reg->obj) {
			struct drm_i915_gem_object *old = reg->obj;

2955
			ret = i915_gem_object_wait_fence(old);
2956 2957 2958
			if (ret)
				return ret;

2959
			i915_gem_object_fence_lost(old);
2960
		}
2961
	} else
2962 2963
		return 0;

2964
	i915_gem_object_update_fence(obj, reg, enable);
2965
	obj->fence_dirty = false;
2966

2967
	return 0;
2968 2969
}

2970 2971 2972 2973 2974 2975 2976 2977
static bool i915_gem_valid_gtt_space(struct drm_device *dev,
				     struct drm_mm_node *gtt_space,
				     unsigned long cache_level)
{
	struct drm_mm_node *other;

	/* On non-LLC machines we have to be careful when putting differing
	 * types of snoopable memory together to avoid the prefetcher
2978
	 * crossing memory domains and dying.
2979 2980 2981 2982
	 */
	if (HAS_LLC(dev))
		return true;

2983
	if (!drm_mm_node_allocated(gtt_space))
2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006
		return true;

	if (list_empty(&gtt_space->node_list))
		return true;

	other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
	if (other->allocated && !other->hole_follows && other->color != cache_level)
		return false;

	other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
	if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
		return false;

	return true;
}

static void i915_gem_verify_gtt(struct drm_device *dev)
{
#if WATCH_GTT
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_gem_object *obj;
	int err = 0;

3007
	list_for_each_entry(obj, &dev_priv->mm.gtt_list, global_list) {
3008 3009 3010 3011 3012 3013 3014 3015
		if (obj->gtt_space == NULL) {
			printk(KERN_ERR "object found on GTT list with no space reserved\n");
			err++;
			continue;
		}

		if (obj->cache_level != obj->gtt_space->color) {
			printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
3016 3017
			       i915_gem_obj_ggtt_offset(obj),
			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3018 3019 3020 3021 3022 3023 3024 3025 3026 3027
			       obj->cache_level,
			       obj->gtt_space->color);
			err++;
			continue;
		}

		if (!i915_gem_valid_gtt_space(dev,
					      obj->gtt_space,
					      obj->cache_level)) {
			printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
3028 3029
			       i915_gem_obj_ggtt_offset(obj),
			       i915_gem_obj_ggtt_offset(obj) + i915_gem_obj_ggtt_size(obj),
3030 3031 3032 3033 3034 3035 3036 3037 3038 3039
			       obj->cache_level);
			err++;
			continue;
		}
	}

	WARN_ON(err);
#endif
}

3040 3041 3042 3043
/**
 * Finds free space in the GTT aperture and binds the object there.
 */
static int
3044
i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
3045
			    unsigned alignment,
3046 3047
			    bool map_and_fenceable,
			    bool nonblocking)
3048
{
3049
	struct drm_device *dev = obj->base.dev;
3050
	drm_i915_private_t *dev_priv = dev->dev_private;
3051
	u32 size, fence_size, fence_alignment, unfenced_alignment;
3052
	bool mappable, fenceable;
3053 3054
	size_t gtt_max = map_and_fenceable ?
		dev_priv->gtt.mappable_end : dev_priv->gtt.total;
3055
	int ret;
3056

3057 3058 3059 3060 3061
	fence_size = i915_gem_get_gtt_size(dev,
					   obj->base.size,
					   obj->tiling_mode);
	fence_alignment = i915_gem_get_gtt_alignment(dev,
						     obj->base.size,
3062
						     obj->tiling_mode, true);
3063
	unfenced_alignment =
3064
		i915_gem_get_gtt_alignment(dev,
3065
						    obj->base.size,
3066
						    obj->tiling_mode, false);
3067

3068
	if (alignment == 0)
3069 3070
		alignment = map_and_fenceable ? fence_alignment :
						unfenced_alignment;
3071
	if (map_and_fenceable && alignment & (fence_alignment - 1)) {
3072 3073 3074 3075
		DRM_ERROR("Invalid object alignment requested %u\n", alignment);
		return -EINVAL;
	}

3076
	size = map_and_fenceable ? fence_size : obj->base.size;
3077

3078 3079 3080
	/* If the object is bigger than the entire aperture, reject it early
	 * before evicting everything in a vain attempt to find space.
	 */
3081
	if (obj->base.size > gtt_max) {
3082
		DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
3083 3084
			  obj->base.size,
			  map_and_fenceable ? "mappable" : "total",
3085
			  gtt_max);
3086 3087 3088
		return -E2BIG;
	}

3089
	ret = i915_gem_object_get_pages(obj);
C
Chris Wilson 已提交
3090 3091 3092
	if (ret)
		return ret;

3093 3094
	i915_gem_object_pin_pages(obj);

3095
search_free:
3096 3097
	ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space,
						  &obj->gtt_space,
3098 3099
						  size, alignment,
						  obj->cache_level, 0, gtt_max);
3100
	if (ret) {
3101
		ret = i915_gem_evict_something(dev, size, alignment,
3102
					       obj->cache_level,
3103 3104
					       map_and_fenceable,
					       nonblocking);
3105 3106
		if (ret == 0)
			goto search_free;
3107

3108 3109
		i915_gem_object_unpin_pages(obj);
		return ret;
3110
	}
3111 3112
	if (WARN_ON(!i915_gem_valid_gtt_space(dev, &obj->gtt_space,
					      obj->cache_level))) {
3113
		i915_gem_object_unpin_pages(obj);
3114
		drm_mm_remove_node(&obj->gtt_space);
3115
		return -EINVAL;
3116 3117
	}

3118
	ret = i915_gem_gtt_prepare_object(obj);
3119
	if (ret) {
3120
		i915_gem_object_unpin_pages(obj);
3121
		drm_mm_remove_node(&obj->gtt_space);
C
Chris Wilson 已提交
3122
		return ret;
3123 3124
	}

3125
	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3126
	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3127

3128
	fenceable =
3129 3130
		i915_gem_obj_ggtt_size(obj) == fence_size &&
		(i915_gem_obj_ggtt_offset(obj) & (fence_alignment - 1)) == 0;
3131

3132 3133
	mappable = i915_gem_obj_ggtt_offset(obj) + obj->base.size <=
		dev_priv->gtt.mappable_end;
3134

3135
	obj->map_and_fenceable = mappable && fenceable;
3136

C
Chris Wilson 已提交
3137
	trace_i915_gem_object_bind(obj, map_and_fenceable);
3138
	i915_gem_verify_gtt(dev);
3139 3140 3141 3142
	return 0;
}

void
3143
i915_gem_clflush_object(struct drm_i915_gem_object *obj)
3144 3145 3146 3147 3148
{
	/* If we don't have a page list set up, then we're not pinned
	 * to GPU, and we can ignore the cache flush because it'll happen
	 * again at bind time.
	 */
3149
	if (obj->pages == NULL)
3150 3151
		return;

3152 3153 3154 3155 3156 3157 3158
	/*
	 * Stolen memory is always coherent with the GPU as it is explicitly
	 * marked as wc by the system, or the system is cache-coherent.
	 */
	if (obj->stolen)
		return;

3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169
	/* If the GPU is snooping the contents of the CPU cache,
	 * we do not need to manually clear the CPU cache lines.  However,
	 * the caches are only snooped when the render cache is
	 * flushed/invalidated.  As we always have to emit invalidations
	 * and flushes when moving into and out of the RENDER domain, correct
	 * snooping behaviour occurs naturally as the result of our domain
	 * tracking.
	 */
	if (obj->cache_level != I915_CACHE_NONE)
		return;

C
Chris Wilson 已提交
3170
	trace_i915_gem_object_clflush(obj);
3171

3172
	drm_clflush_sg(obj->pages);
3173 3174 3175 3176
}

/** Flushes the GTT write domain for the object if it's dirty. */
static void
3177
i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3178
{
C
Chris Wilson 已提交
3179 3180
	uint32_t old_write_domain;

3181
	if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3182 3183
		return;

3184
	/* No actual flushing is required for the GTT write domain.  Writes
3185 3186
	 * to it immediately go to main memory as far as we know, so there's
	 * no chipset flush.  It also doesn't land in render cache.
3187 3188 3189 3190
	 *
	 * However, we do have to enforce the order so that all writes through
	 * the GTT land before any writes to the device, such as updates to
	 * the GATT itself.
3191
	 */
3192 3193
	wmb();

3194 3195
	old_write_domain = obj->base.write_domain;
	obj->base.write_domain = 0;
C
Chris Wilson 已提交
3196 3197

	trace_i915_gem_object_change_domain(obj,
3198
					    obj->base.read_domains,
C
Chris Wilson 已提交
3199
					    old_write_domain);
3200 3201 3202 3203
}

/** Flushes the CPU write domain for the object if it's dirty. */
static void
3204
i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3205
{
C
Chris Wilson 已提交
3206
	uint32_t old_write_domain;
3207

3208
	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3209 3210 3211
		return;

	i915_gem_clflush_object(obj);
3212
	i915_gem_chipset_flush(obj->base.dev);
3213 3214
	old_write_domain = obj->base.write_domain;
	obj->base.write_domain = 0;
C
Chris Wilson 已提交
3215 3216

	trace_i915_gem_object_change_domain(obj,
3217
					    obj->base.read_domains,
C
Chris Wilson 已提交
3218
					    old_write_domain);
3219 3220
}

3221 3222 3223 3224 3225 3226
/**
 * Moves a single object to the GTT read, and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
J
Jesse Barnes 已提交
3227
int
3228
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3229
{
3230
	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
C
Chris Wilson 已提交
3231
	uint32_t old_write_domain, old_read_domains;
3232
	int ret;
3233

3234
	/* Not valid to be called on unbound objects. */
3235
	if (!i915_gem_obj_ggtt_bound(obj))
3236 3237
		return -EINVAL;

3238 3239 3240
	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		return 0;

3241
	ret = i915_gem_object_wait_rendering(obj, !write);
3242 3243 3244
	if (ret)
		return ret;

3245
	i915_gem_object_flush_cpu_write_domain(obj);
C
Chris Wilson 已提交
3246

3247 3248 3249 3250 3251 3252 3253
	/* Serialise direct access to this object with the barriers for
	 * coherent writes from the GPU, by effectively invalidating the
	 * GTT domain upon first access.
	 */
	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		mb();

3254 3255
	old_write_domain = obj->base.write_domain;
	old_read_domains = obj->base.read_domains;
C
Chris Wilson 已提交
3256

3257 3258 3259
	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3260 3261
	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3262
	if (write) {
3263 3264 3265
		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
		obj->dirty = 1;
3266 3267
	}

C
Chris Wilson 已提交
3268 3269 3270 3271
	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);

3272 3273 3274 3275
	/* And bump the LRU for this access */
	if (i915_gem_object_is_inactive(obj))
		list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);

3276 3277 3278
	return 0;
}

3279 3280 3281
int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
				    enum i915_cache_level cache_level)
{
3282 3283
	struct drm_device *dev = obj->base.dev;
	drm_i915_private_t *dev_priv = dev->dev_private;
3284 3285 3286 3287 3288 3289 3290 3291 3292 3293
	int ret;

	if (obj->cache_level == cache_level)
		return 0;

	if (obj->pin_count) {
		DRM_DEBUG("can not change the cache level of pinned objects\n");
		return -EBUSY;
	}

3294
	if (!i915_gem_valid_gtt_space(dev, &obj->gtt_space, cache_level)) {
3295 3296 3297 3298 3299
		ret = i915_gem_object_unbind(obj);
		if (ret)
			return ret;
	}

3300
	if (i915_gem_obj_ggtt_bound(obj)) {
3301 3302 3303 3304 3305 3306 3307 3308 3309 3310
		ret = i915_gem_object_finish_gpu(obj);
		if (ret)
			return ret;

		i915_gem_object_finish_gtt(obj);

		/* Before SandyBridge, you could not use tiling or fence
		 * registers with snooped memory, so relinquish any fences
		 * currently pointing to our region in the aperture.
		 */
3311
		if (INTEL_INFO(dev)->gen < 6) {
3312 3313 3314 3315 3316
			ret = i915_gem_object_put_fence(obj);
			if (ret)
				return ret;
		}

3317 3318
		if (obj->has_global_gtt_mapping)
			i915_gem_gtt_bind_object(obj, cache_level);
3319 3320 3321
		if (obj->has_aliasing_ppgtt_mapping)
			i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
					       obj, cache_level);
3322

3323
		i915_gem_obj_ggtt_set_color(obj, cache_level);
3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349
	}

	if (cache_level == I915_CACHE_NONE) {
		u32 old_read_domains, old_write_domain;

		/* If we're coming from LLC cached, then we haven't
		 * actually been tracking whether the data is in the
		 * CPU cache or not, since we only allow one bit set
		 * in obj->write_domain and have been skipping the clflushes.
		 * Just set it to the CPU cache for now.
		 */
		WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
		WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);

		old_read_domains = obj->base.read_domains;
		old_write_domain = obj->base.write_domain;

		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		obj->base.write_domain = I915_GEM_DOMAIN_CPU;

		trace_i915_gem_object_change_domain(obj,
						    old_read_domains,
						    old_write_domain);
	}

	obj->cache_level = cache_level;
3350
	i915_gem_verify_gtt(dev);
3351 3352 3353
	return 0;
}

B
Ben Widawsky 已提交
3354 3355
int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3356
{
B
Ben Widawsky 已提交
3357
	struct drm_i915_gem_caching *args = data;
3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370
	struct drm_i915_gem_object *obj;
	int ret;

	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
	if (&obj->base == NULL) {
		ret = -ENOENT;
		goto unlock;
	}

B
Ben Widawsky 已提交
3371
	args->caching = obj->cache_level != I915_CACHE_NONE;
3372 3373 3374 3375 3376 3377 3378

	drm_gem_object_unreference(&obj->base);
unlock:
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

B
Ben Widawsky 已提交
3379 3380
int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3381
{
B
Ben Widawsky 已提交
3382
	struct drm_i915_gem_caching *args = data;
3383 3384 3385 3386
	struct drm_i915_gem_object *obj;
	enum i915_cache_level level;
	int ret;

B
Ben Widawsky 已提交
3387 3388
	switch (args->caching) {
	case I915_CACHING_NONE:
3389 3390
		level = I915_CACHE_NONE;
		break;
B
Ben Widawsky 已提交
3391
	case I915_CACHING_CACHED:
3392 3393 3394 3395 3396 3397
		level = I915_CACHE_LLC;
		break;
	default:
		return -EINVAL;
	}

B
Ben Widawsky 已提交
3398 3399 3400 3401
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
	if (&obj->base == NULL) {
		ret = -ENOENT;
		goto unlock;
	}

	ret = i915_gem_object_set_cache_level(obj, level);

	drm_gem_object_unreference(&obj->base);
unlock:
	mutex_unlock(&dev->struct_mutex);
	return ret;
}

3416
/*
3417 3418 3419
 * Prepare buffer for display plane (scanout, cursors, etc).
 * Can be called from an uninterruptible phase (modesetting) and allows
 * any flushes to be pipelined (for pageflips).
3420 3421
 */
int
3422 3423
i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
				     u32 alignment,
3424
				     struct intel_ring_buffer *pipelined)
3425
{
3426
	u32 old_read_domains, old_write_domain;
3427 3428
	int ret;

3429
	if (pipelined != obj->ring) {
3430 3431
		ret = i915_gem_object_sync(obj, pipelined);
		if (ret)
3432 3433 3434
			return ret;
	}

3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447
	/* The display engine is not coherent with the LLC cache on gen6.  As
	 * a result, we make sure that the pinning that is about to occur is
	 * done with uncached PTEs. This is lowest common denominator for all
	 * chipsets.
	 *
	 * However for gen6+, we could do better by using the GFDT bit instead
	 * of uncaching, which would allow us to flush all the LLC-cached data
	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
	 */
	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
	if (ret)
		return ret;

3448 3449 3450 3451
	/* As the user may map the buffer once pinned in the display plane
	 * (e.g. libkms for the bootup splash), we have to ensure that we
	 * always use map_and_fenceable for all scanout buffers.
	 */
3452
	ret = i915_gem_object_pin(obj, alignment, true, false);
3453 3454 3455
	if (ret)
		return ret;

3456 3457
	i915_gem_object_flush_cpu_write_domain(obj);

3458
	old_write_domain = obj->base.write_domain;
3459
	old_read_domains = obj->base.read_domains;
3460 3461 3462 3463

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3464
	obj->base.write_domain = 0;
3465
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3466 3467 3468

	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
3469
					    old_write_domain);
3470 3471 3472 3473

	return 0;
}

3474
int
3475
i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3476
{
3477 3478
	int ret;

3479
	if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3480 3481
		return 0;

3482
	ret = i915_gem_object_wait_rendering(obj, false);
3483 3484 3485
	if (ret)
		return ret;

3486 3487
	/* Ensure that we invalidate the GPU's caches and TLBs. */
	obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3488
	return 0;
3489 3490
}

3491 3492 3493 3494 3495 3496
/**
 * Moves a single object to the CPU read, and possibly write domain.
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
3497
int
3498
i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3499
{
C
Chris Wilson 已提交
3500
	uint32_t old_write_domain, old_read_domains;
3501 3502
	int ret;

3503 3504 3505
	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		return 0;

3506
	ret = i915_gem_object_wait_rendering(obj, !write);
3507 3508 3509
	if (ret)
		return ret;

3510
	i915_gem_object_flush_gtt_write_domain(obj);
3511

3512 3513
	old_write_domain = obj->base.write_domain;
	old_read_domains = obj->base.read_domains;
C
Chris Wilson 已提交
3514

3515
	/* Flush the CPU cache if it's still invalid. */
3516
	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3517 3518
		i915_gem_clflush_object(obj);

3519
		obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3520 3521 3522 3523 3524
	}

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3525
	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3526 3527 3528 3529 3530

	/* If we're writing through the CPU, then the GPU read domains will
	 * need to be invalidated at next use.
	 */
	if (write) {
3531 3532
		obj->base.read_domains = I915_GEM_DOMAIN_CPU;
		obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3533
	}
3534

C
Chris Wilson 已提交
3535 3536 3537 3538
	trace_i915_gem_object_change_domain(obj,
					    old_read_domains,
					    old_write_domain);

3539 3540 3541
	return 0;
}

3542 3543 3544
/* Throttle our rendering by waiting until the ring has completed our requests
 * emitted over 20 msec ago.
 *
3545 3546 3547 3548
 * Note that if we were to use the current jiffies each time around the loop,
 * we wouldn't escape the function with any frames outstanding if the time to
 * render a frame was over 20ms.
 *
3549 3550 3551
 * This should get us reasonable parallelism between CPU and GPU but also
 * relatively low latency when blocking on a particular request to finish.
 */
3552
static int
3553
i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3554
{
3555 3556
	struct drm_i915_private *dev_priv = dev->dev_private;
	struct drm_i915_file_private *file_priv = file->driver_priv;
3557
	unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3558 3559
	struct drm_i915_gem_request *request;
	struct intel_ring_buffer *ring = NULL;
3560
	unsigned reset_counter;
3561 3562
	u32 seqno = 0;
	int ret;
3563

3564 3565 3566 3567 3568 3569 3570
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
	if (ret)
		return ret;

	ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
	if (ret)
		return ret;
3571

3572
	spin_lock(&file_priv->mm.lock);
3573
	list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3574 3575
		if (time_after_eq(request->emitted_jiffies, recent_enough))
			break;
3576

3577 3578
		ring = request->ring;
		seqno = request->seqno;
3579
	}
3580
	reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3581
	spin_unlock(&file_priv->mm.lock);
3582

3583 3584
	if (seqno == 0)
		return 0;
3585

3586
	ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
3587 3588
	if (ret == 0)
		queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3589 3590 3591 3592

	return ret;
}

3593
int
3594 3595
i915_gem_object_pin(struct drm_i915_gem_object *obj,
		    uint32_t alignment,
3596 3597
		    bool map_and_fenceable,
		    bool nonblocking)
3598 3599 3600
{
	int ret;

3601 3602
	if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
		return -EBUSY;
3603

3604 3605
	if (i915_gem_obj_ggtt_bound(obj)) {
		if ((alignment && i915_gem_obj_ggtt_offset(obj) & (alignment - 1)) ||
3606 3607
		    (map_and_fenceable && !obj->map_and_fenceable)) {
			WARN(obj->pin_count,
3608
			     "bo is already pinned with incorrect alignment:"
3609
			     " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
3610
			     " obj->map_and_fenceable=%d\n",
3611
			     i915_gem_obj_ggtt_offset(obj), alignment,
3612
			     map_and_fenceable,
3613
			     obj->map_and_fenceable);
3614 3615 3616 3617 3618 3619
			ret = i915_gem_object_unbind(obj);
			if (ret)
				return ret;
		}
	}

3620
	if (!i915_gem_obj_ggtt_bound(obj)) {
3621 3622
		struct drm_i915_private *dev_priv = obj->base.dev->dev_private;

3623
		ret = i915_gem_object_bind_to_gtt(obj, alignment,
3624 3625
						  map_and_fenceable,
						  nonblocking);
3626
		if (ret)
3627
			return ret;
3628 3629 3630

		if (!dev_priv->mm.aliasing_ppgtt)
			i915_gem_gtt_bind_object(obj, obj->cache_level);
3631
	}
J
Jesse Barnes 已提交
3632

3633 3634 3635
	if (!obj->has_global_gtt_mapping && map_and_fenceable)
		i915_gem_gtt_bind_object(obj, obj->cache_level);

3636
	obj->pin_count++;
3637
	obj->pin_mappable |= map_and_fenceable;
3638 3639 3640 3641 3642

	return 0;
}

void
3643
i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3644
{
3645
	BUG_ON(obj->pin_count == 0);
3646
	BUG_ON(!i915_gem_obj_ggtt_bound(obj));
3647

3648
	if (--obj->pin_count == 0)
3649
		obj->pin_mappable = false;
3650 3651 3652 3653
}

int
i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3654
		   struct drm_file *file)
3655 3656
{
	struct drm_i915_gem_pin *args = data;
3657
	struct drm_i915_gem_object *obj;
3658 3659
	int ret;

3660 3661 3662
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;
3663

3664
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3665
	if (&obj->base == NULL) {
3666 3667
		ret = -ENOENT;
		goto unlock;
3668 3669
	}

3670
	if (obj->madv != I915_MADV_WILLNEED) {
C
Chris Wilson 已提交
3671
		DRM_ERROR("Attempting to pin a purgeable buffer\n");
3672 3673
		ret = -EINVAL;
		goto out;
3674 3675
	}

3676
	if (obj->pin_filp != NULL && obj->pin_filp != file) {
J
Jesse Barnes 已提交
3677 3678
		DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
			  args->handle);
3679 3680
		ret = -EINVAL;
		goto out;
J
Jesse Barnes 已提交
3681 3682
	}

3683
	if (obj->user_pin_count == 0) {
3684
		ret = i915_gem_object_pin(obj, args->alignment, true, false);
3685 3686
		if (ret)
			goto out;
3687 3688
	}

3689 3690 3691
	obj->user_pin_count++;
	obj->pin_filp = file;

3692 3693 3694
	/* XXX - flush the CPU caches for pinned objects
	 * as the X server doesn't manage domains yet
	 */
3695
	i915_gem_object_flush_cpu_write_domain(obj);
3696
	args->offset = i915_gem_obj_ggtt_offset(obj);
3697
out:
3698
	drm_gem_object_unreference(&obj->base);
3699
unlock:
3700
	mutex_unlock(&dev->struct_mutex);
3701
	return ret;
3702 3703 3704 3705
}

int
i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3706
		     struct drm_file *file)
3707 3708
{
	struct drm_i915_gem_pin *args = data;
3709
	struct drm_i915_gem_object *obj;
3710
	int ret;
3711

3712 3713 3714
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;
3715

3716
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3717
	if (&obj->base == NULL) {
3718 3719
		ret = -ENOENT;
		goto unlock;
3720
	}
3721

3722
	if (obj->pin_filp != file) {
J
Jesse Barnes 已提交
3723 3724
		DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
			  args->handle);
3725 3726
		ret = -EINVAL;
		goto out;
J
Jesse Barnes 已提交
3727
	}
3728 3729 3730
	obj->user_pin_count--;
	if (obj->user_pin_count == 0) {
		obj->pin_filp = NULL;
J
Jesse Barnes 已提交
3731 3732
		i915_gem_object_unpin(obj);
	}
3733

3734
out:
3735
	drm_gem_object_unreference(&obj->base);
3736
unlock:
3737
	mutex_unlock(&dev->struct_mutex);
3738
	return ret;
3739 3740 3741 3742
}

int
i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3743
		    struct drm_file *file)
3744 3745
{
	struct drm_i915_gem_busy *args = data;
3746
	struct drm_i915_gem_object *obj;
3747 3748
	int ret;

3749
	ret = i915_mutex_lock_interruptible(dev);
3750
	if (ret)
3751
		return ret;
3752

3753
	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3754
	if (&obj->base == NULL) {
3755 3756
		ret = -ENOENT;
		goto unlock;
3757
	}
3758

3759 3760 3761 3762
	/* Count all active objects as busy, even if they are currently not used
	 * by the gpu. Users of this interface expect objects to eventually
	 * become non-busy without any further actions, therefore emit any
	 * necessary flushes here.
3763
	 */
3764
	ret = i915_gem_object_flush_active(obj);
3765

3766
	args->busy = obj->active;
3767 3768 3769 3770
	if (obj->ring) {
		BUILD_BUG_ON(I915_NUM_RINGS > 16);
		args->busy |= intel_ring_flag(obj->ring) << 16;
	}
3771

3772
	drm_gem_object_unreference(&obj->base);
3773
unlock:
3774
	mutex_unlock(&dev->struct_mutex);
3775
	return ret;
3776 3777 3778 3779 3780 3781
}

int
i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file_priv)
{
3782
	return i915_gem_ring_throttle(dev, file_priv);
3783 3784
}

3785 3786 3787 3788 3789
int
i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
	struct drm_i915_gem_madvise *args = data;
3790
	struct drm_i915_gem_object *obj;
3791
	int ret;
3792 3793 3794 3795 3796 3797 3798 3799 3800

	switch (args->madv) {
	case I915_MADV_DONTNEED:
	case I915_MADV_WILLNEED:
	    break;
	default:
	    return -EINVAL;
	}

3801 3802 3803 3804
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		return ret;

3805
	obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3806
	if (&obj->base == NULL) {
3807 3808
		ret = -ENOENT;
		goto unlock;
3809 3810
	}

3811
	if (obj->pin_count) {
3812 3813
		ret = -EINVAL;
		goto out;
3814 3815
	}

3816 3817
	if (obj->madv != __I915_MADV_PURGED)
		obj->madv = args->madv;
3818

C
Chris Wilson 已提交
3819 3820
	/* if the object is no longer attached, discard its backing storage */
	if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
3821 3822
		i915_gem_object_truncate(obj);

3823
	args->retained = obj->madv != __I915_MADV_PURGED;
C
Chris Wilson 已提交
3824

3825
out:
3826
	drm_gem_object_unreference(&obj->base);
3827
unlock:
3828
	mutex_unlock(&dev->struct_mutex);
3829
	return ret;
3830 3831
}

3832 3833
void i915_gem_object_init(struct drm_i915_gem_object *obj,
			  const struct drm_i915_gem_object_ops *ops)
3834 3835
{
	INIT_LIST_HEAD(&obj->mm_list);
3836
	INIT_LIST_HEAD(&obj->global_list);
3837 3838 3839
	INIT_LIST_HEAD(&obj->ring_list);
	INIT_LIST_HEAD(&obj->exec_list);

3840 3841
	obj->ops = ops;

3842 3843 3844 3845 3846 3847 3848 3849
	obj->fence_reg = I915_FENCE_REG_NONE;
	obj->madv = I915_MADV_WILLNEED;
	/* Avoid an unnecessary call to unbind on the first bind. */
	obj->map_and_fenceable = true;

	i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
}

3850 3851 3852 3853 3854
static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
	.get_pages = i915_gem_object_get_pages_gtt,
	.put_pages = i915_gem_object_put_pages_gtt,
};

3855 3856
struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
						  size_t size)
3857
{
3858
	struct drm_i915_gem_object *obj;
3859
	struct address_space *mapping;
D
Daniel Vetter 已提交
3860
	gfp_t mask;
3861

3862
	obj = i915_gem_object_alloc(dev);
3863 3864
	if (obj == NULL)
		return NULL;
3865

3866
	if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3867
		i915_gem_object_free(obj);
3868 3869
		return NULL;
	}
3870

3871 3872 3873 3874 3875 3876 3877
	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
	if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
		/* 965gm cannot relocate objects above 4GiB. */
		mask &= ~__GFP_HIGHMEM;
		mask |= __GFP_DMA32;
	}

A
Al Viro 已提交
3878
	mapping = file_inode(obj->base.filp)->i_mapping;
3879
	mapping_set_gfp_mask(mapping, mask);
3880

3881
	i915_gem_object_init(obj, &i915_gem_object_ops);
3882

3883 3884
	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3885

3886 3887
	if (HAS_LLC(dev)) {
		/* On some devices, we can have the GPU use the LLC (the CPU
3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902
		 * cache) for about a 10% performance improvement
		 * compared to uncached.  Graphics requests other than
		 * display scanout are coherent with the CPU in
		 * accessing this cache.  This means in this mode we
		 * don't need to clflush on the CPU side, and on the
		 * GPU side we only need to flush internal caches to
		 * get data visible to the CPU.
		 *
		 * However, we maintain the display planes as UC, and so
		 * need to rebind when first used as such.
		 */
		obj->cache_level = I915_CACHE_LLC;
	} else
		obj->cache_level = I915_CACHE_NONE;

3903
	return obj;
3904 3905 3906 3907 3908
}

int i915_gem_init_object(struct drm_gem_object *obj)
{
	BUG();
3909

3910 3911 3912
	return 0;
}

3913
void i915_gem_free_object(struct drm_gem_object *gem_obj)
3914
{
3915
	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3916
	struct drm_device *dev = obj->base.dev;
3917
	drm_i915_private_t *dev_priv = dev->dev_private;
3918

3919 3920
	trace_i915_gem_object_destroy(obj);

3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935
	if (obj->phys_obj)
		i915_gem_detach_phys_object(dev, obj);

	obj->pin_count = 0;
	if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
		bool was_interruptible;

		was_interruptible = dev_priv->mm.interruptible;
		dev_priv->mm.interruptible = false;

		WARN_ON(i915_gem_object_unbind(obj));

		dev_priv->mm.interruptible = was_interruptible;
	}

B
Ben Widawsky 已提交
3936 3937 3938 3939 3940
	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
	 * before progressing. */
	if (obj->stolen)
		i915_gem_object_unpin_pages(obj);

B
Ben Widawsky 已提交
3941 3942
	if (WARN_ON(obj->pages_pin_count))
		obj->pages_pin_count = 0;
3943
	i915_gem_object_put_pages(obj);
3944
	i915_gem_object_free_mmap_offset(obj);
3945
	i915_gem_object_release_stolen(obj);
3946

3947 3948
	BUG_ON(obj->pages);

3949 3950
	if (obj->base.import_attach)
		drm_prime_gem_destroy(&obj->base, NULL);
3951

3952 3953
	drm_gem_object_release(&obj->base);
	i915_gem_info_remove_obj(dev_priv, obj->base.size);
3954

3955
	kfree(obj->bit_17);
3956
	i915_gem_object_free(obj);
3957 3958
}

3959 3960 3961 3962 3963
int
i915_gem_idle(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret;
3964

3965
	if (dev_priv->ums.mm_suspended) {
3966 3967
		mutex_unlock(&dev->struct_mutex);
		return 0;
3968 3969
	}

3970
	ret = i915_gpu_idle(dev);
3971 3972
	if (ret) {
		mutex_unlock(&dev->struct_mutex);
3973
		return ret;
3974
	}
3975
	i915_gem_retire_requests(dev);
3976

3977
	/* Under UMS, be paranoid and evict. */
3978
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
C
Chris Wilson 已提交
3979
		i915_gem_evict_everything(dev);
3980

3981
	del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
3982 3983

	i915_kernel_lost_context(dev);
3984
	i915_gem_cleanup_ringbuffer(dev);
3985 3986 3987 3988

	/* Cancel the retire work handler, which should be idle now. */
	cancel_delayed_work_sync(&dev_priv->mm.retire_work);

3989 3990 3991
	return 0;
}

B
Ben Widawsky 已提交
3992 3993 3994 3995 3996 3997
void i915_gem_l3_remap(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	u32 misccpctl;
	int i;

3998
	if (!HAS_L3_GPU_CACHE(dev))
B
Ben Widawsky 已提交
3999 4000
		return;

4001
	if (!dev_priv->l3_parity.remap_info)
B
Ben Widawsky 已提交
4002 4003 4004 4005 4006 4007 4008 4009
		return;

	misccpctl = I915_READ(GEN7_MISCCPCTL);
	I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
	POSTING_READ(GEN7_MISCCPCTL);

	for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
		u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
4010
		if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
B
Ben Widawsky 已提交
4011 4012
			DRM_DEBUG("0x%x was already programmed to %x\n",
				  GEN7_L3LOG_BASE + i, remap);
4013
		if (remap && !dev_priv->l3_parity.remap_info[i/4])
B
Ben Widawsky 已提交
4014
			DRM_DEBUG_DRIVER("Clearing remapped register\n");
4015
		I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
B
Ben Widawsky 已提交
4016 4017 4018 4019 4020 4021 4022 4023
	}

	/* Make sure all the writes land before disabling dop clock gating */
	POSTING_READ(GEN7_L3LOG_BASE);

	I915_WRITE(GEN7_MISCCPCTL, misccpctl);
}

4024 4025 4026 4027
void i915_gem_init_swizzling(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;

4028
	if (INTEL_INFO(dev)->gen < 5 ||
4029 4030 4031 4032 4033 4034
	    dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		return;

	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
				 DISP_TILE_SURFACE_SWIZZLING);

4035 4036 4037
	if (IS_GEN5(dev))
		return;

4038 4039
	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
	if (IS_GEN6(dev))
4040
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4041
	else if (IS_GEN7(dev))
4042
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4043 4044
	else
		BUG();
4045
}
D
Daniel Vetter 已提交
4046

4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062
static bool
intel_enable_blt(struct drm_device *dev)
{
	if (!HAS_BLT(dev))
		return false;

	/* The blitter was dysfunctional on early prototypes */
	if (IS_GEN6(dev) && dev->pdev->revision < 8) {
		DRM_INFO("BLT not supported on this pre-production hardware;"
			 " graphics performance will be degraded.\n");
		return false;
	}

	return true;
}

4063
static int i915_gem_init_rings(struct drm_device *dev)
4064
{
4065
	struct drm_i915_private *dev_priv = dev->dev_private;
4066
	int ret;
4067

4068
	ret = intel_init_render_ring_buffer(dev);
4069
	if (ret)
4070
		return ret;
4071 4072

	if (HAS_BSD(dev)) {
4073
		ret = intel_init_bsd_ring_buffer(dev);
4074 4075
		if (ret)
			goto cleanup_render_ring;
4076
	}
4077

4078
	if (intel_enable_blt(dev)) {
4079 4080 4081 4082 4083
		ret = intel_init_blt_ring_buffer(dev);
		if (ret)
			goto cleanup_bsd_ring;
	}

B
Ben Widawsky 已提交
4084 4085 4086 4087 4088 4089 4090
	if (HAS_VEBOX(dev)) {
		ret = intel_init_vebox_ring_buffer(dev);
		if (ret)
			goto cleanup_blt_ring;
	}


4091
	ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
4092
	if (ret)
B
Ben Widawsky 已提交
4093
		goto cleanup_vebox_ring;
4094 4095 4096

	return 0;

B
Ben Widawsky 已提交
4097 4098
cleanup_vebox_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120
cleanup_blt_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
cleanup_bsd_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
cleanup_render_ring:
	intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);

	return ret;
}

int
i915_gem_init_hw(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret;

	if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
		return -EIO;

	if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
		I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);

4121 4122 4123 4124 4125 4126
	if (HAS_PCH_NOP(dev)) {
		u32 temp = I915_READ(GEN7_MSG_CTL);
		temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
		I915_WRITE(GEN7_MSG_CTL, temp);
	}

4127 4128 4129 4130 4131
	i915_gem_l3_remap(dev);

	i915_gem_init_swizzling(dev);

	ret = i915_gem_init_rings(dev);
4132 4133 4134
	if (ret)
		return ret;

4135 4136 4137 4138 4139
	/*
	 * XXX: There was some w/a described somewhere suggesting loading
	 * contexts before PPGTT.
	 */
	i915_gem_context_init(dev);
4140 4141 4142 4143 4144 4145 4146
	if (dev_priv->mm.aliasing_ppgtt) {
		ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
		if (ret) {
			i915_gem_cleanup_aliasing_ppgtt(dev);
			DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
		}
	}
D
Daniel Vetter 已提交
4147

4148
	return 0;
4149 4150
}

4151 4152 4153 4154 4155 4156
int i915_gem_init(struct drm_device *dev)
{
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

	mutex_lock(&dev->struct_mutex);
4157 4158 4159 4160 4161 4162 4163 4164

	if (IS_VALLEYVIEW(dev)) {
		/* VLVA0 (potential hack), BIOS isn't actually waking us */
		I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
		if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
			DRM_DEBUG_DRIVER("allow wake ack timed out\n");
	}

4165
	i915_gem_init_global_gtt(dev);
4166

4167 4168 4169 4170 4171 4172 4173
	ret = i915_gem_init_hw(dev);
	mutex_unlock(&dev->struct_mutex);
	if (ret) {
		i915_gem_cleanup_aliasing_ppgtt(dev);
		return ret;
	}

4174 4175 4176
	/* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		dev_priv->dri1.allow_batchbuffer = 1;
4177 4178 4179
	return 0;
}

4180 4181 4182 4183
void
i915_gem_cleanup_ringbuffer(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
4184
	struct intel_ring_buffer *ring;
4185
	int i;
4186

4187 4188
	for_each_ring(ring, dev_priv, i)
		intel_cleanup_ring_buffer(ring);
4189 4190
}

4191 4192 4193 4194
int
i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
4195
	struct drm_i915_private *dev_priv = dev->dev_private;
4196
	int ret;
4197

J
Jesse Barnes 已提交
4198 4199 4200
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return 0;

4201
	if (i915_reset_in_progress(&dev_priv->gpu_error)) {
4202
		DRM_ERROR("Reenabling wedged hardware, good luck\n");
4203
		atomic_set(&dev_priv->gpu_error.reset_counter, 0);
4204 4205 4206
	}

	mutex_lock(&dev->struct_mutex);
4207
	dev_priv->ums.mm_suspended = 0;
4208

4209
	ret = i915_gem_init_hw(dev);
4210 4211
	if (ret != 0) {
		mutex_unlock(&dev->struct_mutex);
4212
		return ret;
4213
	}
4214

4215
	BUG_ON(!list_empty(&dev_priv->mm.active_list));
4216
	mutex_unlock(&dev->struct_mutex);
4217

4218 4219 4220
	ret = drm_irq_install(dev);
	if (ret)
		goto cleanup_ringbuffer;
4221

4222
	return 0;
4223 4224 4225 4226

cleanup_ringbuffer:
	mutex_lock(&dev->struct_mutex);
	i915_gem_cleanup_ringbuffer(dev);
4227
	dev_priv->ums.mm_suspended = 1;
4228 4229 4230
	mutex_unlock(&dev->struct_mutex);

	return ret;
4231 4232 4233 4234 4235 4236
}

int
i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
4237 4238 4239
	struct drm_i915_private *dev_priv = dev->dev_private;
	int ret;

J
Jesse Barnes 已提交
4240 4241 4242
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return 0;

4243
	drm_irq_uninstall(dev);
4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256

	mutex_lock(&dev->struct_mutex);
	ret =  i915_gem_idle(dev);

	/* Hack!  Don't let anybody do execbuf while we don't control the chip.
	 * We need to replace this with a semaphore, or something.
	 * And not confound ums.mm_suspended!
	 */
	if (ret != 0)
		dev_priv->ums.mm_suspended = 1;
	mutex_unlock(&dev->struct_mutex);

	return ret;
4257 4258 4259 4260 4261 4262 4263
}

void
i915_gem_lastclose(struct drm_device *dev)
{
	int ret;

4264 4265 4266
	if (drm_core_check_feature(dev, DRIVER_MODESET))
		return;

4267
	mutex_lock(&dev->struct_mutex);
4268 4269 4270
	ret = i915_gem_idle(dev);
	if (ret)
		DRM_ERROR("failed to idle hardware: %d\n", ret);
4271
	mutex_unlock(&dev->struct_mutex);
4272 4273
}

4274 4275 4276 4277 4278 4279 4280
static void
init_ring_lists(struct intel_ring_buffer *ring)
{
	INIT_LIST_HEAD(&ring->active_list);
	INIT_LIST_HEAD(&ring->request_list);
}

4281 4282 4283 4284
void
i915_gem_load(struct drm_device *dev)
{
	drm_i915_private_t *dev_priv = dev->dev_private;
4285 4286 4287 4288 4289 4290 4291
	int i;

	dev_priv->slab =
		kmem_cache_create("i915_gem_object",
				  sizeof(struct drm_i915_gem_object), 0,
				  SLAB_HWCACHE_ALIGN,
				  NULL);
4292

4293
	INIT_LIST_HEAD(&dev_priv->mm.active_list);
4294
	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
C
Chris Wilson 已提交
4295 4296
	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4297
	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4298 4299
	for (i = 0; i < I915_NUM_RINGS; i++)
		init_ring_lists(&dev_priv->ring[i]);
4300
	for (i = 0; i < I915_MAX_NUM_FENCES; i++)
4301
		INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4302 4303
	INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
			  i915_gem_retire_work_handler);
4304
	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4305

4306 4307
	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
	if (IS_GEN3(dev)) {
4308 4309
		I915_WRITE(MI_ARB_STATE,
			   _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4310 4311
	}

4312 4313
	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;

4314
	/* Old X drivers will take 0-2 for front, back, depth buffers */
4315 4316
	if (!drm_core_check_feature(dev, DRIVER_MODESET))
		dev_priv->fence_reg_start = 3;
4317

4318 4319 4320
	if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
		dev_priv->num_fence_regs = 32;
	else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4321 4322 4323 4324
		dev_priv->num_fence_regs = 16;
	else
		dev_priv->num_fence_regs = 8;

4325
	/* Initialize fence registers to zero */
4326 4327
	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
	i915_gem_restore_fences(dev);
4328

4329
	i915_gem_detect_bit_6_swizzle(dev);
4330
	init_waitqueue_head(&dev_priv->pending_flip_queue);
4331

4332 4333
	dev_priv->mm.interruptible = true;

4334 4335 4336
	dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
	dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
	register_shrinker(&dev_priv->mm.inactive_shrinker);
4337
}
4338 4339 4340 4341 4342

/*
 * Create a physically contiguous memory object for this object
 * e.g. for cursor + overlay regs
 */
4343 4344
static int i915_gem_init_phys_object(struct drm_device *dev,
				     int id, int size, int align)
4345 4346 4347 4348 4349 4350 4351 4352
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_i915_gem_phys_object *phys_obj;
	int ret;

	if (dev_priv->mm.phys_objs[id - 1] || !size)
		return 0;

4353
	phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4354 4355 4356 4357 4358
	if (!phys_obj)
		return -ENOMEM;

	phys_obj->id = id;

4359
	phys_obj->handle = drm_pci_alloc(dev, size, align);
4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371
	if (!phys_obj->handle) {
		ret = -ENOMEM;
		goto kfree_obj;
	}
#ifdef CONFIG_X86
	set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
#endif

	dev_priv->mm.phys_objs[id - 1] = phys_obj;

	return 0;
kfree_obj:
4372
	kfree(phys_obj);
4373 4374 4375
	return ret;
}

4376
static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400
{
	drm_i915_private_t *dev_priv = dev->dev_private;
	struct drm_i915_gem_phys_object *phys_obj;

	if (!dev_priv->mm.phys_objs[id - 1])
		return;

	phys_obj = dev_priv->mm.phys_objs[id - 1];
	if (phys_obj->cur_obj) {
		i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
	}

#ifdef CONFIG_X86
	set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
#endif
	drm_pci_free(dev, phys_obj->handle);
	kfree(phys_obj);
	dev_priv->mm.phys_objs[id - 1] = NULL;
}

void i915_gem_free_all_phys_object(struct drm_device *dev)
{
	int i;

4401
	for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4402 4403 4404 4405
		i915_gem_free_phys_object(dev, i);
}

void i915_gem_detach_phys_object(struct drm_device *dev,
4406
				 struct drm_i915_gem_object *obj)
4407
{
A
Al Viro 已提交
4408
	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4409
	char *vaddr;
4410 4411 4412
	int i;
	int page_count;

4413
	if (!obj->phys_obj)
4414
		return;
4415
	vaddr = obj->phys_obj->handle->vaddr;
4416

4417
	page_count = obj->base.size / PAGE_SIZE;
4418
	for (i = 0; i < page_count; i++) {
4419
		struct page *page = shmem_read_mapping_page(mapping, i);
4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430
		if (!IS_ERR(page)) {
			char *dst = kmap_atomic(page);
			memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
			kunmap_atomic(dst);

			drm_clflush_pages(&page, 1);

			set_page_dirty(page);
			mark_page_accessed(page);
			page_cache_release(page);
		}
4431
	}
4432
	i915_gem_chipset_flush(dev);
4433

4434 4435
	obj->phys_obj->cur_obj = NULL;
	obj->phys_obj = NULL;
4436 4437 4438 4439
}

int
i915_gem_attach_phys_object(struct drm_device *dev,
4440
			    struct drm_i915_gem_object *obj,
4441 4442
			    int id,
			    int align)
4443
{
A
Al Viro 已提交
4444
	struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
4445 4446 4447 4448 4449 4450 4451 4452
	drm_i915_private_t *dev_priv = dev->dev_private;
	int ret = 0;
	int page_count;
	int i;

	if (id > I915_MAX_PHYS_OBJECT)
		return -EINVAL;

4453 4454
	if (obj->phys_obj) {
		if (obj->phys_obj->id == id)
4455 4456 4457 4458 4459 4460 4461
			return 0;
		i915_gem_detach_phys_object(dev, obj);
	}

	/* create a new object */
	if (!dev_priv->mm.phys_objs[id - 1]) {
		ret = i915_gem_init_phys_object(dev, id,
4462
						obj->base.size, align);
4463
		if (ret) {
4464 4465
			DRM_ERROR("failed to init phys object %d size: %zu\n",
				  id, obj->base.size);
4466
			return ret;
4467 4468 4469 4470
		}
	}

	/* bind to the object */
4471 4472
	obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
	obj->phys_obj->cur_obj = obj;
4473

4474
	page_count = obj->base.size / PAGE_SIZE;
4475 4476

	for (i = 0; i < page_count; i++) {
4477 4478 4479
		struct page *page;
		char *dst, *src;

4480
		page = shmem_read_mapping_page(mapping, i);
4481 4482
		if (IS_ERR(page))
			return PTR_ERR(page);
4483

4484
		src = kmap_atomic(page);
4485
		dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4486
		memcpy(dst, src, PAGE_SIZE);
P
Peter Zijlstra 已提交
4487
		kunmap_atomic(src);
4488

4489 4490 4491
		mark_page_accessed(page);
		page_cache_release(page);
	}
4492

4493 4494 4495 4496
	return 0;
}

static int
4497 4498
i915_gem_phys_pwrite(struct drm_device *dev,
		     struct drm_i915_gem_object *obj,
4499 4500 4501
		     struct drm_i915_gem_pwrite *args,
		     struct drm_file *file_priv)
{
4502
	void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
V
Ville Syrjälä 已提交
4503
	char __user *user_data = to_user_ptr(args->data_ptr);
4504

4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517
	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
		unsigned long unwritten;

		/* The physical object once assigned is fixed for the lifetime
		 * of the obj, so we can safely drop the lock and continue
		 * to access vaddr.
		 */
		mutex_unlock(&dev->struct_mutex);
		unwritten = copy_from_user(vaddr, user_data, args->size);
		mutex_lock(&dev->struct_mutex);
		if (unwritten)
			return -EFAULT;
	}
4518

4519
	i915_gem_chipset_flush(dev);
4520 4521
	return 0;
}
4522

4523
void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4524
{
4525
	struct drm_i915_file_private *file_priv = file->driver_priv;
4526 4527 4528 4529 4530

	/* Clean up our request list when the client is going away, so that
	 * later retire_requests won't dereference our soon-to-be-gone
	 * file_priv.
	 */
4531
	spin_lock(&file_priv->mm.lock);
4532 4533 4534 4535 4536 4537 4538 4539 4540
	while (!list_empty(&file_priv->mm.request_list)) {
		struct drm_i915_gem_request *request;

		request = list_first_entry(&file_priv->mm.request_list,
					   struct drm_i915_gem_request,
					   client_list);
		list_del(&request->client_list);
		request->file_priv = NULL;
	}
4541
	spin_unlock(&file_priv->mm.lock);
4542
}
4543

4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556
static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
{
	if (!mutex_is_locked(mutex))
		return false;

#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
	return mutex->owner == task;
#else
	/* Since UP may be pre-empted, we cannot assume that we own the lock */
	return false;
#endif
}

4557
static int
4558
i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
4559
{
4560 4561 4562 4563 4564
	struct drm_i915_private *dev_priv =
		container_of(shrinker,
			     struct drm_i915_private,
			     mm.inactive_shrinker);
	struct drm_device *dev = dev_priv->dev;
C
Chris Wilson 已提交
4565
	struct drm_i915_gem_object *obj;
4566
	int nr_to_scan = sc->nr_to_scan;
4567
	bool unlock = true;
4568 4569
	int cnt;

4570 4571 4572 4573
	if (!mutex_trylock(&dev->struct_mutex)) {
		if (!mutex_is_locked_by(&dev->struct_mutex, current))
			return 0;

4574 4575 4576
		if (dev_priv->mm.shrinker_no_lock_stealing)
			return 0;

4577 4578
		unlock = false;
	}
4579

C
Chris Wilson 已提交
4580 4581
	if (nr_to_scan) {
		nr_to_scan -= i915_gem_purge(dev_priv, nr_to_scan);
4582 4583 4584
		if (nr_to_scan > 0)
			nr_to_scan -= __i915_gem_shrink(dev_priv, nr_to_scan,
							false);
C
Chris Wilson 已提交
4585 4586
		if (nr_to_scan > 0)
			i915_gem_shrink_all(dev_priv);
4587 4588
	}

4589
	cnt = 0;
4590
	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
4591 4592
		if (obj->pages_pin_count == 0)
			cnt += obj->base.size >> PAGE_SHIFT;
4593
	list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list)
4594
		if (obj->pin_count == 0 && obj->pages_pin_count == 0)
C
Chris Wilson 已提交
4595
			cnt += obj->base.size >> PAGE_SHIFT;
4596

4597 4598
	if (unlock)
		mutex_unlock(&dev->struct_mutex);
C
Chris Wilson 已提交
4599
	return cnt;
4600
}