i915_gem.c 145.0 KB
Newer Older
1
/*
2
 * Copyright © 2008-2015 Intel Corporation
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *
 */

28
#include <drm/drmP.h>
29
#include <drm/drm_vma_manager.h>
30
#include <drm/i915_drm.h>
31
#include "i915_drv.h"
32
#include "i915_gem_clflush.h"
33
#include "i915_vgpu.h"
C
Chris Wilson 已提交
34
#include "i915_trace.h"
35
#include "intel_drv.h"
36
#include "intel_frontbuffer.h"
37
#include "intel_mocs.h"
M
Matthew Auld 已提交
38
#include "i915_gemfs.h"
39
#include <linux/dma-fence-array.h>
40
#include <linux/kthread.h>
41
#include <linux/reservation.h>
42
#include <linux/shmem_fs.h>
43
#include <linux/slab.h>
44
#include <linux/stop_machine.h>
45
#include <linux/swap.h>
J
Jesse Barnes 已提交
46
#include <linux/pci.h>
47
#include <linux/dma-buf.h>
48

49
static void i915_gem_flush_free_objects(struct drm_i915_private *i915);
50

51 52
static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
{
53
	if (obj->cache_dirty)
54 55
		return false;

56
	if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
57 58 59 60 61
		return true;

	return obj->pin_display;
}

62
static int
63
insert_mappable_node(struct i915_ggtt *ggtt,
64 65 66
                     struct drm_mm_node *node, u32 size)
{
	memset(node, 0, sizeof(*node));
67 68 69 70
	return drm_mm_insert_node_in_range(&ggtt->base.mm, node,
					   size, 0, I915_COLOR_UNEVICTABLE,
					   0, ggtt->mappable_end,
					   DRM_MM_INSERT_LOW);
71 72 73 74 75 76 77 78
}

static void
remove_mappable_node(struct drm_mm_node *node)
{
	drm_mm_remove_node(node);
}

79 80
/* some bookkeeping */
static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
81
				  u64 size)
82
{
83
	spin_lock(&dev_priv->mm.object_stat_lock);
84 85
	dev_priv->mm.object_count++;
	dev_priv->mm.object_memory += size;
86
	spin_unlock(&dev_priv->mm.object_stat_lock);
87 88 89
}

static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
90
				     u64 size)
91
{
92
	spin_lock(&dev_priv->mm.object_stat_lock);
93 94
	dev_priv->mm.object_count--;
	dev_priv->mm.object_memory -= size;
95
	spin_unlock(&dev_priv->mm.object_stat_lock);
96 97
}

98
static int
99
i915_gem_wait_for_error(struct i915_gpu_error *error)
100 101 102
{
	int ret;

103 104
	might_sleep();

105 106 107 108 109
	/*
	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
	 * userspace. If it takes that long something really bad is going on and
	 * we should simply try to bail out and fail as gracefully as possible.
	 */
110
	ret = wait_event_interruptible_timeout(error->reset_queue,
111
					       !i915_reset_backoff(error),
112
					       I915_RESET_TIMEOUT);
113 114 115 116
	if (ret == 0) {
		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
		return -EIO;
	} else if (ret < 0) {
117
		return ret;
118 119
	} else {
		return 0;
120
	}
121 122
}

123
int i915_mutex_lock_interruptible(struct drm_device *dev)
124
{
125
	struct drm_i915_private *dev_priv = to_i915(dev);
126 127
	int ret;

128
	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
129 130 131 132 133 134 135 136 137
	if (ret)
		return ret;

	ret = mutex_lock_interruptible(&dev->struct_mutex);
	if (ret)
		return ret;

	return 0;
}
138

139 140
int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
141
			    struct drm_file *file)
142
{
143
	struct drm_i915_private *dev_priv = to_i915(dev);
144
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
145
	struct drm_i915_gem_get_aperture *args = data;
146
	struct i915_vma *vma;
147
	u64 pinned;
148

149
	pinned = ggtt->base.reserved;
150
	mutex_lock(&dev->struct_mutex);
151
	list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
152
		if (i915_vma_is_pinned(vma))
153
			pinned += vma->node.size;
154
	list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
155
		if (i915_vma_is_pinned(vma))
156
			pinned += vma->node.size;
157
	mutex_unlock(&dev->struct_mutex);
158

159
	args->aper_size = ggtt->base.total;
160
	args->aper_available_size = args->aper_size - pinned;
161

162 163 164
	return 0;
}

165
static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
166
{
167
	struct address_space *mapping = obj->base.filp->f_mapping;
168
	drm_dma_handle_t *phys;
169 170
	struct sg_table *st;
	struct scatterlist *sg;
171
	char *vaddr;
172
	int i;
173
	int err;
174

175
	if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
176
		return -EINVAL;
177

178 179 180 181 182
	/* Always aligning to the object size, allows a single allocation
	 * to handle all possible callers, and given typical object sizes,
	 * the alignment of the buddy allocation will naturally match.
	 */
	phys = drm_pci_alloc(obj->base.dev,
183
			     roundup_pow_of_two(obj->base.size),
184 185
			     roundup_pow_of_two(obj->base.size));
	if (!phys)
186
		return -ENOMEM;
187 188

	vaddr = phys->vaddr;
189 190 191 192 193
	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
		struct page *page;
		char *src;

		page = shmem_read_mapping_page(mapping, i);
194
		if (IS_ERR(page)) {
195
			err = PTR_ERR(page);
196 197
			goto err_phys;
		}
198 199 200 201 202 203

		src = kmap_atomic(page);
		memcpy(vaddr, src, PAGE_SIZE);
		drm_clflush_virt_range(vaddr, PAGE_SIZE);
		kunmap_atomic(src);

204
		put_page(page);
205 206 207
		vaddr += PAGE_SIZE;
	}

208
	i915_gem_chipset_flush(to_i915(obj->base.dev));
209 210

	st = kmalloc(sizeof(*st), GFP_KERNEL);
211
	if (!st) {
212
		err = -ENOMEM;
213 214
		goto err_phys;
	}
215 216 217

	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
		kfree(st);
218
		err = -ENOMEM;
219
		goto err_phys;
220 221 222 223 224
	}

	sg = st->sgl;
	sg->offset = 0;
	sg->length = obj->base.size;
225

226
	sg_dma_address(sg) = phys->busaddr;
227 228
	sg_dma_len(sg) = obj->base.size;

229
	obj->phys_handle = phys;
230

231
	__i915_gem_object_set_pages(obj, st, sg->length);
232 233

	return 0;
234 235 236

err_phys:
	drm_pci_free(obj->base.dev, phys);
237 238

	return err;
239 240
}

241 242 243 244 245 246 247 248
static void __start_cpu_write(struct drm_i915_gem_object *obj)
{
	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	if (cpu_write_needs_clflush(obj))
		obj->cache_dirty = true;
}

249
static void
250
__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
251 252
				struct sg_table *pages,
				bool needs_clflush)
253
{
C
Chris Wilson 已提交
254
	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
255

C
Chris Wilson 已提交
256 257
	if (obj->mm.madv == I915_MADV_DONTNEED)
		obj->mm.dirty = false;
258

259 260
	if (needs_clflush &&
	    (obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
261
	    !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
262
		drm_clflush_sg(pages);
263

264
	__start_cpu_write(obj);
265 266 267 268 269 270
}

static void
i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
			       struct sg_table *pages)
{
271
	__i915_gem_object_release_shmem(obj, pages, false);
272

C
Chris Wilson 已提交
273
	if (obj->mm.dirty) {
274
		struct address_space *mapping = obj->base.filp->f_mapping;
275
		char *vaddr = obj->phys_handle->vaddr;
276 277 278
		int i;

		for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
279 280 281 282 283 284 285 286 287 288 289 290 291
			struct page *page;
			char *dst;

			page = shmem_read_mapping_page(mapping, i);
			if (IS_ERR(page))
				continue;

			dst = kmap_atomic(page);
			drm_clflush_virt_range(vaddr, PAGE_SIZE);
			memcpy(dst, vaddr, PAGE_SIZE);
			kunmap_atomic(dst);

			set_page_dirty(page);
C
Chris Wilson 已提交
292
			if (obj->mm.madv == I915_MADV_WILLNEED)
293
				mark_page_accessed(page);
294
			put_page(page);
295 296
			vaddr += PAGE_SIZE;
		}
C
Chris Wilson 已提交
297
		obj->mm.dirty = false;
298 299
	}

300 301
	sg_free_table(pages);
	kfree(pages);
302 303

	drm_pci_free(obj->base.dev, obj->phys_handle);
304 305 306 307 308
}

static void
i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
{
C
Chris Wilson 已提交
309
	i915_gem_object_unpin_pages(obj);
310 311 312 313 314 315 316 317
}

static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
	.get_pages = i915_gem_object_get_pages_phys,
	.put_pages = i915_gem_object_put_pages_phys,
	.release = i915_gem_object_release_phys,
};

318 319
static const struct drm_i915_gem_object_ops i915_gem_object_ops;

320
int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
321 322 323
{
	struct i915_vma *vma;
	LIST_HEAD(still_in_list);
324 325 326
	int ret;

	lockdep_assert_held(&obj->base.dev->struct_mutex);
327

328 329 330 331
	/* Closed vma are removed from the obj->vma_list - but they may
	 * still have an active binding on the object. To remove those we
	 * must wait for all rendering to complete to the object (as unbinding
	 * must anyway), and retire the requests.
332
	 */
333 334 335 336 337 338
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE |
				   I915_WAIT_LOCKED |
				   I915_WAIT_ALL,
				   MAX_SCHEDULE_TIMEOUT,
				   NULL);
339 340 341 342 343
	if (ret)
		return ret;

	i915_gem_retire_requests(to_i915(obj->base.dev));

344 345 346 347 348 349 350 351 352 353 354 355 356
	while ((vma = list_first_entry_or_null(&obj->vma_list,
					       struct i915_vma,
					       obj_link))) {
		list_move_tail(&vma->obj_link, &still_in_list);
		ret = i915_vma_unbind(vma);
		if (ret)
			break;
	}
	list_splice(&still_in_list, &obj->vma_list);

	return ret;
}

357 358 359 360 361
static long
i915_gem_object_wait_fence(struct dma_fence *fence,
			   unsigned int flags,
			   long timeout,
			   struct intel_rps_client *rps)
362
{
363
	struct drm_i915_gem_request *rq;
364

365
	BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
366

367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395
	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
		return timeout;

	if (!dma_fence_is_i915(fence))
		return dma_fence_wait_timeout(fence,
					      flags & I915_WAIT_INTERRUPTIBLE,
					      timeout);

	rq = to_request(fence);
	if (i915_gem_request_completed(rq))
		goto out;

	/* This client is about to stall waiting for the GPU. In many cases
	 * this is undesirable and limits the throughput of the system, as
	 * many clients cannot continue processing user input/output whilst
	 * blocked. RPS autotuning may take tens of milliseconds to respond
	 * to the GPU load and thus incurs additional latency for the client.
	 * We can circumvent that by promoting the GPU frequency to maximum
	 * before we wait. This makes the GPU throttle up much more quickly
	 * (good for benchmarks and user experience, e.g. window animations),
	 * but at a cost of spending more power processing the workload
	 * (bad for battery). Not all clients even want their results
	 * immediately and for them we should just let the GPU select its own
	 * frequency to maximise efficiency. To prevent a single client from
	 * forcing the clocks too high for the whole system, we only allow
	 * each client to waitboost once in a busy period.
	 */
	if (rps) {
		if (INTEL_GEN(rq->i915) >= 6)
396
			gen6_rps_boost(rq, rps);
397 398
		else
			rps = NULL;
399 400
	}

401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
	timeout = i915_wait_request(rq, flags, timeout);

out:
	if (flags & I915_WAIT_LOCKED && i915_gem_request_completed(rq))
		i915_gem_request_retire_upto(rq);

	return timeout;
}

static long
i915_gem_object_wait_reservation(struct reservation_object *resv,
				 unsigned int flags,
				 long timeout,
				 struct intel_rps_client *rps)
{
416
	unsigned int seq = __read_seqcount_begin(&resv->seq);
417
	struct dma_fence *excl;
418
	bool prune_fences = false;
419 420 421 422

	if (flags & I915_WAIT_ALL) {
		struct dma_fence **shared;
		unsigned int count, i;
423 424
		int ret;

425 426
		ret = reservation_object_get_fences_rcu(resv,
							&excl, &count, &shared);
427 428 429
		if (ret)
			return ret;

430 431 432 433
		for (i = 0; i < count; i++) {
			timeout = i915_gem_object_wait_fence(shared[i],
							     flags, timeout,
							     rps);
434
			if (timeout < 0)
435
				break;
436

437 438 439 440 441 442
			dma_fence_put(shared[i]);
		}

		for (; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);
443 444

		prune_fences = count && timeout >= 0;
445 446
	} else {
		excl = reservation_object_get_excl_rcu(resv);
447 448
	}

449
	if (excl && timeout >= 0) {
450
		timeout = i915_gem_object_wait_fence(excl, flags, timeout, rps);
451 452
		prune_fences = timeout >= 0;
	}
453 454 455

	dma_fence_put(excl);

456 457 458 459
	/* Oportunistically prune the fences iff we know they have *all* been
	 * signaled and that the reservation object has not been changed (i.e.
	 * no new fences have been added).
	 */
460
	if (prune_fences && !__read_seqcount_retry(&resv->seq, seq)) {
461 462 463 464 465
		if (reservation_object_trylock(resv)) {
			if (!__read_seqcount_retry(&resv->seq, seq))
				reservation_object_add_excl_fence(resv, NULL);
			reservation_object_unlock(resv);
		}
466 467
	}

468
	return timeout;
469 470
}

471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534
static void __fence_set_priority(struct dma_fence *fence, int prio)
{
	struct drm_i915_gem_request *rq;
	struct intel_engine_cs *engine;

	if (!dma_fence_is_i915(fence))
		return;

	rq = to_request(fence);
	engine = rq->engine;
	if (!engine->schedule)
		return;

	engine->schedule(rq, prio);
}

static void fence_set_priority(struct dma_fence *fence, int prio)
{
	/* Recurse once into a fence-array */
	if (dma_fence_is_array(fence)) {
		struct dma_fence_array *array = to_dma_fence_array(fence);
		int i;

		for (i = 0; i < array->num_fences; i++)
			__fence_set_priority(array->fences[i], prio);
	} else {
		__fence_set_priority(fence, prio);
	}
}

int
i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
			      unsigned int flags,
			      int prio)
{
	struct dma_fence *excl;

	if (flags & I915_WAIT_ALL) {
		struct dma_fence **shared;
		unsigned int count, i;
		int ret;

		ret = reservation_object_get_fences_rcu(obj->resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			fence_set_priority(shared[i], prio);
			dma_fence_put(shared[i]);
		}

		kfree(shared);
	} else {
		excl = reservation_object_get_excl_rcu(obj->resv);
	}

	if (excl) {
		fence_set_priority(excl, prio);
		dma_fence_put(excl);
	}
	return 0;
}

535 536 537 538 539 540
/**
 * Waits for rendering to the object to be completed
 * @obj: i915 gem object
 * @flags: how to wait (under a lock, for all rendering or just for writes etc)
 * @timeout: how long to wait
 * @rps: client (user process) to charge for any waitboosting
541
 */
542 543 544 545 546
int
i915_gem_object_wait(struct drm_i915_gem_object *obj,
		     unsigned int flags,
		     long timeout,
		     struct intel_rps_client *rps)
547
{
548 549 550 551 552 553 554
	might_sleep();
#if IS_ENABLED(CONFIG_LOCKDEP)
	GEM_BUG_ON(debug_locks &&
		   !!lockdep_is_held(&obj->base.dev->struct_mutex) !=
		   !!(flags & I915_WAIT_LOCKED));
#endif
	GEM_BUG_ON(timeout < 0);
555

556 557 558
	timeout = i915_gem_object_wait_reservation(obj->resv,
						   flags, timeout,
						   rps);
559
	return timeout < 0 ? timeout : 0;
560 561 562 563 564 565 566 567 568
}

static struct intel_rps_client *to_rps_client(struct drm_file *file)
{
	struct drm_i915_file_private *fpriv = file->driver_priv;

	return &fpriv->rps;
}

569 570 571
static int
i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pwrite *args,
572
		     struct drm_file *file)
573 574
{
	void *vaddr = obj->phys_handle->vaddr + args->offset;
575
	char __user *user_data = u64_to_user_ptr(args->data_ptr);
576 577 578 579

	/* We manually control the domain here and pretend that it
	 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
	 */
580
	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
581 582
	if (copy_from_user(vaddr, user_data, args->size))
		return -EFAULT;
583

584
	drm_clflush_virt_range(vaddr, args->size);
585
	i915_gem_chipset_flush(to_i915(obj->base.dev));
586

587
	intel_fb_obj_flush(obj, ORIGIN_CPU);
588
	return 0;
589 590
}

591
void *i915_gem_object_alloc(struct drm_i915_private *dev_priv)
592
{
593
	return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
594 595 596 597
}

void i915_gem_object_free(struct drm_i915_gem_object *obj)
{
598
	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
599
	kmem_cache_free(dev_priv->objects, obj);
600 601
}

602 603
static int
i915_gem_create(struct drm_file *file,
604
		struct drm_i915_private *dev_priv,
605 606
		uint64_t size,
		uint32_t *handle_p)
607
{
608
	struct drm_i915_gem_object *obj;
609 610
	int ret;
	u32 handle;
611

612
	size = roundup(size, PAGE_SIZE);
613 614
	if (size == 0)
		return -EINVAL;
615 616

	/* Allocate the new object */
617
	obj = i915_gem_object_create(dev_priv, size);
618 619
	if (IS_ERR(obj))
		return PTR_ERR(obj);
620

621
	ret = drm_gem_handle_create(file, &obj->base, &handle);
622
	/* drop reference from allocate - handle holds it now */
C
Chris Wilson 已提交
623
	i915_gem_object_put(obj);
624 625
	if (ret)
		return ret;
626

627
	*handle_p = handle;
628 629 630
	return 0;
}

631 632 633 634 635 636
int
i915_gem_dumb_create(struct drm_file *file,
		     struct drm_device *dev,
		     struct drm_mode_create_dumb *args)
{
	/* have to work out size/pitch and return them */
637
	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
638
	args->size = args->pitch * args->height;
639
	return i915_gem_create(file, to_i915(dev),
640
			       args->size, &args->handle);
641 642
}

643 644 645 646 647 648
static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
{
	return !(obj->cache_level == I915_CACHE_NONE ||
		 obj->cache_level == I915_CACHE_WT);
}

649 650
/**
 * Creates a new mm object and returns a handle to it.
651 652 653
 * @dev: drm device pointer
 * @data: ioctl data blob
 * @file: drm file pointer
654 655 656 657 658
 */
int
i915_gem_create_ioctl(struct drm_device *dev, void *data,
		      struct drm_file *file)
{
659
	struct drm_i915_private *dev_priv = to_i915(dev);
660
	struct drm_i915_gem_create *args = data;
661

662
	i915_gem_flush_free_objects(dev_priv);
663

664
	return i915_gem_create(file, dev_priv,
665
			       args->size, &args->handle);
666 667
}

668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
static inline enum fb_op_origin
fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain)
{
	return (domain == I915_GEM_DOMAIN_GTT ?
		obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
}

static void
flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
{
	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);

	if (!(obj->base.write_domain & flush_domains))
		return;

	/* No actual flushing is required for the GTT write domain.  Writes
	 * to it "immediately" go to main memory as far as we know, so there's
	 * no chipset flush.  It also doesn't land in render cache.
	 *
	 * However, we do have to enforce the order so that all writes through
	 * the GTT land before any writes to the device, such as updates to
	 * the GATT itself.
	 *
	 * We also have to wait a bit for the writes to land from the GTT.
	 * An uncached read (i.e. mmio) seems to be ideal for the round-trip
	 * timing. This issue has only been observed when switching quickly
	 * between GTT writes and CPU reads from inside the kernel on recent hw,
	 * and it appears to only affect discrete GTT blocks (i.e. on LLC
	 * system agents we cannot reproduce this behaviour).
	 */
	wmb();

	switch (obj->base.write_domain) {
	case I915_GEM_DOMAIN_GTT:
702
		if (!HAS_LLC(dev_priv)) {
703 704
			intel_runtime_pm_get(dev_priv);
			spin_lock_irq(&dev_priv->uncore.lock);
705
			POSTING_READ_FW(RING_HEAD(dev_priv->engine[RCS]->mmio_base));
706 707
			spin_unlock_irq(&dev_priv->uncore.lock);
			intel_runtime_pm_put(dev_priv);
708 709 710 711 712 713 714 715 716
		}

		intel_fb_obj_flush(obj,
				   fb_write_origin(obj, I915_GEM_DOMAIN_GTT));
		break;

	case I915_GEM_DOMAIN_CPU:
		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
		break;
717 718 719 720 721

	case I915_GEM_DOMAIN_RENDER:
		if (gpu_write_needs_clflush(obj))
			obj->cache_dirty = true;
		break;
722 723 724 725 726
	}

	obj->base.write_domain = 0;
}

727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
static inline int
__copy_to_user_swizzled(char __user *cpu_vaddr,
			const char *gpu_vaddr, int gpu_offset,
			int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_to_user(cpu_vaddr + cpu_offset,
				     gpu_vaddr + swizzled_gpu_offset,
				     this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

753
static inline int
754 755
__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
			  const char __user *cpu_vaddr,
756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778
			  int length)
{
	int ret, cpu_offset = 0;

	while (length > 0) {
		int cacheline_end = ALIGN(gpu_offset + 1, 64);
		int this_length = min(cacheline_end - gpu_offset, length);
		int swizzled_gpu_offset = gpu_offset ^ 64;

		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
				       cpu_vaddr + cpu_offset,
				       this_length);
		if (ret)
			return ret + length;

		cpu_offset += this_length;
		gpu_offset += this_length;
		length -= this_length;
	}

	return 0;
}

779 780 781 782 783 784
/*
 * Pins the specified object's pages and synchronizes the object with
 * GPU accesses. Sets needs_clflush to non-zero if the caller should
 * flush the object from the CPU cache.
 */
int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
785
				    unsigned int *needs_clflush)
786 787 788
{
	int ret;

789
	lockdep_assert_held(&obj->base.dev->struct_mutex);
790

791
	*needs_clflush = 0;
792 793
	if (!i915_gem_object_has_struct_page(obj))
		return -ENODEV;
794

795 796 797 798 799
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE |
				   I915_WAIT_LOCKED,
				   MAX_SCHEDULE_TIMEOUT,
				   NULL);
800 801 802
	if (ret)
		return ret;

C
Chris Wilson 已提交
803
	ret = i915_gem_object_pin_pages(obj);
804 805 806
	if (ret)
		return ret;

807 808
	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
809 810 811 812 813 814 815
		ret = i915_gem_object_set_to_cpu_domain(obj, false);
		if (ret)
			goto err_unpin;
		else
			goto out;
	}

816
	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
817

818 819 820 821 822
	/* If we're not in the cpu read domain, set ourself into the gtt
	 * read domain and manually flush cachelines (if required). This
	 * optimizes for the case when the gpu will dirty the data
	 * anyway again before the next pread happens.
	 */
823 824
	if (!obj->cache_dirty &&
	    !(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
825
		*needs_clflush = CLFLUSH_BEFORE;
826

827
out:
828
	/* return with the pages pinned */
829
	return 0;
830 831 832 833

err_unpin:
	i915_gem_object_unpin_pages(obj);
	return ret;
834 835 836 837 838 839 840
}

int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
				     unsigned int *needs_clflush)
{
	int ret;

841 842
	lockdep_assert_held(&obj->base.dev->struct_mutex);

843 844 845 846
	*needs_clflush = 0;
	if (!i915_gem_object_has_struct_page(obj))
		return -ENODEV;

847 848 849 850 851 852
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE |
				   I915_WAIT_LOCKED |
				   I915_WAIT_ALL,
				   MAX_SCHEDULE_TIMEOUT,
				   NULL);
853 854 855
	if (ret)
		return ret;

C
Chris Wilson 已提交
856
	ret = i915_gem_object_pin_pages(obj);
857 858 859
	if (ret)
		return ret;

860 861
	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
862 863 864 865 866 867 868
		ret = i915_gem_object_set_to_cpu_domain(obj, true);
		if (ret)
			goto err_unpin;
		else
			goto out;
	}

869
	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
870

871 872 873 874 875
	/* If we're not in the cpu write domain, set ourself into the
	 * gtt write domain and manually flush cachelines (as required).
	 * This optimizes for the case when the gpu will use the data
	 * right away and we therefore have to clflush anyway.
	 */
876
	if (!obj->cache_dirty) {
877
		*needs_clflush |= CLFLUSH_AFTER;
878

879 880 881 882 883 884 885
		/*
		 * Same trick applies to invalidate partially written
		 * cachelines read before writing.
		 */
		if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU))
			*needs_clflush |= CLFLUSH_BEFORE;
	}
886

887
out:
888
	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
C
Chris Wilson 已提交
889
	obj->mm.dirty = true;
890
	/* return with the pages pinned */
891
	return 0;
892 893 894 895

err_unpin:
	i915_gem_object_unpin_pages(obj);
	return ret;
896 897
}

898 899 900 901
static void
shmem_clflush_swizzled_range(char *addr, unsigned long length,
			     bool swizzled)
{
902
	if (unlikely(swizzled)) {
903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
		unsigned long start = (unsigned long) addr;
		unsigned long end = (unsigned long) addr + length;

		/* For swizzling simply ensure that we always flush both
		 * channels. Lame, but simple and it works. Swizzled
		 * pwrite/pread is far from a hotpath - current userspace
		 * doesn't use it at all. */
		start = round_down(start, 128);
		end = round_up(end, 128);

		drm_clflush_virt_range((void *)start, end - start);
	} else {
		drm_clflush_virt_range(addr, length);
	}

}

920 921 922
/* Only difference to the fast-path function is that this can handle bit17
 * and uses non-atomic copy and kmap functions. */
static int
923
shmem_pread_slow(struct page *page, int offset, int length,
924 925 926 927 928 929 930 931
		 char __user *user_data,
		 bool page_do_bit17_swizzling, bool needs_clflush)
{
	char *vaddr;
	int ret;

	vaddr = kmap(page);
	if (needs_clflush)
932
		shmem_clflush_swizzled_range(vaddr + offset, length,
933
					     page_do_bit17_swizzling);
934 935

	if (page_do_bit17_swizzling)
936
		ret = __copy_to_user_swizzled(user_data, vaddr, offset, length);
937
	else
938
		ret = __copy_to_user(user_data, vaddr + offset, length);
939 940
	kunmap(page);

941
	return ret ? - EFAULT : 0;
942 943
}

944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019
static int
shmem_pread(struct page *page, int offset, int length, char __user *user_data,
	    bool page_do_bit17_swizzling, bool needs_clflush)
{
	int ret;

	ret = -ENODEV;
	if (!page_do_bit17_swizzling) {
		char *vaddr = kmap_atomic(page);

		if (needs_clflush)
			drm_clflush_virt_range(vaddr + offset, length);
		ret = __copy_to_user_inatomic(user_data, vaddr + offset, length);
		kunmap_atomic(vaddr);
	}
	if (ret == 0)
		return 0;

	return shmem_pread_slow(page, offset, length, user_data,
				page_do_bit17_swizzling, needs_clflush);
}

static int
i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
		     struct drm_i915_gem_pread *args)
{
	char __user *user_data;
	u64 remain;
	unsigned int obj_do_bit17_swizzling;
	unsigned int needs_clflush;
	unsigned int idx, offset;
	int ret;

	obj_do_bit17_swizzling = 0;
	if (i915_gem_object_needs_bit17_swizzle(obj))
		obj_do_bit17_swizzling = BIT(17);

	ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
	if (ret)
		return ret;

	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
	mutex_unlock(&obj->base.dev->struct_mutex);
	if (ret)
		return ret;

	remain = args->size;
	user_data = u64_to_user_ptr(args->data_ptr);
	offset = offset_in_page(args->offset);
	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
		struct page *page = i915_gem_object_get_page(obj, idx);
		int length;

		length = remain;
		if (offset + length > PAGE_SIZE)
			length = PAGE_SIZE - offset;

		ret = shmem_pread(page, offset, length, user_data,
				  page_to_phys(page) & obj_do_bit17_swizzling,
				  needs_clflush);
		if (ret)
			break;

		remain -= length;
		user_data += length;
		offset = 0;
	}

	i915_gem_obj_finish_shmem_access(obj);
	return ret;
}

static inline bool
gtt_user_read(struct io_mapping *mapping,
	      loff_t base, int offset,
	      char __user *user_data, int length)
1020
{
1021
	void __iomem *vaddr;
1022
	unsigned long unwritten;
1023 1024

	/* We can use the cpu mem copy function because this is X86. */
1025 1026 1027 1028
	vaddr = io_mapping_map_atomic_wc(mapping, base);
	unwritten = __copy_to_user_inatomic(user_data,
					    (void __force *)vaddr + offset,
					    length);
1029 1030
	io_mapping_unmap_atomic(vaddr);
	if (unwritten) {
1031 1032 1033 1034
		vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
		unwritten = copy_to_user(user_data,
					 (void __force *)vaddr + offset,
					 length);
1035 1036
		io_mapping_unmap(vaddr);
	}
1037 1038 1039 1040
	return unwritten;
}

static int
1041 1042
i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
		   const struct drm_i915_gem_pread *args)
1043
{
1044 1045
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
	struct i915_ggtt *ggtt = &i915->ggtt;
1046
	struct drm_mm_node node;
1047 1048 1049
	struct i915_vma *vma;
	void __user *user_data;
	u64 remain, offset;
1050 1051
	int ret;

1052 1053 1054 1055 1056 1057 1058
	ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
	if (ret)
		return ret;

	intel_runtime_pm_get(i915);
	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
				       PIN_MAPPABLE | PIN_NONBLOCK);
1059 1060 1061
	if (!IS_ERR(vma)) {
		node.start = i915_ggtt_offset(vma);
		node.allocated = false;
1062
		ret = i915_vma_put_fence(vma);
1063 1064 1065 1066 1067
		if (ret) {
			i915_vma_unpin(vma);
			vma = ERR_PTR(ret);
		}
	}
C
Chris Wilson 已提交
1068
	if (IS_ERR(vma)) {
1069
		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1070
		if (ret)
1071 1072
			goto out_unlock;
		GEM_BUG_ON(!node.allocated);
1073 1074 1075 1076 1077 1078
	}

	ret = i915_gem_object_set_to_gtt_domain(obj, false);
	if (ret)
		goto out_unpin;

1079
	mutex_unlock(&i915->drm.struct_mutex);
1080

1081 1082 1083
	user_data = u64_to_user_ptr(args->data_ptr);
	remain = args->size;
	offset = args->offset;
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099

	while (remain > 0) {
		/* Operation in this page
		 *
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
		 */
		u32 page_base = node.start;
		unsigned page_offset = offset_in_page(offset);
		unsigned page_length = PAGE_SIZE - page_offset;
		page_length = remain < page_length ? remain : page_length;
		if (node.allocated) {
			wmb();
			ggtt->base.insert_page(&ggtt->base,
					       i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1100
					       node.start, I915_CACHE_NONE, 0);
1101 1102 1103 1104
			wmb();
		} else {
			page_base += offset & PAGE_MASK;
		}
1105 1106 1107

		if (gtt_user_read(&ggtt->mappable, page_base, page_offset,
				  user_data, page_length)) {
1108 1109 1110 1111 1112 1113 1114 1115 1116
			ret = -EFAULT;
			break;
		}

		remain -= page_length;
		user_data += page_length;
		offset += page_length;
	}

1117
	mutex_lock(&i915->drm.struct_mutex);
1118 1119 1120 1121
out_unpin:
	if (node.allocated) {
		wmb();
		ggtt->base.clear_range(&ggtt->base,
1122
				       node.start, node.size);
1123 1124
		remove_mappable_node(&node);
	} else {
C
Chris Wilson 已提交
1125
		i915_vma_unpin(vma);
1126
	}
1127 1128 1129
out_unlock:
	intel_runtime_pm_put(i915);
	mutex_unlock(&i915->drm.struct_mutex);
1130

1131 1132 1133
	return ret;
}

1134 1135
/**
 * Reads data from the object referenced by handle.
1136 1137 1138
 * @dev: drm device pointer
 * @data: ioctl data blob
 * @file: drm file pointer
1139 1140 1141 1142 1143
 *
 * On error, the contents of *data are undefined.
 */
int
i915_gem_pread_ioctl(struct drm_device *dev, void *data,
1144
		     struct drm_file *file)
1145 1146
{
	struct drm_i915_gem_pread *args = data;
1147
	struct drm_i915_gem_object *obj;
1148
	int ret;
1149

1150 1151 1152 1153
	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_WRITE,
1154
		       u64_to_user_ptr(args->data_ptr),
1155 1156 1157
		       args->size))
		return -EFAULT;

1158
	obj = i915_gem_object_lookup(file, args->handle);
1159 1160
	if (!obj)
		return -ENOENT;
1161

1162
	/* Bounds check source.  */
1163
	if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
C
Chris Wilson 已提交
1164
		ret = -EINVAL;
1165
		goto out;
C
Chris Wilson 已提交
1166 1167
	}

C
Chris Wilson 已提交
1168 1169
	trace_i915_gem_object_pread(obj, args->offset, args->size);

1170 1171 1172 1173
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE,
				   MAX_SCHEDULE_TIMEOUT,
				   to_rps_client(file));
1174
	if (ret)
1175
		goto out;
1176

1177
	ret = i915_gem_object_pin_pages(obj);
1178
	if (ret)
1179
		goto out;
1180

1181
	ret = i915_gem_shmem_pread(obj, args);
1182
	if (ret == -EFAULT || ret == -ENODEV)
1183
		ret = i915_gem_gtt_pread(obj, args);
1184

1185 1186
	i915_gem_object_unpin_pages(obj);
out:
C
Chris Wilson 已提交
1187
	i915_gem_object_put(obj);
1188
	return ret;
1189 1190
}

1191 1192
/* This is the fast write path which cannot handle
 * page faults in the source data
1193
 */
1194

1195 1196 1197 1198
static inline bool
ggtt_write(struct io_mapping *mapping,
	   loff_t base, int offset,
	   char __user *user_data, int length)
1199
{
1200
	void __iomem *vaddr;
1201
	unsigned long unwritten;
1202

1203
	/* We can use the cpu mem copy function because this is X86. */
1204 1205
	vaddr = io_mapping_map_atomic_wc(mapping, base);
	unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
1206
						      user_data, length);
1207 1208
	io_mapping_unmap_atomic(vaddr);
	if (unwritten) {
1209 1210 1211
		vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
		unwritten = copy_from_user((void __force *)vaddr + offset,
					   user_data, length);
1212 1213
		io_mapping_unmap(vaddr);
	}
1214 1215 1216 1217

	return unwritten;
}

1218 1219 1220
/**
 * This is the fast pwrite path, where we copy the data directly from the
 * user into the GTT, uncached.
1221
 * @obj: i915 GEM object
1222
 * @args: pwrite arguments structure
1223
 */
1224
static int
1225 1226
i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
			 const struct drm_i915_gem_pwrite *args)
1227
{
1228
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
1229 1230
	struct i915_ggtt *ggtt = &i915->ggtt;
	struct drm_mm_node node;
1231 1232 1233
	struct i915_vma *vma;
	u64 remain, offset;
	void __user *user_data;
1234
	int ret;
1235

1236 1237 1238
	ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
	if (ret)
		return ret;
D
Daniel Vetter 已提交
1239

1240
	intel_runtime_pm_get(i915);
C
Chris Wilson 已提交
1241
	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1242
				       PIN_MAPPABLE | PIN_NONBLOCK);
1243 1244 1245
	if (!IS_ERR(vma)) {
		node.start = i915_ggtt_offset(vma);
		node.allocated = false;
1246
		ret = i915_vma_put_fence(vma);
1247 1248 1249 1250 1251
		if (ret) {
			i915_vma_unpin(vma);
			vma = ERR_PTR(ret);
		}
	}
C
Chris Wilson 已提交
1252
	if (IS_ERR(vma)) {
1253
		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1254
		if (ret)
1255 1256
			goto out_unlock;
		GEM_BUG_ON(!node.allocated);
1257
	}
D
Daniel Vetter 已提交
1258 1259 1260 1261 1262

	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		goto out_unpin;

1263 1264
	mutex_unlock(&i915->drm.struct_mutex);

1265
	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1266

1267 1268 1269 1270
	user_data = u64_to_user_ptr(args->data_ptr);
	offset = args->offset;
	remain = args->size;
	while (remain) {
1271 1272
		/* Operation in this page
		 *
1273 1274 1275
		 * page_base = page offset within aperture
		 * page_offset = offset within page
		 * page_length = bytes to copy for this page
1276
		 */
1277
		u32 page_base = node.start;
1278 1279
		unsigned int page_offset = offset_in_page(offset);
		unsigned int page_length = PAGE_SIZE - page_offset;
1280 1281 1282 1283 1284 1285 1286 1287 1288 1289
		page_length = remain < page_length ? remain : page_length;
		if (node.allocated) {
			wmb(); /* flush the write before we modify the GGTT */
			ggtt->base.insert_page(&ggtt->base,
					       i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
					       node.start, I915_CACHE_NONE, 0);
			wmb(); /* flush modifications to the GGTT (insert_page) */
		} else {
			page_base += offset & PAGE_MASK;
		}
1290
		/* If we get a fault while copying data, then (presumably) our
1291 1292
		 * source page isn't available.  Return the error and we'll
		 * retry in the slow path.
1293 1294
		 * If the object is non-shmem backed, we retry again with the
		 * path that handles page fault.
1295
		 */
1296 1297 1298 1299
		if (ggtt_write(&ggtt->mappable, page_base, page_offset,
			       user_data, page_length)) {
			ret = -EFAULT;
			break;
D
Daniel Vetter 已提交
1300
		}
1301

1302 1303 1304
		remain -= page_length;
		user_data += page_length;
		offset += page_length;
1305
	}
1306
	intel_fb_obj_flush(obj, ORIGIN_CPU);
1307 1308

	mutex_lock(&i915->drm.struct_mutex);
D
Daniel Vetter 已提交
1309
out_unpin:
1310 1311 1312
	if (node.allocated) {
		wmb();
		ggtt->base.clear_range(&ggtt->base,
1313
				       node.start, node.size);
1314 1315
		remove_mappable_node(&node);
	} else {
C
Chris Wilson 已提交
1316
		i915_vma_unpin(vma);
1317
	}
1318
out_unlock:
1319
	intel_runtime_pm_put(i915);
1320
	mutex_unlock(&i915->drm.struct_mutex);
1321
	return ret;
1322 1323
}

1324
static int
1325
shmem_pwrite_slow(struct page *page, int offset, int length,
1326 1327 1328 1329
		  char __user *user_data,
		  bool page_do_bit17_swizzling,
		  bool needs_clflush_before,
		  bool needs_clflush_after)
1330
{
1331 1332
	char *vaddr;
	int ret;
1333

1334
	vaddr = kmap(page);
1335
	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1336
		shmem_clflush_swizzled_range(vaddr + offset, length,
1337
					     page_do_bit17_swizzling);
1338
	if (page_do_bit17_swizzling)
1339 1340
		ret = __copy_from_user_swizzled(vaddr, offset, user_data,
						length);
1341
	else
1342
		ret = __copy_from_user(vaddr + offset, user_data, length);
1343
	if (needs_clflush_after)
1344
		shmem_clflush_swizzled_range(vaddr + offset, length,
1345
					     page_do_bit17_swizzling);
1346
	kunmap(page);
1347

1348
	return ret ? -EFAULT : 0;
1349 1350
}

1351 1352 1353 1354 1355
/* Per-page copy function for the shmem pwrite fastpath.
 * Flushes invalid cachelines before writing to the target if
 * needs_clflush_before is set and flushes out any written cachelines after
 * writing if needs_clflush is set.
 */
1356
static int
1357 1358 1359 1360
shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
	     bool page_do_bit17_swizzling,
	     bool needs_clflush_before,
	     bool needs_clflush_after)
1361
{
1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
	int ret;

	ret = -ENODEV;
	if (!page_do_bit17_swizzling) {
		char *vaddr = kmap_atomic(page);

		if (needs_clflush_before)
			drm_clflush_virt_range(vaddr + offset, len);
		ret = __copy_from_user_inatomic(vaddr + offset, user_data, len);
		if (needs_clflush_after)
			drm_clflush_virt_range(vaddr + offset, len);

		kunmap_atomic(vaddr);
	}
	if (ret == 0)
		return ret;

	return shmem_pwrite_slow(page, offset, len, user_data,
				 page_do_bit17_swizzling,
				 needs_clflush_before,
				 needs_clflush_after);
}

static int
i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
		      const struct drm_i915_gem_pwrite *args)
{
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
	void __user *user_data;
	u64 remain;
	unsigned int obj_do_bit17_swizzling;
	unsigned int partial_cacheline_write;
1394
	unsigned int needs_clflush;
1395 1396
	unsigned int offset, idx;
	int ret;
1397

1398
	ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1399 1400 1401
	if (ret)
		return ret;

1402 1403 1404 1405
	ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
	mutex_unlock(&i915->drm.struct_mutex);
	if (ret)
		return ret;
1406

1407 1408 1409
	obj_do_bit17_swizzling = 0;
	if (i915_gem_object_needs_bit17_swizzle(obj))
		obj_do_bit17_swizzling = BIT(17);
1410

1411 1412 1413 1414 1415 1416 1417
	/* If we don't overwrite a cacheline completely we need to be
	 * careful to have up-to-date data by first clflushing. Don't
	 * overcomplicate things and flush the entire patch.
	 */
	partial_cacheline_write = 0;
	if (needs_clflush & CLFLUSH_BEFORE)
		partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
1418

1419 1420 1421 1422 1423 1424
	user_data = u64_to_user_ptr(args->data_ptr);
	remain = args->size;
	offset = offset_in_page(args->offset);
	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
		struct page *page = i915_gem_object_get_page(obj, idx);
		int length;
1425

1426 1427 1428
		length = remain;
		if (offset + length > PAGE_SIZE)
			length = PAGE_SIZE - offset;
1429

1430 1431 1432 1433
		ret = shmem_pwrite(page, offset, length, user_data,
				   page_to_phys(page) & obj_do_bit17_swizzling,
				   (offset | length) & partial_cacheline_write,
				   needs_clflush & CLFLUSH_AFTER);
1434
		if (ret)
1435
			break;
1436

1437 1438 1439
		remain -= length;
		user_data += length;
		offset = 0;
1440
	}
1441

1442
	intel_fb_obj_flush(obj, ORIGIN_CPU);
1443
	i915_gem_obj_finish_shmem_access(obj);
1444
	return ret;
1445 1446 1447 1448
}

/**
 * Writes data to the object referenced by handle.
1449 1450 1451
 * @dev: drm device
 * @data: ioctl data blob
 * @file: drm file
1452 1453 1454 1455 1456
 *
 * On error, the contents of the buffer that were to be modified are undefined.
 */
int
i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1457
		      struct drm_file *file)
1458 1459
{
	struct drm_i915_gem_pwrite *args = data;
1460
	struct drm_i915_gem_object *obj;
1461 1462 1463 1464 1465 1466
	int ret;

	if (args->size == 0)
		return 0;

	if (!access_ok(VERIFY_READ,
1467
		       u64_to_user_ptr(args->data_ptr),
1468 1469 1470
		       args->size))
		return -EFAULT;

1471
	obj = i915_gem_object_lookup(file, args->handle);
1472 1473
	if (!obj)
		return -ENOENT;
1474

1475
	/* Bounds check destination. */
1476
	if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
C
Chris Wilson 已提交
1477
		ret = -EINVAL;
1478
		goto err;
C
Chris Wilson 已提交
1479 1480
	}

C
Chris Wilson 已提交
1481 1482
	trace_i915_gem_object_pwrite(obj, args->offset, args->size);

1483 1484 1485 1486 1487 1488
	ret = -ENODEV;
	if (obj->ops->pwrite)
		ret = obj->ops->pwrite(obj, args);
	if (ret != -ENODEV)
		goto err;

1489 1490 1491 1492 1493
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE |
				   I915_WAIT_ALL,
				   MAX_SCHEDULE_TIMEOUT,
				   to_rps_client(file));
1494 1495 1496
	if (ret)
		goto err;

1497
	ret = i915_gem_object_pin_pages(obj);
1498
	if (ret)
1499
		goto err;
1500

D
Daniel Vetter 已提交
1501
	ret = -EFAULT;
1502 1503 1504 1505 1506 1507
	/* We can only do the GTT pwrite on untiled buffers, as otherwise
	 * it would end up going through the fenced access, and we'll get
	 * different detiling behavior between reading and writing.
	 * pread/pwrite currently are reading and writing from the CPU
	 * perspective, requiring manual detiling by the client.
	 */
1508
	if (!i915_gem_object_has_struct_page(obj) ||
1509
	    cpu_write_needs_clflush(obj))
D
Daniel Vetter 已提交
1510 1511
		/* Note that the gtt paths might fail with non-page-backed user
		 * pointers (e.g. gtt mappings when moving data between
1512 1513
		 * textures). Fallback to the shmem path in that case.
		 */
1514
		ret = i915_gem_gtt_pwrite_fast(obj, args);
1515

1516
	if (ret == -EFAULT || ret == -ENOSPC) {
1517 1518
		if (obj->phys_handle)
			ret = i915_gem_phys_pwrite(obj, args, file);
1519
		else
1520
			ret = i915_gem_shmem_pwrite(obj, args);
1521
	}
1522

1523
	i915_gem_object_unpin_pages(obj);
1524
err:
C
Chris Wilson 已提交
1525
	i915_gem_object_put(obj);
1526
	return ret;
1527 1528
}

1529 1530 1531 1532 1533 1534 1535 1536
static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
{
	struct drm_i915_private *i915;
	struct list_head *list;
	struct i915_vma *vma;

	list_for_each_entry(vma, &obj->vma_list, obj_link) {
		if (!i915_vma_is_ggtt(vma))
1537
			break;
1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549

		if (i915_vma_is_active(vma))
			continue;

		if (!drm_mm_node_allocated(&vma->node))
			continue;

		list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
	}

	i915 = to_i915(obj->base.dev);
	list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list;
1550
	list_move_tail(&obj->global_link, list);
1551 1552
}

1553
/**
1554 1555
 * Called when user space prepares to use an object with the CPU, either
 * through the mmap ioctl's mapping or a GTT mapping.
1556 1557 1558
 * @dev: drm device
 * @data: ioctl data blob
 * @file: drm file
1559 1560 1561
 */
int
i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1562
			  struct drm_file *file)
1563 1564
{
	struct drm_i915_gem_set_domain *args = data;
1565
	struct drm_i915_gem_object *obj;
1566 1567
	uint32_t read_domains = args->read_domains;
	uint32_t write_domain = args->write_domain;
1568
	int err;
1569

1570
	/* Only handle setting domains to types used by the CPU. */
1571
	if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
1572 1573 1574 1575 1576 1577 1578 1579
		return -EINVAL;

	/* Having something in the write domain implies it's in the read
	 * domain, and only that read domain.  Enforce that in the request.
	 */
	if (write_domain != 0 && read_domains != write_domain)
		return -EINVAL;

1580
	obj = i915_gem_object_lookup(file, args->handle);
1581 1582
	if (!obj)
		return -ENOENT;
1583

1584 1585 1586 1587
	/* Try to flush the object off the GPU without holding the lock.
	 * We will repeat the flush holding the lock in the normal manner
	 * to catch cases where we are gazumped.
	 */
1588
	err = i915_gem_object_wait(obj,
1589 1590 1591 1592
				   I915_WAIT_INTERRUPTIBLE |
				   (write_domain ? I915_WAIT_ALL : 0),
				   MAX_SCHEDULE_TIMEOUT,
				   to_rps_client(file));
1593
	if (err)
C
Chris Wilson 已提交
1594
		goto out;
1595

1596 1597 1598 1599 1600 1601 1602 1603 1604 1605
	/* Flush and acquire obj->pages so that we are coherent through
	 * direct access in memory with previous cached writes through
	 * shmemfs and that our cache domain tracking remains valid.
	 * For example, if the obj->filp was moved to swap without us
	 * being notified and releasing the pages, we would mistakenly
	 * continue to assume that the obj remained out of the CPU cached
	 * domain.
	 */
	err = i915_gem_object_pin_pages(obj);
	if (err)
C
Chris Wilson 已提交
1606
		goto out;
1607 1608 1609

	err = i915_mutex_lock_interruptible(dev);
	if (err)
C
Chris Wilson 已提交
1610
		goto out_unpin;
1611

1612 1613 1614 1615
	if (read_domains & I915_GEM_DOMAIN_WC)
		err = i915_gem_object_set_to_wc_domain(obj, write_domain);
	else if (read_domains & I915_GEM_DOMAIN_GTT)
		err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
1616
	else
1617
		err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
1618

1619 1620
	/* And bump the LRU for this access */
	i915_gem_object_bump_inactive_ggtt(obj);
1621

1622
	mutex_unlock(&dev->struct_mutex);
1623

1624
	if (write_domain != 0)
1625 1626
		intel_fb_obj_invalidate(obj,
					fb_write_origin(obj, write_domain));
1627

C
Chris Wilson 已提交
1628
out_unpin:
1629
	i915_gem_object_unpin_pages(obj);
C
Chris Wilson 已提交
1630 1631
out:
	i915_gem_object_put(obj);
1632
	return err;
1633 1634 1635 1636
}

/**
 * Called when user space has done writes to this buffer
1637 1638 1639
 * @dev: drm device
 * @data: ioctl data blob
 * @file: drm file
1640 1641 1642
 */
int
i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1643
			 struct drm_file *file)
1644 1645
{
	struct drm_i915_gem_sw_finish *args = data;
1646
	struct drm_i915_gem_object *obj;
1647

1648
	obj = i915_gem_object_lookup(file, args->handle);
1649 1650
	if (!obj)
		return -ENOENT;
1651 1652

	/* Pinned buffers may be scanout, so flush the cache */
1653
	i915_gem_object_flush_if_display(obj);
C
Chris Wilson 已提交
1654
	i915_gem_object_put(obj);
1655 1656

	return 0;
1657 1658 1659
}

/**
1660 1661 1662 1663 1664
 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
 *			 it is mapped to.
 * @dev: drm device
 * @data: ioctl data blob
 * @file: drm file
1665 1666 1667
 *
 * While the mapping holds a reference on the contents of the object, it doesn't
 * imply a ref on the object itself.
1668 1669 1670 1671 1672 1673 1674 1675 1676 1677
 *
 * IMPORTANT:
 *
 * DRM driver writers who look a this function as an example for how to do GEM
 * mmap support, please don't implement mmap support like here. The modern way
 * to implement DRM mmap support is with an mmap offset ioctl (like
 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
 * That way debug tooling like valgrind will understand what's going on, hiding
 * the mmap call in a driver private ioctl will break that. The i915 driver only
 * does cpu mmaps this way because we didn't know better.
1678 1679 1680
 */
int
i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1681
		    struct drm_file *file)
1682 1683
{
	struct drm_i915_gem_mmap *args = data;
1684
	struct drm_i915_gem_object *obj;
1685 1686
	unsigned long addr;

1687 1688 1689
	if (args->flags & ~(I915_MMAP_WC))
		return -EINVAL;

1690
	if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1691 1692
		return -ENODEV;

1693 1694
	obj = i915_gem_object_lookup(file, args->handle);
	if (!obj)
1695
		return -ENOENT;
1696

1697 1698 1699
	/* prime objects have no backing filp to GEM mmap
	 * pages from.
	 */
1700
	if (!obj->base.filp) {
C
Chris Wilson 已提交
1701
		i915_gem_object_put(obj);
1702 1703 1704
		return -EINVAL;
	}

1705
	addr = vm_mmap(obj->base.filp, 0, args->size,
1706 1707
		       PROT_READ | PROT_WRITE, MAP_SHARED,
		       args->offset);
1708 1709 1710 1711
	if (args->flags & I915_MMAP_WC) {
		struct mm_struct *mm = current->mm;
		struct vm_area_struct *vma;

1712
		if (down_write_killable(&mm->mmap_sem)) {
C
Chris Wilson 已提交
1713
			i915_gem_object_put(obj);
1714 1715
			return -EINTR;
		}
1716 1717 1718 1719 1720 1721 1722
		vma = find_vma(mm, addr);
		if (vma)
			vma->vm_page_prot =
				pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
		else
			addr = -ENOMEM;
		up_write(&mm->mmap_sem);
1723 1724

		/* This may race, but that's ok, it only gets set */
1725
		WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU);
1726
	}
C
Chris Wilson 已提交
1727
	i915_gem_object_put(obj);
1728 1729 1730 1731 1732 1733 1734 1735
	if (IS_ERR((void *)addr))
		return addr;

	args->addr_ptr = (uint64_t) addr;

	return 0;
}

1736 1737
static unsigned int tile_row_pages(struct drm_i915_gem_object *obj)
{
1738
	return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
1739 1740
}

1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
/**
 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
 *
 * A history of the GTT mmap interface:
 *
 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
 *     aligned and suitable for fencing, and still fit into the available
 *     mappable space left by the pinned display objects. A classic problem
 *     we called the page-fault-of-doom where we would ping-pong between
 *     two objects that could not fit inside the GTT and so the memcpy
 *     would page one object in at the expense of the other between every
 *     single byte.
 *
 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
 *     as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
 *     object is too large for the available space (or simply too large
 *     for the mappable aperture!), a view is created instead and faulted
 *     into userspace. (This view is aligned and sized appropriately for
 *     fenced access.)
 *
1761 1762 1763
 * 2 - Recognise WC as a separate cache domain so that we can flush the
 *     delayed writes via GTT before performing direct access via WC.
 *
1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790
 * Restrictions:
 *
 *  * snoopable objects cannot be accessed via the GTT. It can cause machine
 *    hangs on some architectures, corruption on others. An attempt to service
 *    a GTT page fault from a snoopable object will generate a SIGBUS.
 *
 *  * the object must be able to fit into RAM (physical memory, though no
 *    limited to the mappable aperture).
 *
 *
 * Caveats:
 *
 *  * a new GTT page fault will synchronize rendering from the GPU and flush
 *    all data to system memory. Subsequent access will not be synchronized.
 *
 *  * all mappings are revoked on runtime device suspend.
 *
 *  * there are only 8, 16 or 32 fence registers to share between all users
 *    (older machines require fence register for display and blitter access
 *    as well). Contention of the fence registers will cause the previous users
 *    to be unmapped and any new access will generate new page faults.
 *
 *  * running out of memory while servicing a fault may generate a SIGBUS,
 *    rather than the expected SIGSEGV.
 */
int i915_gem_mmap_gtt_version(void)
{
1791
	return 2;
1792 1793
}

1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804
static inline struct i915_ggtt_view
compute_partial_view(struct drm_i915_gem_object *obj,
		     pgoff_t page_offset,
		     unsigned int chunk)
{
	struct i915_ggtt_view view;

	if (i915_gem_object_is_tiled(obj))
		chunk = roundup(chunk, tile_row_pages(obj));

	view.type = I915_GGTT_VIEW_PARTIAL;
1805 1806
	view.partial.offset = rounddown(page_offset, chunk);
	view.partial.size =
1807
		min_t(unsigned int, chunk,
1808
		      (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
1809 1810 1811 1812 1813 1814 1815 1816

	/* If the partial covers the entire object, just create a normal VMA. */
	if (chunk >= obj->base.size >> PAGE_SHIFT)
		view.type = I915_GGTT_VIEW_NORMAL;

	return view;
}

1817 1818
/**
 * i915_gem_fault - fault a page into the GTT
1819
 * @vmf: fault info
1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830
 *
 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
 * from userspace.  The fault handler takes care of binding the object to
 * the GTT (if needed), allocating and programming a fence register (again,
 * only if needed based on whether the old reg is still valid or the object
 * is tiled) and inserting a new PTE into the faulting process.
 *
 * Note that the faulting process may involve evicting existing objects
 * from the GTT and/or fence registers to make room.  So performance may
 * suffer if the GTT working set is large or there are few fence registers
 * left.
1831 1832 1833
 *
 * The current feature set supported by i915_gem_fault() and thus GTT mmaps
 * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version).
1834
 */
1835
int i915_gem_fault(struct vm_fault *vmf)
1836
{
1837
#define MIN_CHUNK_PAGES ((1 << 20) >> PAGE_SHIFT) /* 1 MiB */
1838
	struct vm_area_struct *area = vmf->vma;
C
Chris Wilson 已提交
1839
	struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
1840
	struct drm_device *dev = obj->base.dev;
1841 1842
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
1843
	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
C
Chris Wilson 已提交
1844
	struct i915_vma *vma;
1845
	pgoff_t page_offset;
1846
	unsigned int flags;
1847
	int ret;
1848

1849
	/* We don't use vmf->pgoff since that has the fake offset */
1850
	page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
1851

C
Chris Wilson 已提交
1852 1853
	trace_i915_gem_object_fault(obj, page_offset, true, write);

1854
	/* Try to flush the object off the GPU first without holding the lock.
1855
	 * Upon acquiring the lock, we will perform our sanity checks and then
1856 1857 1858
	 * repeat the flush holding the lock in the normal manner to catch cases
	 * where we are gazumped.
	 */
1859 1860 1861 1862
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE,
				   MAX_SCHEDULE_TIMEOUT,
				   NULL);
1863
	if (ret)
1864 1865
		goto err;

1866 1867 1868 1869
	ret = i915_gem_object_pin_pages(obj);
	if (ret)
		goto err;

1870 1871 1872 1873 1874
	intel_runtime_pm_get(dev_priv);

	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto err_rpm;
1875

1876
	/* Access to snoopable pages through the GTT is incoherent. */
1877
	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) {
1878
		ret = -EFAULT;
1879
		goto err_unlock;
1880 1881
	}

1882 1883 1884 1885 1886 1887 1888 1889
	/* If the object is smaller than a couple of partial vma, it is
	 * not worth only creating a single partial vma - we may as well
	 * clear enough space for the full object.
	 */
	flags = PIN_MAPPABLE;
	if (obj->base.size > 2 * MIN_CHUNK_PAGES << PAGE_SHIFT)
		flags |= PIN_NONBLOCK | PIN_NONFAULT;

1890
	/* Now pin it into the GTT as needed */
1891
	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, flags);
1892 1893
	if (IS_ERR(vma)) {
		/* Use a partial view if it is bigger than available space */
1894
		struct i915_ggtt_view view =
1895
			compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
1896

1897 1898 1899 1900 1901
		/* Userspace is now writing through an untracked VMA, abandon
		 * all hope that the hardware is able to track future writes.
		 */
		obj->frontbuffer_ggtt_origin = ORIGIN_CPU;

1902 1903
		vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
	}
C
Chris Wilson 已提交
1904 1905
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
1906
		goto err_unlock;
C
Chris Wilson 已提交
1907
	}
1908

1909 1910
	ret = i915_gem_object_set_to_gtt_domain(obj, write);
	if (ret)
1911
		goto err_unpin;
1912

1913
	ret = i915_vma_get_fence(vma);
1914
	if (ret)
1915
		goto err_unpin;
1916

1917
	/* Mark as being mmapped into userspace for later revocation */
1918
	assert_rpm_wakelock_held(dev_priv);
1919 1920 1921
	if (list_empty(&obj->userfault_link))
		list_add(&obj->userfault_link, &dev_priv->mm.userfault_list);

1922
	/* Finally, remap it using the new GTT offset */
1923
	ret = remap_io_mapping(area,
1924
			       area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
1925 1926 1927
			       (ggtt->mappable_base + vma->node.start) >> PAGE_SHIFT,
			       min_t(u64, vma->size, area->vm_end - area->vm_start),
			       &ggtt->mappable);
1928

1929
err_unpin:
C
Chris Wilson 已提交
1930
	__i915_vma_unpin(vma);
1931
err_unlock:
1932
	mutex_unlock(&dev->struct_mutex);
1933 1934
err_rpm:
	intel_runtime_pm_put(dev_priv);
1935
	i915_gem_object_unpin_pages(obj);
1936
err:
1937
	switch (ret) {
1938
	case -EIO:
1939 1940 1941 1942 1943 1944 1945
		/*
		 * We eat errors when the gpu is terminally wedged to avoid
		 * userspace unduly crashing (gl has no provisions for mmaps to
		 * fail). But any other -EIO isn't ours (e.g. swap in failure)
		 * and so needs to be reported.
		 */
		if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1946 1947 1948
			ret = VM_FAULT_SIGBUS;
			break;
		}
1949
	case -EAGAIN:
D
Daniel Vetter 已提交
1950 1951 1952 1953
		/*
		 * EAGAIN means the gpu is hung and we'll wait for the error
		 * handler to reset everything when re-faulting in
		 * i915_mutex_lock_interruptible.
1954
		 */
1955 1956
	case 0:
	case -ERESTARTSYS:
1957
	case -EINTR:
1958 1959 1960 1961 1962
	case -EBUSY:
		/*
		 * EBUSY is ok: this just means that another thread
		 * already did the job.
		 */
1963 1964
		ret = VM_FAULT_NOPAGE;
		break;
1965
	case -ENOMEM:
1966 1967
		ret = VM_FAULT_OOM;
		break;
1968
	case -ENOSPC:
1969
	case -EFAULT:
1970 1971
		ret = VM_FAULT_SIGBUS;
		break;
1972
	default:
1973
		WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1974 1975
		ret = VM_FAULT_SIGBUS;
		break;
1976
	}
1977
	return ret;
1978 1979
}

1980 1981 1982 1983
/**
 * i915_gem_release_mmap - remove physical page mappings
 * @obj: obj in question
 *
1984
 * Preserve the reservation of the mmapping with the DRM core code, but
1985 1986 1987 1988 1989 1990 1991 1992 1993
 * relinquish ownership of the pages back to the system.
 *
 * It is vital that we remove the page mapping if we have mapped a tiled
 * object through the GTT and then lose the fence register due to
 * resource pressure. Similarly if the object has been moved out of the
 * aperture, than pages mapped into userspace must be revoked. Removing the
 * mapping will then trigger a page fault on the next user access, allowing
 * fixup by i915_gem_fault().
 */
1994
void
1995
i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1996
{
1997 1998
	struct drm_i915_private *i915 = to_i915(obj->base.dev);

1999 2000 2001
	/* Serialisation between user GTT access and our code depends upon
	 * revoking the CPU's PTE whilst the mutex is held. The next user
	 * pagefault then has to wait until we release the mutex.
2002 2003 2004 2005
	 *
	 * Note that RPM complicates somewhat by adding an additional
	 * requirement that operations to the GGTT be made holding the RPM
	 * wakeref.
2006
	 */
2007
	lockdep_assert_held(&i915->drm.struct_mutex);
2008
	intel_runtime_pm_get(i915);
2009

2010
	if (list_empty(&obj->userfault_link))
2011
		goto out;
2012

2013
	list_del_init(&obj->userfault_link);
2014 2015
	drm_vma_node_unmap(&obj->base.vma_node,
			   obj->base.dev->anon_inode->i_mapping);
2016 2017 2018 2019 2020 2021 2022 2023 2024

	/* Ensure that the CPU's PTE are revoked and there are not outstanding
	 * memory transactions from userspace before we return. The TLB
	 * flushing implied above by changing the PTE above *should* be
	 * sufficient, an extra barrier here just provides us with a bit
	 * of paranoid documentation about our requirement to serialise
	 * memory writes before touching registers / GSM.
	 */
	wmb();
2025 2026 2027

out:
	intel_runtime_pm_put(i915);
2028 2029
}

2030
void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv)
2031
{
2032
	struct drm_i915_gem_object *obj, *on;
2033
	int i;
2034

2035 2036 2037 2038 2039 2040
	/*
	 * Only called during RPM suspend. All users of the userfault_list
	 * must be holding an RPM wakeref to ensure that this can not
	 * run concurrently with themselves (and use the struct_mutex for
	 * protection between themselves).
	 */
2041

2042 2043 2044
	list_for_each_entry_safe(obj, on,
				 &dev_priv->mm.userfault_list, userfault_link) {
		list_del_init(&obj->userfault_link);
2045 2046 2047
		drm_vma_node_unmap(&obj->base.vma_node,
				   obj->base.dev->anon_inode->i_mapping);
	}
2048 2049 2050 2051 2052 2053 2054 2055

	/* The fence will be lost when the device powers down. If any were
	 * in use by hardware (i.e. they are pinned), we should not be powering
	 * down! All other fences will be reacquired by the user upon waking.
	 */
	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];

2056 2057 2058 2059 2060 2061 2062 2063 2064 2065
		/* Ideally we want to assert that the fence register is not
		 * live at this point (i.e. that no piece of code will be
		 * trying to write through fence + GTT, as that both violates
		 * our tracking of activity and associated locking/barriers,
		 * but also is illegal given that the hw is powered down).
		 *
		 * Previously we used reg->pin_count as a "liveness" indicator.
		 * That is not sufficient, and we need a more fine-grained
		 * tool if we want to have a sanity check here.
		 */
2066 2067 2068 2069 2070 2071 2072

		if (!reg->vma)
			continue;

		GEM_BUG_ON(!list_empty(&reg->vma->obj->userfault_link));
		reg->dirty = true;
	}
2073 2074
}

2075 2076
static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
{
2077
	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2078
	int err;
2079

2080
	err = drm_gem_create_mmap_offset(&obj->base);
2081
	if (likely(!err))
2082
		return 0;
2083

2084 2085 2086 2087 2088
	/* Attempt to reap some mmap space from dead objects */
	do {
		err = i915_gem_wait_for_idle(dev_priv, I915_WAIT_INTERRUPTIBLE);
		if (err)
			break;
2089

2090
		i915_gem_drain_freed_objects(dev_priv);
2091
		err = drm_gem_create_mmap_offset(&obj->base);
2092 2093 2094 2095
		if (!err)
			break;

	} while (flush_delayed_work(&dev_priv->gt.retire_work));
2096

2097
	return err;
2098 2099 2100 2101 2102 2103 2104
}

static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
{
	drm_gem_free_mmap_offset(&obj->base);
}

2105
int
2106 2107
i915_gem_mmap_gtt(struct drm_file *file,
		  struct drm_device *dev,
2108
		  uint32_t handle,
2109
		  uint64_t *offset)
2110
{
2111
	struct drm_i915_gem_object *obj;
2112 2113
	int ret;

2114
	obj = i915_gem_object_lookup(file, handle);
2115 2116
	if (!obj)
		return -ENOENT;
2117

2118
	ret = i915_gem_object_create_mmap_offset(obj);
2119 2120
	if (ret == 0)
		*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2121

C
Chris Wilson 已提交
2122
	i915_gem_object_put(obj);
2123
	return ret;
2124 2125
}

2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146
/**
 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
 * @dev: DRM device
 * @data: GTT mapping ioctl data
 * @file: GEM object info
 *
 * Simply returns the fake offset to userspace so it can mmap it.
 * The mmap call will end up in drm_gem_mmap(), which will set things
 * up so we can get faults in the handler above.
 *
 * The fault handler will take care of binding the object into the GTT
 * (since it may have been evicted to make room for something), allocating
 * a fence register, and mapping the appropriate aperture address into
 * userspace.
 */
int
i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file)
{
	struct drm_i915_gem_mmap_gtt *args = data;

2147
	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2148 2149
}

D
Daniel Vetter 已提交
2150 2151 2152
/* Immediately discard the backing storage */
static void
i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2153
{
2154
	i915_gem_object_free_mmap_offset(obj);
2155

2156 2157
	if (obj->base.filp == NULL)
		return;
2158

D
Daniel Vetter 已提交
2159 2160 2161 2162 2163
	/* Our goal here is to return as much of the memory as
	 * is possible back to the system as we are called from OOM.
	 * To do this we must instruct the shmfs to drop all of its
	 * backing pages, *now*.
	 */
2164
	shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
C
Chris Wilson 已提交
2165
	obj->mm.madv = __I915_MADV_PURGED;
2166
	obj->mm.pages = ERR_PTR(-EFAULT);
D
Daniel Vetter 已提交
2167
}
2168

2169
/* Try to discard unwanted pages */
2170
void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
D
Daniel Vetter 已提交
2171
{
2172 2173
	struct address_space *mapping;

2174 2175 2176
	lockdep_assert_held(&obj->mm.lock);
	GEM_BUG_ON(obj->mm.pages);

C
Chris Wilson 已提交
2177
	switch (obj->mm.madv) {
2178 2179 2180 2181 2182 2183 2184 2185 2186
	case I915_MADV_DONTNEED:
		i915_gem_object_truncate(obj);
	case __I915_MADV_PURGED:
		return;
	}

	if (obj->base.filp == NULL)
		return;

2187
	mapping = obj->base.filp->f_mapping,
2188
	invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2189 2190
}

2191
static void
2192 2193
i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
			      struct sg_table *pages)
2194
{
2195 2196
	struct sgt_iter sgt_iter;
	struct page *page;
2197

2198
	__i915_gem_object_release_shmem(obj, pages, true);
2199

2200
	i915_gem_gtt_finish_pages(obj, pages);
I
Imre Deak 已提交
2201

2202
	if (i915_gem_object_needs_bit17_swizzle(obj))
2203
		i915_gem_object_save_bit_17_swizzle(obj, pages);
2204

2205
	for_each_sgt_page(page, sgt_iter, pages) {
C
Chris Wilson 已提交
2206
		if (obj->mm.dirty)
2207
			set_page_dirty(page);
2208

C
Chris Wilson 已提交
2209
		if (obj->mm.madv == I915_MADV_WILLNEED)
2210
			mark_page_accessed(page);
2211

2212
		put_page(page);
2213
	}
C
Chris Wilson 已提交
2214
	obj->mm.dirty = false;
2215

2216 2217
	sg_free_table(pages);
	kfree(pages);
2218
}
C
Chris Wilson 已提交
2219

2220 2221 2222
static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
{
	struct radix_tree_iter iter;
2223
	void __rcu **slot;
2224

C
Chris Wilson 已提交
2225 2226
	radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
		radix_tree_delete(&obj->mm.get_page.radix, iter.index);
2227 2228
}

2229 2230
void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
				 enum i915_mm_subclass subclass)
2231
{
2232
	struct sg_table *pages;
2233

C
Chris Wilson 已提交
2234
	if (i915_gem_object_has_pinned_pages(obj))
2235
		return;
2236

2237
	GEM_BUG_ON(obj->bind_count);
2238 2239 2240 2241
	if (!READ_ONCE(obj->mm.pages))
		return;

	/* May be called by shrinker from within get_pages() (on another bo) */
2242
	mutex_lock_nested(&obj->mm.lock, subclass);
2243 2244
	if (unlikely(atomic_read(&obj->mm.pages_pin_count)))
		goto unlock;
B
Ben Widawsky 已提交
2245

2246 2247 2248
	/* ->put_pages might need to allocate memory for the bit17 swizzle
	 * array, hence protect them from being reaped by removing them from gtt
	 * lists early. */
2249 2250
	pages = fetch_and_zero(&obj->mm.pages);
	GEM_BUG_ON(!pages);
2251

C
Chris Wilson 已提交
2252
	if (obj->mm.mapping) {
2253 2254
		void *ptr;

2255
		ptr = page_mask_bits(obj->mm.mapping);
2256 2257
		if (is_vmalloc_addr(ptr))
			vunmap(ptr);
2258
		else
2259 2260
			kunmap(kmap_to_page(ptr));

C
Chris Wilson 已提交
2261
		obj->mm.mapping = NULL;
2262 2263
	}

2264 2265
	__i915_gem_object_reset_page_iter(obj);

2266 2267 2268
	if (!IS_ERR(pages))
		obj->ops->put_pages(obj, pages);

2269 2270
	obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;

2271 2272
unlock:
	mutex_unlock(&obj->mm.lock);
C
Chris Wilson 已提交
2273 2274
}

2275
static bool i915_sg_trim(struct sg_table *orig_st)
2276 2277 2278 2279 2280 2281
{
	struct sg_table new_st;
	struct scatterlist *sg, *new_sg;
	unsigned int i;

	if (orig_st->nents == orig_st->orig_nents)
2282
		return false;
2283

2284
	if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN))
2285
		return false;
2286 2287 2288 2289 2290 2291 2292

	new_sg = new_st.sgl;
	for_each_sg(orig_st->sgl, sg, orig_st->nents, i) {
		sg_set_page(new_sg, sg_page(sg), sg->length, 0);
		/* called before being DMA mapped, no need to copy sg->dma_* */
		new_sg = sg_next(new_sg);
	}
2293
	GEM_BUG_ON(new_sg); /* Should walk exactly nents and hit the end */
2294 2295 2296 2297

	sg_free_table(orig_st);

	*orig_st = new_st;
2298
	return true;
2299 2300
}

2301
static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2302
{
2303
	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2304 2305
	const unsigned long page_count = obj->base.size / PAGE_SIZE;
	unsigned long i;
2306
	struct address_space *mapping;
2307 2308
	struct sg_table *st;
	struct scatterlist *sg;
2309
	struct sgt_iter sgt_iter;
2310
	struct page *page;
2311
	unsigned long last_pfn = 0;	/* suppress gcc warning */
2312
	unsigned int max_segment = i915_sg_segment_size();
2313
	unsigned int sg_mask;
2314
	gfp_t noreclaim;
I
Imre Deak 已提交
2315
	int ret;
2316

C
Chris Wilson 已提交
2317 2318 2319 2320
	/* Assert that the object is not currently in any GPU domain. As it
	 * wasn't in the GTT, there shouldn't be any way it could have been in
	 * a GPU cache
	 */
2321 2322
	GEM_BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
	GEM_BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
C
Chris Wilson 已提交
2323

2324 2325
	st = kmalloc(sizeof(*st), GFP_KERNEL);
	if (st == NULL)
2326
		return -ENOMEM;
2327

2328
rebuild_st:
2329 2330
	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
		kfree(st);
2331
		return -ENOMEM;
2332
	}
2333

2334 2335 2336 2337 2338
	/* Get the list of pages out of our struct file.  They'll be pinned
	 * at this point until we release them.
	 *
	 * Fail silently without starting the shrinker
	 */
2339
	mapping = obj->base.filp->f_mapping;
2340
	noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
2341 2342
	noreclaim |= __GFP_NORETRY | __GFP_NOWARN;

2343 2344
	sg = st->sgl;
	st->nents = 0;
2345
	sg_mask = 0;
2346
	for (i = 0; i < page_count; i++) {
2347 2348 2349 2350 2351 2352 2353
		const unsigned int shrink[] = {
			I915_SHRINK_BOUND | I915_SHRINK_UNBOUND | I915_SHRINK_PURGEABLE,
			0,
		}, *s = shrink;
		gfp_t gfp = noreclaim;

		do {
C
Chris Wilson 已提交
2354
			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2355 2356 2357 2358 2359 2360 2361 2362
			if (likely(!IS_ERR(page)))
				break;

			if (!*s) {
				ret = PTR_ERR(page);
				goto err_sg;
			}

2363
			i915_gem_shrink(dev_priv, 2 * page_count, NULL, *s++);
2364
			cond_resched();
2365

C
Chris Wilson 已提交
2366 2367 2368
			/* We've tried hard to allocate the memory by reaping
			 * our own buffer, now let the real VM do its job and
			 * go down in flames if truly OOM.
2369 2370 2371 2372
			 *
			 * However, since graphics tend to be disposable,
			 * defer the oom here by reporting the ENOMEM back
			 * to userspace.
C
Chris Wilson 已提交
2373
			 */
2374 2375 2376
			if (!*s) {
				/* reclaim and warn, but no oom */
				gfp = mapping_gfp_mask(mapping);
2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388

				/* Our bo are always dirty and so we require
				 * kswapd to reclaim our pages (direct reclaim
				 * does not effectively begin pageout of our
				 * buffers on its own). However, direct reclaim
				 * only waits for kswapd when under allocation
				 * congestion. So as a result __GFP_RECLAIM is
				 * unreliable and fails to actually reclaim our
				 * dirty pages -- unless you try over and over
				 * again with !__GFP_NORETRY. However, we still
				 * want to fail this allocation rather than
				 * trigger the out-of-memory killer and for
M
Michal Hocko 已提交
2389
				 * this we want __GFP_RETRY_MAYFAIL.
2390
				 */
M
Michal Hocko 已提交
2391
				gfp |= __GFP_RETRY_MAYFAIL;
I
Imre Deak 已提交
2392
			}
2393 2394
		} while (1);

2395 2396 2397
		if (!i ||
		    sg->length >= max_segment ||
		    page_to_pfn(page) != last_pfn + 1) {
2398 2399
			if (i) {
				sg_mask |= sg->length;
2400
				sg = sg_next(sg);
2401
			}
2402 2403 2404 2405 2406 2407
			st->nents++;
			sg_set_page(sg, page, PAGE_SIZE, 0);
		} else {
			sg->length += PAGE_SIZE;
		}
		last_pfn = page_to_pfn(page);
2408 2409 2410

		/* Check that the i965g/gm workaround works. */
		WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2411
	}
2412 2413
	if (sg) { /* loop terminated early; short sg table */
		sg_mask |= sg->length;
2414
		sg_mark_end(sg);
2415
	}
2416

2417 2418 2419
	/* Trim unused sg entries to avoid wasting memory. */
	i915_sg_trim(st);

2420
	ret = i915_gem_gtt_prepare_pages(obj, st);
2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439
	if (ret) {
		/* DMA remapping failed? One possible cause is that
		 * it could not reserve enough large entries, asking
		 * for PAGE_SIZE chunks instead may be helpful.
		 */
		if (max_segment > PAGE_SIZE) {
			for_each_sgt_page(page, sgt_iter, st)
				put_page(page);
			sg_free_table(st);

			max_segment = PAGE_SIZE;
			goto rebuild_st;
		} else {
			dev_warn(&dev_priv->drm.pdev->dev,
				 "Failed to DMA remap %lu pages\n",
				 page_count);
			goto err_pages;
		}
	}
I
Imre Deak 已提交
2440

2441
	if (i915_gem_object_needs_bit17_swizzle(obj))
2442
		i915_gem_object_do_bit_17_swizzle(obj, st);
2443

2444
	__i915_gem_object_set_pages(obj, st, sg_mask);
2445 2446

	return 0;
2447

2448
err_sg:
2449
	sg_mark_end(sg);
2450
err_pages:
2451 2452
	for_each_sgt_page(page, sgt_iter, st)
		put_page(page);
2453 2454
	sg_free_table(st);
	kfree(st);
2455 2456 2457 2458 2459 2460 2461 2462 2463

	/* shmemfs first checks if there is enough memory to allocate the page
	 * and reports ENOSPC should there be insufficient, along with the usual
	 * ENOMEM for a genuine allocation failure.
	 *
	 * We use ENOSPC in our driver to mean that we have run out of aperture
	 * space and so want to translate the error from shmemfs back to our
	 * usual understanding of ENOMEM.
	 */
I
Imre Deak 已提交
2464 2465 2466
	if (ret == -ENOSPC)
		ret = -ENOMEM;

2467
	return ret;
2468 2469 2470
}

void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
2471 2472
				 struct sg_table *pages,
				 unsigned int sg_mask)
2473
{
2474 2475 2476 2477
	struct drm_i915_private *i915 = to_i915(obj->base.dev);
	unsigned long supported = INTEL_INFO(i915)->page_sizes;
	int i;

2478
	lockdep_assert_held(&obj->mm.lock);
2479 2480 2481 2482 2483

	obj->mm.get_page.sg_pos = pages->sgl;
	obj->mm.get_page.sg_idx = 0;

	obj->mm.pages = pages;
2484 2485 2486 2487 2488 2489 2490

	if (i915_gem_object_is_tiled(obj) &&
	    to_i915(obj->base.dev)->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
		GEM_BUG_ON(obj->mm.quirked);
		__i915_gem_object_pin_pages(obj);
		obj->mm.quirked = true;
	}
2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509

	GEM_BUG_ON(!sg_mask);
	obj->mm.page_sizes.phys = sg_mask;

	/*
	 * Calculate the supported page-sizes which fit into the given sg_mask.
	 * This will give us the page-sizes which we may be able to use
	 * opportunistically when later inserting into the GTT. For example if
	 * phys=2G, then in theory we should be able to use 1G, 2M, 64K or 4K
	 * pages, although in practice this will depend on a number of other
	 * factors.
	 */
	obj->mm.page_sizes.sg = 0;
	for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
		if (obj->mm.page_sizes.phys & ~0u << i)
			obj->mm.page_sizes.sg |= BIT(i);
	}

	GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
2510 2511 2512 2513
}

static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
{
2514
	int err;
2515 2516 2517 2518 2519 2520

	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
		DRM_DEBUG("Attempting to obtain a purgeable object\n");
		return -EFAULT;
	}

2521 2522
	err = obj->ops->get_pages(obj);
	GEM_BUG_ON(!err && IS_ERR_OR_NULL(obj->mm.pages));
2523

2524
	return err;
2525 2526
}

2527
/* Ensure that the associated pages are gathered from the backing storage
2528
 * and pinned into our object. i915_gem_object_pin_pages() may be called
2529
 * multiple times before they are released by a single call to
2530
 * i915_gem_object_unpin_pages() - once the pages are no longer referenced
2531 2532 2533
 * either as a result of memory pressure (reaping pages under the shrinker)
 * or as the object is itself released.
 */
C
Chris Wilson 已提交
2534
int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2535
{
2536
	int err;
2537

2538 2539 2540
	err = mutex_lock_interruptible(&obj->mm.lock);
	if (err)
		return err;
2541

2542
	if (unlikely(IS_ERR_OR_NULL(obj->mm.pages))) {
2543 2544
		GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));

2545 2546 2547
		err = ____i915_gem_object_get_pages(obj);
		if (err)
			goto unlock;
2548

2549 2550 2551
		smp_mb__before_atomic();
	}
	atomic_inc(&obj->mm.pages_pin_count);
2552

2553 2554
unlock:
	mutex_unlock(&obj->mm.lock);
2555
	return err;
2556 2557
}

2558
/* The 'mapping' part of i915_gem_object_pin_map() below */
2559 2560
static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
				 enum i915_map_type type)
2561 2562
{
	unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
C
Chris Wilson 已提交
2563
	struct sg_table *sgt = obj->mm.pages;
2564 2565
	struct sgt_iter sgt_iter;
	struct page *page;
2566 2567
	struct page *stack_pages[32];
	struct page **pages = stack_pages;
2568
	unsigned long i = 0;
2569
	pgprot_t pgprot;
2570 2571 2572
	void *addr;

	/* A single page can always be kmapped */
2573
	if (n_pages == 1 && type == I915_MAP_WB)
2574 2575
		return kmap(sg_page(sgt->sgl));

2576 2577
	if (n_pages > ARRAY_SIZE(stack_pages)) {
		/* Too big for stack -- allocate temporary array instead */
2578
		pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
2579 2580 2581
		if (!pages)
			return NULL;
	}
2582

2583 2584
	for_each_sgt_page(page, sgt_iter, sgt)
		pages[i++] = page;
2585 2586 2587 2588

	/* Check that we have the expected number of pages */
	GEM_BUG_ON(i != n_pages);

2589
	switch (type) {
2590 2591 2592
	default:
		MISSING_CASE(type);
		/* fallthrough to use PAGE_KERNEL anyway */
2593 2594 2595 2596 2597 2598 2599 2600
	case I915_MAP_WB:
		pgprot = PAGE_KERNEL;
		break;
	case I915_MAP_WC:
		pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
		break;
	}
	addr = vmap(pages, n_pages, 0, pgprot);
2601

2602
	if (pages != stack_pages)
M
Michal Hocko 已提交
2603
		kvfree(pages);
2604 2605 2606 2607 2608

	return addr;
}

/* get, pin, and map the pages of the object into kernel space */
2609 2610
void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
			      enum i915_map_type type)
2611
{
2612 2613 2614
	enum i915_map_type has_type;
	bool pinned;
	void *ptr;
2615 2616
	int ret;

2617
	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
2618

2619
	ret = mutex_lock_interruptible(&obj->mm.lock);
2620 2621 2622
	if (ret)
		return ERR_PTR(ret);

2623 2624 2625
	pinned = !(type & I915_MAP_OVERRIDE);
	type &= ~I915_MAP_OVERRIDE;

2626
	if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
2627
		if (unlikely(IS_ERR_OR_NULL(obj->mm.pages))) {
2628 2629
			GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));

2630 2631 2632
			ret = ____i915_gem_object_get_pages(obj);
			if (ret)
				goto err_unlock;
2633

2634 2635 2636
			smp_mb__before_atomic();
		}
		atomic_inc(&obj->mm.pages_pin_count);
2637 2638 2639
		pinned = false;
	}
	GEM_BUG_ON(!obj->mm.pages);
2640

2641
	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
2642 2643 2644
	if (ptr && has_type != type) {
		if (pinned) {
			ret = -EBUSY;
2645
			goto err_unpin;
2646
		}
2647 2648 2649 2650 2651 2652

		if (is_vmalloc_addr(ptr))
			vunmap(ptr);
		else
			kunmap(kmap_to_page(ptr));

C
Chris Wilson 已提交
2653
		ptr = obj->mm.mapping = NULL;
2654 2655
	}

2656 2657 2658 2659
	if (!ptr) {
		ptr = i915_gem_object_map(obj, type);
		if (!ptr) {
			ret = -ENOMEM;
2660
			goto err_unpin;
2661 2662
		}

2663
		obj->mm.mapping = page_pack_bits(ptr, type);
2664 2665
	}

2666 2667
out_unlock:
	mutex_unlock(&obj->mm.lock);
2668 2669
	return ptr;

2670 2671 2672 2673 2674
err_unpin:
	atomic_dec(&obj->mm.pages_pin_count);
err_unlock:
	ptr = ERR_PTR(ret);
	goto out_unlock;
2675 2676
}

2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745
static int
i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
			   const struct drm_i915_gem_pwrite *arg)
{
	struct address_space *mapping = obj->base.filp->f_mapping;
	char __user *user_data = u64_to_user_ptr(arg->data_ptr);
	u64 remain, offset;
	unsigned int pg;

	/* Before we instantiate/pin the backing store for our use, we
	 * can prepopulate the shmemfs filp efficiently using a write into
	 * the pagecache. We avoid the penalty of instantiating all the
	 * pages, important if the user is just writing to a few and never
	 * uses the object on the GPU, and using a direct write into shmemfs
	 * allows it to avoid the cost of retrieving a page (either swapin
	 * or clearing-before-use) before it is overwritten.
	 */
	if (READ_ONCE(obj->mm.pages))
		return -ENODEV;

	/* Before the pages are instantiated the object is treated as being
	 * in the CPU domain. The pages will be clflushed as required before
	 * use, and we can freely write into the pages directly. If userspace
	 * races pwrite with any other operation; corruption will ensue -
	 * that is userspace's prerogative!
	 */

	remain = arg->size;
	offset = arg->offset;
	pg = offset_in_page(offset);

	do {
		unsigned int len, unwritten;
		struct page *page;
		void *data, *vaddr;
		int err;

		len = PAGE_SIZE - pg;
		if (len > remain)
			len = remain;

		err = pagecache_write_begin(obj->base.filp, mapping,
					    offset, len, 0,
					    &page, &data);
		if (err < 0)
			return err;

		vaddr = kmap(page);
		unwritten = copy_from_user(vaddr + pg, user_data, len);
		kunmap(page);

		err = pagecache_write_end(obj->base.filp, mapping,
					  offset, len, len - unwritten,
					  page, data);
		if (err < 0)
			return err;

		if (unwritten)
			return -EFAULT;

		remain -= len;
		user_data += len;
		offset += len;
		pg = 0;
	} while (remain);

	return 0;
}

2746 2747
static bool ban_context(const struct i915_gem_context *ctx,
			unsigned int score)
2748
{
2749
	return (i915_gem_context_is_bannable(ctx) &&
2750
		score >= CONTEXT_SCORE_BAN_THRESHOLD);
2751 2752
}

2753
static void i915_gem_context_mark_guilty(struct i915_gem_context *ctx)
2754
{
2755 2756
	unsigned int score;
	bool banned;
2757

2758
	atomic_inc(&ctx->guilty_count);
2759

2760 2761 2762 2763 2764
	score = atomic_add_return(CONTEXT_SCORE_GUILTY, &ctx->ban_score);
	banned = ban_context(ctx, score);
	DRM_DEBUG_DRIVER("context %s marked guilty (score %d) banned? %s\n",
			 ctx->name, score, yesno(banned));
	if (!banned)
2765 2766
		return;

2767 2768 2769 2770 2771 2772
	i915_gem_context_set_banned(ctx);
	if (!IS_ERR_OR_NULL(ctx->file_priv)) {
		atomic_inc(&ctx->file_priv->context_bans);
		DRM_DEBUG_DRIVER("client %s has had %d context banned\n",
				 ctx->name, atomic_read(&ctx->file_priv->context_bans));
	}
2773 2774 2775 2776
}

static void i915_gem_context_mark_innocent(struct i915_gem_context *ctx)
{
2777
	atomic_inc(&ctx->active_count);
2778 2779
}

2780
struct drm_i915_gem_request *
2781
i915_gem_find_active_request(struct intel_engine_cs *engine)
2782
{
2783 2784
	struct drm_i915_gem_request *request, *active = NULL;
	unsigned long flags;
2785

2786 2787 2788 2789 2790 2791 2792 2793
	/* We are called by the error capture and reset at a random
	 * point in time. In particular, note that neither is crucially
	 * ordered with an interrupt. After a hang, the GPU is dead and we
	 * assume that no more writes can happen (we waited long enough for
	 * all writes that were in transaction to be flushed) - adding an
	 * extra delay for a recent interrupt is pointless. Hence, we do
	 * not need an engine->irq_seqno_barrier() before the seqno reads.
	 */
2794
	spin_lock_irqsave(&engine->timeline->lock, flags);
2795
	list_for_each_entry(request, &engine->timeline->requests, link) {
2796 2797
		if (__i915_gem_request_completed(request,
						 request->global_seqno))
2798
			continue;
2799

2800
		GEM_BUG_ON(request->engine != engine);
2801 2802
		GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
				    &request->fence.flags));
2803 2804 2805

		active = request;
		break;
2806
	}
2807
	spin_unlock_irqrestore(&engine->timeline->lock, flags);
2808

2809
	return active;
2810 2811
}

2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825
static bool engine_stalled(struct intel_engine_cs *engine)
{
	if (!engine->hangcheck.stalled)
		return false;

	/* Check for possible seqno movement after hang declaration */
	if (engine->hangcheck.seqno != intel_engine_get_seqno(engine)) {
		DRM_DEBUG_DRIVER("%s pardoned\n", engine->name);
		return false;
	}

	return true;
}

2826 2827 2828 2829 2830 2831 2832 2833 2834
/*
 * Ensure irq handler finishes, and not run again.
 * Also return the active request so that we only search for it once.
 */
struct drm_i915_gem_request *
i915_gem_reset_prepare_engine(struct intel_engine_cs *engine)
{
	struct drm_i915_gem_request *request = NULL;

2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845
	/*
	 * During the reset sequence, we must prevent the engine from
	 * entering RC6. As the context state is undefined until we restart
	 * the engine, if it does enter RC6 during the reset, the state
	 * written to the powercontext is undefined and so we may lose
	 * GPU state upon resume, i.e. fail to restart after a reset.
	 */
	intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL);

	/*
	 * Prevent the signaler thread from updating the request
2846 2847 2848 2849 2850 2851 2852 2853 2854 2855
	 * state (by calling dma_fence_signal) as we are processing
	 * the reset. The write from the GPU of the seqno is
	 * asynchronous and the signaler thread may see a different
	 * value to us and declare the request complete, even though
	 * the reset routine have picked that request as the active
	 * (incomplete) request. This conflict is not handled
	 * gracefully!
	 */
	kthread_park(engine->breadcrumbs.signaler);

2856 2857
	/*
	 * Prevent request submission to the hardware until we have
2858 2859 2860 2861 2862 2863 2864
	 * completed the reset in i915_gem_reset_finish(). If a request
	 * is completed by one engine, it may then queue a request
	 * to a second via its engine->irq_tasklet *just* as we are
	 * calling engine->init_hw() and also writing the ELSP.
	 * Turning off the engine->irq_tasklet until the reset is over
	 * prevents the race.
	 */
2865 2866
	tasklet_kill(&engine->execlists.irq_tasklet);
	tasklet_disable(&engine->execlists.irq_tasklet);
2867 2868 2869 2870

	if (engine->irq_seqno_barrier)
		engine->irq_seqno_barrier(engine);

2871 2872 2873
	request = i915_gem_find_active_request(engine);
	if (request && request->fence.error == -EIO)
		request = ERR_PTR(-EIO); /* Previous reset failed! */
2874 2875 2876 2877

	return request;
}

2878
int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
2879 2880
{
	struct intel_engine_cs *engine;
2881
	struct drm_i915_gem_request *request;
2882
	enum intel_engine_id id;
2883
	int err = 0;
2884

2885
	for_each_engine(engine, dev_priv, id) {
2886 2887 2888 2889
		request = i915_gem_reset_prepare_engine(engine);
		if (IS_ERR(request)) {
			err = PTR_ERR(request);
			continue;
2890
		}
2891 2892

		engine->hangcheck.active_request = request;
2893 2894
	}

2895
	i915_gem_revoke_fences(dev_priv);
2896 2897

	return err;
2898 2899
}

2900
static void skip_request(struct drm_i915_gem_request *request)
2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914
{
	void *vaddr = request->ring->vaddr;
	u32 head;

	/* As this request likely depends on state from the lost
	 * context, clear out all the user operations leaving the
	 * breadcrumb at the end (so we get the fence notifications).
	 */
	head = request->head;
	if (request->postfix < head) {
		memset(vaddr + head, 0, request->ring->size - head);
		head = 0;
	}
	memset(vaddr + head, 0, request->postfix - head);
2915 2916

	dma_fence_set_error(&request->fence, -EIO);
2917 2918
}

2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941
static void engine_skip_context(struct drm_i915_gem_request *request)
{
	struct intel_engine_cs *engine = request->engine;
	struct i915_gem_context *hung_ctx = request->ctx;
	struct intel_timeline *timeline;
	unsigned long flags;

	timeline = i915_gem_context_lookup_timeline(hung_ctx, engine);

	spin_lock_irqsave(&engine->timeline->lock, flags);
	spin_lock(&timeline->lock);

	list_for_each_entry_continue(request, &engine->timeline->requests, link)
		if (request->ctx == hung_ctx)
			skip_request(request);

	list_for_each_entry(request, &timeline->requests, link)
		skip_request(request);

	spin_unlock(&timeline->lock);
	spin_unlock_irqrestore(&engine->timeline->lock, flags);
}

2942 2943 2944 2945
/* Returns the request if it was guilty of the hang */
static struct drm_i915_gem_request *
i915_gem_reset_request(struct intel_engine_cs *engine,
		       struct drm_i915_gem_request *request)
2946
{
2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967
	/* The guilty request will get skipped on a hung engine.
	 *
	 * Users of client default contexts do not rely on logical
	 * state preserved between batches so it is safe to execute
	 * queued requests following the hang. Non default contexts
	 * rely on preserved state, so skipping a batch loses the
	 * evolution of the state and it needs to be considered corrupted.
	 * Executing more queued batches on top of corrupted state is
	 * risky. But we take the risk by trying to advance through
	 * the queued requests in order to make the client behaviour
	 * more predictable around resets, by not throwing away random
	 * amount of batches it has prepared for execution. Sophisticated
	 * clients can use gem_reset_stats_ioctl and dma fence status
	 * (exported via sync_file info ioctl on explicit fences) to observe
	 * when it loses the context state and should rebuild accordingly.
	 *
	 * The context ban, and ultimately the client ban, mechanism are safety
	 * valves if client submission ends up resulting in nothing more than
	 * subsequent hangs.
	 */

2968
	if (engine_stalled(engine)) {
2969 2970
		i915_gem_context_mark_guilty(request->ctx);
		skip_request(request);
2971 2972 2973 2974

		/* If this context is now banned, skip all pending requests. */
		if (i915_gem_context_is_banned(request->ctx))
			engine_skip_context(request);
2975
	} else {
2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992
		/*
		 * Since this is not the hung engine, it may have advanced
		 * since the hang declaration. Double check by refinding
		 * the active request at the time of the reset.
		 */
		request = i915_gem_find_active_request(engine);
		if (request) {
			i915_gem_context_mark_innocent(request->ctx);
			dma_fence_set_error(&request->fence, -EAGAIN);

			/* Rewind the engine to replay the incomplete rq */
			spin_lock_irq(&engine->timeline->lock);
			request = list_prev_entry(request, link);
			if (&request->link == &engine->timeline->requests)
				request = NULL;
			spin_unlock_irq(&engine->timeline->lock);
		}
2993 2994
	}

2995
	return request;
2996 2997
}

2998 2999
void i915_gem_reset_engine(struct intel_engine_cs *engine,
			   struct drm_i915_gem_request *request)
3000
{
3001 3002
	engine->irq_posted = 0;

3003 3004 3005 3006
	if (request)
		request = i915_gem_reset_request(engine, request);

	if (request) {
3007 3008 3009
		DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n",
				 engine->name, request->global_seqno);
	}
3010 3011 3012

	/* Setup the CS to resume from the breadcrumb of the hung request */
	engine->reset_hw(engine, request);
3013
}
3014

3015
void i915_gem_reset(struct drm_i915_private *dev_priv)
3016
{
3017
	struct intel_engine_cs *engine;
3018
	enum intel_engine_id id;
3019

3020 3021
	lockdep_assert_held(&dev_priv->drm.struct_mutex);

3022 3023
	i915_gem_retire_requests(dev_priv);

3024 3025 3026
	for_each_engine(engine, dev_priv, id) {
		struct i915_gem_context *ctx;

3027
		i915_gem_reset_engine(engine, engine->hangcheck.active_request);
3028 3029 3030 3031
		ctx = fetch_and_zero(&engine->last_retired_context);
		if (ctx)
			engine->context_unpin(engine, ctx);
	}
3032

3033
	i915_gem_restore_fences(dev_priv);
3034 3035 3036 3037 3038 3039 3040

	if (dev_priv->gt.awake) {
		intel_sanitize_gt_powersave(dev_priv);
		intel_enable_gt_powersave(dev_priv);
		if (INTEL_GEN(dev_priv) >= 6)
			gen6_rps_busy(dev_priv);
	}
3041 3042
}

3043 3044
void i915_gem_reset_finish_engine(struct intel_engine_cs *engine)
{
3045
	tasklet_enable(&engine->execlists.irq_tasklet);
3046
	kthread_unpark(engine->breadcrumbs.signaler);
3047 3048

	intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL);
3049 3050
}

3051 3052
void i915_gem_reset_finish(struct drm_i915_private *dev_priv)
{
3053 3054 3055
	struct intel_engine_cs *engine;
	enum intel_engine_id id;

3056
	lockdep_assert_held(&dev_priv->drm.struct_mutex);
3057

3058
	for_each_engine(engine, dev_priv, id) {
3059
		engine->hangcheck.active_request = NULL;
3060
		i915_gem_reset_finish_engine(engine);
3061
	}
3062 3063
}

3064 3065
static void nop_submit_request(struct drm_i915_gem_request *request)
{
3066 3067
	unsigned long flags;

3068
	GEM_BUG_ON(!i915_terminally_wedged(&request->i915->gpu_error));
3069
	dma_fence_set_error(&request->fence, -EIO);
3070 3071 3072

	spin_lock_irqsave(&request->engine->timeline->lock, flags);
	__i915_gem_request_submit(request);
3073
	intel_engine_init_global_seqno(request->engine, request->global_seqno);
3074
	spin_unlock_irqrestore(&request->engine->timeline->lock, flags);
3075 3076
}

3077
static void engine_set_wedged(struct intel_engine_cs *engine)
3078
{
3079 3080 3081 3082 3083 3084
	/* We need to be sure that no thread is running the old callback as
	 * we install the nop handler (otherwise we would submit a request
	 * to hardware that will never complete). In order to prevent this
	 * race, we wait until the machine is idle before making the swap
	 * (using stop_machine()).
	 */
3085
	engine->submit_request = nop_submit_request;
3086

3087
	/* Mark all executing requests as skipped */
3088
	engine->cancel_requests(engine);
3089 3090 3091 3092 3093 3094 3095

	/* Mark all pending requests as complete so that any concurrent
	 * (lockless) lookup doesn't try and wait upon the request as we
	 * reset it.
	 */
	intel_engine_init_global_seqno(engine,
				       intel_engine_last_submit(engine));
3096 3097
}

3098
static int __i915_gem_set_wedged_BKL(void *data)
3099
{
3100
	struct drm_i915_private *i915 = data;
3101
	struct intel_engine_cs *engine;
3102
	enum intel_engine_id id;
3103

3104
	for_each_engine(engine, i915, id)
3105
		engine_set_wedged(engine);
3106

3107 3108 3109
	set_bit(I915_WEDGED, &i915->gpu_error.flags);
	wake_up_all(&i915->gpu_error.reset_queue);

3110 3111 3112 3113 3114 3115
	return 0;
}

void i915_gem_set_wedged(struct drm_i915_private *dev_priv)
{
	stop_machine(__i915_gem_set_wedged_BKL, dev_priv, NULL);
3116 3117
}

3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169
bool i915_gem_unset_wedged(struct drm_i915_private *i915)
{
	struct i915_gem_timeline *tl;
	int i;

	lockdep_assert_held(&i915->drm.struct_mutex);
	if (!test_bit(I915_WEDGED, &i915->gpu_error.flags))
		return true;

	/* Before unwedging, make sure that all pending operations
	 * are flushed and errored out - we may have requests waiting upon
	 * third party fences. We marked all inflight requests as EIO, and
	 * every execbuf since returned EIO, for consistency we want all
	 * the currently pending requests to also be marked as EIO, which
	 * is done inside our nop_submit_request - and so we must wait.
	 *
	 * No more can be submitted until we reset the wedged bit.
	 */
	list_for_each_entry(tl, &i915->gt.timelines, link) {
		for (i = 0; i < ARRAY_SIZE(tl->engine); i++) {
			struct drm_i915_gem_request *rq;

			rq = i915_gem_active_peek(&tl->engine[i].last_request,
						  &i915->drm.struct_mutex);
			if (!rq)
				continue;

			/* We can't use our normal waiter as we want to
			 * avoid recursively trying to handle the current
			 * reset. The basic dma_fence_default_wait() installs
			 * a callback for dma_fence_signal(), which is
			 * triggered by our nop handler (indirectly, the
			 * callback enables the signaler thread which is
			 * woken by the nop_submit_request() advancing the seqno
			 * and when the seqno passes the fence, the signaler
			 * then signals the fence waking us up).
			 */
			if (dma_fence_default_wait(&rq->fence, true,
						   MAX_SCHEDULE_TIMEOUT) < 0)
				return false;
		}
	}

	/* Undo nop_submit_request. We prevent all new i915 requests from
	 * being queued (by disallowing execbuf whilst wedged) so having
	 * waited for all active requests above, we know the system is idle
	 * and do not have to worry about a thread being inside
	 * engine->submit_request() as we swap over. So unlike installing
	 * the nop_submit_request on reset, we can do this from normal
	 * context and do not require stop_machine().
	 */
	intel_engines_reset_default_submission(i915);
3170
	i915_gem_contexts_lost(i915);
3171 3172 3173 3174 3175 3176 3177

	smp_mb__before_atomic(); /* complete takeover before enabling execbuf */
	clear_bit(I915_WEDGED, &i915->gpu_error.flags);

	return true;
}

3178
static void
3179 3180
i915_gem_retire_work_handler(struct work_struct *work)
{
3181
	struct drm_i915_private *dev_priv =
3182
		container_of(work, typeof(*dev_priv), gt.retire_work.work);
3183
	struct drm_device *dev = &dev_priv->drm;
3184

3185
	/* Come back later if the device is busy... */
3186
	if (mutex_trylock(&dev->struct_mutex)) {
3187
		i915_gem_retire_requests(dev_priv);
3188
		mutex_unlock(&dev->struct_mutex);
3189
	}
3190 3191 3192 3193 3194

	/* Keep the retire handler running until we are finally idle.
	 * We do not need to do this test under locking as in the worst-case
	 * we queue the retire worker once too often.
	 */
3195 3196
	if (READ_ONCE(dev_priv->gt.awake)) {
		i915_queue_hangcheck(dev_priv);
3197 3198
		queue_delayed_work(dev_priv->wq,
				   &dev_priv->gt.retire_work,
3199
				   round_jiffies_up_relative(HZ));
3200
	}
3201
}
3202

3203 3204 3205 3206
static void
i915_gem_idle_work_handler(struct work_struct *work)
{
	struct drm_i915_private *dev_priv =
3207
		container_of(work, typeof(*dev_priv), gt.idle_work.work);
3208
	struct drm_device *dev = &dev_priv->drm;
3209 3210 3211 3212 3213
	bool rearm_hangcheck;

	if (!READ_ONCE(dev_priv->gt.awake))
		return;

3214 3215 3216 3217
	/*
	 * Wait for last execlists context complete, but bail out in case a
	 * new request is submitted.
	 */
3218
	wait_for(intel_engines_are_idle(dev_priv), 10);
3219
	if (READ_ONCE(dev_priv->gt.active_requests))
3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232
		return;

	rearm_hangcheck =
		cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);

	if (!mutex_trylock(&dev->struct_mutex)) {
		/* Currently busy, come back later */
		mod_delayed_work(dev_priv->wq,
				 &dev_priv->gt.idle_work,
				 msecs_to_jiffies(50));
		goto out_rearm;
	}

3233 3234 3235 3236 3237 3238 3239
	/*
	 * New request retired after this work handler started, extend active
	 * period until next instance of the work.
	 */
	if (work_pending(work))
		goto out_unlock;

3240
	if (dev_priv->gt.active_requests)
3241
		goto out_unlock;
3242

3243
	if (wait_for(intel_engines_are_idle(dev_priv), 10))
3244 3245
		DRM_ERROR("Timeout waiting for engines to idle\n");

3246
	intel_engines_mark_idle(dev_priv);
3247
	i915_gem_timelines_mark_idle(dev_priv);
3248

3249 3250 3251
	GEM_BUG_ON(!dev_priv->gt.awake);
	dev_priv->gt.awake = false;
	rearm_hangcheck = false;
3252

3253 3254 3255 3256 3257
	if (INTEL_GEN(dev_priv) >= 6)
		gen6_rps_idle(dev_priv);
	intel_runtime_pm_put(dev_priv);
out_unlock:
	mutex_unlock(&dev->struct_mutex);
3258

3259 3260 3261 3262
out_rearm:
	if (rearm_hangcheck) {
		GEM_BUG_ON(!dev_priv->gt.awake);
		i915_queue_hangcheck(dev_priv);
3263
	}
3264 3265
}

3266 3267
void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
{
3268
	struct drm_i915_private *i915 = to_i915(gem->dev);
3269 3270
	struct drm_i915_gem_object *obj = to_intel_bo(gem);
	struct drm_i915_file_private *fpriv = file->driver_priv;
3271
	struct i915_lut_handle *lut, *ln;
3272

3273 3274 3275 3276 3277 3278
	mutex_lock(&i915->drm.struct_mutex);

	list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) {
		struct i915_gem_context *ctx = lut->ctx;
		struct i915_vma *vma;

3279
		GEM_BUG_ON(ctx->file_priv == ERR_PTR(-EBADF));
3280 3281 3282 3283
		if (ctx->file_priv != fpriv)
			continue;

		vma = radix_tree_delete(&ctx->handles_vma, lut->handle);
3284 3285 3286 3287 3288 3289 3290
		GEM_BUG_ON(vma->obj != obj);

		/* We allow the process to have multiple handles to the same
		 * vma, in the same fd namespace, by virtue of flink/open.
		 */
		GEM_BUG_ON(!vma->open_count);
		if (!--vma->open_count && !i915_vma_is_ggtt(vma))
3291
			i915_vma_close(vma);
3292

3293 3294
		list_del(&lut->obj_link);
		list_del(&lut->ctx_link);
3295

3296 3297
		kmem_cache_free(i915->luts, lut);
		__i915_gem_object_release_unless_active(obj);
3298
	}
3299 3300

	mutex_unlock(&i915->drm.struct_mutex);
3301 3302
}

3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313
static unsigned long to_wait_timeout(s64 timeout_ns)
{
	if (timeout_ns < 0)
		return MAX_SCHEDULE_TIMEOUT;

	if (timeout_ns == 0)
		return 0;

	return nsecs_to_jiffies_timeout(timeout_ns);
}

3314 3315
/**
 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
3316 3317 3318
 * @dev: drm device pointer
 * @data: ioctl data blob
 * @file: drm file pointer
3319 3320 3321 3322 3323 3324 3325
 *
 * Returns 0 if successful, else an error is returned with the remaining time in
 * the timeout parameter.
 *  -ETIME: object is still busy after timeout
 *  -ERESTARTSYS: signal interrupted the wait
 *  -ENONENT: object doesn't exist
 * Also possible, but rare:
3326
 *  -EAGAIN: incomplete, restart syscall
3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342
 *  -ENOMEM: damn
 *  -ENODEV: Internal IRQ fail
 *  -E?: The add request failed
 *
 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
 * non-zero timeout parameter the wait ioctl will wait for the given number of
 * nanoseconds on an object becoming unbusy. Since the wait itself does so
 * without holding struct_mutex the object may become re-busied before this
 * function completes. A similar but shorter * race condition exists in the busy
 * ioctl
 */
int
i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
	struct drm_i915_gem_wait *args = data;
	struct drm_i915_gem_object *obj;
3343 3344
	ktime_t start;
	long ret;
3345

3346 3347 3348
	if (args->flags != 0)
		return -EINVAL;

3349
	obj = i915_gem_object_lookup(file, args->bo_handle);
3350
	if (!obj)
3351 3352
		return -ENOENT;

3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363
	start = ktime_get();

	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL,
				   to_wait_timeout(args->timeout_ns),
				   to_rps_client(file));

	if (args->timeout_ns > 0) {
		args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
		if (args->timeout_ns < 0)
			args->timeout_ns = 0;
3364 3365 3366 3367 3368 3369 3370 3371 3372 3373

		/*
		 * Apparently ktime isn't accurate enough and occasionally has a
		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
		 * things up to make the test happy. We allow up to 1 jiffy.
		 *
		 * This is a regression from the timespec->ktime conversion.
		 */
		if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
			args->timeout_ns = 0;
3374 3375 3376 3377

		/* Asked to wait beyond the jiffie/scheduler precision? */
		if (ret == -ETIME && args->timeout_ns)
			ret = -EAGAIN;
3378 3379
	}

C
Chris Wilson 已提交
3380
	i915_gem_object_put(obj);
3381
	return ret;
3382 3383
}

3384
static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags)
3385
{
3386
	int ret, i;
3387

3388 3389 3390 3391 3392
	for (i = 0; i < ARRAY_SIZE(tl->engine); i++) {
		ret = i915_gem_active_wait(&tl->engine[i].last_request, flags);
		if (ret)
			return ret;
	}
3393

3394 3395 3396
	return 0;
}

3397 3398
static int wait_for_engines(struct drm_i915_private *i915)
{
3399 3400 3401 3402
	if (wait_for(intel_engines_are_idle(i915), 50)) {
		DRM_ERROR("Failed to idle engines, declaring wedged!\n");
		i915_gem_set_wedged(i915);
		return -EIO;
3403 3404 3405 3406 3407
	}

	return 0;
}

3408 3409 3410 3411
int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
{
	int ret;

3412 3413 3414 3415
	/* If the device is asleep, we have no requests outstanding */
	if (!READ_ONCE(i915->gt.awake))
		return 0;

3416 3417 3418 3419 3420 3421 3422 3423 3424 3425
	if (flags & I915_WAIT_LOCKED) {
		struct i915_gem_timeline *tl;

		lockdep_assert_held(&i915->drm.struct_mutex);

		list_for_each_entry(tl, &i915->gt.timelines, link) {
			ret = wait_for_timeline(tl, flags);
			if (ret)
				return ret;
		}
3426 3427 3428

		i915_gem_retire_requests(i915);
		GEM_BUG_ON(i915->gt.active_requests);
3429 3430

		ret = wait_for_engines(i915);
3431 3432
	} else {
		ret = wait_for_timeline(&i915->gt.global_timeline, flags);
3433
	}
3434

3435
	return ret;
3436 3437
}

3438 3439
static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
{
3440 3441 3442 3443 3444 3445 3446
	/*
	 * We manually flush the CPU domain so that we can override and
	 * force the flush for the display, and perform it asyncrhonously.
	 */
	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
	if (obj->cache_dirty)
		i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459
	obj->base.write_domain = 0;
}

void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
{
	if (!READ_ONCE(obj->pin_display))
		return;

	mutex_lock(&obj->base.dev->struct_mutex);
	__i915_gem_object_flush_for_display(obj);
	mutex_unlock(&obj->base.dev->struct_mutex);
}

3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522
/**
 * Moves a single object to the WC read, and possibly write domain.
 * @obj: object to act on
 * @write: ask for write access or read only
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
int
i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
{
	int ret;

	lockdep_assert_held(&obj->base.dev->struct_mutex);

	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE |
				   I915_WAIT_LOCKED |
				   (write ? I915_WAIT_ALL : 0),
				   MAX_SCHEDULE_TIMEOUT,
				   NULL);
	if (ret)
		return ret;

	if (obj->base.write_domain == I915_GEM_DOMAIN_WC)
		return 0;

	/* Flush and acquire obj->pages so that we are coherent through
	 * direct access in memory with previous cached writes through
	 * shmemfs and that our cache domain tracking remains valid.
	 * For example, if the obj->filp was moved to swap without us
	 * being notified and releasing the pages, we would mistakenly
	 * continue to assume that the obj remained out of the CPU cached
	 * domain.
	 */
	ret = i915_gem_object_pin_pages(obj);
	if (ret)
		return ret;

	flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);

	/* Serialise direct access to this object with the barriers for
	 * coherent writes from the GPU, by effectively invalidating the
	 * WC domain upon first access.
	 */
	if ((obj->base.read_domains & I915_GEM_DOMAIN_WC) == 0)
		mb();

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
	GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_WC) != 0);
	obj->base.read_domains |= I915_GEM_DOMAIN_WC;
	if (write) {
		obj->base.read_domains = I915_GEM_DOMAIN_WC;
		obj->base.write_domain = I915_GEM_DOMAIN_WC;
		obj->mm.dirty = true;
	}

	i915_gem_object_unpin_pages(obj);
	return 0;
}

3523 3524
/**
 * Moves a single object to the GTT read, and possibly write domain.
3525 3526
 * @obj: object to act on
 * @write: ask for write access or read only
3527 3528 3529 3530
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
J
Jesse Barnes 已提交
3531
int
3532
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3533
{
3534
	int ret;
3535

3536
	lockdep_assert_held(&obj->base.dev->struct_mutex);
3537

3538 3539 3540 3541 3542 3543
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE |
				   I915_WAIT_LOCKED |
				   (write ? I915_WAIT_ALL : 0),
				   MAX_SCHEDULE_TIMEOUT,
				   NULL);
3544 3545 3546
	if (ret)
		return ret;

3547 3548 3549
	if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
		return 0;

3550 3551 3552 3553 3554 3555 3556 3557
	/* Flush and acquire obj->pages so that we are coherent through
	 * direct access in memory with previous cached writes through
	 * shmemfs and that our cache domain tracking remains valid.
	 * For example, if the obj->filp was moved to swap without us
	 * being notified and releasing the pages, we would mistakenly
	 * continue to assume that the obj remained out of the CPU cached
	 * domain.
	 */
C
Chris Wilson 已提交
3558
	ret = i915_gem_object_pin_pages(obj);
3559 3560 3561
	if (ret)
		return ret;

3562
	flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
C
Chris Wilson 已提交
3563

3564 3565 3566 3567 3568 3569 3570
	/* Serialise direct access to this object with the barriers for
	 * coherent writes from the GPU, by effectively invalidating the
	 * GTT domain upon first access.
	 */
	if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
		mb();

3571 3572 3573
	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3574
	GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3575
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3576
	if (write) {
3577 3578
		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
C
Chris Wilson 已提交
3579
		obj->mm.dirty = true;
3580 3581
	}

C
Chris Wilson 已提交
3582
	i915_gem_object_unpin_pages(obj);
3583 3584 3585
	return 0;
}

3586 3587
/**
 * Changes the cache-level of an object across all VMA.
3588 3589
 * @obj: object to act on
 * @cache_level: new cache level to set for the object
3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600
 *
 * After this function returns, the object will be in the new cache-level
 * across all GTT and the contents of the backing storage will be coherent,
 * with respect to the new cache-level. In order to keep the backing storage
 * coherent for all users, we only allow a single cache level to be set
 * globally on the object and prevent it from being changed whilst the
 * hardware is reading from the object. That is if the object is currently
 * on the scanout it will be set to uncached (or equivalent display
 * cache coherency) and all non-MOCS GPU access will also be uncached so
 * that all direct access to the scanout remains coherent.
 */
3601 3602 3603
int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
				    enum i915_cache_level cache_level)
{
3604
	struct i915_vma *vma;
3605
	int ret;
3606

3607 3608
	lockdep_assert_held(&obj->base.dev->struct_mutex);

3609
	if (obj->cache_level == cache_level)
3610
		return 0;
3611

3612 3613 3614 3615 3616
	/* Inspect the list of currently bound VMA and unbind any that would
	 * be invalid given the new cache-level. This is principally to
	 * catch the issue of the CS prefetch crossing page boundaries and
	 * reading an invalid PTE on older architectures.
	 */
3617 3618
restart:
	list_for_each_entry(vma, &obj->vma_list, obj_link) {
3619 3620 3621
		if (!drm_mm_node_allocated(&vma->node))
			continue;

3622
		if (i915_vma_is_pinned(vma)) {
3623 3624 3625 3626
			DRM_DEBUG("can not change the cache level of pinned objects\n");
			return -EBUSY;
		}

3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638
		if (i915_gem_valid_gtt_space(vma, cache_level))
			continue;

		ret = i915_vma_unbind(vma);
		if (ret)
			return ret;

		/* As unbinding may affect other elements in the
		 * obj->vma_list (due to side-effects from retiring
		 * an active vma), play safe and restart the iterator.
		 */
		goto restart;
3639 3640
	}

3641 3642 3643 3644 3645 3646 3647
	/* We can reuse the existing drm_mm nodes but need to change the
	 * cache-level on the PTE. We could simply unbind them all and
	 * rebind with the correct cache-level on next use. However since
	 * we already have a valid slot, dma mapping, pages etc, we may as
	 * rewrite the PTE in the belief that doing so tramples upon less
	 * state and so involves less work.
	 */
3648
	if (obj->bind_count) {
3649 3650 3651 3652
		/* Before we change the PTE, the GPU must not be accessing it.
		 * If we wait upon the object, we know that all the bound
		 * VMA are no longer active.
		 */
3653 3654 3655 3656 3657 3658
		ret = i915_gem_object_wait(obj,
					   I915_WAIT_INTERRUPTIBLE |
					   I915_WAIT_LOCKED |
					   I915_WAIT_ALL,
					   MAX_SCHEDULE_TIMEOUT,
					   NULL);
3659 3660 3661
		if (ret)
			return ret;

3662 3663
		if (!HAS_LLC(to_i915(obj->base.dev)) &&
		    cache_level != I915_CACHE_NONE) {
3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679
			/* Access to snoopable pages through the GTT is
			 * incoherent and on some machines causes a hard
			 * lockup. Relinquish the CPU mmaping to force
			 * userspace to refault in the pages and we can
			 * then double check if the GTT mapping is still
			 * valid for that pointer access.
			 */
			i915_gem_release_mmap(obj);

			/* As we no longer need a fence for GTT access,
			 * we can relinquish it now (and so prevent having
			 * to steal a fence from someone else on the next
			 * fence request). Note GPU activity would have
			 * dropped the fence as all snoopable access is
			 * supposed to be linear.
			 */
3680 3681 3682 3683 3684
			list_for_each_entry(vma, &obj->vma_list, obj_link) {
				ret = i915_vma_put_fence(vma);
				if (ret)
					return ret;
			}
3685 3686 3687 3688 3689 3690 3691 3692
		} else {
			/* We either have incoherent backing store and
			 * so no GTT access or the architecture is fully
			 * coherent. In such cases, existing GTT mmaps
			 * ignore the cache bit in the PTE and we can
			 * rewrite it without confusing the GPU or having
			 * to force userspace to fault back in its mmaps.
			 */
3693 3694
		}

3695
		list_for_each_entry(vma, &obj->vma_list, obj_link) {
3696 3697 3698 3699 3700 3701 3702
			if (!drm_mm_node_allocated(&vma->node))
				continue;

			ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
			if (ret)
				return ret;
		}
3703 3704
	}

3705
	list_for_each_entry(vma, &obj->vma_list, obj_link)
3706
		vma->node.color = cache_level;
3707
	i915_gem_object_set_cache_coherency(obj, cache_level);
3708
	obj->cache_dirty = true; /* Always invalidate stale cachelines */
3709

3710 3711 3712
	return 0;
}

B
Ben Widawsky 已提交
3713 3714
int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3715
{
B
Ben Widawsky 已提交
3716
	struct drm_i915_gem_caching *args = data;
3717
	struct drm_i915_gem_object *obj;
3718
	int err = 0;
3719

3720 3721 3722 3723 3724 3725
	rcu_read_lock();
	obj = i915_gem_object_lookup_rcu(file, args->handle);
	if (!obj) {
		err = -ENOENT;
		goto out;
	}
3726

3727 3728 3729 3730 3731 3732
	switch (obj->cache_level) {
	case I915_CACHE_LLC:
	case I915_CACHE_L3_LLC:
		args->caching = I915_CACHING_CACHED;
		break;

3733 3734 3735 3736
	case I915_CACHE_WT:
		args->caching = I915_CACHING_DISPLAY;
		break;

3737 3738 3739 3740
	default:
		args->caching = I915_CACHING_NONE;
		break;
	}
3741 3742 3743
out:
	rcu_read_unlock();
	return err;
3744 3745
}

B
Ben Widawsky 已提交
3746 3747
int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
3748
{
3749
	struct drm_i915_private *i915 = to_i915(dev);
B
Ben Widawsky 已提交
3750
	struct drm_i915_gem_caching *args = data;
3751 3752
	struct drm_i915_gem_object *obj;
	enum i915_cache_level level;
3753
	int ret = 0;
3754

B
Ben Widawsky 已提交
3755 3756
	switch (args->caching) {
	case I915_CACHING_NONE:
3757 3758
		level = I915_CACHE_NONE;
		break;
B
Ben Widawsky 已提交
3759
	case I915_CACHING_CACHED:
3760 3761 3762 3763 3764 3765
		/*
		 * Due to a HW issue on BXT A stepping, GPU stores via a
		 * snooped mapping may leave stale data in a corresponding CPU
		 * cacheline, whereas normally such cachelines would get
		 * invalidated.
		 */
3766
		if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
3767 3768
			return -ENODEV;

3769 3770
		level = I915_CACHE_LLC;
		break;
3771
	case I915_CACHING_DISPLAY:
3772
		level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
3773
		break;
3774 3775 3776 3777
	default:
		return -EINVAL;
	}

3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788
	obj = i915_gem_object_lookup(file, args->handle);
	if (!obj)
		return -ENOENT;

	if (obj->cache_level == level)
		goto out;

	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE,
				   MAX_SCHEDULE_TIMEOUT,
				   to_rps_client(file));
B
Ben Widawsky 已提交
3789
	if (ret)
3790
		goto out;
B
Ben Widawsky 已提交
3791

3792 3793 3794
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto out;
3795 3796 3797

	ret = i915_gem_object_set_cache_level(obj, level);
	mutex_unlock(&dev->struct_mutex);
3798 3799 3800

out:
	i915_gem_object_put(obj);
3801 3802 3803
	return ret;
}

3804
/*
3805 3806 3807
 * Prepare buffer for display plane (scanout, cursors, etc).
 * Can be called from an uninterruptible phase (modesetting) and allows
 * any flushes to be pipelined (for pageflips).
3808
 */
C
Chris Wilson 已提交
3809
struct i915_vma *
3810 3811
i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
				     u32 alignment,
3812
				     const struct i915_ggtt_view *view)
3813
{
C
Chris Wilson 已提交
3814
	struct i915_vma *vma;
3815 3816
	int ret;

3817 3818
	lockdep_assert_held(&obj->base.dev->struct_mutex);

3819 3820 3821
	/* Mark the pin_display early so that we account for the
	 * display coherency whilst setting up the cache domains.
	 */
3822
	obj->pin_display++;
3823

3824 3825 3826 3827 3828 3829 3830 3831 3832
	/* The display engine is not coherent with the LLC cache on gen6.  As
	 * a result, we make sure that the pinning that is about to occur is
	 * done with uncached PTEs. This is lowest common denominator for all
	 * chipsets.
	 *
	 * However for gen6+, we could do better by using the GFDT bit instead
	 * of uncaching, which would allow us to flush all the LLC-cached data
	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
	 */
3833
	ret = i915_gem_object_set_cache_level(obj,
3834 3835
					      HAS_WT(to_i915(obj->base.dev)) ?
					      I915_CACHE_WT : I915_CACHE_NONE);
C
Chris Wilson 已提交
3836 3837
	if (ret) {
		vma = ERR_PTR(ret);
3838
		goto err_unpin_display;
C
Chris Wilson 已提交
3839
	}
3840

3841 3842
	/* As the user may map the buffer once pinned in the display plane
	 * (e.g. libkms for the bootup splash), we have to ensure that we
3843 3844 3845 3846
	 * always use map_and_fenceable for all scanout buffers. However,
	 * it may simply be too big to fit into mappable, in which case
	 * put it anyway and hope that userspace can cope (but always first
	 * try to preserve the existing ABI).
3847
	 */
3848
	vma = ERR_PTR(-ENOSPC);
3849
	if (!view || view->type == I915_GGTT_VIEW_NORMAL)
3850 3851
		vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
					       PIN_MAPPABLE | PIN_NONBLOCK);
3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867
	if (IS_ERR(vma)) {
		struct drm_i915_private *i915 = to_i915(obj->base.dev);
		unsigned int flags;

		/* Valleyview is definitely limited to scanning out the first
		 * 512MiB. Lets presume this behaviour was inherited from the
		 * g4x display engine and that all earlier gen are similarly
		 * limited. Testing suggests that it is a little more
		 * complicated than this. For example, Cherryview appears quite
		 * happy to scanout from anywhere within its global aperture.
		 */
		flags = 0;
		if (HAS_GMCH_DISPLAY(i915))
			flags = PIN_MAPPABLE;
		vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags);
	}
C
Chris Wilson 已提交
3868
	if (IS_ERR(vma))
3869
		goto err_unpin_display;
3870

3871 3872
	vma->display_alignment = max_t(u64, vma->display_alignment, alignment);

3873
	/* Treat this as an end-of-frame, like intel_user_framebuffer_dirty() */
3874
	__i915_gem_object_flush_for_display(obj);
3875
	intel_fb_obj_flush(obj, ORIGIN_DIRTYFB);
3876

3877 3878 3879
	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3880
	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3881

C
Chris Wilson 已提交
3882
	return vma;
3883 3884

err_unpin_display:
3885
	obj->pin_display--;
C
Chris Wilson 已提交
3886
	return vma;
3887 3888 3889
}

void
C
Chris Wilson 已提交
3890
i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
3891
{
3892
	lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
3893

C
Chris Wilson 已提交
3894
	if (WARN_ON(vma->obj->pin_display == 0))
3895 3896
		return;

3897
	if (--vma->obj->pin_display == 0)
3898
		vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
3899

3900
	/* Bump the LRU to try and avoid premature eviction whilst flipping  */
3901
	i915_gem_object_bump_inactive_ggtt(vma->obj);
3902

C
Chris Wilson 已提交
3903
	i915_vma_unpin(vma);
3904 3905
}

3906 3907
/**
 * Moves a single object to the CPU read, and possibly write domain.
3908 3909
 * @obj: object to act on
 * @write: requesting write or read-only access
3910 3911 3912 3913
 *
 * This function returns when the move is complete, including waiting on
 * flushes to occur.
 */
3914
int
3915
i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3916 3917 3918
{
	int ret;

3919
	lockdep_assert_held(&obj->base.dev->struct_mutex);
3920

3921 3922 3923 3924 3925 3926
	ret = i915_gem_object_wait(obj,
				   I915_WAIT_INTERRUPTIBLE |
				   I915_WAIT_LOCKED |
				   (write ? I915_WAIT_ALL : 0),
				   MAX_SCHEDULE_TIMEOUT,
				   NULL);
3927 3928 3929
	if (ret)
		return ret;

3930
	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
3931

3932
	/* Flush the CPU cache if it's still invalid. */
3933
	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3934
		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
3935
		obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3936 3937 3938 3939 3940
	}

	/* It should now be out of any other write domains, and we can update
	 * the domain values for our changes.
	 */
3941
	GEM_BUG_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3942 3943 3944 3945

	/* If we're writing through the CPU, then the GPU read domains will
	 * need to be invalidated at next use.
	 */
3946 3947
	if (write)
		__start_cpu_write(obj);
3948 3949 3950 3951

	return 0;
}

3952 3953 3954
/* Throttle our rendering by waiting until the ring has completed our requests
 * emitted over 20 msec ago.
 *
3955 3956 3957 3958
 * Note that if we were to use the current jiffies each time around the loop,
 * we wouldn't escape the function with any frames outstanding if the time to
 * render a frame was over 20ms.
 *
3959 3960 3961
 * This should get us reasonable parallelism between CPU and GPU but also
 * relatively low latency when blocking on a particular request to finish.
 */
3962
static int
3963
i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3964
{
3965
	struct drm_i915_private *dev_priv = to_i915(dev);
3966
	struct drm_i915_file_private *file_priv = file->driver_priv;
3967
	unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
3968
	struct drm_i915_gem_request *request, *target = NULL;
3969
	long ret;
3970

3971 3972 3973
	/* ABI: return -EIO if already wedged */
	if (i915_terminally_wedged(&dev_priv->gpu_error))
		return -EIO;
3974

3975
	spin_lock(&file_priv->mm.lock);
3976
	list_for_each_entry(request, &file_priv->mm.request_list, client_link) {
3977 3978
		if (time_after_eq(request->emitted_jiffies, recent_enough))
			break;
3979

3980 3981 3982 3983
		if (target) {
			list_del(&target->client_link);
			target->file_priv = NULL;
		}
3984

3985
		target = request;
3986
	}
3987
	if (target)
3988
		i915_gem_request_get(target);
3989
	spin_unlock(&file_priv->mm.lock);
3990

3991
	if (target == NULL)
3992
		return 0;
3993

3994 3995 3996
	ret = i915_wait_request(target,
				I915_WAIT_INTERRUPTIBLE,
				MAX_SCHEDULE_TIMEOUT);
3997
	i915_gem_request_put(target);
3998

3999
	return ret < 0 ? ret : 0;
4000 4001
}

C
Chris Wilson 已提交
4002
struct i915_vma *
4003 4004
i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
			 const struct i915_ggtt_view *view,
4005
			 u64 size,
4006 4007
			 u64 alignment,
			 u64 flags)
4008
{
4009 4010
	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
	struct i915_address_space *vm = &dev_priv->ggtt.base;
4011 4012
	struct i915_vma *vma;
	int ret;
4013

4014 4015
	lockdep_assert_held(&obj->base.dev->struct_mutex);

4016
	vma = i915_vma_instance(obj, vm, view);
4017
	if (unlikely(IS_ERR(vma)))
C
Chris Wilson 已提交
4018
		return vma;
4019 4020 4021 4022

	if (i915_vma_misplaced(vma, size, alignment, flags)) {
		if (flags & PIN_NONBLOCK &&
		    (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)))
C
Chris Wilson 已提交
4023
			return ERR_PTR(-ENOSPC);
4024

4025 4026 4027 4028 4029 4030 4031 4032
		if (flags & PIN_MAPPABLE) {
			/* If the required space is larger than the available
			 * aperture, we will not able to find a slot for the
			 * object and unbinding the object now will be in
			 * vain. Worse, doing so may cause us to ping-pong
			 * the object in and out of the Global GTT and
			 * waste a lot of cycles under the mutex.
			 */
4033
			if (vma->fence_size > dev_priv->ggtt.mappable_end)
4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051
				return ERR_PTR(-E2BIG);

			/* If NONBLOCK is set the caller is optimistically
			 * trying to cache the full object within the mappable
			 * aperture, and *must* have a fallback in place for
			 * situations where we cannot bind the object. We
			 * can be a little more lax here and use the fallback
			 * more often to avoid costly migrations of ourselves
			 * and other objects within the aperture.
			 *
			 * Half-the-aperture is used as a simple heuristic.
			 * More interesting would to do search for a free
			 * block prior to making the commitment to unbind.
			 * That caters for the self-harm case, and with a
			 * little more heuristics (e.g. NOFAULT, NOEVICT)
			 * we could try to minimise harm to others.
			 */
			if (flags & PIN_NONBLOCK &&
4052
			    vma->fence_size > dev_priv->ggtt.mappable_end / 2)
4053 4054 4055
				return ERR_PTR(-ENOSPC);
		}

4056 4057
		WARN(i915_vma_is_pinned(vma),
		     "bo is already pinned in ggtt with incorrect alignment:"
4058 4059 4060
		     " offset=%08x, req.alignment=%llx,"
		     " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n",
		     i915_ggtt_offset(vma), alignment,
4061
		     !!(flags & PIN_MAPPABLE),
4062
		     i915_vma_is_map_and_fenceable(vma));
4063 4064
		ret = i915_vma_unbind(vma);
		if (ret)
C
Chris Wilson 已提交
4065
			return ERR_PTR(ret);
4066 4067
	}

C
Chris Wilson 已提交
4068 4069 4070
	ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
	if (ret)
		return ERR_PTR(ret);
4071

C
Chris Wilson 已提交
4072
	return vma;
4073 4074
}

4075
static __always_inline unsigned int __busy_read_flag(unsigned int id)
4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089
{
	/* Note that we could alias engines in the execbuf API, but
	 * that would be very unwise as it prevents userspace from
	 * fine control over engine selection. Ahem.
	 *
	 * This should be something like EXEC_MAX_ENGINE instead of
	 * I915_NUM_ENGINES.
	 */
	BUILD_BUG_ON(I915_NUM_ENGINES > 16);
	return 0x10000 << id;
}

static __always_inline unsigned int __busy_write_id(unsigned int id)
{
4090 4091 4092 4093 4094 4095 4096 4097 4098
	/* The uABI guarantees an active writer is also amongst the read
	 * engines. This would be true if we accessed the activity tracking
	 * under the lock, but as we perform the lookup of the object and
	 * its activity locklessly we can not guarantee that the last_write
	 * being active implies that we have set the same engine flag from
	 * last_read - hence we always set both read and write busy for
	 * last_write.
	 */
	return id | __busy_read_flag(id);
4099 4100
}

4101
static __always_inline unsigned int
4102
__busy_set_if_active(const struct dma_fence *fence,
4103 4104
		     unsigned int (*flag)(unsigned int id))
{
4105
	struct drm_i915_gem_request *rq;
4106

4107 4108 4109 4110
	/* We have to check the current hw status of the fence as the uABI
	 * guarantees forward progress. We could rely on the idle worker
	 * to eventually flush us, but to minimise latency just ask the
	 * hardware.
4111
	 *
4112
	 * Note we only report on the status of native fences.
4113
	 */
4114 4115 4116 4117 4118 4119 4120 4121
	if (!dma_fence_is_i915(fence))
		return 0;

	/* opencode to_request() in order to avoid const warnings */
	rq = container_of(fence, struct drm_i915_gem_request, fence);
	if (i915_gem_request_completed(rq))
		return 0;

4122
	return flag(rq->engine->uabi_id);
4123 4124
}

4125
static __always_inline unsigned int
4126
busy_check_reader(const struct dma_fence *fence)
4127
{
4128
	return __busy_set_if_active(fence, __busy_read_flag);
4129 4130
}

4131
static __always_inline unsigned int
4132
busy_check_writer(const struct dma_fence *fence)
4133
{
4134 4135 4136 4137
	if (!fence)
		return 0;

	return __busy_set_if_active(fence, __busy_write_id);
4138 4139
}

4140 4141
int
i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4142
		    struct drm_file *file)
4143 4144
{
	struct drm_i915_gem_busy *args = data;
4145
	struct drm_i915_gem_object *obj;
4146 4147
	struct reservation_object_list *list;
	unsigned int seq;
4148
	int err;
4149

4150
	err = -ENOENT;
4151 4152
	rcu_read_lock();
	obj = i915_gem_object_lookup_rcu(file, args->handle);
4153
	if (!obj)
4154
		goto out;
4155

4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173
	/* A discrepancy here is that we do not report the status of
	 * non-i915 fences, i.e. even though we may report the object as idle,
	 * a call to set-domain may still stall waiting for foreign rendering.
	 * This also means that wait-ioctl may report an object as busy,
	 * where busy-ioctl considers it idle.
	 *
	 * We trade the ability to warn of foreign fences to report on which
	 * i915 engines are active for the object.
	 *
	 * Alternatively, we can trade that extra information on read/write
	 * activity with
	 *	args->busy =
	 *		!reservation_object_test_signaled_rcu(obj->resv, true);
	 * to report the overall busyness. This is what the wait-ioctl does.
	 *
	 */
retry:
	seq = raw_read_seqcount(&obj->resv->seq);
4174

4175 4176
	/* Translate the exclusive fence to the READ *and* WRITE engine */
	args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl));
4177

4178 4179 4180 4181
	/* Translate shared fences to READ set of engines */
	list = rcu_dereference(obj->resv->fence);
	if (list) {
		unsigned int shared_count = list->shared_count, i;
4182

4183 4184 4185 4186 4187 4188
		for (i = 0; i < shared_count; ++i) {
			struct dma_fence *fence =
				rcu_dereference(list->shared[i]);

			args->busy |= busy_check_reader(fence);
		}
4189
	}
4190

4191 4192 4193 4194
	if (args->busy && read_seqcount_retry(&obj->resv->seq, seq))
		goto retry;

	err = 0;
4195 4196 4197
out:
	rcu_read_unlock();
	return err;
4198 4199 4200 4201 4202 4203
}

int
i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
			struct drm_file *file_priv)
{
4204
	return i915_gem_ring_throttle(dev, file_priv);
4205 4206
}

4207 4208 4209 4210
int
i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
		       struct drm_file *file_priv)
{
4211
	struct drm_i915_private *dev_priv = to_i915(dev);
4212
	struct drm_i915_gem_madvise *args = data;
4213
	struct drm_i915_gem_object *obj;
4214
	int err;
4215 4216 4217 4218 4219 4220 4221 4222 4223

	switch (args->madv) {
	case I915_MADV_DONTNEED:
	case I915_MADV_WILLNEED:
	    break;
	default:
	    return -EINVAL;
	}

4224
	obj = i915_gem_object_lookup(file_priv, args->handle);
4225 4226 4227 4228 4229 4230
	if (!obj)
		return -ENOENT;

	err = mutex_lock_interruptible(&obj->mm.lock);
	if (err)
		goto out;
4231

C
Chris Wilson 已提交
4232
	if (obj->mm.pages &&
4233
	    i915_gem_object_is_tiled(obj) &&
4234
	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4235 4236
		if (obj->mm.madv == I915_MADV_WILLNEED) {
			GEM_BUG_ON(!obj->mm.quirked);
C
Chris Wilson 已提交
4237
			__i915_gem_object_unpin_pages(obj);
4238 4239 4240
			obj->mm.quirked = false;
		}
		if (args->madv == I915_MADV_WILLNEED) {
4241
			GEM_BUG_ON(obj->mm.quirked);
C
Chris Wilson 已提交
4242
			__i915_gem_object_pin_pages(obj);
4243 4244
			obj->mm.quirked = true;
		}
4245 4246
	}

C
Chris Wilson 已提交
4247 4248
	if (obj->mm.madv != __I915_MADV_PURGED)
		obj->mm.madv = args->madv;
4249

C
Chris Wilson 已提交
4250
	/* if the object is no longer attached, discard its backing storage */
C
Chris Wilson 已提交
4251
	if (obj->mm.madv == I915_MADV_DONTNEED && !obj->mm.pages)
4252 4253
		i915_gem_object_truncate(obj);

C
Chris Wilson 已提交
4254
	args->retained = obj->mm.madv != __I915_MADV_PURGED;
4255
	mutex_unlock(&obj->mm.lock);
C
Chris Wilson 已提交
4256

4257
out:
4258
	i915_gem_object_put(obj);
4259
	return err;
4260 4261
}

4262 4263 4264 4265 4266 4267 4268
static void
frontbuffer_retire(struct i915_gem_active *active,
		   struct drm_i915_gem_request *request)
{
	struct drm_i915_gem_object *obj =
		container_of(active, typeof(*obj), frontbuffer_write);

4269
	intel_fb_obj_flush(obj, ORIGIN_CS);
4270 4271
}

4272 4273
void i915_gem_object_init(struct drm_i915_gem_object *obj,
			  const struct drm_i915_gem_object_ops *ops)
4274
{
4275 4276
	mutex_init(&obj->mm.lock);

4277
	INIT_LIST_HEAD(&obj->global_link);
4278
	INIT_LIST_HEAD(&obj->userfault_link);
B
Ben Widawsky 已提交
4279
	INIT_LIST_HEAD(&obj->vma_list);
4280
	INIT_LIST_HEAD(&obj->lut_list);
4281
	INIT_LIST_HEAD(&obj->batch_pool_link);
4282

4283 4284
	obj->ops = ops;

4285 4286 4287
	reservation_object_init(&obj->__builtin_resv);
	obj->resv = &obj->__builtin_resv;

4288
	obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
4289
	init_request_active(&obj->frontbuffer_write, frontbuffer_retire);
C
Chris Wilson 已提交
4290 4291 4292 4293

	obj->mm.madv = I915_MADV_WILLNEED;
	INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);
	mutex_init(&obj->mm.get_page.lock);
4294

4295
	i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
4296 4297
}

4298
static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4299 4300
	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
		 I915_GEM_OBJECT_IS_SHRINKABLE,
4301

4302 4303
	.get_pages = i915_gem_object_get_pages_gtt,
	.put_pages = i915_gem_object_put_pages_gtt,
4304 4305

	.pwrite = i915_gem_object_pwrite_gtt,
4306 4307
};

M
Matthew Auld 已提交
4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331
static int i915_gem_object_create_shmem(struct drm_device *dev,
					struct drm_gem_object *obj,
					size_t size)
{
	struct drm_i915_private *i915 = to_i915(dev);
	unsigned long flags = VM_NORESERVE;
	struct file *filp;

	drm_gem_private_object_init(dev, obj, size);

	if (i915->mm.gemfs)
		filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
						 flags);
	else
		filp = shmem_file_setup("i915", size, flags);

	if (IS_ERR(filp))
		return PTR_ERR(filp);

	obj->filp = filp;

	return 0;
}

4332
struct drm_i915_gem_object *
4333
i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size)
4334
{
4335
	struct drm_i915_gem_object *obj;
4336
	struct address_space *mapping;
4337
	unsigned int cache_level;
D
Daniel Vetter 已提交
4338
	gfp_t mask;
4339
	int ret;
4340

4341 4342 4343 4344 4345
	/* There is a prevalence of the assumption that we fit the object's
	 * page count inside a 32bit _signed_ variable. Let's document this and
	 * catch if we ever need to fix it. In the meantime, if you do spot
	 * such a local variable, please consider fixing!
	 */
4346
	if (size >> PAGE_SHIFT > INT_MAX)
4347 4348 4349 4350 4351
		return ERR_PTR(-E2BIG);

	if (overflows_type(size, obj->base.size))
		return ERR_PTR(-E2BIG);

4352
	obj = i915_gem_object_alloc(dev_priv);
4353
	if (obj == NULL)
4354
		return ERR_PTR(-ENOMEM);
4355

M
Matthew Auld 已提交
4356
	ret = i915_gem_object_create_shmem(&dev_priv->drm, &obj->base, size);
4357 4358
	if (ret)
		goto fail;
4359

4360
	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4361
	if (IS_I965GM(dev_priv) || IS_I965G(dev_priv)) {
4362 4363 4364 4365 4366
		/* 965gm cannot relocate objects above 4GiB. */
		mask &= ~__GFP_HIGHMEM;
		mask |= __GFP_DMA32;
	}

4367
	mapping = obj->base.filp->f_mapping;
4368
	mapping_set_gfp_mask(mapping, mask);
4369
	GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
4370

4371
	i915_gem_object_init(obj, &i915_gem_object_ops);
4372

4373 4374
	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4375

4376
	if (HAS_LLC(dev_priv))
4377
		/* On some devices, we can have the GPU use the LLC (the CPU
4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388
		 * cache) for about a 10% performance improvement
		 * compared to uncached.  Graphics requests other than
		 * display scanout are coherent with the CPU in
		 * accessing this cache.  This means in this mode we
		 * don't need to clflush on the CPU side, and on the
		 * GPU side we only need to flush internal caches to
		 * get data visible to the CPU.
		 *
		 * However, we maintain the display planes as UC, and so
		 * need to rebind when first used as such.
		 */
4389 4390 4391
		cache_level = I915_CACHE_LLC;
	else
		cache_level = I915_CACHE_NONE;
4392

4393
	i915_gem_object_set_cache_coherency(obj, cache_level);
4394

4395 4396
	trace_i915_gem_object_create(obj);

4397
	return obj;
4398 4399 4400 4401

fail:
	i915_gem_object_free(obj);
	return ERR_PTR(ret);
4402 4403
}

4404 4405 4406 4407 4408 4409 4410 4411
static bool discard_backing_storage(struct drm_i915_gem_object *obj)
{
	/* If we are the last user of the backing storage (be it shmemfs
	 * pages or stolen etc), we know that the pages are going to be
	 * immediately released. In this case, we can then skip copying
	 * back the contents from the GPU.
	 */

C
Chris Wilson 已提交
4412
	if (obj->mm.madv != I915_MADV_WILLNEED)
4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427
		return false;

	if (obj->base.filp == NULL)
		return true;

	/* At first glance, this looks racy, but then again so would be
	 * userspace racing mmap against close. However, the first external
	 * reference to the filp can only be obtained through the
	 * i915_gem_mmap_ioctl() which safeguards us against the user
	 * acquiring such a reference whilst we are in the middle of
	 * freeing the object.
	 */
	return atomic_long_read(&obj->base.filp->f_count) == 1;
}

4428 4429
static void __i915_gem_free_objects(struct drm_i915_private *i915,
				    struct llist_node *freed)
4430
{
4431
	struct drm_i915_gem_object *obj, *on;
4432

4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446
	mutex_lock(&i915->drm.struct_mutex);
	intel_runtime_pm_get(i915);
	llist_for_each_entry(obj, freed, freed) {
		struct i915_vma *vma, *vn;

		trace_i915_gem_object_destroy(obj);

		GEM_BUG_ON(i915_gem_object_is_active(obj));
		list_for_each_entry_safe(vma, vn,
					 &obj->vma_list, obj_link) {
			GEM_BUG_ON(i915_vma_is_active(vma));
			vma->flags &= ~I915_VMA_PIN_MASK;
			i915_vma_close(vma);
		}
4447 4448
		GEM_BUG_ON(!list_empty(&obj->vma_list));
		GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree));
4449

4450
		list_del(&obj->global_link);
4451 4452 4453 4454
	}
	intel_runtime_pm_put(i915);
	mutex_unlock(&i915->drm.struct_mutex);

4455 4456
	cond_resched();

4457 4458 4459
	llist_for_each_entry_safe(obj, on, freed, freed) {
		GEM_BUG_ON(obj->bind_count);
		GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits));
4460
		GEM_BUG_ON(!list_empty(&obj->lut_list));
4461 4462 4463

		if (obj->ops->release)
			obj->ops->release(obj);
4464

4465 4466
		if (WARN_ON(i915_gem_object_has_pinned_pages(obj)))
			atomic_set(&obj->mm.pages_pin_count, 0);
4467
		__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
4468 4469 4470 4471 4472
		GEM_BUG_ON(obj->mm.pages);

		if (obj->base.import_attach)
			drm_prime_gem_destroy(&obj->base, NULL);

4473
		reservation_object_fini(&obj->__builtin_resv);
4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495
		drm_gem_object_release(&obj->base);
		i915_gem_info_remove_obj(i915, obj->base.size);

		kfree(obj->bit_17);
		i915_gem_object_free(obj);
	}
}

static void i915_gem_flush_free_objects(struct drm_i915_private *i915)
{
	struct llist_node *freed;

	freed = llist_del_all(&i915->mm.free_list);
	if (unlikely(freed))
		__i915_gem_free_objects(i915, freed);
}

static void __i915_gem_free_work(struct work_struct *work)
{
	struct drm_i915_private *i915 =
		container_of(work, struct drm_i915_private, mm.free_work);
	struct llist_node *freed;
4496

4497 4498 4499 4500 4501 4502 4503
	/* All file-owned VMA should have been released by this point through
	 * i915_gem_close_object(), or earlier by i915_gem_context_close().
	 * However, the object may also be bound into the global GTT (e.g.
	 * older GPUs without per-process support, or for direct access through
	 * the GTT either for the user or for scanout). Those VMA still need to
	 * unbound now.
	 */
4504

4505
	while ((freed = llist_del_all(&i915->mm.free_list))) {
4506
		__i915_gem_free_objects(i915, freed);
4507 4508 4509
		if (need_resched())
			break;
	}
4510
}
4511

4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525
static void __i915_gem_free_object_rcu(struct rcu_head *head)
{
	struct drm_i915_gem_object *obj =
		container_of(head, typeof(*obj), rcu);
	struct drm_i915_private *i915 = to_i915(obj->base.dev);

	/* We can't simply use call_rcu() from i915_gem_free_object()
	 * as we need to block whilst unbinding, and the call_rcu
	 * task may be called from softirq context. So we take a
	 * detour through a worker.
	 */
	if (llist_add(&obj->freed, &i915->mm.free_list))
		schedule_work(&i915->mm.free_work);
}
4526

4527 4528 4529
void i915_gem_free_object(struct drm_gem_object *gem_obj)
{
	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
C
Chris Wilson 已提交
4530

4531 4532 4533
	if (obj->mm.quirked)
		__i915_gem_object_unpin_pages(obj);

4534
	if (discard_backing_storage(obj))
C
Chris Wilson 已提交
4535
		obj->mm.madv = I915_MADV_DONTNEED;
4536

4537 4538 4539 4540 4541 4542
	/* Before we free the object, make sure any pure RCU-only
	 * read-side critical sections are complete, e.g.
	 * i915_gem_busy_ioctl(). For the corresponding synchronized
	 * lookup see i915_gem_object_lookup_rcu().
	 */
	call_rcu(&obj->rcu, __i915_gem_free_object_rcu);
4543 4544
}

4545 4546 4547 4548
void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
{
	lockdep_assert_held(&obj->base.dev->struct_mutex);

4549 4550
	if (!i915_gem_object_has_active_reference(obj) &&
	    i915_gem_object_is_active(obj))
4551 4552 4553 4554 4555
		i915_gem_object_set_active_reference(obj);
	else
		i915_gem_object_put(obj);
}

4556 4557 4558 4559 4560 4561
static void assert_kernel_context_is_current(struct drm_i915_private *dev_priv)
{
	struct intel_engine_cs *engine;
	enum intel_engine_id id;

	for_each_engine(engine, dev_priv, id)
4562 4563
		GEM_BUG_ON(engine->last_retired_context &&
			   !i915_gem_context_is_kernel(engine->last_retired_context));
4564 4565
}

4566 4567
void i915_gem_sanitize(struct drm_i915_private *i915)
{
4568 4569 4570 4571 4572 4573
	if (i915_terminally_wedged(&i915->gpu_error)) {
		mutex_lock(&i915->drm.struct_mutex);
		i915_gem_unset_wedged(i915);
		mutex_unlock(&i915->drm.struct_mutex);
	}

4574 4575 4576 4577 4578 4579
	/*
	 * If we inherit context state from the BIOS or earlier occupants
	 * of the GPU, the GPU may be in an inconsistent state when we
	 * try to take over. The only way to remove the earlier state
	 * is by resetting. However, resetting on earlier gen is tricky as
	 * it may impact the display and we are uncertain about the stability
4580
	 * of the reset, so this could be applied to even earlier gen.
4581
	 */
4582
	if (INTEL_GEN(i915) >= 5) {
4583 4584 4585 4586 4587
		int reset = intel_gpu_reset(i915, ALL_ENGINES);
		WARN_ON(reset && reset != -ENODEV);
	}
}

4588
int i915_gem_suspend(struct drm_i915_private *dev_priv)
4589
{
4590
	struct drm_device *dev = &dev_priv->drm;
4591
	int ret;
4592

4593
	intel_runtime_pm_get(dev_priv);
4594 4595
	intel_suspend_gt_powersave(dev_priv);

4596
	mutex_lock(&dev->struct_mutex);
4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607

	/* We have to flush all the executing contexts to main memory so
	 * that they can saved in the hibernation image. To ensure the last
	 * context image is coherent, we have to switch away from it. That
	 * leaves the dev_priv->kernel_context still active when
	 * we actually suspend, and its image in memory may not match the GPU
	 * state. Fortunately, the kernel_context is disposable and we do
	 * not rely on its state.
	 */
	ret = i915_gem_switch_to_kernel_context(dev_priv);
	if (ret)
4608
		goto err_unlock;
4609

4610 4611 4612
	ret = i915_gem_wait_for_idle(dev_priv,
				     I915_WAIT_INTERRUPTIBLE |
				     I915_WAIT_LOCKED);
4613
	if (ret && ret != -EIO)
4614
		goto err_unlock;
4615

4616
	assert_kernel_context_is_current(dev_priv);
4617
	i915_gem_contexts_lost(dev_priv);
4618 4619
	mutex_unlock(&dev->struct_mutex);

4620 4621
	intel_guc_suspend(dev_priv);

4622
	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4623
	cancel_delayed_work_sync(&dev_priv->gt.retire_work);
4624 4625 4626 4627

	/* As the idle_work is rearming if it detects a race, play safe and
	 * repeat the flush until it is definitely idle.
	 */
4628
	drain_delayed_work(&dev_priv->gt.idle_work);
4629

4630 4631 4632
	/* Assert that we sucessfully flushed all the work and
	 * reset the GPU back to its idle, low power state.
	 */
4633
	WARN_ON(dev_priv->gt.awake);
4634 4635
	if (WARN_ON(!intel_engines_are_idle(dev_priv)))
		i915_gem_set_wedged(dev_priv); /* no hope, discard everything */
4636

4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655
	/*
	 * Neither the BIOS, ourselves or any other kernel
	 * expects the system to be in execlists mode on startup,
	 * so we need to reset the GPU back to legacy mode. And the only
	 * known way to disable logical contexts is through a GPU reset.
	 *
	 * So in order to leave the system in a known default configuration,
	 * always reset the GPU upon unload and suspend. Afterwards we then
	 * clean up the GEM state tracking, flushing off the requests and
	 * leaving the system in a known idle state.
	 *
	 * Note that is of the upmost importance that the GPU is idle and
	 * all stray writes are flushed *before* we dismantle the backing
	 * storage for the pinned objects.
	 *
	 * However, since we are uncertain that resetting the GPU on older
	 * machines is a good idea, we don't - just in case it leaves the
	 * machine in an unusable condition.
	 */
4656
	i915_gem_sanitize(dev_priv);
4657 4658 4659

	intel_runtime_pm_put(dev_priv);
	return 0;
4660

4661
err_unlock:
4662
	mutex_unlock(&dev->struct_mutex);
4663
	intel_runtime_pm_put(dev_priv);
4664
	return ret;
4665 4666
}

4667
void i915_gem_resume(struct drm_i915_private *dev_priv)
4668
{
4669
	struct drm_device *dev = &dev_priv->drm;
4670

4671 4672
	WARN_ON(dev_priv->gt.awake);

4673
	mutex_lock(&dev->struct_mutex);
4674
	i915_gem_restore_gtt_mappings(dev_priv);
4675
	i915_gem_restore_fences(dev_priv);
4676 4677 4678 4679 4680

	/* As we didn't flush the kernel context before suspend, we cannot
	 * guarantee that the context image is complete. So let's just reset
	 * it and start again.
	 */
4681
	dev_priv->gt.resume(dev_priv);
4682 4683 4684 4685

	mutex_unlock(&dev->struct_mutex);
}

4686
void i915_gem_init_swizzling(struct drm_i915_private *dev_priv)
4687
{
4688
	if (INTEL_GEN(dev_priv) < 5 ||
4689 4690 4691 4692 4693 4694
	    dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
		return;

	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
				 DISP_TILE_SURFACE_SWIZZLING);

4695
	if (IS_GEN5(dev_priv))
4696 4697
		return;

4698
	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4699
	if (IS_GEN6(dev_priv))
4700
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4701
	else if (IS_GEN7(dev_priv))
4702
		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4703
	else if (IS_GEN8(dev_priv))
B
Ben Widawsky 已提交
4704
		I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4705 4706
	else
		BUG();
4707
}
D
Daniel Vetter 已提交
4708

4709
static void init_unused_ring(struct drm_i915_private *dev_priv, u32 base)
4710 4711 4712 4713 4714 4715 4716
{
	I915_WRITE(RING_CTL(base), 0);
	I915_WRITE(RING_HEAD(base), 0);
	I915_WRITE(RING_TAIL(base), 0);
	I915_WRITE(RING_START(base), 0);
}

4717
static void init_unused_rings(struct drm_i915_private *dev_priv)
4718
{
4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730
	if (IS_I830(dev_priv)) {
		init_unused_ring(dev_priv, PRB1_BASE);
		init_unused_ring(dev_priv, SRB0_BASE);
		init_unused_ring(dev_priv, SRB1_BASE);
		init_unused_ring(dev_priv, SRB2_BASE);
		init_unused_ring(dev_priv, SRB3_BASE);
	} else if (IS_GEN2(dev_priv)) {
		init_unused_ring(dev_priv, SRB0_BASE);
		init_unused_ring(dev_priv, SRB1_BASE);
	} else if (IS_GEN3(dev_priv)) {
		init_unused_ring(dev_priv, PRB1_BASE);
		init_unused_ring(dev_priv, PRB2_BASE);
4731 4732 4733
	}
}

4734
static int __i915_gem_restart_engines(void *data)
4735
{
4736
	struct drm_i915_private *i915 = data;
4737
	struct intel_engine_cs *engine;
4738
	enum intel_engine_id id;
4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751
	int err;

	for_each_engine(engine, i915, id) {
		err = engine->init_hw(engine);
		if (err)
			return err;
	}

	return 0;
}

int i915_gem_init_hw(struct drm_i915_private *dev_priv)
{
C
Chris Wilson 已提交
4752
	int ret;
4753

4754 4755
	dev_priv->gt.last_init_time = ktime_get();

4756 4757 4758
	/* Double layer security blanket, see i915_gem_init() */
	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);

4759
	if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9)
4760
		I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4761

4762
	if (IS_HASWELL(dev_priv))
4763
		I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ?
4764
			   LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4765

4766
	if (HAS_PCH_NOP(dev_priv)) {
4767
		if (IS_IVYBRIDGE(dev_priv)) {
4768 4769 4770
			u32 temp = I915_READ(GEN7_MSG_CTL);
			temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
			I915_WRITE(GEN7_MSG_CTL, temp);
4771
		} else if (INTEL_GEN(dev_priv) >= 7) {
4772 4773 4774 4775
			u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
			temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
			I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
		}
4776 4777
	}

4778
	i915_gem_init_swizzling(dev_priv);
4779

4780 4781 4782 4783 4784 4785
	/*
	 * At least 830 can leave some of the unused rings
	 * "active" (ie. head != tail) after resume which
	 * will prevent c3 entry. Makes sure all unused rings
	 * are totally idle.
	 */
4786
	init_unused_rings(dev_priv);
4787

4788
	BUG_ON(!dev_priv->kernel_context);
4789

4790
	ret = i915_ppgtt_init_hw(dev_priv);
4791 4792 4793 4794 4795 4796
	if (ret) {
		DRM_ERROR("PPGTT enable HW failed %d\n", ret);
		goto out;
	}

	/* Need to do basic initialisation of all rings first: */
4797 4798 4799
	ret = __i915_gem_restart_engines(dev_priv);
	if (ret)
		goto out;
4800

4801
	intel_mocs_init_l3cc_table(dev_priv);
4802

4803 4804 4805 4806
	/* We can't enable contexts until all firmware is loaded */
	ret = intel_uc_init_hw(dev_priv);
	if (ret)
		goto out;
4807

4808 4809
out:
	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4810
	return ret;
4811 4812
}

4813 4814 4815 4816 4817 4818
bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
{
	if (INTEL_INFO(dev_priv)->gen < 6)
		return false;

	/* TODO: make semaphores and Execlists play nicely together */
4819
	if (i915_modparams.enable_execlists)
4820 4821 4822 4823 4824 4825
		return false;

	if (value >= 0)
		return value;

	/* Enable semaphores on SNB when IO remapping is off */
4826
	if (IS_GEN6(dev_priv) && intel_vtd_active())
4827 4828 4829 4830 4831
		return false;

	return true;
}

4832
int i915_gem_init(struct drm_i915_private *dev_priv)
4833 4834 4835
{
	int ret;

4836
	mutex_lock(&dev_priv->drm.struct_mutex);
4837

4838 4839 4840 4841 4842 4843 4844 4845 4846
	/*
	 * We need to fallback to 4K pages since gvt gtt handling doesn't
	 * support huge page entries - we will need to check either hypervisor
	 * mm can support huge guest page or just do emulation in gvt.
	 */
	if (intel_vgpu_active(dev_priv))
		mkwrite_device_info(dev_priv)->page_sizes =
			I915_GTT_PAGE_SIZE_4K;

4847
	dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1);
4848

4849
	if (!i915_modparams.enable_execlists) {
4850
		dev_priv->gt.resume = intel_legacy_submission_resume;
4851
		dev_priv->gt.cleanup_engine = intel_engine_cleanup;
4852
	} else {
4853
		dev_priv->gt.resume = intel_lr_context_resume;
4854
		dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
4855 4856
	}

4857 4858 4859 4860 4861 4862 4863 4864
	/* This is just a security blanket to placate dragons.
	 * On some systems, we very sporadically observe that the first TLBs
	 * used by the CS may be stale, despite us poking the TLB reset. If
	 * we hold the forcewake during initialisation these problems
	 * just magically go away.
	 */
	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);

4865 4866 4867
	ret = i915_gem_init_userptr(dev_priv);
	if (ret)
		goto out_unlock;
4868 4869 4870 4871

	ret = i915_gem_init_ggtt(dev_priv);
	if (ret)
		goto out_unlock;
4872

4873
	ret = i915_gem_contexts_init(dev_priv);
4874 4875
	if (ret)
		goto out_unlock;
4876

4877
	ret = intel_engines_init(dev_priv);
D
Daniel Vetter 已提交
4878
	if (ret)
4879
		goto out_unlock;
4880

4881
	ret = i915_gem_init_hw(dev_priv);
4882
	if (ret == -EIO) {
4883
		/* Allow engine initialisation to fail by marking the GPU as
4884 4885 4886 4887
		 * wedged. But we only want to do this where the GPU is angry,
		 * for all other failure, such as an allocation failure, bail.
		 */
		DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4888
		i915_gem_set_wedged(dev_priv);
4889
		ret = 0;
4890
	}
4891 4892

out_unlock:
4893
	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4894
	mutex_unlock(&dev_priv->drm.struct_mutex);
4895

4896
	return ret;
4897 4898
}

4899 4900 4901 4902 4903
void i915_gem_init_mmio(struct drm_i915_private *i915)
{
	i915_gem_sanitize(i915);
}

4904
void
4905
i915_gem_cleanup_engines(struct drm_i915_private *dev_priv)
4906
{
4907
	struct intel_engine_cs *engine;
4908
	enum intel_engine_id id;
4909

4910
	for_each_engine(engine, dev_priv, id)
4911
		dev_priv->gt.cleanup_engine(engine);
4912 4913
}

4914 4915 4916
void
i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
{
4917
	int i;
4918 4919 4920 4921

	if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
	    !IS_CHERRYVIEW(dev_priv))
		dev_priv->num_fence_regs = 32;
4922 4923 4924
	else if (INTEL_INFO(dev_priv)->gen >= 4 ||
		 IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
		 IS_G33(dev_priv) || IS_PINEVIEW(dev_priv))
4925 4926 4927 4928
		dev_priv->num_fence_regs = 16;
	else
		dev_priv->num_fence_regs = 8;

4929
	if (intel_vgpu_active(dev_priv))
4930 4931 4932 4933
		dev_priv->num_fence_regs =
				I915_READ(vgtif_reg(avail_rs.fence_num));

	/* Initialize fence registers to zero */
4934 4935 4936 4937 4938 4939 4940
	for (i = 0; i < dev_priv->num_fence_regs; i++) {
		struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i];

		fence->i915 = dev_priv;
		fence->id = i;
		list_add_tail(&fence->link, &dev_priv->mm.fence_list);
	}
4941
	i915_gem_restore_fences(dev_priv);
4942

4943
	i915_gem_detect_bit_6_swizzle(dev_priv);
4944 4945
}

4946
int
4947
i915_gem_load_init(struct drm_i915_private *dev_priv)
4948
{
4949
	int err = -ENOMEM;
4950

4951 4952
	dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
	if (!dev_priv->objects)
4953 4954
		goto err_out;

4955 4956
	dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
	if (!dev_priv->vmas)
4957 4958
		goto err_objects;

4959 4960 4961 4962
	dev_priv->luts = KMEM_CACHE(i915_lut_handle, 0);
	if (!dev_priv->luts)
		goto err_vmas;

4963 4964 4965
	dev_priv->requests = KMEM_CACHE(drm_i915_gem_request,
					SLAB_HWCACHE_ALIGN |
					SLAB_RECLAIM_ACCOUNT |
4966
					SLAB_TYPESAFE_BY_RCU);
4967
	if (!dev_priv->requests)
4968
		goto err_luts;
4969

4970 4971 4972 4973 4974 4975
	dev_priv->dependencies = KMEM_CACHE(i915_dependency,
					    SLAB_HWCACHE_ALIGN |
					    SLAB_RECLAIM_ACCOUNT);
	if (!dev_priv->dependencies)
		goto err_requests;

4976 4977 4978 4979
	dev_priv->priorities = KMEM_CACHE(i915_priolist, SLAB_HWCACHE_ALIGN);
	if (!dev_priv->priorities)
		goto err_dependencies;

4980 4981
	mutex_lock(&dev_priv->drm.struct_mutex);
	INIT_LIST_HEAD(&dev_priv->gt.timelines);
4982
	err = i915_gem_timeline_init__global(dev_priv);
4983 4984
	mutex_unlock(&dev_priv->drm.struct_mutex);
	if (err)
4985
		goto err_priorities;
4986

4987 4988
	INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work);
	init_llist_head(&dev_priv->mm.free_list);
C
Chris Wilson 已提交
4989 4990
	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
4991
	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4992
	INIT_LIST_HEAD(&dev_priv->mm.userfault_list);
4993
	INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
4994
			  i915_gem_retire_work_handler);
4995
	INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
4996
			  i915_gem_idle_work_handler);
4997
	init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
4998
	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
4999

5000 5001
	atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);

5002
	spin_lock_init(&dev_priv->fb_tracking.lock);
5003

M
Matthew Auld 已提交
5004 5005 5006 5007
	err = i915_gemfs_init(dev_priv);
	if (err)
		DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", err);

5008 5009
	return 0;

5010 5011
err_priorities:
	kmem_cache_destroy(dev_priv->priorities);
5012 5013
err_dependencies:
	kmem_cache_destroy(dev_priv->dependencies);
5014 5015
err_requests:
	kmem_cache_destroy(dev_priv->requests);
5016 5017
err_luts:
	kmem_cache_destroy(dev_priv->luts);
5018 5019 5020 5021 5022 5023
err_vmas:
	kmem_cache_destroy(dev_priv->vmas);
err_objects:
	kmem_cache_destroy(dev_priv->objects);
err_out:
	return err;
5024
}
5025

5026
void i915_gem_load_cleanup(struct drm_i915_private *dev_priv)
5027
{
5028
	i915_gem_drain_freed_objects(dev_priv);
5029
	WARN_ON(!llist_empty(&dev_priv->mm.free_list));
5030
	WARN_ON(dev_priv->mm.object_count);
5031

5032 5033 5034 5035 5036
	mutex_lock(&dev_priv->drm.struct_mutex);
	i915_gem_timeline_fini(&dev_priv->gt.global_timeline);
	WARN_ON(!list_empty(&dev_priv->gt.timelines));
	mutex_unlock(&dev_priv->drm.struct_mutex);

5037
	kmem_cache_destroy(dev_priv->priorities);
5038
	kmem_cache_destroy(dev_priv->dependencies);
5039
	kmem_cache_destroy(dev_priv->requests);
5040
	kmem_cache_destroy(dev_priv->luts);
5041 5042
	kmem_cache_destroy(dev_priv->vmas);
	kmem_cache_destroy(dev_priv->objects);
5043 5044 5045

	/* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
	rcu_barrier();
M
Matthew Auld 已提交
5046 5047

	i915_gemfs_fini(dev_priv);
5048 5049
}

5050 5051
int i915_gem_freeze(struct drm_i915_private *dev_priv)
{
5052 5053 5054
	/* Discard all purgeable objects, let userspace recover those as
	 * required after resuming.
	 */
5055 5056 5057 5058 5059
	i915_gem_shrink_all(dev_priv);

	return 0;
}

5060 5061 5062
int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
{
	struct drm_i915_gem_object *obj;
5063 5064 5065 5066 5067
	struct list_head *phases[] = {
		&dev_priv->mm.unbound_list,
		&dev_priv->mm.bound_list,
		NULL
	}, **p;
5068 5069 5070 5071 5072 5073 5074 5075 5076 5077

	/* Called just before we write the hibernation image.
	 *
	 * We need to update the domain tracking to reflect that the CPU
	 * will be accessing all the pages to create and restore from the
	 * hibernation, and so upon restoration those pages will be in the
	 * CPU domain.
	 *
	 * To make sure the hibernation image contains the latest state,
	 * we update that state just before writing out the image.
5078 5079
	 *
	 * To try and reduce the hibernation image, we manually shrink
5080
	 * the objects as well, see i915_gem_freeze()
5081 5082
	 */

5083
	i915_gem_shrink(dev_priv, -1UL, NULL, I915_SHRINK_UNBOUND);
5084
	i915_gem_drain_freed_objects(dev_priv);
5085

5086
	mutex_lock(&dev_priv->drm.struct_mutex);
5087
	for (p = phases; *p; p++) {
5088 5089
		list_for_each_entry(obj, *p, global_link)
			__start_cpu_write(obj);
5090
	}
5091
	mutex_unlock(&dev_priv->drm.struct_mutex);
5092 5093 5094 5095

	return 0;
}

5096
void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5097
{
5098
	struct drm_i915_file_private *file_priv = file->driver_priv;
5099
	struct drm_i915_gem_request *request;
5100 5101 5102 5103 5104

	/* Clean up our request list when the client is going away, so that
	 * later retire_requests won't dereference our soon-to-be-gone
	 * file_priv.
	 */
5105
	spin_lock(&file_priv->mm.lock);
5106
	list_for_each_entry(request, &file_priv->mm.request_list, client_link)
5107
		request->file_priv = NULL;
5108
	spin_unlock(&file_priv->mm.lock);
5109 5110
}

5111
int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
5112 5113
{
	struct drm_i915_file_private *file_priv;
5114
	int ret;
5115

5116
	DRM_DEBUG("\n");
5117 5118 5119 5120 5121 5122

	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
	if (!file_priv)
		return -ENOMEM;

	file->driver_priv = file_priv;
5123
	file_priv->dev_priv = i915;
5124
	file_priv->file = file;
5125 5126 5127 5128

	spin_lock_init(&file_priv->mm.lock);
	INIT_LIST_HEAD(&file_priv->mm.request_list);

5129
	file_priv->bsd_engine = -1;
5130

5131
	ret = i915_gem_context_open(i915, file);
5132 5133
	if (ret)
		kfree(file_priv);
5134

5135
	return ret;
5136 5137
}

5138 5139
/**
 * i915_gem_track_fb - update frontbuffer tracking
5140 5141 5142
 * @old: current GEM buffer for the frontbuffer slots
 * @new: new GEM buffer for the frontbuffer slots
 * @frontbuffer_bits: bitmask of frontbuffer slots
5143 5144 5145 5146
 *
 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
 * from @old and setting them in @new. Both @old and @new can be NULL.
 */
5147 5148 5149 5150
void i915_gem_track_fb(struct drm_i915_gem_object *old,
		       struct drm_i915_gem_object *new,
		       unsigned frontbuffer_bits)
{
5151 5152 5153 5154 5155 5156 5157 5158 5159
	/* Control of individual bits within the mask are guarded by
	 * the owning plane->mutex, i.e. we can never see concurrent
	 * manipulation of individual bits. But since the bitfield as a whole
	 * is updated using RMW, we need to use atomics in order to update
	 * the bits.
	 */
	BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
		     sizeof(atomic_t) * BITS_PER_BYTE);

5160
	if (old) {
5161 5162
		WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits));
		atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits);
5163 5164 5165
	}

	if (new) {
5166 5167
		WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits);
		atomic_or(frontbuffer_bits, &new->frontbuffer_bits);
5168 5169 5170
	}
}

5171 5172
/* Allocate a new GEM object and fill it with the supplied data */
struct drm_i915_gem_object *
5173
i915_gem_object_create_from_data(struct drm_i915_private *dev_priv,
5174 5175 5176
			         const void *data, size_t size)
{
	struct drm_i915_gem_object *obj;
5177 5178 5179
	struct file *file;
	size_t offset;
	int err;
5180

5181
	obj = i915_gem_object_create(dev_priv, round_up(size, PAGE_SIZE));
5182
	if (IS_ERR(obj))
5183 5184
		return obj;

5185
	GEM_BUG_ON(obj->base.write_domain != I915_GEM_DOMAIN_CPU);
5186

5187 5188 5189 5190 5191 5192
	file = obj->base.filp;
	offset = 0;
	do {
		unsigned int len = min_t(typeof(size), size, PAGE_SIZE);
		struct page *page;
		void *pgdata, *vaddr;
5193

5194 5195 5196 5197 5198
		err = pagecache_write_begin(file, file->f_mapping,
					    offset, len, 0,
					    &page, &pgdata);
		if (err < 0)
			goto fail;
5199

5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213
		vaddr = kmap(page);
		memcpy(vaddr, data, len);
		kunmap(page);

		err = pagecache_write_end(file, file->f_mapping,
					  offset, len, len,
					  page, pgdata);
		if (err < 0)
			goto fail;

		size -= len;
		data += len;
		offset += len;
	} while (size);
5214 5215 5216 5217

	return obj;

fail:
5218
	i915_gem_object_put(obj);
5219
	return ERR_PTR(err);
5220
}
5221 5222 5223 5224 5225 5226

struct scatterlist *
i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
		       unsigned int n,
		       unsigned int *offset)
{
C
Chris Wilson 已提交
5227
	struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
5228 5229 5230 5231 5232
	struct scatterlist *sg;
	unsigned int idx, count;

	might_sleep();
	GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
C
Chris Wilson 已提交
5233
	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357

	/* As we iterate forward through the sg, we record each entry in a
	 * radixtree for quick repeated (backwards) lookups. If we have seen
	 * this index previously, we will have an entry for it.
	 *
	 * Initial lookup is O(N), but this is amortized to O(1) for
	 * sequential page access (where each new request is consecutive
	 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
	 * i.e. O(1) with a large constant!
	 */
	if (n < READ_ONCE(iter->sg_idx))
		goto lookup;

	mutex_lock(&iter->lock);

	/* We prefer to reuse the last sg so that repeated lookup of this
	 * (or the subsequent) sg are fast - comparing against the last
	 * sg is faster than going through the radixtree.
	 */

	sg = iter->sg_pos;
	idx = iter->sg_idx;
	count = __sg_page_count(sg);

	while (idx + count <= n) {
		unsigned long exception, i;
		int ret;

		/* If we cannot allocate and insert this entry, or the
		 * individual pages from this range, cancel updating the
		 * sg_idx so that on this lookup we are forced to linearly
		 * scan onwards, but on future lookups we will try the
		 * insertion again (in which case we need to be careful of
		 * the error return reporting that we have already inserted
		 * this index).
		 */
		ret = radix_tree_insert(&iter->radix, idx, sg);
		if (ret && ret != -EEXIST)
			goto scan;

		exception =
			RADIX_TREE_EXCEPTIONAL_ENTRY |
			idx << RADIX_TREE_EXCEPTIONAL_SHIFT;
		for (i = 1; i < count; i++) {
			ret = radix_tree_insert(&iter->radix, idx + i,
						(void *)exception);
			if (ret && ret != -EEXIST)
				goto scan;
		}

		idx += count;
		sg = ____sg_next(sg);
		count = __sg_page_count(sg);
	}

scan:
	iter->sg_pos = sg;
	iter->sg_idx = idx;

	mutex_unlock(&iter->lock);

	if (unlikely(n < idx)) /* insertion completed by another thread */
		goto lookup;

	/* In case we failed to insert the entry into the radixtree, we need
	 * to look beyond the current sg.
	 */
	while (idx + count <= n) {
		idx += count;
		sg = ____sg_next(sg);
		count = __sg_page_count(sg);
	}

	*offset = n - idx;
	return sg;

lookup:
	rcu_read_lock();

	sg = radix_tree_lookup(&iter->radix, n);
	GEM_BUG_ON(!sg);

	/* If this index is in the middle of multi-page sg entry,
	 * the radixtree will contain an exceptional entry that points
	 * to the start of that range. We will return the pointer to
	 * the base page and the offset of this page within the
	 * sg entry's range.
	 */
	*offset = 0;
	if (unlikely(radix_tree_exception(sg))) {
		unsigned long base =
			(unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT;

		sg = radix_tree_lookup(&iter->radix, base);
		GEM_BUG_ON(!sg);

		*offset = n - base;
	}

	rcu_read_unlock();

	return sg;
}

struct page *
i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
{
	struct scatterlist *sg;
	unsigned int offset;

	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));

	sg = i915_gem_object_get_sg(obj, n, &offset);
	return nth_page(sg_page(sg), offset);
}

/* Like i915_gem_object_get_page(), but mark the returned page dirty */
struct page *
i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
			       unsigned int n)
{
	struct page *page;

	page = i915_gem_object_get_page(obj, n);
C
Chris Wilson 已提交
5358
	if (!obj->mm.dirty)
5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373
		set_page_dirty(page);

	return page;
}

dma_addr_t
i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
				unsigned long n)
{
	struct scatterlist *sg;
	unsigned int offset;

	sg = i915_gem_object_get_sg(obj, n, &offset);
	return sg_dma_address(sg) + (offset << PAGE_SHIFT);
}
5374

5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412
int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
{
	struct sg_table *pages;
	int err;

	if (align > obj->base.size)
		return -EINVAL;

	if (obj->ops == &i915_gem_phys_ops)
		return 0;

	if (obj->ops != &i915_gem_object_ops)
		return -EINVAL;

	err = i915_gem_object_unbind(obj);
	if (err)
		return err;

	mutex_lock(&obj->mm.lock);

	if (obj->mm.madv != I915_MADV_WILLNEED) {
		err = -EFAULT;
		goto err_unlock;
	}

	if (obj->mm.quirked) {
		err = -EFAULT;
		goto err_unlock;
	}

	if (obj->mm.mapping) {
		err = -EBUSY;
		goto err_unlock;
	}

	pages = obj->mm.pages;
	obj->ops = &i915_gem_phys_ops;

5413
	err = ____i915_gem_object_get_pages(obj);
5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432
	if (err)
		goto err_xfer;

	/* Perma-pin (until release) the physical set of pages */
	__i915_gem_object_pin_pages(obj);

	if (!IS_ERR_OR_NULL(pages))
		i915_gem_object_ops.put_pages(obj, pages);
	mutex_unlock(&obj->mm.lock);
	return 0;

err_xfer:
	obj->ops = &i915_gem_object_ops;
	obj->mm.pages = pages;
err_unlock:
	mutex_unlock(&obj->mm.lock);
	return err;
}

5433 5434
#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
#include "selftests/scatterlist.c"
5435
#include "selftests/mock_gem_device.c"
5436
#include "selftests/huge_gem_object.c"
M
Matthew Auld 已提交
5437
#include "selftests/huge_pages.c"
5438
#include "selftests/i915_gem_object.c"
5439
#include "selftests/i915_gem_coherency.c"
5440
#endif