i915_gem_execbuffer.c 49.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
/*
 * Copyright © 2008,2010 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Eric Anholt <eric@anholt.net>
 *    Chris Wilson <chris@chris-wilson.co.uk>
 *
 */

29 30
#include <drm/drmP.h>
#include <drm/i915_drm.h>
31 32 33
#include "i915_drv.h"
#include "i915_trace.h"
#include "intel_drv.h"
34
#include <linux/dma_remapping.h>
35
#include <linux/uaccess.h>
36

37 38 39 40 41
#define  __EXEC_OBJECT_HAS_PIN		(1<<31)
#define  __EXEC_OBJECT_HAS_FENCE	(1<<30)
#define  __EXEC_OBJECT_NEEDS_MAP	(1<<29)
#define  __EXEC_OBJECT_NEEDS_BIAS	(1<<28)
#define  __EXEC_OBJECT_INTERNAL_FLAGS (0xf<<28) /* all of the above */
42 43

#define BATCH_OFFSET_BIAS (256*1024)
44

45 46
struct eb_vmas {
	struct list_head vmas;
47
	int and;
48
	union {
49
		struct i915_vma *lut[0];
50 51
		struct hlist_head buckets[0];
	};
52 53
};

54
static struct eb_vmas *
B
Ben Widawsky 已提交
55
eb_create(struct drm_i915_gem_execbuffer2 *args)
56
{
57
	struct eb_vmas *eb = NULL;
58 59

	if (args->flags & I915_EXEC_HANDLE_LUT) {
60
		unsigned size = args->buffer_count;
61 62
		size *= sizeof(struct i915_vma *);
		size += sizeof(struct eb_vmas);
63 64 65 66
		eb = kmalloc(size, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
	}

	if (eb == NULL) {
67 68
		unsigned size = args->buffer_count;
		unsigned count = PAGE_SIZE / sizeof(struct hlist_head) / 2;
L
Lauri Kasanen 已提交
69
		BUILD_BUG_ON_NOT_POWER_OF_2(PAGE_SIZE / sizeof(struct hlist_head));
70 71 72
		while (count > 2*size)
			count >>= 1;
		eb = kzalloc(count*sizeof(struct hlist_head) +
73
			     sizeof(struct eb_vmas),
74 75 76 77 78 79 80 81
			     GFP_TEMPORARY);
		if (eb == NULL)
			return eb;

		eb->and = count - 1;
	} else
		eb->and = -args->buffer_count;

82
	INIT_LIST_HEAD(&eb->vmas);
83 84 85 86
	return eb;
}

static void
87
eb_reset(struct eb_vmas *eb)
88
{
89 90
	if (eb->and >= 0)
		memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head));
91 92
}

93
static int
94 95 96 97 98
eb_lookup_vmas(struct eb_vmas *eb,
	       struct drm_i915_gem_exec_object2 *exec,
	       const struct drm_i915_gem_execbuffer2 *args,
	       struct i915_address_space *vm,
	       struct drm_file *file)
99
{
100 101
	struct drm_i915_gem_object *obj;
	struct list_head objects;
102
	int i, ret;
103

104
	INIT_LIST_HEAD(&objects);
105
	spin_lock(&file->table_lock);
106 107
	/* Grab a reference to the object and release the lock so we can lookup
	 * or create the VMA without using GFP_ATOMIC */
108
	for (i = 0; i < args->buffer_count; i++) {
109 110 111 112 113
		obj = to_intel_bo(idr_find(&file->object_idr, exec[i].handle));
		if (obj == NULL) {
			spin_unlock(&file->table_lock);
			DRM_DEBUG("Invalid object handle %d at index %d\n",
				   exec[i].handle, i);
114
			ret = -ENOENT;
115
			goto err;
116 117
		}

118
		if (!list_empty(&obj->obj_exec_link)) {
119 120 121
			spin_unlock(&file->table_lock);
			DRM_DEBUG("Object %p [handle %d, index %d] appears more than once in object list\n",
				   obj, exec[i].handle, i);
122
			ret = -EINVAL;
123
			goto err;
124 125
		}

126
		i915_gem_object_get(obj);
127 128 129
		list_add_tail(&obj->obj_exec_link, &objects);
	}
	spin_unlock(&file->table_lock);
130

131
	i = 0;
132
	while (!list_empty(&objects)) {
133
		struct i915_vma *vma;
134

135 136 137 138
		obj = list_first_entry(&objects,
				       struct drm_i915_gem_object,
				       obj_exec_link);

139 140 141 142 143 144 145 146
		/*
		 * NOTE: We can leak any vmas created here when something fails
		 * later on. But that's no issue since vma_unbind can deal with
		 * vmas which are not actually bound. And since only
		 * lookup_or_create exists as an interface to get at the vma
		 * from the (obj, vm) we don't run the risk of creating
		 * duplicated vmas for the same vm.
		 */
147
		vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
148 149 150
		if (IS_ERR(vma)) {
			DRM_DEBUG("Failed to lookup VMA\n");
			ret = PTR_ERR(vma);
151
			goto err;
152 153
		}

154
		/* Transfer ownership from the objects list to the vmas list. */
155
		list_add_tail(&vma->exec_list, &eb->vmas);
156
		list_del_init(&obj->obj_exec_link);
157 158

		vma->exec_entry = &exec[i];
159
		if (eb->and < 0) {
160
			eb->lut[i] = vma;
161 162
		} else {
			uint32_t handle = args->flags & I915_EXEC_HANDLE_LUT ? i : exec[i].handle;
163 164
			vma->exec_handle = handle;
			hlist_add_head(&vma->exec_node,
165 166
				       &eb->buckets[handle & eb->and]);
		}
167
		++i;
168 169
	}

170
	return 0;
171 172


173
err:
174 175 176 177 178
	while (!list_empty(&objects)) {
		obj = list_first_entry(&objects,
				       struct drm_i915_gem_object,
				       obj_exec_link);
		list_del_init(&obj->obj_exec_link);
179
		i915_gem_object_put(obj);
180
	}
181 182 183 184 185
	/*
	 * Objects already transfered to the vmas list will be unreferenced by
	 * eb_destroy.
	 */

186
	return ret;
187 188
}

D
Dave Gordon 已提交
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
static inline struct i915_vma *
eb_get_batch_vma(struct eb_vmas *eb)
{
	/* The batch is always the LAST item in the VMA list */
	struct i915_vma *vma = list_last_entry(&eb->vmas, typeof(*vma), exec_list);

	return vma;
}

static struct drm_i915_gem_object *
eb_get_batch(struct eb_vmas *eb)
{
	struct i915_vma *vma = eb_get_batch_vma(eb);

	/*
	 * SNA is doing fancy tricks with compressing batch buffers, which leads
	 * to negative relocation deltas. Usually that works out ok since the
	 * relocate address is still positive, except when the batch is placed
	 * very low in the GTT. Ensure this doesn't happen.
	 *
	 * Note that actual hangs have only been observed on gen7, but for
	 * paranoia do it everywhere.
	 */
	if ((vma->exec_entry->flags & EXEC_OBJECT_PINNED) == 0)
		vma->exec_entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;

	return vma->obj;
}

218
static struct i915_vma *eb_get_vma(struct eb_vmas *eb, unsigned long handle)
219
{
220 221 222 223 224 225
	if (eb->and < 0) {
		if (handle >= -eb->and)
			return NULL;
		return eb->lut[handle];
	} else {
		struct hlist_head *head;
226
		struct i915_vma *vma;
227

228
		head = &eb->buckets[handle & eb->and];
229
		hlist_for_each_entry(vma, head, exec_node) {
230 231
			if (vma->exec_handle == handle)
				return vma;
232 233 234
		}
		return NULL;
	}
235 236
}

237 238 239 240 241 242 243 244 245 246 247 248 249 250 251
static void
i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
{
	struct drm_i915_gem_exec_object2 *entry;
	struct drm_i915_gem_object *obj = vma->obj;

	if (!drm_mm_node_allocated(&vma->node))
		return;

	entry = vma->exec_entry;

	if (entry->flags & __EXEC_OBJECT_HAS_FENCE)
		i915_gem_object_unpin_fence(obj);

	if (entry->flags & __EXEC_OBJECT_HAS_PIN)
252
		vma->pin_count--;
253

C
Chris Wilson 已提交
254
	entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
255 256 257 258
}

static void eb_destroy(struct eb_vmas *eb)
{
259 260
	while (!list_empty(&eb->vmas)) {
		struct i915_vma *vma;
261

262 263
		vma = list_first_entry(&eb->vmas,
				       struct i915_vma,
264
				       exec_list);
265
		list_del_init(&vma->exec_list);
266
		i915_gem_execbuffer_unreserve_vma(vma);
267
		i915_gem_object_put(vma->obj);
268
	}
269 270 271
	kfree(eb);
}

272 273
static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
{
274 275
	return (HAS_LLC(obj->base.dev) ||
		obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
276 277 278
		obj->cache_level != I915_CACHE_NONE);
}

279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
/* Used to convert any address to canonical form.
 * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
 * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
 * addresses to be in a canonical form:
 * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
 * canonical form [63:48] == [47]."
 */
#define GEN8_HIGH_ADDRESS_BIT 47
static inline uint64_t gen8_canonical_addr(uint64_t address)
{
	return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
}

static inline uint64_t gen8_noncanonical_addr(uint64_t address)
{
	return address & ((1ULL << (GEN8_HIGH_ADDRESS_BIT + 1)) - 1);
}

static inline uint64_t
relocation_target(struct drm_i915_gem_relocation_entry *reloc,
		  uint64_t target_offset)
{
	return gen8_canonical_addr((int)reloc->delta + target_offset);
}

304 305
static int
relocate_entry_cpu(struct drm_i915_gem_object *obj,
B
Ben Widawsky 已提交
306 307
		   struct drm_i915_gem_relocation_entry *reloc,
		   uint64_t target_offset)
308
{
309
	struct drm_device *dev = obj->base.dev;
310
	uint32_t page_offset = offset_in_page(reloc->offset);
311
	uint64_t delta = relocation_target(reloc, target_offset);
312
	char *vaddr;
313
	int ret;
314

315
	ret = i915_gem_object_set_to_cpu_domain(obj, true);
316 317 318
	if (ret)
		return ret;

319
	vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
320
				reloc->offset >> PAGE_SHIFT));
B
Ben Widawsky 已提交
321
	*(uint32_t *)(vaddr + page_offset) = lower_32_bits(delta);
322 323 324 325 326 327

	if (INTEL_INFO(dev)->gen >= 8) {
		page_offset = offset_in_page(page_offset + sizeof(uint32_t));

		if (page_offset == 0) {
			kunmap_atomic(vaddr);
328
			vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
329 330 331
			    (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
		}

B
Ben Widawsky 已提交
332
		*(uint32_t *)(vaddr + page_offset) = upper_32_bits(delta);
333 334
	}

335 336 337 338 339 340 341
	kunmap_atomic(vaddr);

	return 0;
}

static int
relocate_entry_gtt(struct drm_i915_gem_object *obj,
B
Ben Widawsky 已提交
342 343
		   struct drm_i915_gem_relocation_entry *reloc,
		   uint64_t target_offset)
344 345
{
	struct drm_device *dev = obj->base.dev;
346 347
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
348
	uint64_t delta = relocation_target(reloc, target_offset);
349
	uint64_t offset;
350
	void __iomem *reloc_page;
351
	int ret;
352 353 354 355 356 357 358 359 360 361

	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		return ret;

	ret = i915_gem_object_put_fence(obj);
	if (ret)
		return ret;

	/* Map the page containing the relocation we're going to perform.  */
362 363
	offset = i915_gem_obj_ggtt_offset(obj);
	offset += reloc->offset;
364
	reloc_page = io_mapping_map_atomic_wc(ggtt->mappable,
365 366
					      offset & PAGE_MASK);
	iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
367 368

	if (INTEL_INFO(dev)->gen >= 8) {
369
		offset += sizeof(uint32_t);
370

371
		if (offset_in_page(offset) == 0) {
372
			io_mapping_unmap_atomic(reloc_page);
373
			reloc_page =
374
				io_mapping_map_atomic_wc(ggtt->mappable,
375
							 offset);
376 377
		}

378 379
		iowrite32(upper_32_bits(delta),
			  reloc_page + offset_in_page(offset));
380 381
	}

382 383 384 385 386
	io_mapping_unmap_atomic(reloc_page);

	return 0;
}

387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402
static void
clflush_write32(void *addr, uint32_t value)
{
	/* This is not a fast path, so KISS. */
	drm_clflush_virt_range(addr, sizeof(uint32_t));
	*(uint32_t *)addr = value;
	drm_clflush_virt_range(addr, sizeof(uint32_t));
}

static int
relocate_entry_clflush(struct drm_i915_gem_object *obj,
		       struct drm_i915_gem_relocation_entry *reloc,
		       uint64_t target_offset)
{
	struct drm_device *dev = obj->base.dev;
	uint32_t page_offset = offset_in_page(reloc->offset);
403
	uint64_t delta = relocation_target(reloc, target_offset);
404 405 406 407 408 409 410
	char *vaddr;
	int ret;

	ret = i915_gem_object_set_to_gtt_domain(obj, true);
	if (ret)
		return ret;

411
	vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
412 413 414 415 416 417 418 419
				reloc->offset >> PAGE_SHIFT));
	clflush_write32(vaddr + page_offset, lower_32_bits(delta));

	if (INTEL_INFO(dev)->gen >= 8) {
		page_offset = offset_in_page(page_offset + sizeof(uint32_t));

		if (page_offset == 0) {
			kunmap_atomic(vaddr);
420
			vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj,
421 422 423 424 425 426 427 428 429 430 431
			    (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
		}

		clflush_write32(vaddr + page_offset, upper_32_bits(delta));
	}

	kunmap_atomic(vaddr);

	return 0;
}

432 433
static int
i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
434
				   struct eb_vmas *eb,
435
				   struct drm_i915_gem_relocation_entry *reloc)
436 437 438
{
	struct drm_device *dev = obj->base.dev;
	struct drm_gem_object *target_obj;
439
	struct drm_i915_gem_object *target_i915_obj;
440
	struct i915_vma *target_vma;
B
Ben Widawsky 已提交
441
	uint64_t target_offset;
442
	int ret;
443

444
	/* we've already hold a reference to all valid objects */
445 446
	target_vma = eb_get_vma(eb, reloc->target_handle);
	if (unlikely(target_vma == NULL))
447
		return -ENOENT;
448 449
	target_i915_obj = target_vma->obj;
	target_obj = &target_vma->obj->base;
450

451
	target_offset = gen8_canonical_addr(target_vma->node.start);
452

453 454 455 456
	/* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
	 * pipe_control writes because the gpu doesn't properly redirect them
	 * through the ppgtt for non_secure batchbuffers. */
	if (unlikely(IS_GEN6(dev) &&
457
	    reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION)) {
458
		ret = i915_vma_bind(target_vma, target_i915_obj->cache_level,
459
				    PIN_GLOBAL);
460 461 462
		if (WARN_ONCE(ret, "Unexpected failure to bind target VMA!"))
			return ret;
	}
463

464
	/* Validate that the target is in a valid r/w GPU domain */
465
	if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
466
		DRM_DEBUG("reloc with multiple write domains: "
467 468 469 470 471 472
			  "obj %p target %d offset %d "
			  "read %08x write %08x",
			  obj, reloc->target_handle,
			  (int) reloc->offset,
			  reloc->read_domains,
			  reloc->write_domain);
473
		return -EINVAL;
474
	}
475 476
	if (unlikely((reloc->write_domain | reloc->read_domains)
		     & ~I915_GEM_GPU_DOMAINS)) {
477
		DRM_DEBUG("reloc with read/write non-GPU domains: "
478 479 480 481 482 483
			  "obj %p target %d offset %d "
			  "read %08x write %08x",
			  obj, reloc->target_handle,
			  (int) reloc->offset,
			  reloc->read_domains,
			  reloc->write_domain);
484
		return -EINVAL;
485 486 487 488 489 490 491 492 493
	}

	target_obj->pending_read_domains |= reloc->read_domains;
	target_obj->pending_write_domain |= reloc->write_domain;

	/* If the relocation already has the right value in it, no
	 * more work needs to be done.
	 */
	if (target_offset == reloc->presumed_offset)
494
		return 0;
495 496

	/* Check that the relocation address is valid... */
497 498
	if (unlikely(reloc->offset >
		obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
499
		DRM_DEBUG("Relocation beyond object bounds: "
500 501 502 503
			  "obj %p target %d offset %d size %d.\n",
			  obj, reloc->target_handle,
			  (int) reloc->offset,
			  (int) obj->base.size);
504
		return -EINVAL;
505
	}
506
	if (unlikely(reloc->offset & 3)) {
507
		DRM_DEBUG("Relocation not 4-byte aligned: "
508 509 510
			  "obj %p target %d offset %d.\n",
			  obj, reloc->target_handle,
			  (int) reloc->offset);
511
		return -EINVAL;
512 513
	}

514
	/* We can't wait for rendering with pagefaults disabled */
515
	if (obj->active && pagefault_disabled())
516 517
		return -EFAULT;

518
	if (use_cpu_reloc(obj))
B
Ben Widawsky 已提交
519
		ret = relocate_entry_cpu(obj, reloc, target_offset);
520
	else if (obj->map_and_fenceable)
B
Ben Widawsky 已提交
521
		ret = relocate_entry_gtt(obj, reloc, target_offset);
522
	else if (static_cpu_has(X86_FEATURE_CLFLUSH))
523 524 525 526 527
		ret = relocate_entry_clflush(obj, reloc, target_offset);
	else {
		WARN_ONCE(1, "Impossible case in relocation handling\n");
		ret = -ENODEV;
	}
528

529 530 531
	if (ret)
		return ret;

532 533 534
	/* and update the user's relocation entry */
	reloc->presumed_offset = target_offset;

535
	return 0;
536 537 538
}

static int
539 540
i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
				 struct eb_vmas *eb)
541
{
542 543
#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
	struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
544
	struct drm_i915_gem_relocation_entry __user *user_relocs;
545
	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
546
	int remain, ret;
547

548
	user_relocs = u64_to_user_ptr(entry->relocs_ptr);
549

550 551 552 553 554 555 556 557 558
	remain = entry->relocation_count;
	while (remain) {
		struct drm_i915_gem_relocation_entry *r = stack_reloc;
		int count = remain;
		if (count > ARRAY_SIZE(stack_reloc))
			count = ARRAY_SIZE(stack_reloc);
		remain -= count;

		if (__copy_from_user_inatomic(r, user_relocs, count*sizeof(r[0])))
559 560
			return -EFAULT;

561 562
		do {
			u64 offset = r->presumed_offset;
563

564
			ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
565 566 567 568
			if (ret)
				return ret;

			if (r->presumed_offset != offset &&
569
			    __put_user(r->presumed_offset, &user_relocs->presumed_offset)) {
570 571 572 573 574 575
				return -EFAULT;
			}

			user_relocs++;
			r++;
		} while (--count);
576 577 578
	}

	return 0;
579
#undef N_RELOC
580 581 582
}

static int
583 584 585
i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
				      struct eb_vmas *eb,
				      struct drm_i915_gem_relocation_entry *relocs)
586
{
587
	const struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
588 589 590
	int i, ret;

	for (i = 0; i < entry->relocation_count; i++) {
591
		ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
592 593 594 595 596 597 598 599
		if (ret)
			return ret;
	}

	return 0;
}

static int
B
Ben Widawsky 已提交
600
i915_gem_execbuffer_relocate(struct eb_vmas *eb)
601
{
602
	struct i915_vma *vma;
603 604 605 606 607 608 609 610 611 612
	int ret = 0;

	/* This is the fast path and we cannot handle a pagefault whilst
	 * holding the struct mutex lest the user pass in the relocations
	 * contained within a mmaped bo. For in such a case we, the page
	 * fault handler would call i915_gem_fault() and we would try to
	 * acquire the struct mutex again. Obviously this is bad and so
	 * lockdep complains vehemently.
	 */
	pagefault_disable();
613 614
	list_for_each_entry(vma, &eb->vmas, exec_list) {
		ret = i915_gem_execbuffer_relocate_vma(vma, eb);
615
		if (ret)
616
			break;
617
	}
618
	pagefault_enable();
619

620
	return ret;
621 622
}

623 624 625 626 627 628
static bool only_mappable_for_reloc(unsigned int flags)
{
	return (flags & (EXEC_OBJECT_NEEDS_FENCE | __EXEC_OBJECT_NEEDS_MAP)) ==
		__EXEC_OBJECT_NEEDS_MAP;
}

629
static int
630
i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
631
				struct intel_engine_cs *engine,
632
				bool *need_reloc)
633
{
634
	struct drm_i915_gem_object *obj = vma->obj;
635
	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
636
	uint64_t flags;
637 638
	int ret;

639
	flags = PIN_USER;
640 641 642
	if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
		flags |= PIN_GLOBAL;

643
	if (!drm_mm_node_allocated(&vma->node)) {
644 645 646 647 648
		/* Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
		 * limit address to the first 4GBs for unflagged objects.
		 */
		if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0)
			flags |= PIN_ZONE_4G;
649 650 651 652
		if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
			flags |= PIN_GLOBAL | PIN_MAPPABLE;
		if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
			flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
653 654
		if (entry->flags & EXEC_OBJECT_PINNED)
			flags |= entry->offset | PIN_OFFSET_FIXED;
655 656
		if ((flags & PIN_MAPPABLE) == 0)
			flags |= PIN_HIGH;
657
	}
658 659

	ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);
660 661 662 663
	if ((ret == -ENOSPC  || ret == -E2BIG) &&
	    only_mappable_for_reloc(entry->flags))
		ret = i915_gem_object_pin(obj, vma->vm,
					  entry->alignment,
664
					  flags & ~PIN_MAPPABLE);
665 666 667
	if (ret)
		return ret;

668 669
	entry->flags |= __EXEC_OBJECT_HAS_PIN;

670 671 672 673
	if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
		ret = i915_gem_object_get_fence(obj);
		if (ret)
			return ret;
674

675 676
		if (i915_gem_object_pin_fence(obj))
			entry->flags |= __EXEC_OBJECT_HAS_FENCE;
677 678
	}

679 680
	if (entry->offset != vma->node.start) {
		entry->offset = vma->node.start;
681 682 683 684 685 686 687 688
		*need_reloc = true;
	}

	if (entry->flags & EXEC_OBJECT_WRITE) {
		obj->base.pending_read_domains = I915_GEM_DOMAIN_RENDER;
		obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
	}

689
	return 0;
690
}
691

692
static bool
693
need_reloc_mappable(struct i915_vma *vma)
694 695 696
{
	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;

697 698 699
	if (entry->relocation_count == 0)
		return false;

700
	if (!vma->is_ggtt)
701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717
		return false;

	/* See also use_cpu_reloc() */
	if (HAS_LLC(vma->obj->base.dev))
		return false;

	if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
		return false;

	return true;
}

static bool
eb_vma_misplaced(struct i915_vma *vma)
{
	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
	struct drm_i915_gem_object *obj = vma->obj;
718

719
	WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP && !vma->is_ggtt);
720 721 722 723 724

	if (entry->alignment &&
	    vma->node.start & (entry->alignment - 1))
		return true;

725 726 727 728
	if (entry->flags & EXEC_OBJECT_PINNED &&
	    vma->node.start != entry->offset)
		return true;

729 730 731 732
	if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
	    vma->node.start < BATCH_OFFSET_BIAS)
		return true;

733 734 735 736
	/* avoid costly ping-pong once a batch bo ended up non-mappable */
	if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
		return !only_mappable_for_reloc(entry->flags);

737 738 739 740
	if ((entry->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) == 0 &&
	    (vma->node.start + vma->node.size - 1) >> 32)
		return true;

741 742 743
	return false;
}

744
static int
745
i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
746
			    struct list_head *vmas,
747
			    struct i915_gem_context *ctx,
748
			    bool *need_relocs)
749
{
750
	struct drm_i915_gem_object *obj;
751
	struct i915_vma *vma;
752
	struct i915_address_space *vm;
753
	struct list_head ordered_vmas;
754
	struct list_head pinned_vmas;
755
	bool has_fenced_gpu_access = INTEL_GEN(engine->i915) < 4;
756
	int retry;
757

758
	i915_gem_retire_requests_ring(engine);
759

760 761
	vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;

762
	INIT_LIST_HEAD(&ordered_vmas);
763
	INIT_LIST_HEAD(&pinned_vmas);
764
	while (!list_empty(vmas)) {
765 766 767
		struct drm_i915_gem_exec_object2 *entry;
		bool need_fence, need_mappable;

768 769 770
		vma = list_first_entry(vmas, struct i915_vma, exec_list);
		obj = vma->obj;
		entry = vma->exec_entry;
771

772 773 774
		if (ctx->flags & CONTEXT_NO_ZEROMAP)
			entry->flags |= __EXEC_OBJECT_NEEDS_BIAS;

775 776
		if (!has_fenced_gpu_access)
			entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
777 778 779
		need_fence =
			entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
			obj->tiling_mode != I915_TILING_NONE;
780
		need_mappable = need_fence || need_reloc_mappable(vma);
781

782 783 784
		if (entry->flags & EXEC_OBJECT_PINNED)
			list_move_tail(&vma->exec_list, &pinned_vmas);
		else if (need_mappable) {
785
			entry->flags |= __EXEC_OBJECT_NEEDS_MAP;
786
			list_move(&vma->exec_list, &ordered_vmas);
787
		} else
788
			list_move_tail(&vma->exec_list, &ordered_vmas);
789

790
		obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
791
		obj->base.pending_write_domain = 0;
792
	}
793
	list_splice(&ordered_vmas, vmas);
794
	list_splice(&pinned_vmas, vmas);
795 796 797 798 799 800 801 802 803 804

	/* Attempt to pin all of the buffers into the GTT.
	 * This is done in 3 phases:
	 *
	 * 1a. Unbind all objects that do not match the GTT constraints for
	 *     the execbuffer (fenceable, mappable, alignment etc).
	 * 1b. Increment pin count for already bound objects.
	 * 2.  Bind new objects.
	 * 3.  Decrement pin count.
	 *
805
	 * This avoid unnecessary unbinding of later objects in order to make
806 807 808 809
	 * room for the earlier objects *unless* we need to defragment.
	 */
	retry = 0;
	do {
810
		int ret = 0;
811 812

		/* Unbind any ill-fitting objects or pin. */
813 814
		list_for_each_entry(vma, vmas, exec_list) {
			if (!drm_mm_node_allocated(&vma->node))
815 816
				continue;

817
			if (eb_vma_misplaced(vma))
818
				ret = i915_vma_unbind(vma);
819
			else
820 821 822
				ret = i915_gem_execbuffer_reserve_vma(vma,
								      engine,
								      need_relocs);
823
			if (ret)
824 825 826 827
				goto err;
		}

		/* Bind fresh objects */
828 829
		list_for_each_entry(vma, vmas, exec_list) {
			if (drm_mm_node_allocated(&vma->node))
830
				continue;
831

832 833
			ret = i915_gem_execbuffer_reserve_vma(vma, engine,
							      need_relocs);
834 835
			if (ret)
				goto err;
836 837
		}

838
err:
C
Chris Wilson 已提交
839
		if (ret != -ENOSPC || retry++)
840 841
			return ret;

842 843 844 845
		/* Decrement pin count for bound objects */
		list_for_each_entry(vma, vmas, exec_list)
			i915_gem_execbuffer_unreserve_vma(vma);

846
		ret = i915_gem_evict_vm(vm, true);
847 848 849 850 851 852 853
		if (ret)
			return ret;
	} while (1);
}

static int
i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
854
				  struct drm_i915_gem_execbuffer2 *args,
855
				  struct drm_file *file,
856
				  struct intel_engine_cs *engine,
857
				  struct eb_vmas *eb,
858
				  struct drm_i915_gem_exec_object2 *exec,
859
				  struct i915_gem_context *ctx)
860 861
{
	struct drm_i915_gem_relocation_entry *reloc;
862 863
	struct i915_address_space *vm;
	struct i915_vma *vma;
864
	bool need_relocs;
865
	int *reloc_offset;
866
	int i, total, ret;
867
	unsigned count = args->buffer_count;
868

869 870
	vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;

871
	/* We may process another execbuffer during the unlock... */
872 873 874
	while (!list_empty(&eb->vmas)) {
		vma = list_first_entry(&eb->vmas, struct i915_vma, exec_list);
		list_del_init(&vma->exec_list);
875
		i915_gem_execbuffer_unreserve_vma(vma);
876
		i915_gem_object_put(vma->obj);
877 878
	}

879 880 881 882
	mutex_unlock(&dev->struct_mutex);

	total = 0;
	for (i = 0; i < count; i++)
883
		total += exec[i].relocation_count;
884

885
	reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset));
886
	reloc = drm_malloc_ab(total, sizeof(*reloc));
887 888 889
	if (reloc == NULL || reloc_offset == NULL) {
		drm_free_large(reloc);
		drm_free_large(reloc_offset);
890 891 892 893 894 895 896
		mutex_lock(&dev->struct_mutex);
		return -ENOMEM;
	}

	total = 0;
	for (i = 0; i < count; i++) {
		struct drm_i915_gem_relocation_entry __user *user_relocs;
897 898
		u64 invalid_offset = (u64)-1;
		int j;
899

900
		user_relocs = u64_to_user_ptr(exec[i].relocs_ptr);
901 902

		if (copy_from_user(reloc+total, user_relocs,
903
				   exec[i].relocation_count * sizeof(*reloc))) {
904 905 906 907 908
			ret = -EFAULT;
			mutex_lock(&dev->struct_mutex);
			goto err;
		}

909 910 911 912 913 914 915 916 917 918
		/* As we do not update the known relocation offsets after
		 * relocating (due to the complexities in lock handling),
		 * we need to mark them as invalid now so that we force the
		 * relocation processing next time. Just in case the target
		 * object is evicted and then rebound into its old
		 * presumed_offset before the next execbuffer - if that
		 * happened we would make the mistake of assuming that the
		 * relocations were valid.
		 */
		for (j = 0; j < exec[i].relocation_count; j++) {
919 920 921
			if (__copy_to_user(&user_relocs[j].presumed_offset,
					   &invalid_offset,
					   sizeof(invalid_offset))) {
922 923 924 925 926 927
				ret = -EFAULT;
				mutex_lock(&dev->struct_mutex);
				goto err;
			}
		}

928
		reloc_offset[i] = total;
929
		total += exec[i].relocation_count;
930 931 932 933 934 935 936 937
	}

	ret = i915_mutex_lock_interruptible(dev);
	if (ret) {
		mutex_lock(&dev->struct_mutex);
		goto err;
	}

938 939
	/* reacquire the objects */
	eb_reset(eb);
940
	ret = eb_lookup_vmas(eb, exec, args, vm, file);
941 942
	if (ret)
		goto err;
943

944
	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
945 946
	ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
					  &need_relocs);
947 948 949
	if (ret)
		goto err;

950 951 952 953
	list_for_each_entry(vma, &eb->vmas, exec_list) {
		int offset = vma->exec_entry - exec;
		ret = i915_gem_execbuffer_relocate_vma_slow(vma, eb,
							    reloc + reloc_offset[offset]);
954 955 956 957 958 959 960 961 962 963 964 965
		if (ret)
			goto err;
	}

	/* Leave the user relocations as are, this is the painfully slow path,
	 * and we want to avoid the complication of dropping the lock whilst
	 * having buffers reserved in the aperture and so causing spurious
	 * ENOSPC for random operations.
	 */

err:
	drm_free_large(reloc);
966
	drm_free_large(reloc_offset);
967 968 969 970
	return ret;
}

static int
971
i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
972
				struct list_head *vmas)
973
{
974
	const unsigned other_rings = ~intel_engine_flag(req->engine);
975
	struct i915_vma *vma;
976
	uint32_t flush_domains = 0;
977
	bool flush_chipset = false;
978
	int ret;
979

980 981
	list_for_each_entry(vma, vmas, exec_list) {
		struct drm_i915_gem_object *obj = vma->obj;
982 983

		if (obj->active & other_rings) {
984
			ret = i915_gem_object_sync(obj, req->engine, &req);
985 986 987
			if (ret)
				return ret;
		}
988 989

		if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
990
			flush_chipset |= i915_gem_clflush_object(obj, false);
991 992

		flush_domains |= obj->base.write_domain;
993 994
	}

995
	if (flush_chipset)
996
		i915_gem_chipset_flush(req->engine->i915);
997 998 999 1000

	if (flush_domains & I915_GEM_DOMAIN_GTT)
		wmb();

1001 1002 1003
	/* Unconditionally invalidate gpu caches and ensure that we do flush
	 * any residual writes from the previous batch.
	 */
1004
	return intel_ring_invalidate_all_caches(req);
1005 1006
}

1007 1008
static bool
i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
1009
{
1010 1011 1012
	if (exec->flags & __I915_EXEC_UNKNOWN_FLAGS)
		return false;

C
Chris Wilson 已提交
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027
	/* Kernel clipping was a DRI1 misfeature */
	if (exec->num_cliprects || exec->cliprects_ptr)
		return false;

	if (exec->DR4 == 0xffffffff) {
		DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
		exec->DR4 = 0;
	}
	if (exec->DR1 || exec->DR4)
		return false;

	if ((exec->batch_start_offset | exec->batch_len) & 0x7)
		return false;

	return true;
1028 1029 1030
}

static int
1031 1032
validate_exec_list(struct drm_device *dev,
		   struct drm_i915_gem_exec_object2 *exec,
1033 1034
		   int count)
{
1035 1036
	unsigned relocs_total = 0;
	unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
1037 1038 1039
	unsigned invalid_flags;
	int i;

1040 1041 1042
	/* INTERNAL flags must not overlap with external ones */
	BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & ~__EXEC_OBJECT_UNKNOWN_FLAGS);

1043 1044 1045
	invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
	if (USES_FULL_PPGTT(dev))
		invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
1046 1047

	for (i = 0; i < count; i++) {
1048
		char __user *ptr = u64_to_user_ptr(exec[i].relocs_ptr);
1049 1050
		int length; /* limited by fault_in_pages_readable() */

1051
		if (exec[i].flags & invalid_flags)
1052 1053
			return -EINVAL;

1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068
		/* Offset can be used as input (EXEC_OBJECT_PINNED), reject
		 * any non-page-aligned or non-canonical addresses.
		 */
		if (exec[i].flags & EXEC_OBJECT_PINNED) {
			if (exec[i].offset !=
			    gen8_canonical_addr(exec[i].offset & PAGE_MASK))
				return -EINVAL;

			/* From drm_mm perspective address space is continuous,
			 * so from this point we're always using non-canonical
			 * form internally.
			 */
			exec[i].offset = gen8_noncanonical_addr(exec[i].offset);
		}

1069 1070 1071
		if (exec[i].alignment && !is_power_of_2(exec[i].alignment))
			return -EINVAL;

1072 1073 1074 1075 1076
		/* First check for malicious input causing overflow in
		 * the worst case where we need to allocate the entire
		 * relocation tree as a single array.
		 */
		if (exec[i].relocation_count > relocs_max - relocs_total)
1077
			return -EINVAL;
1078
		relocs_total += exec[i].relocation_count;
1079 1080 1081

		length = exec[i].relocation_count *
			sizeof(struct drm_i915_gem_relocation_entry);
1082 1083 1084 1085 1086
		/*
		 * We must check that the entire relocation array is safe
		 * to read, but since we may need to update the presumed
		 * offsets during execution, check for full write access.
		 */
1087 1088 1089
		if (!access_ok(VERIFY_WRITE, ptr, length))
			return -EFAULT;

1090
		if (likely(!i915.prefault_disable)) {
1091 1092 1093
			if (fault_in_multipages_readable(ptr, length))
				return -EFAULT;
		}
1094 1095 1096 1097 1098
	}

	return 0;
}

1099
static struct i915_gem_context *
1100
i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
1101
			  struct intel_engine_cs *engine, const u32 ctx_id)
1102
{
1103
	struct i915_gem_context *ctx = NULL;
1104 1105
	struct i915_ctx_hang_stats *hs;

1106
	if (engine->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
1107 1108
		return ERR_PTR(-EINVAL);

1109
	ctx = i915_gem_context_lookup(file->driver_priv, ctx_id);
1110
	if (IS_ERR(ctx))
1111
		return ctx;
1112

1113
	hs = &ctx->hang_stats;
1114 1115
	if (hs->banned) {
		DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
1116
		return ERR_PTR(-EIO);
1117 1118
	}

1119
	return ctx;
1120 1121
}

1122
void
1123
i915_gem_execbuffer_move_to_active(struct list_head *vmas,
1124
				   struct drm_i915_gem_request *req)
1125
{
1126
	struct intel_engine_cs *engine = i915_gem_request_get_engine(req);
1127
	struct i915_vma *vma;
1128

1129
	list_for_each_entry(vma, vmas, exec_list) {
1130
		struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
1131
		struct drm_i915_gem_object *obj = vma->obj;
1132 1133
		u32 old_read = obj->base.read_domains;
		u32 old_write = obj->base.write_domain;
C
Chris Wilson 已提交
1134

1135
		obj->dirty = 1; /* be paranoid  */
1136
		obj->base.write_domain = obj->base.pending_write_domain;
1137 1138 1139
		if (obj->base.write_domain == 0)
			obj->base.pending_read_domains |= obj->base.read_domains;
		obj->base.read_domains = obj->base.pending_read_domains;
1140

1141
		i915_vma_move_to_active(vma, req);
1142
		if (obj->base.write_domain) {
1143
			i915_gem_request_assign(&obj->last_write_req, req);
1144

1145
			intel_fb_obj_invalidate(obj, ORIGIN_CS);
1146 1147 1148

			/* update for the implicit flush after a batch */
			obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1149
		}
1150
		if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
1151
			i915_gem_request_assign(&obj->last_fenced_req, req);
1152
			if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
1153
				struct drm_i915_private *dev_priv = engine->i915;
1154 1155 1156 1157
				list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
					       &dev_priv->mm.fence_list);
			}
		}
1158

C
Chris Wilson 已提交
1159
		trace_i915_gem_object_change_domain(obj, old_read, old_write);
1160 1161 1162
	}
}

1163
static void
1164
i915_gem_execbuffer_retire_commands(struct i915_execbuffer_params *params)
1165
{
1166
	/* Unconditionally force add_request to emit a full flush. */
1167
	params->engine->gpu_caches_dirty = true;
1168

1169
	/* Add a breadcrumb for the completion of the batch buffer */
1170
	__i915_add_request(params->request, params->batch_obj, true);
1171
}
1172

1173 1174
static int
i915_reset_gen7_sol_offsets(struct drm_device *dev,
1175
			    struct drm_i915_gem_request *req)
1176
{
1177
	struct intel_engine_cs *engine = req->engine;
1178
	struct drm_i915_private *dev_priv = to_i915(dev);
1179 1180
	int ret, i;

1181
	if (!IS_GEN7(dev) || engine != &dev_priv->engine[RCS]) {
1182 1183 1184
		DRM_DEBUG("sol reset is gen7/rcs only\n");
		return -EINVAL;
	}
1185

1186
	ret = intel_ring_begin(req, 4 * 3);
1187 1188 1189 1190
	if (ret)
		return ret;

	for (i = 0; i < 4; i++) {
1191 1192 1193
		intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
		intel_ring_emit_reg(engine, GEN7_SO_WRITE_OFFSET(i));
		intel_ring_emit(engine, 0);
1194 1195
	}

1196
	intel_ring_advance(engine);
1197 1198 1199 1200

	return 0;
}

1201
static struct drm_i915_gem_object*
1202
i915_gem_execbuffer_parse(struct intel_engine_cs *engine,
1203 1204 1205 1206 1207
			  struct drm_i915_gem_exec_object2 *shadow_exec_entry,
			  struct eb_vmas *eb,
			  struct drm_i915_gem_object *batch_obj,
			  u32 batch_start_offset,
			  u32 batch_len,
1208
			  bool is_master)
1209 1210
{
	struct drm_i915_gem_object *shadow_batch_obj;
1211
	struct i915_vma *vma;
1212 1213
	int ret;

1214
	shadow_batch_obj = i915_gem_batch_pool_get(&engine->batch_pool,
1215
						   PAGE_ALIGN(batch_len));
1216 1217 1218
	if (IS_ERR(shadow_batch_obj))
		return shadow_batch_obj;

1219
	ret = i915_parse_cmds(engine,
1220 1221 1222 1223 1224
			      batch_obj,
			      shadow_batch_obj,
			      batch_start_offset,
			      batch_len,
			      is_master);
1225 1226
	if (ret)
		goto err;
1227

1228 1229 1230
	ret = i915_gem_obj_ggtt_pin(shadow_batch_obj, 0, 0);
	if (ret)
		goto err;
1231

C
Chris Wilson 已提交
1232 1233
	i915_gem_object_unpin_pages(shadow_batch_obj);

1234
	memset(shadow_exec_entry, 0, sizeof(*shadow_exec_entry));
1235

1236 1237
	vma = i915_gem_obj_to_ggtt(shadow_batch_obj);
	vma->exec_entry = shadow_exec_entry;
C
Chris Wilson 已提交
1238
	vma->exec_entry->flags = __EXEC_OBJECT_HAS_PIN;
1239
	i915_gem_object_get(shadow_batch_obj);
1240
	list_add_tail(&vma->exec_list, &eb->vmas);
1241

1242 1243 1244
	shadow_batch_obj->base.pending_read_domains = I915_GEM_DOMAIN_COMMAND;

	return shadow_batch_obj;
1245

1246
err:
C
Chris Wilson 已提交
1247
	i915_gem_object_unpin_pages(shadow_batch_obj);
1248 1249 1250 1251
	if (ret == -EACCES) /* unhandled chained batch */
		return batch_obj;
	else
		return ERR_PTR(ret);
1252
}
1253

1254
int
1255
i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
1256
			       struct drm_i915_gem_execbuffer2 *args,
1257
			       struct list_head *vmas)
1258
{
1259
	struct drm_device *dev = params->dev;
1260
	struct intel_engine_cs *engine = params->engine;
1261
	struct drm_i915_private *dev_priv = to_i915(dev);
1262
	u64 exec_start, exec_len;
1263 1264
	int instp_mode;
	u32 instp_mask;
C
Chris Wilson 已提交
1265
	int ret;
1266

1267
	ret = i915_gem_execbuffer_move_to_gpu(params->request, vmas);
1268
	if (ret)
C
Chris Wilson 已提交
1269
		return ret;
1270

1271
	ret = i915_switch_context(params->request);
1272
	if (ret)
C
Chris Wilson 已提交
1273
		return ret;
1274

1275 1276
	WARN(params->ctx->ppgtt && params->ctx->ppgtt->pd_dirty_rings & (1<<engine->id),
	     "%s didn't clear reload\n", engine->name);
1277

1278 1279 1280 1281 1282 1283
	instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
	instp_mask = I915_EXEC_CONSTANTS_MASK;
	switch (instp_mode) {
	case I915_EXEC_CONSTANTS_REL_GENERAL:
	case I915_EXEC_CONSTANTS_ABSOLUTE:
	case I915_EXEC_CONSTANTS_REL_SURFACE:
1284
		if (instp_mode != 0 && engine != &dev_priv->engine[RCS]) {
1285
			DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
C
Chris Wilson 已提交
1286
			return -EINVAL;
1287 1288 1289 1290 1291
		}

		if (instp_mode != dev_priv->relative_constants_mode) {
			if (INTEL_INFO(dev)->gen < 4) {
				DRM_DEBUG("no rel constants on pre-gen4\n");
C
Chris Wilson 已提交
1292
				return -EINVAL;
1293 1294 1295 1296 1297
			}

			if (INTEL_INFO(dev)->gen > 5 &&
			    instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
				DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
C
Chris Wilson 已提交
1298
				return -EINVAL;
1299 1300 1301 1302 1303 1304 1305 1306 1307
			}

			/* The HW changed the meaning on this bit on gen6 */
			if (INTEL_INFO(dev)->gen >= 6)
				instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
		}
		break;
	default:
		DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
C
Chris Wilson 已提交
1308
		return -EINVAL;
1309 1310
	}

1311
	if (engine == &dev_priv->engine[RCS] &&
C
Chris Wilson 已提交
1312
	    instp_mode != dev_priv->relative_constants_mode) {
1313
		ret = intel_ring_begin(params->request, 4);
1314
		if (ret)
C
Chris Wilson 已提交
1315
			return ret;
1316

1317 1318 1319 1320 1321
		intel_ring_emit(engine, MI_NOOP);
		intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
		intel_ring_emit_reg(engine, INSTPM);
		intel_ring_emit(engine, instp_mask << 16 | instp_mode);
		intel_ring_advance(engine);
1322 1323 1324 1325 1326

		dev_priv->relative_constants_mode = instp_mode;
	}

	if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
1327
		ret = i915_reset_gen7_sol_offsets(dev, params->request);
1328
		if (ret)
C
Chris Wilson 已提交
1329
			return ret;
1330 1331
	}

1332 1333 1334 1335
	exec_len   = args->batch_len;
	exec_start = params->batch_obj_vm_offset +
		     params->args_batch_start_offset;

1336 1337 1338
	if (exec_len == 0)
		exec_len = params->batch_obj->base.size;

1339
	ret = engine->dispatch_execbuffer(params->request,
C
Chris Wilson 已提交
1340 1341 1342 1343
					exec_start, exec_len,
					params->dispatch_flags);
	if (ret)
		return ret;
1344

1345
	trace_i915_gem_ring_dispatch(params->request, params->dispatch_flags);
1346

1347
	i915_gem_execbuffer_move_to_active(vmas, params->request);
1348

C
Chris Wilson 已提交
1349
	return 0;
1350 1351
}

1352 1353
/**
 * Find one BSD ring to dispatch the corresponding BSD command.
1354
 * The ring index is returned.
1355
 */
1356 1357
static unsigned int
gen8_dispatch_bsd_ring(struct drm_i915_private *dev_priv, struct drm_file *file)
1358 1359 1360
{
	struct drm_i915_file_private *file_priv = file->driver_priv;

1361 1362 1363
	/* Check whether the file_priv has already selected one ring. */
	if ((int)file_priv->bsd_ring < 0) {
		/* If not, use the ping-pong mechanism to select one. */
1364
		mutex_lock(&dev_priv->drm.struct_mutex);
1365 1366
		file_priv->bsd_ring = dev_priv->mm.bsd_ring_dispatch_index;
		dev_priv->mm.bsd_ring_dispatch_index ^= 1;
1367
		mutex_unlock(&dev_priv->drm.struct_mutex);
1368
	}
1369 1370

	return file_priv->bsd_ring;
1371 1372
}

1373 1374
#define I915_USER_RINGS (4)

1375
static const enum intel_engine_id user_ring_map[I915_USER_RINGS + 1] = {
1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
	[I915_EXEC_DEFAULT]	= RCS,
	[I915_EXEC_RENDER]	= RCS,
	[I915_EXEC_BLT]		= BCS,
	[I915_EXEC_BSD]		= VCS,
	[I915_EXEC_VEBOX]	= VECS
};

static int
eb_select_ring(struct drm_i915_private *dev_priv,
	       struct drm_file *file,
	       struct drm_i915_gem_execbuffer2 *args,
	       struct intel_engine_cs **ring)
{
	unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;

	if (user_ring_id > I915_USER_RINGS) {
		DRM_DEBUG("execbuf with unknown ring: %u\n", user_ring_id);
		return -EINVAL;
	}

	if ((user_ring_id != I915_EXEC_BSD) &&
	    ((args->flags & I915_EXEC_BSD_MASK) != 0)) {
		DRM_DEBUG("execbuf with non bsd ring but with invalid "
			  "bsd dispatch flags: %d\n", (int)(args->flags));
		return -EINVAL;
	}

	if (user_ring_id == I915_EXEC_BSD && HAS_BSD2(dev_priv)) {
		unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;

		if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
			bsd_idx = gen8_dispatch_bsd_ring(dev_priv, file);
		} else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
			   bsd_idx <= I915_EXEC_BSD_RING2) {
1410
			bsd_idx >>= I915_EXEC_BSD_SHIFT;
1411 1412 1413 1414 1415 1416 1417
			bsd_idx--;
		} else {
			DRM_DEBUG("execbuf with unknown bsd ring: %u\n",
				  bsd_idx);
			return -EINVAL;
		}

1418
		*ring = &dev_priv->engine[_VCS(bsd_idx)];
1419
	} else {
1420
		*ring = &dev_priv->engine[user_ring_map[user_ring_id]];
1421 1422
	}

1423
	if (!intel_engine_initialized(*ring)) {
1424 1425 1426 1427 1428 1429 1430
		DRM_DEBUG("execbuf with invalid ring: %u\n", user_ring_id);
		return -EINVAL;
	}

	return 0;
}

1431 1432 1433 1434
static int
i915_gem_do_execbuffer(struct drm_device *dev, void *data,
		       struct drm_file *file,
		       struct drm_i915_gem_execbuffer2 *args,
1435
		       struct drm_i915_gem_exec_object2 *exec)
1436
{
1437 1438
	struct drm_i915_private *dev_priv = to_i915(dev);
	struct i915_ggtt *ggtt = &dev_priv->ggtt;
1439
	struct drm_i915_gem_request *req = NULL;
1440
	struct eb_vmas *eb;
1441
	struct drm_i915_gem_object *batch_obj;
1442
	struct drm_i915_gem_exec_object2 shadow_exec_entry;
1443
	struct intel_engine_cs *engine;
1444
	struct i915_gem_context *ctx;
1445
	struct i915_address_space *vm;
1446 1447
	struct i915_execbuffer_params params_master; /* XXX: will be removed later */
	struct i915_execbuffer_params *params = &params_master;
1448
	const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
1449
	u32 dispatch_flags;
1450
	int ret;
1451
	bool need_relocs;
1452

1453
	if (!i915_gem_check_execbuffer(args))
1454 1455
		return -EINVAL;

1456
	ret = validate_exec_list(dev, exec, args->buffer_count);
1457 1458 1459
	if (ret)
		return ret;

1460
	dispatch_flags = 0;
1461
	if (args->flags & I915_EXEC_SECURE) {
1462
		if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
1463 1464
		    return -EPERM;

1465
		dispatch_flags |= I915_DISPATCH_SECURE;
1466
	}
1467
	if (args->flags & I915_EXEC_IS_PINNED)
1468
		dispatch_flags |= I915_DISPATCH_PINNED;
1469

1470
	ret = eb_select_ring(dev_priv, file, args, &engine);
1471 1472
	if (ret)
		return ret;
1473 1474

	if (args->buffer_count < 1) {
1475
		DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1476 1477 1478
		return -EINVAL;
	}

1479 1480 1481 1482 1483
	if (args->flags & I915_EXEC_RESOURCE_STREAMER) {
		if (!HAS_RESOURCE_STREAMER(dev)) {
			DRM_DEBUG("RS is only allowed for Haswell, Gen8 and above\n");
			return -EINVAL;
		}
1484
		if (engine->id != RCS) {
1485
			DRM_DEBUG("RS is not available on %s\n",
1486
				 engine->name);
1487 1488 1489 1490 1491 1492
			return -EINVAL;
		}

		dispatch_flags |= I915_DISPATCH_RS;
	}

1493 1494 1495 1496 1497 1498
	/* Take a local wakeref for preparing to dispatch the execbuf as
	 * we expect to access the hardware fairly frequently in the
	 * process. Upon first dispatch, we acquire another prolonged
	 * wakeref that we hold until the GPU has been idle for at least
	 * 100ms.
	 */
1499 1500
	intel_runtime_pm_get(dev_priv);

1501 1502 1503 1504
	ret = i915_mutex_lock_interruptible(dev);
	if (ret)
		goto pre_mutex_err;

1505
	ctx = i915_gem_validate_context(dev, file, engine, ctx_id);
1506
	if (IS_ERR(ctx)) {
1507
		mutex_unlock(&dev->struct_mutex);
1508
		ret = PTR_ERR(ctx);
1509
		goto pre_mutex_err;
1510
	}
1511

1512
	i915_gem_context_get(ctx);
1513

1514 1515 1516
	if (ctx->ppgtt)
		vm = &ctx->ppgtt->base;
	else
1517
		vm = &ggtt->base;
1518

1519 1520
	memset(&params_master, 0x00, sizeof(params_master));

B
Ben Widawsky 已提交
1521
	eb = eb_create(args);
1522
	if (eb == NULL) {
1523
		i915_gem_context_put(ctx);
1524 1525 1526 1527 1528
		mutex_unlock(&dev->struct_mutex);
		ret = -ENOMEM;
		goto pre_mutex_err;
	}

1529
	/* Look up object handles */
1530
	ret = eb_lookup_vmas(eb, exec, args, vm, file);
1531 1532
	if (ret)
		goto err;
1533

1534
	/* take note of the batch buffer before we might reorder the lists */
1535
	batch_obj = eb_get_batch(eb);
1536

1537
	/* Move the objects en-masse into the GTT, evicting if necessary. */
1538
	need_relocs = (args->flags & I915_EXEC_NO_RELOC) == 0;
1539 1540
	ret = i915_gem_execbuffer_reserve(engine, &eb->vmas, ctx,
					  &need_relocs);
1541 1542 1543 1544
	if (ret)
		goto err;

	/* The objects are in their final locations, apply the relocations. */
1545
	if (need_relocs)
B
Ben Widawsky 已提交
1546
		ret = i915_gem_execbuffer_relocate(eb);
1547 1548
	if (ret) {
		if (ret == -EFAULT) {
1549 1550
			ret = i915_gem_execbuffer_relocate_slow(dev, args, file,
								engine,
1551
								eb, exec, ctx);
1552 1553 1554 1555 1556 1557 1558 1559
			BUG_ON(!mutex_is_locked(&dev->struct_mutex));
		}
		if (ret)
			goto err;
	}

	/* Set the pending read domains for the batch buffer to COMMAND */
	if (batch_obj->base.pending_write_domain) {
1560
		DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
1561 1562 1563 1564
		ret = -EINVAL;
		goto err;
	}

1565
	params->args_batch_start_offset = args->batch_start_offset;
1566
	if (i915_needs_cmd_parser(engine) && args->batch_len) {
1567 1568
		struct drm_i915_gem_object *parsed_batch_obj;

1569 1570 1571 1572 1573 1574
		parsed_batch_obj = i915_gem_execbuffer_parse(engine,
							     &shadow_exec_entry,
							     eb,
							     batch_obj,
							     args->batch_start_offset,
							     args->batch_len,
1575
							     drm_is_current_master(file));
1576 1577
		if (IS_ERR(parsed_batch_obj)) {
			ret = PTR_ERR(parsed_batch_obj);
1578 1579
			goto err;
		}
1580 1581

		/*
1582 1583
		 * parsed_batch_obj == batch_obj means batch not fully parsed:
		 * Accept, but don't promote to secure.
1584 1585
		 */

1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596
		if (parsed_batch_obj != batch_obj) {
			/*
			 * Batch parsed and accepted:
			 *
			 * Set the DISPATCH_SECURE bit to remove the NON_SECURE
			 * bit from MI_BATCH_BUFFER_START commands issued in
			 * the dispatch_execbuffer implementations. We
			 * specifically don't want that set on batches the
			 * command parser has accepted.
			 */
			dispatch_flags |= I915_DISPATCH_SECURE;
1597
			params->args_batch_start_offset = 0;
1598 1599
			batch_obj = parsed_batch_obj;
		}
1600 1601
	}

1602 1603
	batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;

1604 1605
	/* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
	 * batch" bit. Hence we need to pin secure batches into the global gtt.
B
Ben Widawsky 已提交
1606
	 * hsw should have this fixed, but bdw mucks it up again. */
1607
	if (dispatch_flags & I915_DISPATCH_SECURE) {
1608 1609 1610 1611 1612 1613
		/*
		 * So on first glance it looks freaky that we pin the batch here
		 * outside of the reservation loop. But:
		 * - The batch is already pinned into the relevant ppgtt, so we
		 *   already have the backing storage fully allocated.
		 * - No other BO uses the global gtt (well contexts, but meh),
1614
		 *   so we don't really have issues with multiple objects not
1615 1616 1617 1618 1619 1620
		 *   fitting due to fragmentation.
		 * So this is actually safe.
		 */
		ret = i915_gem_obj_ggtt_pin(batch_obj, 0, 0);
		if (ret)
			goto err;
1621

1622
		params->batch_obj_vm_offset = i915_gem_obj_ggtt_offset(batch_obj);
1623
	} else
1624
		params->batch_obj_vm_offset = i915_gem_obj_offset(batch_obj, vm);
1625

1626
	/* Allocate a request for this batch buffer nice and early. */
1627
	req = i915_gem_request_alloc(engine, ctx);
1628 1629
	if (IS_ERR(req)) {
		ret = PTR_ERR(req);
1630
		goto err_batch_unpin;
1631
	}
1632

1633
	ret = i915_gem_request_add_to_client(req, file);
1634
	if (ret)
1635
		goto err_request;
1636

1637 1638 1639 1640 1641 1642 1643 1644
	/*
	 * Save assorted stuff away to pass through to *_submission().
	 * NB: This data should be 'persistent' and not local as it will
	 * kept around beyond the duration of the IOCTL once the GPU
	 * scheduler arrives.
	 */
	params->dev                     = dev;
	params->file                    = file;
1645
	params->engine                    = engine;
1646 1647 1648
	params->dispatch_flags          = dispatch_flags;
	params->batch_obj               = batch_obj;
	params->ctx                     = ctx;
1649
	params->request                 = req;
1650 1651

	ret = dev_priv->gt.execbuf_submit(params, args, &eb->vmas);
1652 1653
err_request:
	i915_gem_execbuffer_retire_commands(params);
1654

1655
err_batch_unpin:
1656 1657 1658 1659 1660 1661
	/*
	 * FIXME: We crucially rely upon the active tracking for the (ppgtt)
	 * batch vma for correctness. For less ugly and less fragility this
	 * needs to be adjusted to also track the ggtt batch vma properly as
	 * active.
	 */
1662
	if (dispatch_flags & I915_DISPATCH_SECURE)
1663
		i915_gem_object_ggtt_unpin(batch_obj);
1664

1665
err:
1666
	/* the request owns the ref now */
1667
	i915_gem_context_put(ctx);
1668
	eb_destroy(eb);
1669 1670 1671 1672

	mutex_unlock(&dev->struct_mutex);

pre_mutex_err:
1673 1674 1675
	/* intel_gpu_busy should also get a ref, so it will free when the device
	 * is really idle. */
	intel_runtime_pm_put(dev_priv);
1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693
	return ret;
}

/*
 * Legacy execbuffer just creates an exec2 list from the original exec object
 * list array and passes it to the real function.
 */
int
i915_gem_execbuffer(struct drm_device *dev, void *data,
		    struct drm_file *file)
{
	struct drm_i915_gem_execbuffer *args = data;
	struct drm_i915_gem_execbuffer2 exec2;
	struct drm_i915_gem_exec_object *exec_list = NULL;
	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
	int ret, i;

	if (args->buffer_count < 1) {
1694
		DRM_DEBUG("execbuf with %d buffers\n", args->buffer_count);
1695 1696 1697 1698 1699 1700 1701
		return -EINVAL;
	}

	/* Copy in the exec list from userland */
	exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
	exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
	if (exec_list == NULL || exec2_list == NULL) {
1702
		DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1703 1704 1705 1706 1707 1708
			  args->buffer_count);
		drm_free_large(exec_list);
		drm_free_large(exec2_list);
		return -ENOMEM;
	}
	ret = copy_from_user(exec_list,
1709
			     u64_to_user_ptr(args->buffers_ptr),
1710 1711
			     sizeof(*exec_list) * args->buffer_count);
	if (ret != 0) {
1712
		DRM_DEBUG("copy %d exec entries failed %d\n",
1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739
			  args->buffer_count, ret);
		drm_free_large(exec_list);
		drm_free_large(exec2_list);
		return -EFAULT;
	}

	for (i = 0; i < args->buffer_count; i++) {
		exec2_list[i].handle = exec_list[i].handle;
		exec2_list[i].relocation_count = exec_list[i].relocation_count;
		exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
		exec2_list[i].alignment = exec_list[i].alignment;
		exec2_list[i].offset = exec_list[i].offset;
		if (INTEL_INFO(dev)->gen < 4)
			exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
		else
			exec2_list[i].flags = 0;
	}

	exec2.buffers_ptr = args->buffers_ptr;
	exec2.buffer_count = args->buffer_count;
	exec2.batch_start_offset = args->batch_start_offset;
	exec2.batch_len = args->batch_len;
	exec2.DR1 = args->DR1;
	exec2.DR4 = args->DR4;
	exec2.num_cliprects = args->num_cliprects;
	exec2.cliprects_ptr = args->cliprects_ptr;
	exec2.flags = I915_EXEC_RENDER;
1740
	i915_execbuffer2_set_context_id(exec2, 0);
1741

1742
	ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
1743
	if (!ret) {
1744
		struct drm_i915_gem_exec_object __user *user_exec_list =
1745
			u64_to_user_ptr(args->buffers_ptr);
1746

1747
		/* Copy the new buffer offsets back to the user's exec list. */
1748
		for (i = 0; i < args->buffer_count; i++) {
1749 1750
			exec2_list[i].offset =
				gen8_canonical_addr(exec2_list[i].offset);
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
			ret = __copy_to_user(&user_exec_list[i].offset,
					     &exec2_list[i].offset,
					     sizeof(user_exec_list[i].offset));
			if (ret) {
				ret = -EFAULT;
				DRM_DEBUG("failed to copy %d exec entries "
					  "back to user (%d)\n",
					  args->buffer_count, ret);
				break;
			}
1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776
		}
	}

	drm_free_large(exec_list);
	drm_free_large(exec2_list);
	return ret;
}

int
i915_gem_execbuffer2(struct drm_device *dev, void *data,
		     struct drm_file *file)
{
	struct drm_i915_gem_execbuffer2 *args = data;
	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
	int ret;

1777 1778
	if (args->buffer_count < 1 ||
	    args->buffer_count > UINT_MAX / sizeof(*exec2_list)) {
1779
		DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
1780 1781 1782
		return -EINVAL;
	}

1783 1784 1785 1786 1787
	if (args->rsvd2 != 0) {
		DRM_DEBUG("dirty rvsd2 field\n");
		return -EINVAL;
	}

1788 1789 1790
	exec2_list = drm_malloc_gfp(args->buffer_count,
				    sizeof(*exec2_list),
				    GFP_TEMPORARY);
1791
	if (exec2_list == NULL) {
1792
		DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
1793 1794 1795 1796
			  args->buffer_count);
		return -ENOMEM;
	}
	ret = copy_from_user(exec2_list,
1797
			     u64_to_user_ptr(args->buffers_ptr),
1798 1799
			     sizeof(*exec2_list) * args->buffer_count);
	if (ret != 0) {
1800
		DRM_DEBUG("copy %d exec entries failed %d\n",
1801 1802 1803 1804 1805
			  args->buffer_count, ret);
		drm_free_large(exec2_list);
		return -EFAULT;
	}

1806
	ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
1807 1808
	if (!ret) {
		/* Copy the new buffer offsets back to the user's exec list. */
1809
		struct drm_i915_gem_exec_object2 __user *user_exec_list =
1810
				   u64_to_user_ptr(args->buffers_ptr);
1811 1812 1813
		int i;

		for (i = 0; i < args->buffer_count; i++) {
1814 1815
			exec2_list[i].offset =
				gen8_canonical_addr(exec2_list[i].offset);
1816 1817 1818 1819 1820 1821 1822 1823 1824 1825
			ret = __copy_to_user(&user_exec_list[i].offset,
					     &exec2_list[i].offset,
					     sizeof(user_exec_list[i].offset));
			if (ret) {
				ret = -EFAULT;
				DRM_DEBUG("failed to copy %d exec entries "
					  "back to user\n",
					  args->buffer_count);
				break;
			}
1826 1827 1828 1829 1830 1831
		}
	}

	drm_free_large(exec2_list);
	return ret;
}