i915_gem_request.c 33.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
/*
 * Copyright © 2008-2015 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

25
#include <linux/prefetch.h>
26
#include <linux/dma-fence-array.h>
27

28 29
#include "i915_drv.h"

30
static const char *i915_fence_get_driver_name(struct dma_fence *fence)
31 32 33 34
{
	return "i915";
}

35
static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
36
{
37
	return to_request(fence)->timeline->common->name;
38 39
}

40
static bool i915_fence_signaled(struct dma_fence *fence)
41 42 43 44
{
	return i915_gem_request_completed(to_request(fence));
}

45
static bool i915_fence_enable_signaling(struct dma_fence *fence)
46 47 48 49 50 51 52 53
{
	if (i915_fence_signaled(fence))
		return false;

	intel_engine_enable_signaling(to_request(fence));
	return true;
}

54
static signed long i915_fence_wait(struct dma_fence *fence,
55
				   bool interruptible,
56
				   signed long timeout)
57
{
58
	return i915_wait_request(to_request(fence), interruptible, timeout);
59 60
}

61
static void i915_fence_release(struct dma_fence *fence)
62 63 64
{
	struct drm_i915_gem_request *req = to_request(fence);

65 66 67 68 69 70 71 72 73
	/* The request is put onto a RCU freelist (i.e. the address
	 * is immediately reused), mark the fences as being freed now.
	 * Otherwise the debugobjects for the fences are only marked as
	 * freed when the slab cache itself is freed, and so we would get
	 * caught trying to reuse dead objects.
	 */
	i915_sw_fence_fini(&req->submit);
	i915_sw_fence_fini(&req->execute);

74 75 76
	kmem_cache_free(req->i915->requests, req);
}

77
const struct dma_fence_ops i915_fence_ops = {
78 79 80 81 82 83 84 85
	.get_driver_name = i915_fence_get_driver_name,
	.get_timeline_name = i915_fence_get_timeline_name,
	.enable_signaling = i915_fence_enable_signaling,
	.signaled = i915_fence_signaled,
	.wait = i915_fence_wait,
	.release = i915_fence_release,
};

86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124
int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
				   struct drm_file *file)
{
	struct drm_i915_private *dev_private;
	struct drm_i915_file_private *file_priv;

	WARN_ON(!req || !file || req->file_priv);

	if (!req || !file)
		return -EINVAL;

	if (req->file_priv)
		return -EINVAL;

	dev_private = req->i915;
	file_priv = file->driver_priv;

	spin_lock(&file_priv->mm.lock);
	req->file_priv = file_priv;
	list_add_tail(&req->client_list, &file_priv->mm.request_list);
	spin_unlock(&file_priv->mm.lock);

	return 0;
}

static inline void
i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
{
	struct drm_i915_file_private *file_priv = request->file_priv;

	if (!file_priv)
		return;

	spin_lock(&file_priv->mm.lock);
	list_del(&request->client_list);
	request->file_priv = NULL;
	spin_unlock(&file_priv->mm.lock);
}

125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
static struct i915_dependency *
i915_dependency_alloc(struct drm_i915_private *i915)
{
	return kmem_cache_alloc(i915->dependencies, GFP_KERNEL);
}

static void
i915_dependency_free(struct drm_i915_private *i915,
		     struct i915_dependency *dep)
{
	kmem_cache_free(i915->dependencies, dep);
}

static void
__i915_priotree_add_dependency(struct i915_priotree *pt,
			       struct i915_priotree *signal,
			       struct i915_dependency *dep,
			       unsigned long flags)
{
144
	INIT_LIST_HEAD(&dep->dfs_link);
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
	list_add(&dep->wait_link, &signal->waiters_list);
	list_add(&dep->signal_link, &pt->signalers_list);
	dep->signaler = signal;
	dep->flags = flags;
}

static int
i915_priotree_add_dependency(struct drm_i915_private *i915,
			     struct i915_priotree *pt,
			     struct i915_priotree *signal)
{
	struct i915_dependency *dep;

	dep = i915_dependency_alloc(i915);
	if (!dep)
		return -ENOMEM;

	__i915_priotree_add_dependency(pt, signal, dep, I915_DEPENDENCY_ALLOC);
	return 0;
}

static void
i915_priotree_fini(struct drm_i915_private *i915, struct i915_priotree *pt)
{
	struct i915_dependency *dep, *next;

171 172
	GEM_BUG_ON(!RB_EMPTY_NODE(&pt->node));

173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
	/* Everyone we depended upon (the fences we wait to be signaled)
	 * should retire before us and remove themselves from our list.
	 * However, retirement is run independently on each timeline and
	 * so we may be called out-of-order.
	 */
	list_for_each_entry_safe(dep, next, &pt->signalers_list, signal_link) {
		list_del(&dep->wait_link);
		if (dep->flags & I915_DEPENDENCY_ALLOC)
			i915_dependency_free(i915, dep);
	}

	/* Remove ourselves from everyone who depends upon us */
	list_for_each_entry_safe(dep, next, &pt->waiters_list, wait_link) {
		list_del(&dep->signal_link);
		if (dep->flags & I915_DEPENDENCY_ALLOC)
			i915_dependency_free(i915, dep);
	}
}

static void
i915_priotree_init(struct i915_priotree *pt)
{
	INIT_LIST_HEAD(&pt->signalers_list);
	INIT_LIST_HEAD(&pt->waiters_list);
197 198
	RB_CLEAR_NODE(&pt->node);
	pt->priority = INT_MIN;
199 200
}

201 202 203 204 205 206
void i915_gem_retire_noop(struct i915_gem_active *active,
			  struct drm_i915_gem_request *request)
{
	/* Space left intentionally blank */
}

207 208
static void i915_gem_request_retire(struct drm_i915_gem_request *request)
{
209 210
	struct i915_gem_active *active, *next;

211
	lockdep_assert_held(&request->i915->drm.struct_mutex);
212 213
	GEM_BUG_ON(!i915_sw_fence_signaled(&request->submit));
	GEM_BUG_ON(!i915_sw_fence_signaled(&request->execute));
214
	GEM_BUG_ON(!i915_gem_request_completed(request));
215
	GEM_BUG_ON(!request->i915->gt.active_requests);
216

217
	trace_i915_gem_request_retire(request);
C
Chris Wilson 已提交
218 219

	spin_lock_irq(&request->engine->timeline->lock);
220
	list_del_init(&request->link);
C
Chris Wilson 已提交
221
	spin_unlock_irq(&request->engine->timeline->lock);
222 223 224 225 226 227 228 229 230

	/* We know the GPU must have read the request to have
	 * sent us the seqno + interrupt, so use the position
	 * of tail of the request to update the last known position
	 * of the GPU head.
	 *
	 * Note this requires that we are always called in request
	 * completion order.
	 */
231
	list_del(&request->ring_link);
232
	request->ring->last_retired_head = request->postfix;
233 234 235 236 237 238
	if (!--request->i915->gt.active_requests) {
		GEM_BUG_ON(!request->i915->gt.awake);
		mod_delayed_work(request->i915->wq,
				 &request->i915->gt.idle_work,
				 msecs_to_jiffies(100));
	}
239

240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
	/* Walk through the active list, calling retire on each. This allows
	 * objects to track their GPU activity and mark themselves as idle
	 * when their *last* active request is completed (updating state
	 * tracking lists for eviction, active references for GEM, etc).
	 *
	 * As the ->retire() may free the node, we decouple it first and
	 * pass along the auxiliary information (to avoid dereferencing
	 * the node after the callback).
	 */
	list_for_each_entry_safe(active, next, &request->active_list, link) {
		/* In microbenchmarks or focusing upon time inside the kernel,
		 * we may spend an inordinate amount of time simply handling
		 * the retirement of requests and processing their callbacks.
		 * Of which, this loop itself is particularly hot due to the
		 * cache misses when jumping around the list of i915_gem_active.
		 * So we try to keep this loop as streamlined as possible and
		 * also prefetch the next i915_gem_active to try and hide
		 * the likely cache miss.
		 */
		prefetchw(next);

		INIT_LIST_HEAD(&active->link);
262
		RCU_INIT_POINTER(active->request, NULL);
263 264 265 266

		active->retire(active, request);
	}

267 268 269 270 271 272 273 274
	i915_gem_request_remove_from_client(request);

	if (request->previous_context) {
		if (i915.enable_execlists)
			intel_lr_context_unpin(request->previous_context,
					       request->engine);
	}

275
	/* Retirement decays the ban score as it is a sign of ctx progress */
276 277
	if (request->ctx->ban_score > 0)
		request->ctx->ban_score--;
278

279
	i915_gem_context_put(request->ctx);
280 281

	dma_fence_signal(&request->fence);
282 283

	i915_priotree_fini(request->i915, &request->priotree);
284
	i915_gem_request_put(request);
285 286 287 288 289 290 291 292
}

void i915_gem_request_retire_upto(struct drm_i915_gem_request *req)
{
	struct intel_engine_cs *engine = req->engine;
	struct drm_i915_gem_request *tmp;

	lockdep_assert_held(&req->i915->drm.struct_mutex);
293 294
	GEM_BUG_ON(!i915_gem_request_completed(req));

295 296
	if (list_empty(&req->link))
		return;
297 298

	do {
299
		tmp = list_first_entry(&engine->timeline->requests,
300
				       typeof(*tmp), link);
301 302 303 304 305

		i915_gem_request_retire(tmp);
	} while (tmp != req);
}

306
static int i915_gem_check_wedge(struct drm_i915_private *dev_priv)
307
{
308 309 310
	struct i915_gpu_error *error = &dev_priv->gpu_error;

	if (i915_terminally_wedged(error))
311 312
		return -EIO;

313
	if (i915_reset_in_progress(error)) {
314 315 316
		/* Non-interruptible callers can't handle -EAGAIN, hence return
		 * -EIO unconditionally for these.
		 */
317
		if (!dev_priv->mm.interruptible)
318 319 320 321 322 323 324 325
			return -EIO;

		return -EAGAIN;
	}

	return 0;
}

326
static int i915_gem_init_global_seqno(struct drm_i915_private *i915, u32 seqno)
327
{
328
	struct i915_gem_timeline *timeline = &i915->gt.global_timeline;
329
	struct intel_engine_cs *engine;
330
	enum intel_engine_id id;
331 332 333
	int ret;

	/* Carefully retire all requests without writing to the rings */
334
	ret = i915_gem_wait_for_idle(i915,
335 336 337 338 339
				     I915_WAIT_INTERRUPTIBLE |
				     I915_WAIT_LOCKED);
	if (ret)
		return ret;

340
	i915_gem_retire_requests(i915);
341
	GEM_BUG_ON(i915->gt.active_requests > 1);
342 343

	/* If the seqno wraps around, we need to clear the breadcrumb rbtree */
344
	if (!i915_seqno_passed(seqno, atomic_read(&timeline->seqno))) {
345 346
		while (intel_breadcrumbs_busy(i915))
			cond_resched(); /* spin until threads are complete */
347
	}
348
	atomic_set(&timeline->seqno, seqno);
349 350

	/* Finally reset hw state */
351
	for_each_engine(engine, i915, id)
352
		intel_engine_init_global_seqno(engine, seqno);
353

354 355 356 357 358 359 360 361
	list_for_each_entry(timeline, &i915->gt.timelines, link) {
		for_each_engine(engine, i915, id) {
			struct intel_timeline *tl = &timeline->engine[id];

			memset(tl->sync_seqno, 0, sizeof(tl->sync_seqno));
		}
	}

362 363 364
	return 0;
}

365
int i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno)
366 367 368
{
	struct drm_i915_private *dev_priv = to_i915(dev);

369 370
	lockdep_assert_held(&dev_priv->drm.struct_mutex);

371 372 373 374 375 376
	if (seqno == 0)
		return -EINVAL;

	/* HWS page needs to be set less than what we
	 * will inject to ring
	 */
377
	return i915_gem_init_global_seqno(dev_priv, seqno - 1);
378 379
}

380
static int reserve_global_seqno(struct drm_i915_private *i915)
381
{
382
	u32 active_requests = ++i915->gt.active_requests;
383
	u32 seqno = atomic_read(&i915->gt.global_timeline.seqno);
384
	int ret;
385

386
	/* Reservation is fine until we need to wrap around */
387
	if (likely(seqno + active_requests > seqno))
388
		return 0;
389

390 391 392 393
	ret = i915_gem_init_global_seqno(i915, 0);
	if (ret) {
		i915->gt.active_requests--;
		return ret;
394 395 396 397 398
	}

	return 0;
}

C
Chris Wilson 已提交
399 400
static u32 __timeline_get_seqno(struct i915_gem_timeline *tl)
{
401 402
	/* seqno only incremented under a mutex */
	return ++tl->seqno.counter;
C
Chris Wilson 已提交
403 404
}

405 406
static u32 timeline_get_seqno(struct i915_gem_timeline *tl)
{
407
	return atomic_inc_return(&tl->seqno);
408 409
}

410
void __i915_gem_request_submit(struct drm_i915_gem_request *request)
411
{
412
	struct intel_engine_cs *engine = request->engine;
413 414
	struct intel_timeline *timeline;
	u32 seqno;
415

C
Chris Wilson 已提交
416 417 418
	/* Transfer from per-context onto the global per-engine timeline */
	timeline = engine->timeline;
	GEM_BUG_ON(timeline == request->timeline);
419
	assert_spin_locked(&timeline->lock);
420

C
Chris Wilson 已提交
421
	seqno = timeline_get_seqno(timeline->common);
422 423 424 425 426 427 428 429 430 431 432 433 434 435 436
	GEM_BUG_ON(!seqno);
	GEM_BUG_ON(i915_seqno_passed(intel_engine_get_seqno(engine), seqno));

	GEM_BUG_ON(i915_seqno_passed(timeline->last_submitted_seqno, seqno));
	request->previous_seqno = timeline->last_submitted_seqno;
	timeline->last_submitted_seqno = seqno;

	/* We may be recursing from the signal callback of another i915 fence */
	spin_lock_nested(&request->lock, SINGLE_DEPTH_NESTING);
	request->global_seqno = seqno;
	if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &request->fence.flags))
		intel_engine_enable_signaling(request);
	spin_unlock(&request->lock);

	GEM_BUG_ON(!request->global_seqno);
C
Chris Wilson 已提交
437 438
	engine->emit_breadcrumb(request,
				request->ring->vaddr + request->postfix);
439

440
	spin_lock(&request->timeline->lock);
C
Chris Wilson 已提交
441 442 443
	list_move_tail(&request->link, &timeline->requests);
	spin_unlock(&request->timeline->lock);

444
	i915_sw_fence_commit(&request->execute);
445 446 447 448 449 450
}

void i915_gem_request_submit(struct drm_i915_gem_request *request)
{
	struct intel_engine_cs *engine = request->engine;
	unsigned long flags;
451

452 453 454 455 456 457 458 459 460 461 462
	/* Will be called from irq-context when using foreign fences. */
	spin_lock_irqsave(&engine->timeline->lock, flags);

	__i915_gem_request_submit(request);

	spin_unlock_irqrestore(&engine->timeline->lock, flags);
}

static int __i915_sw_fence_call
submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
{
463 464
	struct drm_i915_gem_request *request =
		container_of(fence, typeof(*request), submit);
465

466 467
	switch (state) {
	case FENCE_COMPLETE:
468
		request->engine->submit_request(request);
469 470 471 472 473
		break;

	case FENCE_FREE:
		i915_gem_request_put(request);
		break;
474
	}
C
Chris Wilson 已提交
475

476 477 478
	return NOTIFY_DONE;
}

479 480 481
static int __i915_sw_fence_call
execute_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
{
482 483 484 485 486 487 488 489 490 491 492 493
	struct drm_i915_gem_request *request =
		container_of(fence, typeof(*request), execute);

	switch (state) {
	case FENCE_COMPLETE:
		break;

	case FENCE_FREE:
		i915_gem_request_put(request);
		break;
	}

494 495 496
	return NOTIFY_DONE;
}

497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
/**
 * i915_gem_request_alloc - allocate a request structure
 *
 * @engine: engine that we wish to issue the request on.
 * @ctx: context that the request will be associated with.
 *       This can be NULL if the request is not directly related to
 *       any specific user context, in which case this function will
 *       choose an appropriate context to use.
 *
 * Returns a pointer to the allocated request if successful,
 * or an error code if not.
 */
struct drm_i915_gem_request *
i915_gem_request_alloc(struct intel_engine_cs *engine,
		       struct i915_gem_context *ctx)
512 513 514 515 516
{
	struct drm_i915_private *dev_priv = engine->i915;
	struct drm_i915_gem_request *req;
	int ret;

517 518
	lockdep_assert_held(&dev_priv->drm.struct_mutex);

519 520 521 522
	/* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
	 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
	 * and restart.
	 */
523
	ret = i915_gem_check_wedge(dev_priv);
524
	if (ret)
525
		return ERR_PTR(ret);
526

527 528 529 530
	ret = reserve_global_seqno(dev_priv);
	if (ret)
		return ERR_PTR(ret);

531
	/* Move the oldest request to the slab-cache (if not in use!) */
532
	req = list_first_entry_or_null(&engine->timeline->requests,
533
				       typeof(*req), link);
C
Chris Wilson 已提交
534
	if (req && __i915_gem_request_completed(req))
535
		i915_gem_request_retire(req);
536

537 538 539 540 541
	/* Beware: Dragons be flying overhead.
	 *
	 * We use RCU to look up requests in flight. The lookups may
	 * race with the request being allocated from the slab freelist.
	 * That is the request we are writing to here, may be in the process
542
	 * of being read by __i915_gem_active_get_rcu(). As such,
543 544
	 * we have to be very careful when overwriting the contents. During
	 * the RCU lookup, we change chase the request->engine pointer,
545
	 * read the request->global_seqno and increment the reference count.
546 547 548 549
	 *
	 * The reference count is incremented atomically. If it is zero,
	 * the lookup knows the request is unallocated and complete. Otherwise,
	 * it is either still in use, or has been reallocated and reset
550 551
	 * with dma_fence_init(). This increment is safe for release as we
	 * check that the request we have a reference to and matches the active
552 553 554 555 556 557 558 559 560 561 562 563 564 565
	 * request.
	 *
	 * Before we increment the refcount, we chase the request->engine
	 * pointer. We must not call kmem_cache_zalloc() or else we set
	 * that pointer to NULL and cause a crash during the lookup. If
	 * we see the request is completed (based on the value of the
	 * old engine and seqno), the lookup is complete and reports NULL.
	 * If we decide the request is not completed (new engine or seqno),
	 * then we grab a reference and double check that it is still the
	 * active request - which it won't be and restart the lookup.
	 *
	 * Do not use kmem_cache_zalloc() here!
	 */
	req = kmem_cache_alloc(dev_priv->requests, GFP_KERNEL);
566 567 568 569
	if (!req) {
		ret = -ENOMEM;
		goto err_unreserve;
	}
570

C
Chris Wilson 已提交
571 572
	req->timeline = i915_gem_context_lookup_timeline(ctx, engine);
	GEM_BUG_ON(req->timeline == engine->timeline);
573

574
	spin_lock_init(&req->lock);
575 576 577
	dma_fence_init(&req->fence,
		       &i915_fence_ops,
		       &req->lock,
578
		       req->timeline->fence_context,
C
Chris Wilson 已提交
579
		       __timeline_get_seqno(req->timeline->common));
580

581 582 583 584
	/* We bump the ref for the fence chain */
	i915_sw_fence_init(&i915_gem_request_get(req)->submit, submit_notify);
	i915_sw_fence_init(&i915_gem_request_get(req)->execute, execute_notify);

585 586 587 588 589
	/* Ensure that the execute fence completes after the submit fence -
	 * as we complete the execute fence from within the submit fence
	 * callback, its completion would otherwise be visible first.
	 */
	i915_sw_fence_await_sw_fence(&req->execute, &req->submit, &req->execq);
590

591 592
	i915_priotree_init(&req->priotree);

593
	INIT_LIST_HEAD(&req->active_list);
594 595
	req->i915 = dev_priv;
	req->engine = engine;
596
	req->ctx = i915_gem_context_get(ctx);
597

598
	/* No zalloc, must clear what we need by hand */
599
	req->global_seqno = 0;
600 601
	req->previous_context = NULL;
	req->file_priv = NULL;
C
Chris Wilson 已提交
602
	req->batch = NULL;
603

604 605 606 607 608 609 610 611
	/*
	 * Reserve space in the ring buffer for all the commands required to
	 * eventually emit this request. This is to guarantee that the
	 * i915_add_request() call can't fail. Note that the reserve may need
	 * to be redone if the request is not actually submitted straight
	 * away, e.g. because a GPU scheduler has deferred it.
	 */
	req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
612
	GEM_BUG_ON(req->reserved_space < engine->emit_breadcrumb_sz);
613 614 615 616 617 618 619 620

	if (i915.enable_execlists)
		ret = intel_logical_ring_alloc_request_extras(req);
	else
		ret = intel_ring_alloc_request_extras(req);
	if (ret)
		goto err_ctx;

621 622 623 624 625 626 627
	/* Record the position of the start of the request so that
	 * should we detect the updated seqno part-way through the
	 * GPU processing the request, we never over-estimate the
	 * position of the head.
	 */
	req->head = req->ring->tail;

628
	return req;
629 630

err_ctx:
631 632 633 634 635
	/* Make sure we didn't add ourselves to external state before freeing */
	GEM_BUG_ON(!list_empty(&req->active_list));
	GEM_BUG_ON(!list_empty(&req->priotree.signalers_list));
	GEM_BUG_ON(!list_empty(&req->priotree.waiters_list));

636
	i915_gem_context_put(ctx);
637
	kmem_cache_free(dev_priv->requests, req);
638 639
err_unreserve:
	dev_priv->gt.active_requests--;
640
	return ERR_PTR(ret);
641 642
}

643 644 645 646
static int
i915_gem_request_await_request(struct drm_i915_gem_request *to,
			       struct drm_i915_gem_request *from)
{
647
	int ret;
648 649 650

	GEM_BUG_ON(to == from);

651 652 653 654 655 656 657 658
	if (to->engine->schedule) {
		ret = i915_priotree_add_dependency(to->i915,
						   &to->priotree,
						   &from->priotree);
		if (ret < 0)
			return ret;
	}

659
	if (to->timeline == from->timeline)
660 661
		return 0;

662 663 664 665 666 667 668
	if (to->engine == from->engine) {
		ret = i915_sw_fence_await_sw_fence_gfp(&to->submit,
						       &from->submit,
						       GFP_KERNEL);
		return ret < 0 ? ret : 0;
	}

669 670 671 672 673 674 675
	if (!from->global_seqno) {
		ret = i915_sw_fence_await_dma_fence(&to->submit,
						    &from->fence, 0,
						    GFP_KERNEL);
		return ret < 0 ? ret : 0;
	}

676
	if (from->global_seqno <= to->timeline->sync_seqno[from->engine->id])
677 678 679 680
		return 0;

	trace_i915_gem_ring_sync_to(to, from);
	if (!i915.semaphores) {
681 682 683 684 685 686 687
		if (!i915_spin_request(from, TASK_INTERRUPTIBLE, 2)) {
			ret = i915_sw_fence_await_dma_fence(&to->submit,
							    &from->fence, 0,
							    GFP_KERNEL);
			if (ret < 0)
				return ret;
		}
688 689 690 691 692 693
	} else {
		ret = to->engine->semaphore.sync_to(to, from);
		if (ret)
			return ret;
	}

694
	to->timeline->sync_seqno[from->engine->id] = from->global_seqno;
695 696 697
	return 0;
}

698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744
int
i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
				 struct dma_fence *fence)
{
	struct dma_fence_array *array;
	int ret;
	int i;

	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
		return 0;

	if (dma_fence_is_i915(fence))
		return i915_gem_request_await_request(req, to_request(fence));

	if (!dma_fence_is_array(fence)) {
		ret = i915_sw_fence_await_dma_fence(&req->submit,
						    fence, I915_FENCE_TIMEOUT,
						    GFP_KERNEL);
		return ret < 0 ? ret : 0;
	}

	/* Note that if the fence-array was created in signal-on-any mode,
	 * we should *not* decompose it into its individual fences. However,
	 * we don't currently store which mode the fence-array is operating
	 * in. Fortunately, the only user of signal-on-any is private to
	 * amdgpu and we should not see any incoming fence-array from
	 * sync-file being in signal-on-any mode.
	 */

	array = to_dma_fence_array(fence);
	for (i = 0; i < array->num_fences; i++) {
		struct dma_fence *child = array->fences[i];

		if (dma_fence_is_i915(child))
			ret = i915_gem_request_await_request(req,
							     to_request(child));
		else
			ret = i915_sw_fence_await_dma_fence(&req->submit,
							    child, I915_FENCE_TIMEOUT,
							    GFP_KERNEL);
		if (ret < 0)
			return ret;
	}

	return 0;
}

745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
/**
 * i915_gem_request_await_object - set this request to (async) wait upon a bo
 *
 * @to: request we are wishing to use
 * @obj: object which may be in use on another ring.
 *
 * This code is meant to abstract object synchronization with the GPU.
 * Conceptually we serialise writes between engines inside the GPU.
 * We only allow one engine to write into a buffer at any time, but
 * multiple readers. To ensure each has a coherent view of memory, we must:
 *
 * - If there is an outstanding write request to the object, the new
 *   request must wait for it to complete (either CPU or in hw, requests
 *   on the same ring will be naturally ordered).
 *
 * - If we are a write request (pending_write_domain is set), the new
 *   request must wait for outstanding read requests to complete.
 *
 * Returns 0 if successful, else propagates up the lower layer error.
 */
int
i915_gem_request_await_object(struct drm_i915_gem_request *to,
			      struct drm_i915_gem_object *obj,
			      bool write)
{
770 771
	struct dma_fence *excl;
	int ret = 0;
772 773

	if (write) {
774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792
		struct dma_fence **shared;
		unsigned int count, i;

		ret = reservation_object_get_fences_rcu(obj->resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			ret = i915_gem_request_await_dma_fence(to, shared[i]);
			if (ret)
				break;

			dma_fence_put(shared[i]);
		}

		for (; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);
793
	} else {
794
		excl = reservation_object_get_excl_rcu(obj->resv);
795 796
	}

797 798 799
	if (excl) {
		if (ret == 0)
			ret = i915_gem_request_await_dma_fence(to, excl);
800

801
		dma_fence_put(excl);
802 803
	}

804
	return ret;
805 806
}

807 808 809 810 811 812 813
static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
{
	struct drm_i915_private *dev_priv = engine->i915;

	if (dev_priv->gt.awake)
		return;

814 815
	GEM_BUG_ON(!dev_priv->gt.active_requests);

816 817 818
	intel_runtime_pm_get_noresume(dev_priv);
	dev_priv->gt.awake = true;

819
	intel_enable_gt_powersave(dev_priv);
820 821 822 823 824 825 826 827 828 829 830 831 832 833
	i915_update_gfx_val(dev_priv);
	if (INTEL_GEN(dev_priv) >= 6)
		gen6_rps_busy(dev_priv);

	queue_delayed_work(dev_priv->wq,
			   &dev_priv->gt.retire_work,
			   round_jiffies_up_relative(HZ));
}

/*
 * NB: This function is not allowed to fail. Doing so would mean the the
 * request is not being tracked for completion but the work itself is
 * going to happen on the hardware. This would be a Bad Thing(tm).
 */
834
void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
835
{
836 837
	struct intel_engine_cs *engine = request->engine;
	struct intel_ring *ring = request->ring;
838
	struct intel_timeline *timeline = request->timeline;
839
	struct drm_i915_gem_request *prev;
C
Chris Wilson 已提交
840
	int err;
841

842
	lockdep_assert_held(&request->i915->drm.struct_mutex);
843 844
	trace_i915_gem_request_add(request);

845 846 847 848 849 850 851 852 853 854 855 856 857 858 859
	/*
	 * To ensure that this call will not fail, space for its emissions
	 * should already have been reserved in the ring buffer. Let the ring
	 * know that it is time to use that space up.
	 */
	request->reserved_space = 0;

	/*
	 * Emit any outstanding flushes - execbuf can fail to emit the flush
	 * after having emitted the batchbuffer command. Hence we need to fix
	 * things up similar to emitting the lazy request. The difference here
	 * is that the flush _must_ happen before the next request, no matter
	 * what.
	 */
	if (flush_caches) {
C
Chris Wilson 已提交
860
		err = engine->emit_flush(request, EMIT_FLUSH);
861

862
		/* Not allowed to fail! */
C
Chris Wilson 已提交
863
		WARN(err, "engine->emit_flush() failed: %d!\n", err);
864 865
	}

866
	/* Record the position of the start of the breadcrumb so that
867 868
	 * should we detect the updated seqno part-way through the
	 * GPU processing the request, we never over-estimate the
869
	 * position of the ring's HEAD.
870
	 */
C
Chris Wilson 已提交
871 872
	err = intel_ring_begin(request, engine->emit_breadcrumb_sz);
	GEM_BUG_ON(err);
873
	request->postfix = ring->tail;
C
Chris Wilson 已提交
874
	ring->tail += engine->emit_breadcrumb_sz * sizeof(u32);
875

876 877 878 879 880
	/* Seal the request and mark it as pending execution. Note that
	 * we may inspect this state, without holding any locks, during
	 * hangcheck. Hence we apply the barrier to ensure that we do not
	 * see a more recent value in the hws than we are tracking.
	 */
881

882
	prev = i915_gem_active_raw(&timeline->last_request,
883
				   &request->i915->drm.struct_mutex);
884
	if (prev) {
885 886
		i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,
					     &request->submitq);
887 888 889 890 891 892
		if (engine->schedule)
			__i915_priotree_add_dependency(&request->priotree,
						       &prev->priotree,
						       &request->dep,
						       0);
	}
893

C
Chris Wilson 已提交
894
	spin_lock_irq(&timeline->lock);
895
	list_add_tail(&request->link, &timeline->requests);
C
Chris Wilson 已提交
896 897 898 899
	spin_unlock_irq(&timeline->lock);

	GEM_BUG_ON(i915_seqno_passed(timeline->last_submitted_seqno,
				     request->fence.seqno));
900

C
Chris Wilson 已提交
901
	timeline->last_submitted_seqno = request->fence.seqno;
902
	i915_gem_active_set(&timeline->last_request, request);
903

904
	list_add_tail(&request->ring_link, &ring->request_list);
905
	request->emitted_jiffies = jiffies;
906

907
	i915_gem_mark_busy(engine);
908

909 910 911 912 913 914 915 916 917 918 919
	/* Let the backend know a new request has arrived that may need
	 * to adjust the existing execution schedule due to a high priority
	 * request - i.e. we may want to preempt the current request in order
	 * to run a high priority dependency chain *before* we can execute this
	 * request.
	 *
	 * This is called before the request is ready to run so that we can
	 * decide whether to preempt the entire chain so that it is ready to
	 * run at the earliest possible convenience.
	 */
	if (engine->schedule)
920
		engine->schedule(request, request->ctx->priority);
921

922 923 924
	local_bh_disable();
	i915_sw_fence_commit(&request->submit);
	local_bh_enable(); /* Kick the execlists tasklet if just scheduled */
925 926
}

927 928 929 930 931 932 933 934 935 936
static void reset_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)
{
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	if (list_empty(&wait->task_list))
		__add_wait_queue(q, wait);
	spin_unlock_irqrestore(&q->lock, flags);
}

937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985
static unsigned long local_clock_us(unsigned int *cpu)
{
	unsigned long t;

	/* Cheaply and approximately convert from nanoseconds to microseconds.
	 * The result and subsequent calculations are also defined in the same
	 * approximate microseconds units. The principal source of timing
	 * error here is from the simple truncation.
	 *
	 * Note that local_clock() is only defined wrt to the current CPU;
	 * the comparisons are no longer valid if we switch CPUs. Instead of
	 * blocking preemption for the entire busywait, we can detect the CPU
	 * switch and use that as indicator of system load and a reason to
	 * stop busywaiting, see busywait_stop().
	 */
	*cpu = get_cpu();
	t = local_clock() >> 10;
	put_cpu();

	return t;
}

static bool busywait_stop(unsigned long timeout, unsigned int cpu)
{
	unsigned int this_cpu;

	if (time_after(local_clock_us(&this_cpu), timeout))
		return true;

	return this_cpu != cpu;
}

bool __i915_spin_request(const struct drm_i915_gem_request *req,
			 int state, unsigned long timeout_us)
{
	unsigned int cpu;

	/* When waiting for high frequency requests, e.g. during synchronous
	 * rendering split between the CPU and GPU, the finite amount of time
	 * required to set up the irq and wait upon it limits the response
	 * rate. By busywaiting on the request completion for a short while we
	 * can service the high frequency waits as quick as possible. However,
	 * if it is a slow request, we want to sleep as quickly as possible.
	 * The tradeoff between waiting and sleeping is roughly the time it
	 * takes to sleep on a request, on the order of a microsecond.
	 */

	timeout_us += local_clock_us(&cpu);
	do {
986
		if (__i915_gem_request_completed(req))
987 988 989 990 991 992 993 994 995 996 997 998 999 1000
			return true;

		if (signal_pending_state(state, current))
			break;

		if (busywait_stop(timeout_us, cpu))
			break;

		cpu_relax_lowlatency();
	} while (!need_resched());

	return false;
}

1001
static long
1002 1003 1004
__i915_request_wait_for_execute(struct drm_i915_gem_request *request,
				unsigned int flags,
				long timeout)
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
{
	const int state = flags & I915_WAIT_INTERRUPTIBLE ?
		TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
	wait_queue_head_t *q = &request->i915->gpu_error.wait_queue;
	DEFINE_WAIT(reset);
	DEFINE_WAIT(wait);

	if (flags & I915_WAIT_LOCKED)
		add_wait_queue(q, &reset);

	do {
1016
		prepare_to_wait(&request->execute.wait, &wait, state);
1017

1018
		if (i915_sw_fence_done(&request->execute))
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
			break;

		if (flags & I915_WAIT_LOCKED &&
		    i915_reset_in_progress(&request->i915->gpu_error)) {
			__set_current_state(TASK_RUNNING);
			i915_reset(request->i915);
			reset_wait_queue(q, &reset);
			continue;
		}

		if (signal_pending_state(state, current)) {
			timeout = -ERESTARTSYS;
			break;
		}

		timeout = io_schedule_timeout(timeout);
	} while (timeout);
1036
	finish_wait(&request->execute.wait, &wait);
1037 1038 1039 1040 1041 1042 1043

	if (flags & I915_WAIT_LOCKED)
		remove_wait_queue(q, &reset);

	return timeout;
}

1044
/**
1045
 * i915_wait_request - wait until execution of request has finished
1046
 * @req: the request to wait upon
1047
 * @flags: how to wait
1048 1049 1050 1051 1052
 * @timeout: how long to wait in jiffies
 *
 * i915_wait_request() waits for the request to be completed, for a
 * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an
 * unbounded wait).
1053
 *
1054 1055 1056
 * If the caller holds the struct_mutex, the caller must pass I915_WAIT_LOCKED
 * in via the flags, and vice versa if the struct_mutex is not held, the caller
 * must not specify that the wait is locked.
1057
 *
1058 1059 1060 1061
 * Returns the remaining time (in jiffies) if the request completed, which may
 * be zero or -ETIME if the request is unfinished after the timeout expires.
 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
 * pending before the request completes.
1062
 */
1063 1064 1065
long i915_wait_request(struct drm_i915_gem_request *req,
		       unsigned int flags,
		       long timeout)
1066
{
1067 1068
	const int state = flags & I915_WAIT_INTERRUPTIBLE ?
		TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1069 1070 1071 1072
	DEFINE_WAIT(reset);
	struct intel_wait wait;

	might_sleep();
1073
#if IS_ENABLED(CONFIG_LOCKDEP)
1074 1075
	GEM_BUG_ON(debug_locks &&
		   !!lockdep_is_held(&req->i915->drm.struct_mutex) !=
1076 1077
		   !!(flags & I915_WAIT_LOCKED));
#endif
1078
	GEM_BUG_ON(timeout < 0);
1079 1080

	if (i915_gem_request_completed(req))
1081
		return timeout;
1082

1083 1084
	if (!timeout)
		return -ETIME;
1085 1086 1087

	trace_i915_gem_request_wait_begin(req);

1088 1089
	if (!i915_sw_fence_done(&req->execute)) {
		timeout = __i915_request_wait_for_execute(req, flags, timeout);
1090 1091 1092
		if (timeout < 0)
			goto complete;

1093
		GEM_BUG_ON(!i915_sw_fence_done(&req->execute));
1094
	}
1095
	GEM_BUG_ON(!i915_sw_fence_done(&req->submit));
1096
	GEM_BUG_ON(!req->global_seqno);
1097

1098
	/* Optimistic short spin before touching IRQs */
1099 1100 1101 1102
	if (i915_spin_request(req, state, 5))
		goto complete;

	set_current_state(state);
1103 1104
	if (flags & I915_WAIT_LOCKED)
		add_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
1105

1106
	intel_wait_init(&wait, req->global_seqno);
1107 1108 1109 1110 1111 1112 1113 1114 1115
	if (intel_engine_add_wait(req->engine, &wait))
		/* In order to check that we haven't missed the interrupt
		 * as we enabled it, we need to kick ourselves to do a
		 * coherent check on the seqno before we sleep.
		 */
		goto wakeup;

	for (;;) {
		if (signal_pending_state(state, current)) {
1116
			timeout = -ERESTARTSYS;
1117 1118 1119
			break;
		}

1120 1121
		if (!timeout) {
			timeout = -ETIME;
1122 1123 1124
			break;
		}

1125 1126
		timeout = io_schedule_timeout(timeout);

1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140
		if (intel_wait_complete(&wait))
			break;

		set_current_state(state);

wakeup:
		/* Carefully check if the request is complete, giving time
		 * for the seqno to be visible following the interrupt.
		 * We also have to check in case we are kicked by the GPU
		 * reset in order to drop the struct_mutex.
		 */
		if (__i915_request_irq_complete(req))
			break;

1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
		/* If the GPU is hung, and we hold the lock, reset the GPU
		 * and then check for completion. On a full reset, the engine's
		 * HW seqno will be advanced passed us and we are complete.
		 * If we do a partial reset, we have to wait for the GPU to
		 * resume and update the breadcrumb.
		 *
		 * If we don't hold the mutex, we can just wait for the worker
		 * to come along and update the breadcrumb (either directly
		 * itself, or indirectly by recovering the GPU).
		 */
		if (flags & I915_WAIT_LOCKED &&
		    i915_reset_in_progress(&req->i915->gpu_error)) {
			__set_current_state(TASK_RUNNING);
			i915_reset(req->i915);
			reset_wait_queue(&req->i915->gpu_error.wait_queue,
					 &reset);
			continue;
		}

1160 1161 1162 1163 1164 1165
		/* Only spin if we know the GPU is processing this request */
		if (i915_spin_request(req, state, 2))
			break;
	}

	intel_engine_remove_wait(req->engine, &wait);
1166 1167
	if (flags & I915_WAIT_LOCKED)
		remove_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
1168
	__set_current_state(TASK_RUNNING);
1169

1170 1171 1172
complete:
	trace_i915_gem_request_wait_end(req);

1173
	return timeout;
1174
}
1175

1176
static void engine_retire_requests(struct intel_engine_cs *engine)
1177 1178 1179
{
	struct drm_i915_gem_request *request, *next;

1180 1181
	list_for_each_entry_safe(request, next,
				 &engine->timeline->requests, link) {
C
Chris Wilson 已提交
1182
		if (!__i915_gem_request_completed(request))
1183
			return;
1184 1185 1186 1187 1188 1189 1190 1191

		i915_gem_request_retire(request);
	}
}

void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
{
	struct intel_engine_cs *engine;
1192
	enum intel_engine_id id;
1193 1194 1195

	lockdep_assert_held(&dev_priv->drm.struct_mutex);

1196
	if (!dev_priv->gt.active_requests)
1197 1198
		return;

1199 1200
	for_each_engine(engine, dev_priv, id)
		engine_retire_requests(engine);
1201
}