i915_gem_request.c 26.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
/*
 * Copyright © 2008-2015 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 */

25
#include <linux/prefetch.h>
26
#include <linux/dma-fence-array.h>
27

28 29
#include "i915_drv.h"

30
static const char *i915_fence_get_driver_name(struct dma_fence *fence)
31 32 33 34
{
	return "i915";
}

35
static const char *i915_fence_get_timeline_name(struct dma_fence *fence)
36 37 38 39 40 41 42 43 44 45
{
	/* Timelines are bound by eviction to a VM. However, since
	 * we only have a global seqno at the moment, we only have
	 * a single timeline. Note that each timeline will have
	 * multiple execution contexts (fence contexts) as we allow
	 * engines within a single timeline to execute in parallel.
	 */
	return "global";
}

46
static bool i915_fence_signaled(struct dma_fence *fence)
47 48 49 50
{
	return i915_gem_request_completed(to_request(fence));
}

51
static bool i915_fence_enable_signaling(struct dma_fence *fence)
52 53 54 55 56 57 58 59
{
	if (i915_fence_signaled(fence))
		return false;

	intel_engine_enable_signaling(to_request(fence));
	return true;
}

60
static signed long i915_fence_wait(struct dma_fence *fence,
61
				   bool interruptible,
62
				   signed long timeout)
63
{
64
	return i915_wait_request(to_request(fence), interruptible, timeout);
65 66
}

67
static void i915_fence_value_str(struct dma_fence *fence, char *str, int size)
68 69 70 71
{
	snprintf(str, size, "%u", fence->seqno);
}

72
static void i915_fence_timeline_value_str(struct dma_fence *fence, char *str,
73 74 75 76 77 78
					  int size)
{
	snprintf(str, size, "%u",
		 intel_engine_get_seqno(to_request(fence)->engine));
}

79
static void i915_fence_release(struct dma_fence *fence)
80 81 82 83 84 85
{
	struct drm_i915_gem_request *req = to_request(fence);

	kmem_cache_free(req->i915->requests, req);
}

86
const struct dma_fence_ops i915_fence_ops = {
87 88 89 90 91 92 93 94 95 96
	.get_driver_name = i915_fence_get_driver_name,
	.get_timeline_name = i915_fence_get_timeline_name,
	.enable_signaling = i915_fence_enable_signaling,
	.signaled = i915_fence_signaled,
	.wait = i915_fence_wait,
	.release = i915_fence_release,
	.fence_value_str = i915_fence_value_str,
	.timeline_value_str = i915_fence_timeline_value_str,
};

97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135
int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
				   struct drm_file *file)
{
	struct drm_i915_private *dev_private;
	struct drm_i915_file_private *file_priv;

	WARN_ON(!req || !file || req->file_priv);

	if (!req || !file)
		return -EINVAL;

	if (req->file_priv)
		return -EINVAL;

	dev_private = req->i915;
	file_priv = file->driver_priv;

	spin_lock(&file_priv->mm.lock);
	req->file_priv = file_priv;
	list_add_tail(&req->client_list, &file_priv->mm.request_list);
	spin_unlock(&file_priv->mm.lock);

	return 0;
}

static inline void
i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
{
	struct drm_i915_file_private *file_priv = request->file_priv;

	if (!file_priv)
		return;

	spin_lock(&file_priv->mm.lock);
	list_del(&request->client_list);
	request->file_priv = NULL;
	spin_unlock(&file_priv->mm.lock);
}

136 137 138 139 140 141
void i915_gem_retire_noop(struct i915_gem_active *active,
			  struct drm_i915_gem_request *request)
{
	/* Space left intentionally blank */
}

142 143
static void i915_gem_request_retire(struct drm_i915_gem_request *request)
{
144 145
	struct i915_gem_active *active, *next;

146 147 148
	lockdep_assert_held(&request->i915->drm.struct_mutex);
	GEM_BUG_ON(!i915_gem_request_completed(request));

149
	trace_i915_gem_request_retire(request);
150
	list_del_init(&request->link);
151 152 153 154 155 156 157 158 159

	/* We know the GPU must have read the request to have
	 * sent us the seqno + interrupt, so use the position
	 * of tail of the request to update the last known position
	 * of the GPU head.
	 *
	 * Note this requires that we are always called in request
	 * completion order.
	 */
160
	list_del(&request->ring_link);
161
	request->ring->last_retired_head = request->postfix;
162

163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
	/* Walk through the active list, calling retire on each. This allows
	 * objects to track their GPU activity and mark themselves as idle
	 * when their *last* active request is completed (updating state
	 * tracking lists for eviction, active references for GEM, etc).
	 *
	 * As the ->retire() may free the node, we decouple it first and
	 * pass along the auxiliary information (to avoid dereferencing
	 * the node after the callback).
	 */
	list_for_each_entry_safe(active, next, &request->active_list, link) {
		/* In microbenchmarks or focusing upon time inside the kernel,
		 * we may spend an inordinate amount of time simply handling
		 * the retirement of requests and processing their callbacks.
		 * Of which, this loop itself is particularly hot due to the
		 * cache misses when jumping around the list of i915_gem_active.
		 * So we try to keep this loop as streamlined as possible and
		 * also prefetch the next i915_gem_active to try and hide
		 * the likely cache miss.
		 */
		prefetchw(next);

		INIT_LIST_HEAD(&active->link);
185
		RCU_INIT_POINTER(active->request, NULL);
186 187 188 189

		active->retire(active, request);
	}

190 191 192 193 194 195 196 197
	i915_gem_request_remove_from_client(request);

	if (request->previous_context) {
		if (i915.enable_execlists)
			intel_lr_context_unpin(request->previous_context,
					       request->engine);
	}

198
	i915_gem_context_put(request->ctx);
199 200

	dma_fence_signal(&request->fence);
201
	i915_gem_request_put(request);
202 203 204 205 206 207 208 209
}

void i915_gem_request_retire_upto(struct drm_i915_gem_request *req)
{
	struct intel_engine_cs *engine = req->engine;
	struct drm_i915_gem_request *tmp;

	lockdep_assert_held(&req->i915->drm.struct_mutex);
210 211
	if (list_empty(&req->link))
		return;
212 213 214

	do {
		tmp = list_first_entry(&engine->request_list,
215
				       typeof(*tmp), link);
216 217 218 219 220

		i915_gem_request_retire(tmp);
	} while (tmp != req);
}

221
static int i915_gem_check_wedge(struct drm_i915_private *dev_priv)
222
{
223 224 225
	struct i915_gpu_error *error = &dev_priv->gpu_error;

	if (i915_terminally_wedged(error))
226 227
		return -EIO;

228
	if (i915_reset_in_progress(error)) {
229 230 231
		/* Non-interruptible callers can't handle -EAGAIN, hence return
		 * -EIO unconditionally for these.
		 */
232
		if (!dev_priv->mm.interruptible)
233 234 235 236 237 238 239 240 241 242 243
			return -EIO;

		return -EAGAIN;
	}

	return 0;
}

static int i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
{
	struct intel_engine_cs *engine;
244
	enum intel_engine_id id;
245 246 247
	int ret;

	/* Carefully retire all requests without writing to the rings */
248
	for_each_engine(engine, dev_priv, id) {
249 250 251
		ret = intel_engine_idle(engine,
					I915_WAIT_INTERRUPTIBLE |
					I915_WAIT_LOCKED);
252 253 254 255 256 257 258 259 260 261 262 263 264
		if (ret)
			return ret;
	}
	i915_gem_retire_requests(dev_priv);

	/* If the seqno wraps around, we need to clear the breadcrumb rbtree */
	if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) {
		while (intel_kick_waiters(dev_priv) ||
		       intel_kick_signalers(dev_priv))
			yield();
	}

	/* Finally reset hw state */
265
	for_each_engine(engine, dev_priv, id)
266
		intel_engine_init_seqno(engine, seqno);
267 268 269 270 271 272 273 274 275

	return 0;
}

int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
{
	struct drm_i915_private *dev_priv = to_i915(dev);
	int ret;

276 277
	lockdep_assert_held(&dev_priv->drm.struct_mutex);

278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
	if (seqno == 0)
		return -EINVAL;

	/* HWS page needs to be set less than what we
	 * will inject to ring
	 */
	ret = i915_gem_init_seqno(dev_priv, seqno - 1);
	if (ret)
		return ret;

	dev_priv->next_seqno = seqno;
	return 0;
}

static int i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno)
{
	/* reserve 0 for non-seqno */
	if (unlikely(dev_priv->next_seqno == 0)) {
		int ret;

		ret = i915_gem_init_seqno(dev_priv, 0);
		if (ret)
			return ret;

		dev_priv->next_seqno = 1;
	}

305
	*seqno = dev_priv->next_seqno++;
306 307 308
	return 0;
}

309 310 311 312 313 314 315 316 317 318
static int __i915_sw_fence_call
submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
{
	struct drm_i915_gem_request *request =
		container_of(fence, typeof(*request), submit);

	/* Will be called from irq-context when using foreign DMA fences */

	switch (state) {
	case FENCE_COMPLETE:
319
		request->engine->last_submitted_seqno = request->fence.seqno;
320 321 322 323 324 325 326 327 328 329
		request->engine->submit_request(request);
		break;

	case FENCE_FREE:
		break;
	}

	return NOTIFY_DONE;
}

330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
/**
 * i915_gem_request_alloc - allocate a request structure
 *
 * @engine: engine that we wish to issue the request on.
 * @ctx: context that the request will be associated with.
 *       This can be NULL if the request is not directly related to
 *       any specific user context, in which case this function will
 *       choose an appropriate context to use.
 *
 * Returns a pointer to the allocated request if successful,
 * or an error code if not.
 */
struct drm_i915_gem_request *
i915_gem_request_alloc(struct intel_engine_cs *engine,
		       struct i915_gem_context *ctx)
345 346 347
{
	struct drm_i915_private *dev_priv = engine->i915;
	struct drm_i915_gem_request *req;
348
	u32 seqno;
349 350 351 352 353 354
	int ret;

	/* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
	 * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
	 * and restart.
	 */
355
	ret = i915_gem_check_wedge(dev_priv);
356
	if (ret)
357
		return ERR_PTR(ret);
358

359
	/* Move the oldest request to the slab-cache (if not in use!) */
360
	req = list_first_entry_or_null(&engine->request_list,
361
				       typeof(*req), link);
362 363
	if (req && i915_gem_request_completed(req))
		i915_gem_request_retire(req);
364

365 366 367 368 369
	/* Beware: Dragons be flying overhead.
	 *
	 * We use RCU to look up requests in flight. The lookups may
	 * race with the request being allocated from the slab freelist.
	 * That is the request we are writing to here, may be in the process
370
	 * of being read by __i915_gem_active_get_rcu(). As such,
371 372 373 374 375 376 377
	 * we have to be very careful when overwriting the contents. During
	 * the RCU lookup, we change chase the request->engine pointer,
	 * read the request->fence.seqno and increment the reference count.
	 *
	 * The reference count is incremented atomically. If it is zero,
	 * the lookup knows the request is unallocated and complete. Otherwise,
	 * it is either still in use, or has been reallocated and reset
378 379
	 * with dma_fence_init(). This increment is safe for release as we
	 * check that the request we have a reference to and matches the active
380 381 382 383 384 385 386 387 388 389 390 391 392 393
	 * request.
	 *
	 * Before we increment the refcount, we chase the request->engine
	 * pointer. We must not call kmem_cache_zalloc() or else we set
	 * that pointer to NULL and cause a crash during the lookup. If
	 * we see the request is completed (based on the value of the
	 * old engine and seqno), the lookup is complete and reports NULL.
	 * If we decide the request is not completed (new engine or seqno),
	 * then we grab a reference and double check that it is still the
	 * active request - which it won't be and restart the lookup.
	 *
	 * Do not use kmem_cache_zalloc() here!
	 */
	req = kmem_cache_alloc(dev_priv->requests, GFP_KERNEL);
394
	if (!req)
395
		return ERR_PTR(-ENOMEM);
396

397
	ret = i915_gem_get_seqno(dev_priv, &seqno);
398 399 400
	if (ret)
		goto err;

401
	spin_lock_init(&req->lock);
402 403 404 405 406
	dma_fence_init(&req->fence,
		       &i915_fence_ops,
		       &req->lock,
		       engine->fence_context,
		       seqno);
407

408 409
	i915_sw_fence_init(&req->submit, submit_notify);

410
	INIT_LIST_HEAD(&req->active_list);
411 412
	req->i915 = dev_priv;
	req->engine = engine;
413
	req->ctx = i915_gem_context_get(ctx);
414

415 416 417
	/* No zalloc, must clear what we need by hand */
	req->previous_context = NULL;
	req->file_priv = NULL;
C
Chris Wilson 已提交
418
	req->batch = NULL;
419

420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
	/*
	 * Reserve space in the ring buffer for all the commands required to
	 * eventually emit this request. This is to guarantee that the
	 * i915_add_request() call can't fail. Note that the reserve may need
	 * to be redone if the request is not actually submitted straight
	 * away, e.g. because a GPU scheduler has deferred it.
	 */
	req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;

	if (i915.enable_execlists)
		ret = intel_logical_ring_alloc_request_extras(req);
	else
		ret = intel_ring_alloc_request_extras(req);
	if (ret)
		goto err_ctx;

436 437 438 439 440 441 442
	/* Record the position of the start of the request so that
	 * should we detect the updated seqno part-way through the
	 * GPU processing the request, we never over-estimate the
	 * position of the head.
	 */
	req->head = req->ring->tail;

443
	return req;
444 445

err_ctx:
446
	i915_gem_context_put(ctx);
447 448
err:
	kmem_cache_free(dev_priv->requests, req);
449
	return ERR_PTR(ret);
450 451
}

452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468
static int
i915_gem_request_await_request(struct drm_i915_gem_request *to,
			       struct drm_i915_gem_request *from)
{
	int idx, ret;

	GEM_BUG_ON(to == from);

	if (to->engine == from->engine)
		return 0;

	idx = intel_engine_sync_index(from->engine, to->engine);
	if (from->fence.seqno <= from->engine->semaphore.sync_seqno[idx])
		return 0;

	trace_i915_gem_ring_sync_to(to, from);
	if (!i915.semaphores) {
469 470 471 472 473 474 475
		if (!i915_spin_request(from, TASK_INTERRUPTIBLE, 2)) {
			ret = i915_sw_fence_await_dma_fence(&to->submit,
							    &from->fence, 0,
							    GFP_KERNEL);
			if (ret < 0)
				return ret;
		}
476 477 478 479 480 481 482 483 484 485
	} else {
		ret = to->engine->semaphore.sync_to(to, from);
		if (ret)
			return ret;
	}

	from->engine->semaphore.sync_seqno[idx] = from->fence.seqno;
	return 0;
}

486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532
int
i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
				 struct dma_fence *fence)
{
	struct dma_fence_array *array;
	int ret;
	int i;

	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
		return 0;

	if (dma_fence_is_i915(fence))
		return i915_gem_request_await_request(req, to_request(fence));

	if (!dma_fence_is_array(fence)) {
		ret = i915_sw_fence_await_dma_fence(&req->submit,
						    fence, I915_FENCE_TIMEOUT,
						    GFP_KERNEL);
		return ret < 0 ? ret : 0;
	}

	/* Note that if the fence-array was created in signal-on-any mode,
	 * we should *not* decompose it into its individual fences. However,
	 * we don't currently store which mode the fence-array is operating
	 * in. Fortunately, the only user of signal-on-any is private to
	 * amdgpu and we should not see any incoming fence-array from
	 * sync-file being in signal-on-any mode.
	 */

	array = to_dma_fence_array(fence);
	for (i = 0; i < array->num_fences; i++) {
		struct dma_fence *child = array->fences[i];

		if (dma_fence_is_i915(child))
			ret = i915_gem_request_await_request(req,
							     to_request(child));
		else
			ret = i915_sw_fence_await_dma_fence(&req->submit,
							    child, I915_FENCE_TIMEOUT,
							    GFP_KERNEL);
		if (ret < 0)
			return ret;
	}

	return 0;
}

533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
/**
 * i915_gem_request_await_object - set this request to (async) wait upon a bo
 *
 * @to: request we are wishing to use
 * @obj: object which may be in use on another ring.
 *
 * This code is meant to abstract object synchronization with the GPU.
 * Conceptually we serialise writes between engines inside the GPU.
 * We only allow one engine to write into a buffer at any time, but
 * multiple readers. To ensure each has a coherent view of memory, we must:
 *
 * - If there is an outstanding write request to the object, the new
 *   request must wait for it to complete (either CPU or in hw, requests
 *   on the same ring will be naturally ordered).
 *
 * - If we are a write request (pending_write_domain is set), the new
 *   request must wait for outstanding read requests to complete.
 *
 * Returns 0 if successful, else propagates up the lower layer error.
 */
int
i915_gem_request_await_object(struct drm_i915_gem_request *to,
			      struct drm_i915_gem_object *obj,
			      bool write)
{
558 559
	struct dma_fence *excl;
	int ret = 0;
560 561

	if (write) {
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580
		struct dma_fence **shared;
		unsigned int count, i;

		ret = reservation_object_get_fences_rcu(obj->resv,
							&excl, &count, &shared);
		if (ret)
			return ret;

		for (i = 0; i < count; i++) {
			ret = i915_gem_request_await_dma_fence(to, shared[i]);
			if (ret)
				break;

			dma_fence_put(shared[i]);
		}

		for (; i < count; i++)
			dma_fence_put(shared[i]);
		kfree(shared);
581
	} else {
582
		excl = reservation_object_get_excl_rcu(obj->resv);
583 584
	}

585 586 587
	if (excl) {
		if (ret == 0)
			ret = i915_gem_request_await_dma_fence(to, excl);
588

589
		dma_fence_put(excl);
590 591
	}

592
	return ret;
593 594
}

595 596 597 598 599 600 601 602 603 604 605
static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
{
	struct drm_i915_private *dev_priv = engine->i915;

	dev_priv->gt.active_engines |= intel_engine_flag(engine);
	if (dev_priv->gt.awake)
		return;

	intel_runtime_pm_get_noresume(dev_priv);
	dev_priv->gt.awake = true;

606
	intel_enable_gt_powersave(dev_priv);
607 608 609 610 611 612 613 614 615 616 617 618 619 620
	i915_update_gfx_val(dev_priv);
	if (INTEL_GEN(dev_priv) >= 6)
		gen6_rps_busy(dev_priv);

	queue_delayed_work(dev_priv->wq,
			   &dev_priv->gt.retire_work,
			   round_jiffies_up_relative(HZ));
}

/*
 * NB: This function is not allowed to fail. Doing so would mean the the
 * request is not being tracked for completion but the work itself is
 * going to happen on the hardware. This would be a Bad Thing(tm).
 */
621
void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
622
{
623 624
	struct intel_engine_cs *engine = request->engine;
	struct intel_ring *ring = request->ring;
625
	struct drm_i915_gem_request *prev;
626 627 628 629
	u32 request_start;
	u32 reserved_tail;
	int ret;

630
	lockdep_assert_held(&request->i915->drm.struct_mutex);
631 632
	trace_i915_gem_request_add(request);

633 634 635 636 637
	/*
	 * To ensure that this call will not fail, space for its emissions
	 * should already have been reserved in the ring buffer. Let the ring
	 * know that it is time to use that space up.
	 */
638
	request_start = ring->tail;
639 640 641 642 643 644 645 646 647 648 649
	reserved_tail = request->reserved_space;
	request->reserved_space = 0;

	/*
	 * Emit any outstanding flushes - execbuf can fail to emit the flush
	 * after having emitted the batchbuffer command. Hence we need to fix
	 * things up similar to emitting the lazy request. The difference here
	 * is that the flush _must_ happen before the next request, no matter
	 * what.
	 */
	if (flush_caches) {
650
		ret = engine->emit_flush(request, EMIT_FLUSH);
651

652
		/* Not allowed to fail! */
653
		WARN(ret, "engine->emit_flush() failed: %d!\n", ret);
654 655
	}

656
	/* Record the position of the start of the breadcrumb so that
657 658
	 * should we detect the updated seqno part-way through the
	 * GPU processing the request, we never over-estimate the
659
	 * position of the ring's HEAD.
660
	 */
661
	request->postfix = ring->tail;
662 663

	/* Not allowed to fail! */
664 665
	ret = engine->emit_request(request);
	WARN(ret, "(%s)->emit_request failed: %d!\n", engine->name, ret);
666

667
	/* Sanity check that the reserved size was large enough. */
668
	ret = ring->tail - request_start;
669
	if (ret < 0)
670
		ret += ring->size;
671 672 673 674 675
	WARN_ONCE(ret > reserved_tail,
		  "Not enough space reserved (%d bytes) "
		  "for adding the request (%d bytes)\n",
		  reserved_tail, ret);

676 677 678 679 680
	/* Seal the request and mark it as pending execution. Note that
	 * we may inspect this state, without holding any locks, during
	 * hangcheck. Hence we apply the barrier to ensure that we do not
	 * see a more recent value in the hws than we are tracking.
	 */
681 682 683 684 685 686 687

	prev = i915_gem_active_raw(&engine->last_request,
				   &request->i915->drm.struct_mutex);
	if (prev)
		i915_sw_fence_await_sw_fence(&request->submit, &prev->submit,
					     &request->submitq);

688
	request->emitted_jiffies = jiffies;
689 690
	request->previous_seqno = engine->last_pending_seqno;
	engine->last_pending_seqno = request->fence.seqno;
691 692 693 694
	i915_gem_active_set(&engine->last_request, request);
	list_add_tail(&request->link, &engine->request_list);
	list_add_tail(&request->ring_link, &ring->request_list);

695
	i915_gem_mark_busy(engine);
696 697 698 699

	local_bh_disable();
	i915_sw_fence_commit(&request->submit);
	local_bh_enable(); /* Kick the execlists tasklet if just scheduled */
700 701
}

702 703 704 705 706 707 708 709 710 711
static void reset_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)
{
	unsigned long flags;

	spin_lock_irqsave(&q->lock, flags);
	if (list_empty(&wait->task_list))
		__add_wait_queue(q, wait);
	spin_unlock_irqrestore(&q->lock, flags);
}

712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776
static unsigned long local_clock_us(unsigned int *cpu)
{
	unsigned long t;

	/* Cheaply and approximately convert from nanoseconds to microseconds.
	 * The result and subsequent calculations are also defined in the same
	 * approximate microseconds units. The principal source of timing
	 * error here is from the simple truncation.
	 *
	 * Note that local_clock() is only defined wrt to the current CPU;
	 * the comparisons are no longer valid if we switch CPUs. Instead of
	 * blocking preemption for the entire busywait, we can detect the CPU
	 * switch and use that as indicator of system load and a reason to
	 * stop busywaiting, see busywait_stop().
	 */
	*cpu = get_cpu();
	t = local_clock() >> 10;
	put_cpu();

	return t;
}

static bool busywait_stop(unsigned long timeout, unsigned int cpu)
{
	unsigned int this_cpu;

	if (time_after(local_clock_us(&this_cpu), timeout))
		return true;

	return this_cpu != cpu;
}

bool __i915_spin_request(const struct drm_i915_gem_request *req,
			 int state, unsigned long timeout_us)
{
	unsigned int cpu;

	/* When waiting for high frequency requests, e.g. during synchronous
	 * rendering split between the CPU and GPU, the finite amount of time
	 * required to set up the irq and wait upon it limits the response
	 * rate. By busywaiting on the request completion for a short while we
	 * can service the high frequency waits as quick as possible. However,
	 * if it is a slow request, we want to sleep as quickly as possible.
	 * The tradeoff between waiting and sleeping is roughly the time it
	 * takes to sleep on a request, on the order of a microsecond.
	 */

	timeout_us += local_clock_us(&cpu);
	do {
		if (i915_gem_request_completed(req))
			return true;

		if (signal_pending_state(state, current))
			break;

		if (busywait_stop(timeout_us, cpu))
			break;

		cpu_relax_lowlatency();
	} while (!need_resched());

	return false;
}

/**
777
 * i915_wait_request - wait until execution of request has finished
778
 * @req: the request to wait upon
779
 * @flags: how to wait
780 781 782 783 784
 * @timeout: how long to wait in jiffies
 *
 * i915_wait_request() waits for the request to be completed, for a
 * maximum of @timeout jiffies (with MAX_SCHEDULE_TIMEOUT implying an
 * unbounded wait).
785
 *
786 787 788
 * If the caller holds the struct_mutex, the caller must pass I915_WAIT_LOCKED
 * in via the flags, and vice versa if the struct_mutex is not held, the caller
 * must not specify that the wait is locked.
789
 *
790 791 792 793
 * Returns the remaining time (in jiffies) if the request completed, which may
 * be zero or -ETIME if the request is unfinished after the timeout expires.
 * May return -EINTR is called with I915_WAIT_INTERRUPTIBLE and a signal is
 * pending before the request completes.
794
 */
795 796 797
long i915_wait_request(struct drm_i915_gem_request *req,
		       unsigned int flags,
		       long timeout)
798
{
799 800
	const int state = flags & I915_WAIT_INTERRUPTIBLE ?
		TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
801 802 803 804
	DEFINE_WAIT(reset);
	struct intel_wait wait;

	might_sleep();
805
#if IS_ENABLED(CONFIG_LOCKDEP)
806 807
	GEM_BUG_ON(debug_locks &&
		   !!lockdep_is_held(&req->i915->drm.struct_mutex) !=
808 809
		   !!(flags & I915_WAIT_LOCKED));
#endif
810
	GEM_BUG_ON(timeout < 0);
811 812

	if (i915_gem_request_completed(req))
813
		return timeout;
814

815 816
	if (!timeout)
		return -ETIME;
817 818 819

	trace_i915_gem_request_wait_begin(req);

820
	/* Optimistic short spin before touching IRQs */
821 822 823 824
	if (i915_spin_request(req, state, 5))
		goto complete;

	set_current_state(state);
825 826
	if (flags & I915_WAIT_LOCKED)
		add_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
827

828
	intel_wait_init(&wait, req->fence.seqno);
829 830 831 832 833 834 835 836 837
	if (intel_engine_add_wait(req->engine, &wait))
		/* In order to check that we haven't missed the interrupt
		 * as we enabled it, we need to kick ourselves to do a
		 * coherent check on the seqno before we sleep.
		 */
		goto wakeup;

	for (;;) {
		if (signal_pending_state(state, current)) {
838
			timeout = -ERESTARTSYS;
839 840 841
			break;
		}

842 843
		if (!timeout) {
			timeout = -ETIME;
844 845 846
			break;
		}

847 848
		timeout = io_schedule_timeout(timeout);

849 850 851 852 853 854 855 856 857 858 859 860 861 862
		if (intel_wait_complete(&wait))
			break;

		set_current_state(state);

wakeup:
		/* Carefully check if the request is complete, giving time
		 * for the seqno to be visible following the interrupt.
		 * We also have to check in case we are kicked by the GPU
		 * reset in order to drop the struct_mutex.
		 */
		if (__i915_request_irq_complete(req))
			break;

863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881
		/* If the GPU is hung, and we hold the lock, reset the GPU
		 * and then check for completion. On a full reset, the engine's
		 * HW seqno will be advanced passed us and we are complete.
		 * If we do a partial reset, we have to wait for the GPU to
		 * resume and update the breadcrumb.
		 *
		 * If we don't hold the mutex, we can just wait for the worker
		 * to come along and update the breadcrumb (either directly
		 * itself, or indirectly by recovering the GPU).
		 */
		if (flags & I915_WAIT_LOCKED &&
		    i915_reset_in_progress(&req->i915->gpu_error)) {
			__set_current_state(TASK_RUNNING);
			i915_reset(req->i915);
			reset_wait_queue(&req->i915->gpu_error.wait_queue,
					 &reset);
			continue;
		}

882 883 884 885 886 887
		/* Only spin if we know the GPU is processing this request */
		if (i915_spin_request(req, state, 2))
			break;
	}

	intel_engine_remove_wait(req->engine, &wait);
888 889
	if (flags & I915_WAIT_LOCKED)
		remove_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
890
	__set_current_state(TASK_RUNNING);
891

892 893 894
complete:
	trace_i915_gem_request_wait_end(req);

895
	return timeout;
896
}
897

898
static bool engine_retire_requests(struct intel_engine_cs *engine)
899 900 901 902 903
{
	struct drm_i915_gem_request *request, *next;

	list_for_each_entry_safe(request, next, &engine->request_list, link) {
		if (!i915_gem_request_completed(request))
904
			return false;
905 906 907

		i915_gem_request_retire(request);
	}
908 909

	return true;
910 911 912 913 914
}

void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
{
	struct intel_engine_cs *engine;
915
	unsigned int tmp;
916 917 918 919 920 921 922 923

	lockdep_assert_held(&dev_priv->drm.struct_mutex);

	if (dev_priv->gt.active_engines == 0)
		return;

	GEM_BUG_ON(!dev_priv->gt.awake);

924
	for_each_engine_masked(engine, dev_priv, dev_priv->gt.active_engines, tmp)
925
		if (engine_retire_requests(engine))
926 927 928 929 930 931 932
			dev_priv->gt.active_engines &= ~intel_engine_flag(engine);

	if (dev_priv->gt.active_engines == 0)
		queue_delayed_work(dev_priv->wq,
				   &dev_priv->gt.idle_work,
				   msecs_to_jiffies(100));
}