scheduler.c 43.2 KB
Newer Older
Z
Zhi Wang 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
/*
 * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * Authors:
 *    Zhi Wang <zhi.a.wang@intel.com>
 *
 * Contributors:
 *    Ping Gao <ping.a.gao@intel.com>
 *    Tina Zhang <tina.zhang@intel.com>
 *    Chanbin Du <changbin.du@intel.com>
 *    Min He <min.he@intel.com>
 *    Bing Niu <bing.niu@intel.com>
 *    Zhenyu Wang <zhenyuw@linux.intel.com>
 *
 */

#include <linux/kthread.h>

38 39 40 41
#include "gem/i915_gem_context.h"
#include "gem/i915_gem_pm.h"
#include "gt/intel_context.h"

42 43 44
#include "i915_drv.h"
#include "gvt.h"

Z
Zhi Wang 已提交
45 46 47
#define RING_CTX_OFF(x) \
	offsetof(struct execlist_ring_context, x)

48 49
static void set_context_pdp_root_pointer(
		struct execlist_ring_context *ring_context,
Z
Zhi Wang 已提交
50 51 52 53 54
		u32 pdp[8])
{
	int i;

	for (i = 0; i < 8; i++)
55
		ring_context->pdps[i].val = pdp[7 - i];
Z
Zhi Wang 已提交
56 57
}

58 59 60
static void update_shadow_pdps(struct intel_vgpu_workload *workload)
{
	struct drm_i915_gem_object *ctx_obj =
61
		workload->req->hw_context->state->obj;
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
	struct execlist_ring_context *shadow_ring_context;
	struct page *page;

	if (WARN_ON(!workload->shadow_mm))
		return;

	if (WARN_ON(!atomic_read(&workload->shadow_mm->pincount)))
		return;

	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
	shadow_ring_context = kmap(page);
	set_context_pdp_root_pointer(shadow_ring_context,
			(void *)workload->shadow_mm->ppgtt_mm.shadow_pdps);
	kunmap(page);
}

78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
/*
 * when populating shadow ctx from guest, we should not overrride oa related
 * registers, so that they will not be overlapped by guest oa configs. Thus
 * made it possible to capture oa data from host for both host and guests.
 */
static void sr_oa_regs(struct intel_vgpu_workload *workload,
		u32 *reg_state, bool save)
{
	struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
	u32 ctx_oactxctrl = dev_priv->perf.oa.ctx_oactxctrl_offset;
	u32 ctx_flexeu0 = dev_priv->perf.oa.ctx_flexeu0_offset;
	int i = 0;
	u32 flex_mmio[] = {
		i915_mmio_reg_offset(EU_PERF_CNTL0),
		i915_mmio_reg_offset(EU_PERF_CNTL1),
		i915_mmio_reg_offset(EU_PERF_CNTL2),
		i915_mmio_reg_offset(EU_PERF_CNTL3),
		i915_mmio_reg_offset(EU_PERF_CNTL4),
		i915_mmio_reg_offset(EU_PERF_CNTL5),
		i915_mmio_reg_offset(EU_PERF_CNTL6),
	};

100
	if (workload->ring_id != RCS0)
101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125
		return;

	if (save) {
		workload->oactxctrl = reg_state[ctx_oactxctrl + 1];

		for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {
			u32 state_offset = ctx_flexeu0 + i * 2;

			workload->flex_mmio[i] = reg_state[state_offset + 1];
		}
	} else {
		reg_state[ctx_oactxctrl] =
			i915_mmio_reg_offset(GEN8_OACTXCONTROL);
		reg_state[ctx_oactxctrl + 1] = workload->oactxctrl;

		for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) {
			u32 state_offset = ctx_flexeu0 + i * 2;
			u32 mmio = flex_mmio[i];

			reg_state[state_offset] = mmio;
			reg_state[state_offset + 1] = workload->flex_mmio[i];
		}
	}
}

Z
Zhi Wang 已提交
126 127 128 129 130 131
static int populate_shadow_context(struct intel_vgpu_workload *workload)
{
	struct intel_vgpu *vgpu = workload->vgpu;
	struct intel_gvt *gvt = vgpu->gvt;
	int ring_id = workload->ring_id;
	struct drm_i915_gem_object *ctx_obj =
132
		workload->req->hw_context->state->obj;
Z
Zhi Wang 已提交
133 134 135 136 137 138 139
	struct execlist_ring_context *shadow_ring_context;
	struct page *page;
	void *dst;
	unsigned long context_gpa, context_page_num;
	int i;

	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
140
	shadow_ring_context = kmap(page);
Z
Zhi Wang 已提交
141

142
	sr_oa_regs(workload, (u32 *)shadow_ring_context, true);
Z
Zhi Wang 已提交
143 144 145
#define COPY_REG(name) \
	intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
		+ RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)
146 147 148 149 150 151
#define COPY_REG_MASKED(name) {\
		intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \
					      + RING_CTX_OFF(name.val),\
					      &shadow_ring_context->name.val, 4);\
		shadow_ring_context->name.val |= 0xffff << 16;\
	}
Z
Zhi Wang 已提交
152

153
	COPY_REG_MASKED(ctx_ctrl);
Z
Zhi Wang 已提交
154 155
	COPY_REG(ctx_timestamp);

156
	if (ring_id == RCS0) {
Z
Zhi Wang 已提交
157 158 159 160 161
		COPY_REG(bb_per_ctx_ptr);
		COPY_REG(rcs_indirect_ctx);
		COPY_REG(rcs_indirect_ctx_offset);
	}
#undef COPY_REG
162
#undef COPY_REG_MASKED
Z
Zhi Wang 已提交
163 164 165 166 167 168

	intel_gvt_hypervisor_read_gpa(vgpu,
			workload->ring_context_gpa +
			sizeof(*shadow_ring_context),
			(void *)shadow_ring_context +
			sizeof(*shadow_ring_context),
Z
Zhi Wang 已提交
169
			I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
Z
Zhi Wang 已提交
170

171
	sr_oa_regs(workload, (u32 *)shadow_ring_context, false);
172
	kunmap(page);
173 174 175 176 177 178 179 180 181 182 183

	if (IS_RESTORE_INHIBIT(shadow_ring_context->ctx_ctrl.val))
		return 0;

	gvt_dbg_sched("ring id %d workload lrca %x", ring_id,
			workload->ctx_desc.lrca);

	context_page_num = gvt->dev_priv->engine[ring_id]->context_size;

	context_page_num = context_page_num >> PAGE_SHIFT;

184
	if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS0)
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
		context_page_num = 19;

	i = 2;
	while (i < context_page_num) {
		context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
				(u32)((workload->ctx_desc.lrca + i) <<
				I915_GTT_PAGE_SHIFT));
		if (context_gpa == INTEL_GVT_INVALID_ADDR) {
			gvt_vgpu_err("Invalid guest context descriptor\n");
			return -EFAULT;
		}

		page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i);
		dst = kmap(page);
		intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst,
				I915_GTT_PAGE_SIZE);
		kunmap(page);
		i++;
	}
Z
Zhi Wang 已提交
204 205 206
	return 0;
}

207
static inline bool is_gvt_request(struct i915_request *req)
208
{
C
Chris Wilson 已提交
209
	return i915_gem_context_force_single_submission(req->gem_context);
210 211
}

212 213 214 215 216 217 218 219 220 221 222 223 224 225
static void save_ring_hw_state(struct intel_vgpu *vgpu, int ring_id)
{
	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
	u32 ring_base = dev_priv->engine[ring_id]->mmio_base;
	i915_reg_t reg;

	reg = RING_INSTDONE(ring_base);
	vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg);
	reg = RING_ACTHD(ring_base);
	vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg);
	reg = RING_ACTHD_UDW(ring_base);
	vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg);
}

Z
Zhi Wang 已提交
226 227 228
static int shadow_context_status_change(struct notifier_block *nb,
		unsigned long action, void *data)
{
229
	struct i915_request *req = data;
230 231 232
	struct intel_gvt *gvt = container_of(nb, struct intel_gvt,
				shadow_ctx_notifier_block[req->engine->id]);
	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
233 234
	enum intel_engine_id ring_id = req->engine->id;
	struct intel_vgpu_workload *workload;
235
	unsigned long flags;
236 237

	if (!is_gvt_request(req)) {
238
		spin_lock_irqsave(&scheduler->mmio_context_lock, flags);
239 240 241 242 243 244 245
		if (action == INTEL_CONTEXT_SCHEDULE_IN &&
		    scheduler->engine_owner[ring_id]) {
			/* Switch ring from vGPU to host. */
			intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
					      NULL, ring_id);
			scheduler->engine_owner[ring_id] = NULL;
		}
246
		spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags);
247 248 249

		return NOTIFY_OK;
	}
Z
Zhi Wang 已提交
250

251 252
	workload = scheduler->current_workload[ring_id];
	if (unlikely(!workload))
253 254
		return NOTIFY_OK;

Z
Zhi Wang 已提交
255 256
	switch (action) {
	case INTEL_CONTEXT_SCHEDULE_IN:
257
		spin_lock_irqsave(&scheduler->mmio_context_lock, flags);
258 259 260 261 262 263 264 265
		if (workload->vgpu != scheduler->engine_owner[ring_id]) {
			/* Switch ring from host to vGPU or vGPU to vGPU. */
			intel_gvt_switch_mmio(scheduler->engine_owner[ring_id],
					      workload->vgpu, ring_id);
			scheduler->engine_owner[ring_id] = workload->vgpu;
		} else
			gvt_dbg_sched("skip ring %d mmio switch for vgpu%d\n",
				      ring_id, workload->vgpu->id);
266
		spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags);
Z
Zhi Wang 已提交
267 268 269
		atomic_set(&workload->shadow_ctx_active, 1);
		break;
	case INTEL_CONTEXT_SCHEDULE_OUT:
270
		save_ring_hw_state(workload->vgpu, ring_id);
Z
Zhi Wang 已提交
271 272
		atomic_set(&workload->shadow_ctx_active, 0);
		break;
273 274 275
	case INTEL_CONTEXT_SCHEDULE_PREEMPTED:
		save_ring_hw_state(workload->vgpu, ring_id);
		break;
Z
Zhi Wang 已提交
276 277 278 279 280 281 282 283
	default:
		WARN_ON(1);
		return NOTIFY_OK;
	}
	wake_up(&workload->shadow_ctx_status_wq);
	return NOTIFY_OK;
}

284 285 286
static void
shadow_context_descriptor_update(struct intel_context *ce,
				 struct intel_vgpu_workload *workload)
287
{
288
	u64 desc = ce->lrc_desc;
289

290 291
	/*
	 * Update bits 0-11 of the context descriptor which includes flags
292 293 294
	 * like GEN8_CTX_* cached in desc_template
	 */
	desc &= U64_MAX << 12;
295
	desc |= ce->gem_context->desc_template & ((1ULL << 12) - 1);
296

297 298 299 300
	desc &= ~(0x3 << GEN8_CTX_ADDRESSING_MODE_SHIFT);
	desc |= workload->ctx_desc.addressing_mode <<
		GEN8_CTX_ADDRESSING_MODE_SHIFT;

301 302 303
	ce->lrc_desc = desc;
}

304 305 306
static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload)
{
	struct intel_vgpu *vgpu = workload->vgpu;
307
	struct i915_request *req = workload->req;
308 309
	void *shadow_ring_buffer_va;
	u32 *cs;
310
	int err;
311

312
	if (IS_GEN(req->i915, 9) && is_inhibit_context(req->hw_context))
313
		intel_vgpu_restore_inhibit_context(vgpu, req);
314

315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
	/*
	 * To track whether a request has started on HW, we can emit a
	 * breadcrumb at the beginning of the request and check its
	 * timeline's HWSP to see if the breadcrumb has advanced past the
	 * start of this request. Actually, the request must have the
	 * init_breadcrumb if its timeline set has_init_bread_crumb, or the
	 * scheduler might get a wrong state of it during reset. Since the
	 * requests from gvt always set the has_init_breadcrumb flag, here
	 * need to do the emit_init_breadcrumb for all the requests.
	 */
	if (req->engine->emit_init_breadcrumb) {
		err = req->engine->emit_init_breadcrumb(req);
		if (err) {
			gvt_vgpu_err("fail to emit init breadcrumb\n");
			return err;
		}
	}

333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354
	/* allocate shadow ring buffer */
	cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32));
	if (IS_ERR(cs)) {
		gvt_vgpu_err("fail to alloc size =%ld shadow  ring buffer\n",
			workload->rb_len);
		return PTR_ERR(cs);
	}

	shadow_ring_buffer_va = workload->shadow_ring_buffer_va;

	/* get shadow ring buffer va */
	workload->shadow_ring_buffer_va = cs;

	memcpy(cs, shadow_ring_buffer_va,
			workload->rb_len);

	cs += workload->rb_len / sizeof(u32);
	intel_ring_advance(workload->req, cs);

	return 0;
}

355
static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
356 357 358 359 360 361
{
	if (!wa_ctx->indirect_ctx.obj)
		return;

	i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj);
	i915_gem_object_put(wa_ctx->indirect_ctx.obj);
362 363 364

	wa_ctx->indirect_ctx.obj = NULL;
	wa_ctx->indirect_ctx.shadow_va = NULL;
365 366
}

367 368 369 370
static int set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload,
					 struct i915_gem_context *ctx)
{
	struct intel_vgpu_mm *mm = workload->shadow_mm;
371
	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(ctx->vm);
372 373 374
	int i = 0;

	if (mm->type != INTEL_GVT_MM_PPGTT || !mm->ppgtt_mm.shadowed)
375
		return -EINVAL;
376 377

	if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
378
		px_dma(ppgtt->pd) = mm->ppgtt_mm.shadow_pdps[0];
379 380
	} else {
		for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) {
381 382 383 384
			struct i915_page_directory * const pd =
				i915_pd_entry(ppgtt->pd, i);

			px_dma(pd) = mm->ppgtt_mm.shadow_pdps[i];
385 386 387 388 389 390
		}
	}

	return 0;
}

391 392 393 394 395 396 397 398 399 400 401
static int
intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload)
{
	struct intel_vgpu *vgpu = workload->vgpu;
	struct intel_vgpu_submission *s = &vgpu->submission;
	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
	struct i915_request *rq;

	lockdep_assert_held(&dev_priv->drm.struct_mutex);

	if (workload->req)
402
		return 0;
403

404
	rq = i915_request_create(s->shadow[workload->ring_id]);
405 406
	if (IS_ERR(rq)) {
		gvt_vgpu_err("fail to allocate gem request\n");
407
		return PTR_ERR(rq);
408
	}
409

410
	workload->req = i915_request_get(rq);
411
	return 0;
412 413
}

414 415 416 417 418 419 420 421 422
/**
 * intel_gvt_scan_and_shadow_workload - audit the workload by scanning and
 * shadow it as well, include ringbuffer,wa_ctx and ctx.
 * @workload: an abstract entity for each execlist submission.
 *
 * This function is called before the workload submitting to i915, to make
 * sure the content of the workload is valid.
 */
int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
Z
Zhi Wang 已提交
423
{
424 425 426
	struct intel_vgpu *vgpu = workload->vgpu;
	struct intel_vgpu_submission *s = &vgpu->submission;
	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
Z
Zhi Wang 已提交
427 428
	int ret;

429 430
	lockdep_assert_held(&dev_priv->drm.struct_mutex);

431
	if (workload->shadow)
432
		return 0;
Z
Zhi Wang 已提交
433

434
	if (!test_and_set_bit(workload->ring_id, s->shadow_ctx_desc_updated))
435 436
		shadow_context_descriptor_update(s->shadow[workload->ring_id],
						 workload);
437

438
	ret = intel_gvt_scan_and_shadow_ringbuffer(workload);
Z
Zhi Wang 已提交
439
	if (ret)
440
		return ret;
Z
Zhi Wang 已提交
441

442
	if (workload->ring_id == RCS0 && workload->wa_ctx.indirect_ctx.size) {
443 444
		ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx);
		if (ret)
445
			goto err_shadow;
446
	}
Z
Zhi Wang 已提交
447

448
	workload->shadow = true;
449
	return 0;
450 451
err_shadow:
	release_shadow_wa_ctx(&workload->wa_ctx);
452 453 454
	return ret;
}

455 456
static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload);

457 458 459 460
static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
{
	struct intel_gvt *gvt = workload->vgpu->gvt;
	const int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd;
461 462 463 464
	struct intel_vgpu_shadow_bb *bb;
	int ret;

	list_for_each_entry(bb, &workload->shadow_bb, list) {
465 466 467 468 469 470 471 472 473 474 475
		/* For privilge batch buffer and not wa_ctx, the bb_start_cmd_va
		 * is only updated into ring_scan_buffer, not real ring address
		 * allocated in later copy_workload_to_ring_buffer. pls be noted
		 * shadow_ring_buffer_va is now pointed to real ring buffer va
		 * in copy_workload_to_ring_buffer.
		 */

		if (bb->bb_offset)
			bb->bb_start_cmd_va = workload->shadow_ring_buffer_va
				+ bb->bb_offset;

476 477 478 479 480 481 482 483 484 485 486 487 488 489
		if (bb->ppgtt) {
			/* for non-priv bb, scan&shadow is only for
			 * debugging purpose, so the content of shadow bb
			 * is the same as original bb. Therefore,
			 * here, rather than switch to shadow bb's gma
			 * address, we directly use original batch buffer's
			 * gma address, and send original bb to hardware
			 * directly
			 */
			if (bb->clflush & CLFLUSH_AFTER) {
				drm_clflush_virt_range(bb->va,
						bb->obj->base.size);
				bb->clflush &= ~CLFLUSH_AFTER;
			}
490
			i915_gem_object_finish_access(bb->obj);
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
			bb->accessing = false;

		} else {
			bb->vma = i915_gem_object_ggtt_pin(bb->obj,
					NULL, 0, 0, 0);
			if (IS_ERR(bb->vma)) {
				ret = PTR_ERR(bb->vma);
				goto err;
			}

			/* relocate shadow batch buffer */
			bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma);
			if (gmadr_bytes == 8)
				bb->bb_start_cmd_va[2] = 0;

			/* No one is going to touch shadow bb from now on. */
			if (bb->clflush & CLFLUSH_AFTER) {
				drm_clflush_virt_range(bb->va,
						bb->obj->base.size);
				bb->clflush &= ~CLFLUSH_AFTER;
			}

			ret = i915_gem_object_set_to_gtt_domain(bb->obj,
514
								false);
515 516 517
			if (ret)
				goto err;

518 519 520 521 522
			ret = i915_vma_move_to_active(bb->vma,
						      workload->req,
						      0);
			if (ret)
				goto err;
523 524 525

			i915_gem_object_finish_access(bb->obj);
			bb->accessing = false;
526
		}
527 528
	}
	return 0;
529 530 531
err:
	release_shadow_batch_buffer(workload);
	return ret;
532 533
}

534
static void update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
535
{
536 537 538 539 540
	struct intel_vgpu_workload *workload =
		container_of(wa_ctx, struct intel_vgpu_workload, wa_ctx);
	struct i915_request *rq = workload->req;
	struct execlist_ring_context *shadow_ring_context =
		(struct execlist_ring_context *)rq->hw_context->lrc_reg_state;
541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580

	shadow_ring_context->bb_per_ctx_ptr.val =
		(shadow_ring_context->bb_per_ctx_ptr.val &
		(~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma;
	shadow_ring_context->rcs_indirect_ctx.val =
		(shadow_ring_context->rcs_indirect_ctx.val &
		(~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma;
}

static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
{
	struct i915_vma *vma;
	unsigned char *per_ctx_va =
		(unsigned char *)wa_ctx->indirect_ctx.shadow_va +
		wa_ctx->indirect_ctx.size;

	if (wa_ctx->indirect_ctx.size == 0)
		return 0;

	vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL,
				       0, CACHELINE_BYTES, 0);
	if (IS_ERR(vma))
		return PTR_ERR(vma);

	/* FIXME: we are not tracking our pinned VMA leaving it
	 * up to the core to fix up the stray pin_count upon
	 * free.
	 */

	wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma);

	wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1);
	memset(per_ctx_va, 0, CACHELINE_BYTES);

	update_wa_ctx_2_shadow_ctx(wa_ctx);
	return 0;
}

static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
{
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595
	struct intel_vgpu *vgpu = workload->vgpu;
	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
	struct intel_vgpu_shadow_bb *bb, *pos;

	if (list_empty(&workload->shadow_bb))
		return;

	bb = list_first_entry(&workload->shadow_bb,
			struct intel_vgpu_shadow_bb, list);

	mutex_lock(&dev_priv->drm.struct_mutex);

	list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) {
		if (bb->obj) {
			if (bb->accessing)
596
				i915_gem_object_finish_access(bb->obj);
597 598 599 600 601 602 603 604

			if (bb->va && !IS_ERR(bb->va))
				i915_gem_object_unpin_map(bb->obj);

			if (bb->vma && !IS_ERR(bb->vma)) {
				i915_vma_unpin(bb->vma);
				i915_vma_close(bb->vma);
			}
605
			i915_gem_object_put(bb->obj);
606
		}
607 608
		list_del(&bb->list);
		kfree(bb);
609
	}
610 611

	mutex_unlock(&dev_priv->drm.struct_mutex);
612 613
}

614 615
static int prepare_workload(struct intel_vgpu_workload *workload)
{
616
	struct intel_vgpu *vgpu = workload->vgpu;
617 618
	int ret = 0;

619 620 621 622 623 624
	ret = intel_vgpu_pin_mm(workload->shadow_mm);
	if (ret) {
		gvt_vgpu_err("fail to vgpu pin mm\n");
		return ret;
	}

625 626
	update_shadow_pdps(workload);

627 628 629 630 631 632 633 634 635 636 637 638
	ret = intel_vgpu_sync_oos_pages(workload->vgpu);
	if (ret) {
		gvt_vgpu_err("fail to vgpu sync oos pages\n");
		goto err_unpin_mm;
	}

	ret = intel_vgpu_flush_post_shadow(workload->vgpu);
	if (ret) {
		gvt_vgpu_err("fail to flush post shadow\n");
		goto err_unpin_mm;
	}

639
	ret = copy_workload_to_ring_buffer(workload);
640 641 642 643 644
	if (ret) {
		gvt_vgpu_err("fail to generate request\n");
		goto err_unpin_mm;
	}

645 646 647 648 649 650 651 652 653 654 655 656 657
	ret = prepare_shadow_batch_buffer(workload);
	if (ret) {
		gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n");
		goto err_unpin_mm;
	}

	ret = prepare_shadow_wa_ctx(&workload->wa_ctx);
	if (ret) {
		gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n");
		goto err_shadow_batch;
	}

	if (workload->prepare) {
658
		ret = workload->prepare(workload);
659 660 661
		if (ret)
			goto err_shadow_wa_ctx;
	}
662

663 664 665 666 667 668 669
	return 0;
err_shadow_wa_ctx:
	release_shadow_wa_ctx(&workload->wa_ctx);
err_shadow_batch:
	release_shadow_batch_buffer(workload);
err_unpin_mm:
	intel_vgpu_unpin_mm(workload->shadow_mm);
670 671 672
	return ret;
}

673 674
static int dispatch_workload(struct intel_vgpu_workload *workload)
{
675 676
	struct intel_vgpu *vgpu = workload->vgpu;
	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
677
	struct intel_vgpu_submission *s = &vgpu->submission;
678
	struct i915_request *rq;
679
	int ring_id = workload->ring_id;
680
	int ret;
681 682 683 684

	gvt_dbg_sched("ring id %d prepare to dispatch workload %p\n",
		ring_id, workload);

685
	mutex_lock(&vgpu->vgpu_lock);
686 687
	mutex_lock(&dev_priv->drm.struct_mutex);

688 689
	ret = set_context_ppgtt_from_shadow(workload,
					    s->shadow[ring_id]->gem_context);
690 691 692 693 694
	if (ret < 0) {
		gvt_vgpu_err("workload shadow ppgtt isn't ready\n");
		goto err_req;
	}

695 696 697 698
	ret = intel_gvt_workload_req_alloc(workload);
	if (ret)
		goto err_req;

699
	ret = intel_gvt_scan_and_shadow_workload(workload);
Z
Zhi Wang 已提交
700
	if (ret)
701
		goto out;
Z
Zhi Wang 已提交
702

703 704 705 706 707
	ret = populate_shadow_context(workload);
	if (ret) {
		release_shadow_wa_ctx(&workload->wa_ctx);
		goto out;
	}
Z
Zhi Wang 已提交
708

709
	ret = prepare_workload(workload);
710
out:
711 712 713 714 715 716 717 718
	if (ret) {
		/* We might still need to add request with
		 * clean ctx to retire it properly..
		 */
		rq = fetch_and_zero(&workload->req);
		i915_request_put(rq);
	}

719 720 721
	if (!IS_ERR_OR_NULL(workload->req)) {
		gvt_dbg_sched("ring id %d submit workload to i915 %p\n",
				ring_id, workload->req);
722
		i915_request_add(workload->req);
723 724
		workload->dispatched = true;
	}
725 726 727
err_req:
	if (ret)
		workload->status = ret;
728
	mutex_unlock(&dev_priv->drm.struct_mutex);
729
	mutex_unlock(&vgpu->vgpu_lock);
Z
Zhi Wang 已提交
730 731 732 733 734 735 736 737 738
	return ret;
}

static struct intel_vgpu_workload *pick_next_workload(
		struct intel_gvt *gvt, int ring_id)
{
	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
	struct intel_vgpu_workload *workload = NULL;

739
	mutex_lock(&gvt->sched_lock);
Z
Zhi Wang 已提交
740 741 742 743 744 745 746 747 748 749 750 751 752 753 754

	/*
	 * no current vgpu / will be scheduled out / no workload
	 * bail out
	 */
	if (!scheduler->current_vgpu) {
		gvt_dbg_sched("ring id %d stop - no current vgpu\n", ring_id);
		goto out;
	}

	if (scheduler->need_reschedule) {
		gvt_dbg_sched("ring id %d stop - will reschedule\n", ring_id);
		goto out;
	}

755 756
	if (!scheduler->current_vgpu->active ||
	    list_empty(workload_q_head(scheduler->current_vgpu, ring_id)))
Z
Zhi Wang 已提交
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783
		goto out;

	/*
	 * still have current workload, maybe the workload disptacher
	 * fail to submit it for some reason, resubmit it.
	 */
	if (scheduler->current_workload[ring_id]) {
		workload = scheduler->current_workload[ring_id];
		gvt_dbg_sched("ring id %d still have current workload %p\n",
				ring_id, workload);
		goto out;
	}

	/*
	 * pick a workload as current workload
	 * once current workload is set, schedule policy routines
	 * will wait the current workload is finished when trying to
	 * schedule out a vgpu.
	 */
	scheduler->current_workload[ring_id] = container_of(
			workload_q_head(scheduler->current_vgpu, ring_id)->next,
			struct intel_vgpu_workload, list);

	workload = scheduler->current_workload[ring_id];

	gvt_dbg_sched("ring id %d pick new workload %p\n", ring_id, workload);

784
	atomic_inc(&workload->vgpu->submission.running_workload_num);
Z
Zhi Wang 已提交
785
out:
786
	mutex_unlock(&gvt->sched_lock);
Z
Zhi Wang 已提交
787 788 789 790 791
	return workload;
}

static void update_guest_context(struct intel_vgpu_workload *workload)
{
792
	struct i915_request *rq = workload->req;
Z
Zhi Wang 已提交
793 794
	struct intel_vgpu *vgpu = workload->vgpu;
	struct intel_gvt *gvt = vgpu->gvt;
795
	struct drm_i915_gem_object *ctx_obj = rq->hw_context->state->obj;
Z
Zhi Wang 已提交
796 797 798 799 800
	struct execlist_ring_context *shadow_ring_context;
	struct page *page;
	void *src;
	unsigned long context_gpa, context_page_num;
	int i;
801 802 803 804
	struct drm_i915_private *dev_priv = gvt->dev_priv;
	u32 ring_base;
	u32 head, tail;
	u16 wrap_count;
Z
Zhi Wang 已提交
805

806 807
	gvt_dbg_sched("ring id %d workload lrca %x\n", rq->engine->id,
		      workload->ctx_desc.lrca);
Z
Zhi Wang 已提交
808

809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825
	head = workload->rb_head;
	tail = workload->rb_tail;
	wrap_count = workload->guest_rb_head >> RB_HEAD_WRAP_CNT_OFF;

	if (tail < head) {
		if (wrap_count == RB_HEAD_WRAP_CNT_MAX)
			wrap_count = 0;
		else
			wrap_count += 1;
	}

	head = (wrap_count << RB_HEAD_WRAP_CNT_OFF) | tail;

	ring_base = dev_priv->engine[workload->ring_id]->mmio_base;
	vgpu_vreg_t(vgpu, RING_TAIL(ring_base)) = tail;
	vgpu_vreg_t(vgpu, RING_HEAD(ring_base)) = head;

826
	context_page_num = rq->engine->context_size;
Z
Zhi Wang 已提交
827 828
	context_page_num = context_page_num >> PAGE_SHIFT;

829
	if (IS_BROADWELL(gvt->dev_priv) && rq->engine->id == RCS0)
Z
Zhi Wang 已提交
830 831 832 833 834 835 836
		context_page_num = 19;

	i = 2;

	while (i < context_page_num) {
		context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
				(u32)((workload->ctx_desc.lrca + i) <<
Z
Zhi Wang 已提交
837
					I915_GTT_PAGE_SHIFT));
Z
Zhi Wang 已提交
838
		if (context_gpa == INTEL_GVT_INVALID_ADDR) {
839
			gvt_vgpu_err("invalid guest context descriptor\n");
Z
Zhi Wang 已提交
840 841 842
			return;
		}

843
		page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i);
844
		src = kmap(page);
Z
Zhi Wang 已提交
845
		intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src,
Z
Zhi Wang 已提交
846
				I915_GTT_PAGE_SIZE);
847
		kunmap(page);
Z
Zhi Wang 已提交
848 849 850 851 852 853 854
		i++;
	}

	intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa +
		RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4);

	page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
855
	shadow_ring_context = kmap(page);
Z
Zhi Wang 已提交
856 857 858 859 860 861 862 863 864 865 866 867 868 869 870

#define COPY_REG(name) \
	intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \
		RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4)

	COPY_REG(ctx_ctrl);
	COPY_REG(ctx_timestamp);

#undef COPY_REG

	intel_gvt_hypervisor_write_gpa(vgpu,
			workload->ring_context_gpa +
			sizeof(*shadow_ring_context),
			(void *)shadow_ring_context +
			sizeof(*shadow_ring_context),
Z
Zhi Wang 已提交
871
			I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
Z
Zhi Wang 已提交
872

873
	kunmap(page);
Z
Zhi Wang 已提交
874 875
}

876
void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu,
877
				intel_engine_mask_t engine_mask)
878 879 880 881 882
{
	struct intel_vgpu_submission *s = &vgpu->submission;
	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
	struct intel_engine_cs *engine;
	struct intel_vgpu_workload *pos, *n;
883
	intel_engine_mask_t tmp;
884 885 886 887 888 889 890 891 892 893 894 895

	/* free the unsubmited workloads in the queues. */
	for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
		list_for_each_entry_safe(pos, n,
			&s->workload_q_head[engine->id], list) {
			list_del_init(&pos->list);
			intel_vgpu_destroy_workload(pos);
		}
		clear_bit(engine->id, s->shadow_ctx_desc_updated);
	}
}

Z
Zhi Wang 已提交
896 897 898
static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
{
	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
899 900 901 902
	struct intel_vgpu_workload *workload =
		scheduler->current_workload[ring_id];
	struct intel_vgpu *vgpu = workload->vgpu;
	struct intel_vgpu_submission *s = &vgpu->submission;
903
	struct i915_request *rq = workload->req;
Z
Zhi Wang 已提交
904
	int event;
Z
Zhi Wang 已提交
905

906
	mutex_lock(&vgpu->vgpu_lock);
907
	mutex_lock(&gvt->sched_lock);
Z
Zhi Wang 已提交
908

909 910 911 912
	/* For the workload w/ request, needs to wait for the context
	 * switch to make sure request is completed.
	 * For the workload w/o request, directly complete the workload.
	 */
913
	if (rq) {
Z
Zhi Wang 已提交
914 915 916
		wait_event(workload->shadow_ctx_status_wq,
			   !atomic_read(&workload->shadow_ctx_active));

917 918 919 920 921 922 923 924 925 926 927 928
		/* If this request caused GPU hang, req->fence.error will
		 * be set to -EIO. Use -EIO to set workload status so
		 * that when this request caused GPU hang, didn't trigger
		 * context switch interrupt to guest.
		 */
		if (likely(workload->status == -EINPROGRESS)) {
			if (workload->req->fence.error == -EIO)
				workload->status = -EIO;
			else
				workload->status = 0;
		}

929 930
		if (!workload->status &&
		    !(vgpu->resetting_eng & BIT(ring_id))) {
931
			update_guest_context(workload);
Z
Zhi Wang 已提交
932

933 934 935 936
			for_each_set_bit(event, workload->pending_events,
					 INTEL_GVT_EVENT_MAX)
				intel_vgpu_trigger_virtual_event(vgpu, event);
		}
937

938
		i915_request_put(fetch_and_zero(&workload->req));
Z
Zhi Wang 已提交
939 940 941 942 943 944 945 946
	}

	gvt_dbg_sched("ring id %d complete workload %p status %d\n",
			ring_id, workload, workload->status);

	scheduler->current_workload[ring_id] = NULL;

	list_del_init(&workload->list);
947

948
	if (workload->status || vgpu->resetting_eng & BIT(ring_id)) {
949 950 951 952 953 954 955 956 957 958 959 960 961
		/* if workload->status is not successful means HW GPU
		 * has occurred GPU hang or something wrong with i915/GVT,
		 * and GVT won't inject context switch interrupt to guest.
		 * So this error is a vGPU hang actually to the guest.
		 * According to this we should emunlate a vGPU hang. If
		 * there are pending workloads which are already submitted
		 * from guest, we should clean them up like HW GPU does.
		 *
		 * if it is in middle of engine resetting, the pending
		 * workloads won't be submitted to HW GPU and will be
		 * cleaned up during the resetting process later, so doing
		 * the workload clean up here doesn't have any impact.
		 **/
962
		intel_vgpu_clean_workloads(vgpu, BIT(ring_id));
963 964
	}

Z
Zhi Wang 已提交
965 966
	workload->complete(workload);

967
	atomic_dec(&s->running_workload_num);
Z
Zhi Wang 已提交
968
	wake_up(&scheduler->workload_complete_wq);
969 970 971 972

	if (gvt->scheduler.need_reschedule)
		intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED);

973
	mutex_unlock(&gvt->sched_lock);
974
	mutex_unlock(&vgpu->vgpu_lock);
Z
Zhi Wang 已提交
975 976 977 978 979 980 981 982 983 984 985 986 987 988
}

struct workload_thread_param {
	struct intel_gvt *gvt;
	int ring_id;
};

static int workload_thread(void *priv)
{
	struct workload_thread_param *p = (struct workload_thread_param *)priv;
	struct intel_gvt *gvt = p->gvt;
	int ring_id = p->ring_id;
	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
	struct intel_vgpu_workload *workload = NULL;
989
	struct intel_vgpu *vgpu = NULL;
Z
Zhi Wang 已提交
990
	int ret;
991
	bool need_force_wake = (INTEL_GEN(gvt->dev_priv) >= 9);
992
	DEFINE_WAIT_FUNC(wait, woken_wake_function);
Z
Zhi Wang 已提交
993 994 995 996 997 998

	kfree(p);

	gvt_dbg_core("workload thread for ring %d started\n", ring_id);

	while (!kthread_should_stop()) {
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
		add_wait_queue(&scheduler->waitq[ring_id], &wait);
		do {
			workload = pick_next_workload(gvt, ring_id);
			if (workload)
				break;
			wait_woken(&wait, TASK_INTERRUPTIBLE,
				   MAX_SCHEDULE_TIMEOUT);
		} while (!kthread_should_stop());
		remove_wait_queue(&scheduler->waitq[ring_id], &wait);

		if (!workload)
Z
Zhi Wang 已提交
1010 1011 1012 1013 1014 1015 1016 1017 1018 1019
			break;

		gvt_dbg_sched("ring id %d next workload %p vgpu %d\n",
				workload->ring_id, workload,
				workload->vgpu->id);

		gvt_dbg_sched("ring id %d will dispatch workload %p\n",
				workload->ring_id, workload);

		if (need_force_wake)
1020
			intel_uncore_forcewake_get(&gvt->dev_priv->uncore,
Z
Zhi Wang 已提交
1021 1022 1023
					FORCEWAKE_ALL);

		ret = dispatch_workload(workload);
1024

Z
Zhi Wang 已提交
1025
		if (ret) {
1026 1027
			vgpu = workload->vgpu;
			gvt_vgpu_err("fail to dispatch workload, skip\n");
Z
Zhi Wang 已提交
1028 1029 1030 1031 1032
			goto complete;
		}

		gvt_dbg_sched("ring id %d wait workload %p\n",
				workload->ring_id, workload);
1033
		i915_request_wait(workload->req, 0, MAX_SCHEDULE_TIMEOUT);
Z
Zhi Wang 已提交
1034 1035

complete:
1036
		gvt_dbg_sched("will complete workload %p, status: %d\n",
Z
Zhi Wang 已提交
1037 1038
				workload, workload->status);

1039 1040
		complete_current_workload(gvt, ring_id);

Z
Zhi Wang 已提交
1041
		if (need_force_wake)
1042
			intel_uncore_forcewake_put(&gvt->dev_priv->uncore,
Z
Zhi Wang 已提交
1043 1044
					FORCEWAKE_ALL);

1045
		if (ret && (vgpu_is_vm_unhealthy(ret)))
1046
			enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
Z
Zhi Wang 已提交
1047 1048 1049 1050 1051 1052
	}
	return 0;
}

void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu)
{
1053
	struct intel_vgpu_submission *s = &vgpu->submission;
Z
Zhi Wang 已提交
1054 1055 1056
	struct intel_gvt *gvt = vgpu->gvt;
	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;

1057
	if (atomic_read(&s->running_workload_num)) {
Z
Zhi Wang 已提交
1058 1059 1060
		gvt_dbg_sched("wait vgpu idle\n");

		wait_event(scheduler->workload_complete_wq,
1061
				!atomic_read(&s->running_workload_num));
Z
Zhi Wang 已提交
1062 1063 1064 1065 1066 1067
	}
}

void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt)
{
	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
1068 1069
	struct intel_engine_cs *engine;
	enum intel_engine_id i;
Z
Zhi Wang 已提交
1070 1071 1072

	gvt_dbg_core("clean workload scheduler\n");

1073 1074 1075 1076 1077
	for_each_engine(engine, gvt->dev_priv, i) {
		atomic_notifier_chain_unregister(
					&engine->context_status_notifier,
					&gvt->shadow_ctx_notifier_block[i]);
		kthread_stop(scheduler->thread[i]);
Z
Zhi Wang 已提交
1078 1079 1080 1081 1082 1083 1084
	}
}

int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt)
{
	struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
	struct workload_thread_param *param = NULL;
1085 1086
	struct intel_engine_cs *engine;
	enum intel_engine_id i;
Z
Zhi Wang 已提交
1087 1088 1089 1090 1091 1092
	int ret;

	gvt_dbg_core("init workload scheduler\n");

	init_waitqueue_head(&scheduler->workload_complete_wq);

1093
	for_each_engine(engine, gvt->dev_priv, i) {
Z
Zhi Wang 已提交
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111
		init_waitqueue_head(&scheduler->waitq[i]);

		param = kzalloc(sizeof(*param), GFP_KERNEL);
		if (!param) {
			ret = -ENOMEM;
			goto err;
		}

		param->gvt = gvt;
		param->ring_id = i;

		scheduler->thread[i] = kthread_run(workload_thread, param,
			"gvt workload %d", i);
		if (IS_ERR(scheduler->thread[i])) {
			gvt_err("fail to create workload thread\n");
			ret = PTR_ERR(scheduler->thread[i]);
			goto err;
		}
1112 1113 1114 1115 1116

		gvt->shadow_ctx_notifier_block[i].notifier_call =
					shadow_context_status_change;
		atomic_notifier_chain_register(&engine->context_status_notifier,
					&gvt->shadow_ctx_notifier_block[i]);
Z
Zhi Wang 已提交
1117 1118 1119 1120 1121 1122 1123 1124 1125
	}
	return 0;
err:
	intel_gvt_clean_workload_scheduler(gvt);
	kfree(param);
	param = NULL;
	return ret;
}

1126
static void
1127
i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s,
1128
				struct i915_ppgtt *ppgtt)
1129 1130 1131
{
	int i;

1132
	if (i915_vm_is_4lvl(&ppgtt->vm)) {
1133
		px_dma(ppgtt->pd) = s->i915_context_pml4;
1134
	} else {
1135 1136 1137 1138 1139 1140
		for (i = 0; i < GEN8_3LVL_PDPES; i++) {
			struct i915_page_directory * const pd =
				i915_pd_entry(ppgtt->pd, i);

			px_dma(pd) = s->i915_context_pdps[i];
		}
1141 1142 1143
	}
}

1144 1145 1146 1147 1148 1149 1150 1151
/**
 * intel_vgpu_clean_submission - free submission-related resource for vGPU
 * @vgpu: a vGPU
 *
 * This function is called when a vGPU is being destroyed.
 *
 */
void intel_vgpu_clean_submission(struct intel_vgpu *vgpu)
Z
Zhi Wang 已提交
1152
{
1153
	struct intel_vgpu_submission *s = &vgpu->submission;
1154 1155
	struct intel_engine_cs *engine;
	enum intel_engine_id id;
1156

1157
	intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0);
1158

1159
	i915_context_ppgtt_root_restore(s, i915_vm_to_ppgtt(s->shadow[0]->gem_context->vm));
1160 1161 1162
	for_each_engine(engine, vgpu->gvt->dev_priv, id)
		intel_context_unpin(s->shadow[id]);

1163
	kmem_cache_destroy(s->workloads);
Z
Zhi Wang 已提交
1164 1165
}

1166 1167 1168 1169 1170 1171 1172 1173 1174 1175

/**
 * intel_vgpu_reset_submission - reset submission-related resource for vGPU
 * @vgpu: a vGPU
 * @engine_mask: engines expected to be reset
 *
 * This function is called when a vGPU is being destroyed.
 *
 */
void intel_vgpu_reset_submission(struct intel_vgpu *vgpu,
1176
				 intel_engine_mask_t engine_mask)
1177 1178 1179 1180 1181 1182
{
	struct intel_vgpu_submission *s = &vgpu->submission;

	if (!s->active)
		return;

1183
	intel_vgpu_clean_workloads(vgpu, engine_mask);
1184 1185 1186
	s->ops->reset(vgpu, engine_mask);
}

1187
static void
1188
i915_context_ppgtt_root_save(struct intel_vgpu_submission *s,
1189
			     struct i915_ppgtt *ppgtt)
1190 1191 1192
{
	int i;

1193
	if (i915_vm_is_4lvl(&ppgtt->vm)) {
1194
		s->i915_context_pml4 = px_dma(ppgtt->pd);
1195
	} else {
1196 1197 1198 1199 1200 1201
		for (i = 0; i < GEN8_3LVL_PDPES; i++) {
			struct i915_page_directory * const pd =
				i915_pd_entry(ppgtt->pd, i);

			s->i915_context_pdps[i] = px_dma(pd);
		}
1202 1203 1204
	}
}

1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215
/**
 * intel_vgpu_setup_submission - setup submission-related resource for vGPU
 * @vgpu: a vGPU
 *
 * This function is called when a vGPU is being created.
 *
 * Returns:
 * Zero on success, negative error code if failed.
 *
 */
int intel_vgpu_setup_submission(struct intel_vgpu *vgpu)
Z
Zhi Wang 已提交
1216
{
1217
	struct intel_vgpu_submission *s = &vgpu->submission;
1218
	struct intel_engine_cs *engine;
1219 1220
	struct i915_gem_context *ctx;
	enum intel_engine_id i;
1221
	int ret;
Z
Zhi Wang 已提交
1222

1223 1224 1225 1226
	ctx = i915_gem_context_create_gvt(&vgpu->gvt->dev_priv->drm);
	if (IS_ERR(ctx))
		return PTR_ERR(ctx);

1227
	i915_context_ppgtt_root_save(s, i915_vm_to_ppgtt(ctx->vm));
Z
Zhi Wang 已提交
1228

1229 1230 1231 1232 1233 1234
	for_each_engine(engine, vgpu->gvt->dev_priv, i) {
		struct intel_context *ce;

		INIT_LIST_HEAD(&s->workload_q_head[i]);
		s->shadow[i] = ERR_PTR(-EINVAL);

1235
		ce = i915_gem_context_get_engine(ctx, i);
1236 1237 1238 1239
		if (IS_ERR(ce)) {
			ret = PTR_ERR(ce);
			goto out_shadow_ctx;
		}
Z
Zhi Wang 已提交
1240

1241 1242 1243 1244 1245
		ret = intel_context_pin(ce);
		intel_context_put(ce);
		if (ret)
			goto out_shadow_ctx;

1246 1247
		s->shadow[i] = ce;
	}
1248

1249
	bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES);
1250

1251 1252 1253 1254 1255 1256
	s->workloads = kmem_cache_create_usercopy("gvt-g_vgpu_workload",
						  sizeof(struct intel_vgpu_workload), 0,
						  SLAB_HWCACHE_ALIGN,
						  offsetof(struct intel_vgpu_workload, rb_tail),
						  sizeof_field(struct intel_vgpu_workload, rb_tail),
						  NULL);
1257

1258
	if (!s->workloads) {
1259 1260 1261 1262
		ret = -ENOMEM;
		goto out_shadow_ctx;
	}

1263
	atomic_set(&s->running_workload_num, 0);
1264
	bitmap_zero(s->tlb_handle_pending, I915_NUM_ENGINES);
1265

1266
	i915_gem_context_put(ctx);
Z
Zhi Wang 已提交
1267
	return 0;
1268 1269

out_shadow_ctx:
1270
	i915_context_ppgtt_root_restore(s, i915_vm_to_ppgtt(ctx->vm));
1271 1272 1273 1274 1275 1276 1277
	for_each_engine(engine, vgpu->gvt->dev_priv, i) {
		if (IS_ERR(s->shadow[i]))
			break;

		intel_context_unpin(s->shadow[i]);
	}
	i915_gem_context_put(ctx);
1278
	return ret;
Z
Zhi Wang 已提交
1279
}
1280

1281 1282 1283
/**
 * intel_vgpu_select_submission_ops - select virtual submission interface
 * @vgpu: a vGPU
1284
 * @engine_mask: either ALL_ENGINES or target engine mask
1285 1286 1287 1288 1289 1290 1291 1292 1293
 * @interface: expected vGPU virtual submission interface
 *
 * This function is called when guest configures submission interface.
 *
 * Returns:
 * Zero on success, negative error code if failed.
 *
 */
int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu,
1294
				     intel_engine_mask_t engine_mask,
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
				     unsigned int interface)
{
	struct intel_vgpu_submission *s = &vgpu->submission;
	const struct intel_vgpu_submission_ops *ops[] = {
		[INTEL_VGPU_EXECLIST_SUBMISSION] =
			&intel_vgpu_execlist_submission_ops,
	};
	int ret;

	if (WARN_ON(interface >= ARRAY_SIZE(ops)))
		return -EINVAL;

1307 1308 1309 1310
	if (WARN_ON(interface == 0 && engine_mask != ALL_ENGINES))
		return -EINVAL;

	if (s->active)
1311
		s->ops->clean(vgpu, engine_mask);
1312 1313 1314 1315

	if (interface == 0) {
		s->ops = NULL;
		s->virtual_submission_interface = 0;
1316 1317
		s->active = false;
		gvt_dbg_core("vgpu%d: remove submission ops\n", vgpu->id);
1318 1319 1320
		return 0;
	}

1321
	ret = ops[interface]->init(vgpu, engine_mask);
1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334
	if (ret)
		return ret;

	s->ops = ops[interface];
	s->virtual_submission_interface = interface;
	s->active = true;

	gvt_dbg_core("vgpu%d: activate ops [ %s ]\n",
			vgpu->id, s->ops->name);

	return 0;
}

1335 1336
/**
 * intel_vgpu_destroy_workload - destroy a vGPU workload
1337
 * @workload: workload to destroy
1338 1339 1340 1341 1342 1343 1344 1345
 *
 * This function is called when destroy a vGPU workload.
 *
 */
void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload)
{
	struct intel_vgpu_submission *s = &workload->vgpu->submission;

1346 1347 1348
	release_shadow_batch_buffer(workload);
	release_shadow_wa_ctx(&workload->wa_ctx);

1349
	if (workload->shadow_mm)
1350
		intel_vgpu_mm_put(workload->shadow_mm);
1351 1352 1353 1354

	kmem_cache_free(s->workloads, workload);
}

1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385
static struct intel_vgpu_workload *
alloc_workload(struct intel_vgpu *vgpu)
{
	struct intel_vgpu_submission *s = &vgpu->submission;
	struct intel_vgpu_workload *workload;

	workload = kmem_cache_zalloc(s->workloads, GFP_KERNEL);
	if (!workload)
		return ERR_PTR(-ENOMEM);

	INIT_LIST_HEAD(&workload->list);
	INIT_LIST_HEAD(&workload->shadow_bb);

	init_waitqueue_head(&workload->shadow_ctx_status_wq);
	atomic_set(&workload->shadow_ctx_active, 0);

	workload->status = -EINPROGRESS;
	workload->vgpu = vgpu;

	return workload;
}

#define RING_CTX_OFF(x) \
	offsetof(struct execlist_ring_context, x)

static void read_guest_pdps(struct intel_vgpu *vgpu,
		u64 ring_context_gpa, u32 pdp[8])
{
	u64 gpa;
	int i;

1386
	gpa = ring_context_gpa + RING_CTX_OFF(pdps[0].val);
1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397

	for (i = 0; i < 8; i++)
		intel_gvt_hypervisor_read_gpa(vgpu,
				gpa + i * 8, &pdp[7 - i], 4);
}

static int prepare_mm(struct intel_vgpu_workload *workload)
{
	struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;
	struct intel_vgpu_mm *mm;
	struct intel_vgpu *vgpu = workload->vgpu;
1398
	enum intel_gvt_gtt_type root_entry_type;
1399
	u64 pdps[GVT_RING_CTX_NR_PDPS];
1400

1401 1402 1403 1404 1405 1406 1407 1408
	switch (desc->addressing_mode) {
	case 1: /* legacy 32-bit */
		root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY;
		break;
	case 3: /* legacy 64-bit */
		root_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY;
		break;
	default:
1409 1410 1411 1412
		gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n");
		return -EINVAL;
	}

1413
	read_guest_pdps(workload->vgpu, workload->ring_context_gpa, (void *)pdps);
1414

1415 1416 1417 1418
	mm = intel_vgpu_get_ppgtt_mm(workload->vgpu, root_entry_type, pdps);
	if (IS_ERR(mm))
		return PTR_ERR(mm);

1419 1420 1421 1422 1423 1424 1425 1426 1427 1428
	workload->shadow_mm = mm;
	return 0;
}

#define same_context(a, b) (((a)->context_id == (b)->context_id) && \
		((a)->lrca == (b)->lrca))

#define get_last_workload(q) \
	(list_empty(q) ? NULL : container_of(q->prev, \
	struct intel_vgpu_workload, list))
1429 1430 1431
/**
 * intel_vgpu_create_workload - create a vGPU workload
 * @vgpu: a vGPU
1432
 * @ring_id: ring index
1433
 * @desc: a guest context descriptor
1434 1435 1436 1437 1438 1439 1440 1441 1442
 *
 * This function is called when creating a vGPU workload.
 *
 * Returns:
 * struct intel_vgpu_workload * on success, negative error code in
 * pointer if failed.
 *
 */
struct intel_vgpu_workload *
1443 1444
intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id,
			   struct execlist_ctx_descriptor_format *desc)
1445 1446
{
	struct intel_vgpu_submission *s = &vgpu->submission;
1447 1448 1449 1450 1451 1452
	struct list_head *q = workload_q_head(vgpu, ring_id);
	struct intel_vgpu_workload *last_workload = get_last_workload(q);
	struct intel_vgpu_workload *workload = NULL;
	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
	u64 ring_context_gpa;
	u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx;
1453
	u32 guest_head;
1454
	int ret;
1455

1456
	ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
Z
Zhi Wang 已提交
1457
			(u32)((desc->lrca + 1) << I915_GTT_PAGE_SHIFT));
1458 1459 1460 1461
	if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) {
		gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca);
		return ERR_PTR(-EINVAL);
	}
1462

1463 1464
	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
			RING_CTX_OFF(ring_header.val), &head, 4);
1465

1466 1467
	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
			RING_CTX_OFF(ring_tail.val), &tail, 4);
1468

1469 1470
	guest_head = head;

1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494
	head &= RB_HEAD_OFF_MASK;
	tail &= RB_TAIL_OFF_MASK;

	if (last_workload && same_context(&last_workload->ctx_desc, desc)) {
		gvt_dbg_el("ring id %d cur workload == last\n", ring_id);
		gvt_dbg_el("ctx head %x real head %lx\n", head,
				last_workload->rb_tail);
		/*
		 * cannot use guest context head pointer here,
		 * as it might not be updated at this time
		 */
		head = last_workload->rb_tail;
	}

	gvt_dbg_el("ring id %d begin a new workload\n", ring_id);

	/* record some ring buffer register values for scan and shadow */
	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
			RING_CTX_OFF(rb_start.val), &start, 4);
	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
			RING_CTX_OFF(rb_ctrl.val), &ctl, 4);
	intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
			RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4);

1495 1496 1497 1498 1499 1500
	if (!intel_gvt_ggtt_validate_range(vgpu, start,
				_RING_CTL_BUF_SIZE(ctl))) {
		gvt_vgpu_err("context contain invalid rb at: 0x%x\n", start);
		return ERR_PTR(-EINVAL);
	}

1501 1502 1503 1504 1505 1506 1507 1508
	workload = alloc_workload(vgpu);
	if (IS_ERR(workload))
		return workload;

	workload->ring_id = ring_id;
	workload->ctx_desc = *desc;
	workload->ring_context_gpa = ring_context_gpa;
	workload->rb_head = head;
1509
	workload->guest_rb_head = guest_head;
1510 1511 1512 1513
	workload->rb_tail = tail;
	workload->rb_start = start;
	workload->rb_ctl = ctl;

1514
	if (ring_id == RCS0) {
1515 1516 1517 1518 1519 1520 1521 1522 1523 1524
		intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
			RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
		intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
			RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4);

		workload->wa_ctx.indirect_ctx.guest_gma =
			indirect_ctx & INDIRECT_CTX_ADDR_MASK;
		workload->wa_ctx.indirect_ctx.size =
			(indirect_ctx & INDIRECT_CTX_SIZE_MASK) *
			CACHELINE_BYTES;
1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536

		if (workload->wa_ctx.indirect_ctx.size != 0) {
			if (!intel_gvt_ggtt_validate_range(vgpu,
				workload->wa_ctx.indirect_ctx.guest_gma,
				workload->wa_ctx.indirect_ctx.size)) {
				kmem_cache_free(s->workloads, workload);
				gvt_vgpu_err("invalid wa_ctx at: 0x%lx\n",
				    workload->wa_ctx.indirect_ctx.guest_gma);
				return ERR_PTR(-EINVAL);
			}
		}

1537 1538 1539
		workload->wa_ctx.per_ctx.guest_gma =
			per_ctx & PER_CTX_ADDR_MASK;
		workload->wa_ctx.per_ctx.valid = per_ctx & 1;
1540 1541 1542 1543 1544 1545 1546 1547 1548 1549
		if (workload->wa_ctx.per_ctx.valid) {
			if (!intel_gvt_ggtt_validate_range(vgpu,
				workload->wa_ctx.per_ctx.guest_gma,
				CACHELINE_BYTES)) {
				kmem_cache_free(s->workloads, workload);
				gvt_vgpu_err("invalid per_ctx at: 0x%lx\n",
					workload->wa_ctx.per_ctx.guest_gma);
				return ERR_PTR(-EINVAL);
			}
		}
1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564
	}

	gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n",
			workload, ring_id, head, tail, start, ctl);

	ret = prepare_mm(workload);
	if (ret) {
		kmem_cache_free(s->workloads, workload);
		return ERR_PTR(ret);
	}

	/* Only scan and shadow the first workload in the queue
	 * as there is only one pre-allocated buf-obj for shadow.
	 */
	if (list_empty(workload_q_head(vgpu, ring_id))) {
1565
		intel_runtime_pm_get(&dev_priv->runtime_pm);
1566 1567 1568
		mutex_lock(&dev_priv->drm.struct_mutex);
		ret = intel_gvt_scan_and_shadow_workload(workload);
		mutex_unlock(&dev_priv->drm.struct_mutex);
1569
		intel_runtime_pm_put_unchecked(&dev_priv->runtime_pm);
1570 1571
	}

1572 1573 1574
	if (ret) {
		if (vgpu_is_vm_unhealthy(ret))
			enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
1575 1576 1577
		intel_vgpu_destroy_workload(workload);
		return ERR_PTR(ret);
	}
1578 1579 1580

	return workload;
}
1581 1582 1583 1584 1585 1586 1587 1588 1589

/**
 * intel_vgpu_queue_workload - Qeue a vGPU workload
 * @workload: the workload to queue in
 */
void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload)
{
	list_add_tail(&workload->list,
		workload_q_head(workload->vgpu, workload->ring_id));
1590
	intel_gvt_kick_schedule(workload->vgpu->gvt);
1591 1592
	wake_up(&workload->vgpu->gvt->scheduler.waitq[workload->ring_id]);
}