mmio_context.c 17.1 KB
Newer Older
Z
Zhi Wang 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 * Authors:
 *    Eddie Dong <eddie.dong@intel.com>
 *    Kevin Tian <kevin.tian@intel.com>
 *
 * Contributors:
 *    Zhi Wang <zhi.a.wang@intel.com>
 *    Changbin Du <changbin.du@intel.com>
 *    Zhenyu Wang <zhenyuw@linux.intel.com>
 *    Tina Zhang <tina.zhang@intel.com>
 *    Bing Niu <bing.niu@intel.com>
 *
 */

#include "i915_drv.h"
37
#include "gvt.h"
38
#include "trace.h"
Z
Zhi Wang 已提交
39

40 41
#define GEN9_MOCS_SIZE		64

42
/* Raw offset is appened to each line for convenience. */
43
static struct engine_mmio gen8_engine_mmio_list[] __cacheline_aligned = {
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
	{RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
	{RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
	{RCS, HWSTAM, 0x0, false}, /* 0x2098 */
	{RCS, INSTPM, 0xffff, true}, /* 0x20c0 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
	{RCS, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
	{RCS, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
	{RCS, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
	{RCS, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
	{RCS, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
	{RCS, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */

	{BCS, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
	{BCS, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
	{BCS, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
	{BCS, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
	{BCS, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */
72
	{RCS, INVALID_MMIO_REG, 0, false } /* Terminated */
Z
Zhi Wang 已提交
73 74
};

75
static struct engine_mmio gen9_engine_mmio_list[] __cacheline_aligned = {
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
	{RCS, GFX_MODE_GEN7, 0xffff, false}, /* 0x229c */
	{RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
	{RCS, HWSTAM, 0x0, false}, /* 0x2098 */
	{RCS, INSTPM, 0xffff, true}, /* 0x20c0 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 0), 0, false}, /* 0x24d0 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 1), 0, false}, /* 0x24d4 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 2), 0, false}, /* 0x24d8 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 3), 0, false}, /* 0x24dc */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 4), 0, false}, /* 0x24e0 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 5), 0, false}, /* 0x24e4 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 6), 0, false}, /* 0x24e8 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 7), 0, false}, /* 0x24ec */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 8), 0, false}, /* 0x24f0 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 9), 0, false}, /* 0x24f4 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 10), 0, false}, /* 0x24f8 */
	{RCS, RING_FORCE_TO_NONPRIV(RENDER_RING_BASE, 11), 0, false}, /* 0x24fc */
	{RCS, CACHE_MODE_1, 0xffff, true}, /* 0x7004 */
	{RCS, GEN7_GT_MODE, 0xffff, true}, /* 0x7008 */
	{RCS, CACHE_MODE_0_GEN7, 0xffff, true}, /* 0x7000 */
	{RCS, GEN7_COMMON_SLICE_CHICKEN1, 0xffff, true}, /* 0x7010 */
	{RCS, HDC_CHICKEN0, 0xffff, true}, /* 0x7300 */
	{RCS, VF_GUARDBAND, 0xffff, true}, /* 0x83a4 */

	{RCS, GEN8_PRIVATE_PAT_LO, 0, false}, /* 0x40e0 */
	{RCS, GEN8_PRIVATE_PAT_HI, 0, false}, /* 0x40e4 */
	{RCS, GEN8_CS_CHICKEN1, 0xffff, true}, /* 0x2580 */
	{RCS, COMMON_SLICE_CHICKEN2, 0xffff, true}, /* 0x7014 */
	{RCS, GEN9_CS_DEBUG_MODE1, 0xffff, false}, /* 0x20ec */
	{RCS, GEN8_L3SQCREG4, 0, false}, /* 0xb118 */
	{RCS, GEN7_HALF_SLICE_CHICKEN1, 0xffff, true}, /* 0xe100 */
	{RCS, HALF_SLICE_CHICKEN2, 0xffff, true}, /* 0xe180 */
	{RCS, HALF_SLICE_CHICKEN3, 0xffff, true}, /* 0xe184 */
	{RCS, GEN9_HALF_SLICE_CHICKEN5, 0xffff, true}, /* 0xe188 */
	{RCS, GEN9_HALF_SLICE_CHICKEN7, 0xffff, true}, /* 0xe194 */
110
	{RCS, GEN8_ROW_CHICKEN, 0xffff, true}, /* 0xe4f0 */
111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
	{RCS, TRVATTL3PTRDW(0), 0, false}, /* 0x4de0 */
	{RCS, TRVATTL3PTRDW(1), 0, false}, /* 0x4de4 */
	{RCS, TRNULLDETCT, 0, false}, /* 0x4de8 */
	{RCS, TRINVTILEDETCT, 0, false}, /* 0x4dec */
	{RCS, TRVADR, 0, false}, /* 0x4df0 */
	{RCS, TRTTE, 0, false}, /* 0x4df4 */

	{BCS, RING_GFX_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2229c */
	{BCS, RING_MI_MODE(BLT_RING_BASE), 0xffff, false}, /* 0x2209c */
	{BCS, RING_INSTPM(BLT_RING_BASE), 0xffff, false}, /* 0x220c0 */
	{BCS, RING_HWSTAM(BLT_RING_BASE), 0x0, false}, /* 0x22098 */
	{BCS, RING_EXCC(BLT_RING_BASE), 0x0, false}, /* 0x22028 */

	{VCS2, RING_EXCC(GEN8_BSD2_RING_BASE), 0xffff, false}, /* 0x1c028 */

	{VECS, RING_EXCC(VEBOX_RING_BASE), 0xffff, false}, /* 0x1a028 */

	{RCS, GEN8_HDC_CHICKEN1, 0xffff, true}, /* 0x7304 */
	{RCS, GEN9_CTX_PREEMPT_REG, 0x0, false}, /* 0x2248 */
	{RCS, GEN7_UCGCTL4, 0x0, false}, /* 0x940c */
	{RCS, GAMT_CHKN_BIT_REG, 0x0, false}, /* 0x4ab8 */

	{RCS, GEN9_GAMT_ECO_REG_RW_IA, 0x0, false}, /* 0x4ab0 */
134
	{RCS, GEN9_CSFE_CHICKEN1_RCS, 0xffff, false}, /* 0x20d4 */
135 136 137 138

	{RCS, GEN8_GARBCNTL, 0x0, false}, /* 0xb004 */
	{RCS, GEN7_FF_THREAD_MODE, 0x0, false}, /* 0x20a0 */
	{RCS, FF_SLICE_CS_CHICKEN2, 0xffff, false}, /* 0x20e4 */
139
	{RCS, INVALID_MMIO_REG, 0, false } /* Terminated */
Z
Zhi Wang 已提交
140 141
};

142 143
static struct {
	bool initialized;
144 145
	u32 control_table[I915_NUM_ENGINES][GEN9_MOCS_SIZE];
	u32 l3cc_table[GEN9_MOCS_SIZE / 2];
146 147 148 149 150 151 152 153 154 155 156 157 158 159
} gen9_render_mocs;

static void load_render_mocs(struct drm_i915_private *dev_priv)
{
	i915_reg_t offset;
	u32 regs[] = {
		[RCS] = 0xc800,
		[VCS] = 0xc900,
		[VCS2] = 0xca00,
		[BCS] = 0xcc00,
		[VECS] = 0xcb00,
	};
	int ring_id, i;

160
	for (ring_id = 0; ring_id < ARRAY_SIZE(regs); ring_id++) {
161 162
		if (!HAS_ENGINE(dev_priv, ring_id))
			continue;
163
		offset.reg = regs[ring_id];
164
		for (i = 0; i < GEN9_MOCS_SIZE; i++) {
165 166 167 168 169 170 171
			gen9_render_mocs.control_table[ring_id][i] =
				I915_READ_FW(offset);
			offset.reg += 4;
		}
	}

	offset.reg = 0xb020;
172
	for (i = 0; i < GEN9_MOCS_SIZE / 2; i++) {
173 174 175 176 177 178
		gen9_render_mocs.l3cc_table[i] =
			I915_READ_FW(offset);
		offset.reg += 4;
	}
	gen9_render_mocs.initialized = true;
}
Z
Zhi Wang 已提交
179

180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326
static int
restore_context_mmio_for_inhibit(struct intel_vgpu *vgpu,
				 struct i915_request *req)
{
	u32 *cs;
	int ret;
	struct engine_mmio *mmio;
	struct intel_gvt *gvt = vgpu->gvt;
	int ring_id = req->engine->id;
	int count = gvt->engine_mmio_list.ctx_mmio_count[ring_id];

	if (count == 0)
		return 0;

	ret = req->engine->emit_flush(req, EMIT_BARRIER);
	if (ret)
		return ret;

	cs = intel_ring_begin(req, count * 2 + 2);
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	*cs++ = MI_LOAD_REGISTER_IMM(count);
	for (mmio = gvt->engine_mmio_list.mmio;
	     i915_mmio_reg_valid(mmio->reg); mmio++) {
		if (mmio->ring_id != ring_id ||
		    !mmio->in_context)
			continue;

		*cs++ = i915_mmio_reg_offset(mmio->reg);
		*cs++ = vgpu_vreg_t(vgpu, mmio->reg) |
				(mmio->mask << 16);
		gvt_dbg_core("add lri reg pair 0x%x:0x%x in inhibit ctx, vgpu:%d, rind_id:%d\n",
			      *(cs-2), *(cs-1), vgpu->id, ring_id);
	}

	*cs++ = MI_NOOP;
	intel_ring_advance(req, cs);

	ret = req->engine->emit_flush(req, EMIT_BARRIER);
	if (ret)
		return ret;

	return 0;
}

static int
restore_render_mocs_control_for_inhibit(struct intel_vgpu *vgpu,
					struct i915_request *req)
{
	unsigned int index;
	u32 *cs;

	cs = intel_ring_begin(req, 2 * GEN9_MOCS_SIZE + 2);
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	*cs++ = MI_LOAD_REGISTER_IMM(GEN9_MOCS_SIZE);

	for (index = 0; index < GEN9_MOCS_SIZE; index++) {
		*cs++ = i915_mmio_reg_offset(GEN9_GFX_MOCS(index));
		*cs++ = vgpu_vreg_t(vgpu, GEN9_GFX_MOCS(index));
		gvt_dbg_core("add lri reg pair 0x%x:0x%x in inhibit ctx, vgpu:%d, rind_id:%d\n",
			      *(cs-2), *(cs-1), vgpu->id, req->engine->id);

	}

	*cs++ = MI_NOOP;
	intel_ring_advance(req, cs);

	return 0;
}

static int
restore_render_mocs_l3cc_for_inhibit(struct intel_vgpu *vgpu,
				     struct i915_request *req)
{
	unsigned int index;
	u32 *cs;

	cs = intel_ring_begin(req, 2 * GEN9_MOCS_SIZE / 2 + 2);
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	*cs++ = MI_LOAD_REGISTER_IMM(GEN9_MOCS_SIZE / 2);

	for (index = 0; index < GEN9_MOCS_SIZE / 2; index++) {
		*cs++ = i915_mmio_reg_offset(GEN9_LNCFCMOCS(index));
		*cs++ = vgpu_vreg_t(vgpu, GEN9_LNCFCMOCS(index));
		gvt_dbg_core("add lri reg pair 0x%x:0x%x in inhibit ctx, vgpu:%d, rind_id:%d\n",
			      *(cs-2), *(cs-1), vgpu->id, req->engine->id);

	}

	*cs++ = MI_NOOP;
	intel_ring_advance(req, cs);

	return 0;
}

/*
 * Use lri command to initialize the mmio which is in context state image for
 * inhibit context, it contains tracked engine mmio, render_mocs and
 * render_mocs_l3cc.
 */
int intel_vgpu_restore_inhibit_context(struct intel_vgpu *vgpu,
				       struct i915_request *req)
{
	int ret;
	u32 *cs;

	cs = intel_ring_begin(req, 2);
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
	*cs++ = MI_NOOP;
	intel_ring_advance(req, cs);

	ret = restore_context_mmio_for_inhibit(vgpu, req);
	if (ret)
		goto out;

	/* no MOCS register in context except render engine */
	if (req->engine->id != RCS)
		goto out;

	ret = restore_render_mocs_control_for_inhibit(vgpu, req);
	if (ret)
		goto out;

	ret = restore_render_mocs_l3cc_for_inhibit(vgpu, req);
	if (ret)
		goto out;

out:
	cs = intel_ring_begin(req, 2);
	if (IS_ERR(cs))
		return PTR_ERR(cs);

	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
	*cs++ = MI_NOOP;
	intel_ring_advance(req, cs);

	return ret;
}

Z
Zhi Wang 已提交
327 328 329
static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
{
	struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
330
	struct intel_vgpu_submission *s = &vgpu->submission;
331
	enum forcewake_domains fw;
Z
Zhi Wang 已提交
332 333 334 335 336 337 338 339 340 341 342 343
	i915_reg_t reg;
	u32 regs[] = {
		[RCS] = 0x4260,
		[VCS] = 0x4264,
		[VCS2] = 0x4268,
		[BCS] = 0x426c,
		[VECS] = 0x4270,
	};

	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
		return;

344
	if (!test_and_clear_bit(ring_id, (void *)s->tlb_handle_pending))
Z
Zhi Wang 已提交
345 346 347 348
		return;

	reg = _MMIO(regs[ring_id]);

349 350 351 352 353 354 355
	/* WaForceWakeRenderDuringMmioTLBInvalidate:skl
	 * we need to put a forcewake when invalidating RCS TLB caches,
	 * otherwise device can go to RC6 state and interrupt invalidation
	 * process
	 */
	fw = intel_uncore_forcewake_for_reg(dev_priv, reg,
					    FW_REG_READ | FW_REG_WRITE);
356 357
	if (ring_id == RCS && (IS_SKYLAKE(dev_priv) ||
			IS_KABYLAKE(dev_priv) || IS_BROXTON(dev_priv)))
358
		fw |= FORCEWAKE_RENDER;
Z
Zhi Wang 已提交
359

360 361 362 363 364
	intel_uncore_forcewake_get(dev_priv, fw);

	I915_WRITE_FW(reg, 0x1);

	if (wait_for_atomic((I915_READ_FW(reg) == 0), 50))
365
		gvt_vgpu_err("timeout in invalidate ring (%d) tlb\n", ring_id);
366
	else
367
		vgpu_vreg_t(vgpu, reg) = 0;
Z
Zhi Wang 已提交
368

369 370
	intel_uncore_forcewake_put(dev_priv, fw);

Z
Zhi Wang 已提交
371 372 373
	gvt_dbg_core("invalidate TLB for ring %d\n", ring_id);
}

374 375
static void switch_mocs(struct intel_vgpu *pre, struct intel_vgpu *next,
			int ring_id)
Z
Zhi Wang 已提交
376
{
377
	struct drm_i915_private *dev_priv;
Z
Zhi Wang 已提交
378
	i915_reg_t offset, l3_offset;
379 380
	u32 old_v, new_v;

Z
Zhi Wang 已提交
381 382 383 384 385 386 387 388 389
	u32 regs[] = {
		[RCS] = 0xc800,
		[VCS] = 0xc900,
		[VCS2] = 0xca00,
		[BCS] = 0xcc00,
		[VECS] = 0xcb00,
	};
	int i;

390
	dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv;
Z
Zhi Wang 已提交
391 392 393
	if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
		return;

394
	if ((IS_KABYLAKE(dev_priv) || IS_BROXTON(dev_priv)) && ring_id == RCS)
395 396
		return;

397 398
	if (!pre && !gen9_render_mocs.initialized)
		load_render_mocs(dev_priv);
Z
Zhi Wang 已提交
399

400
	offset.reg = regs[ring_id];
401
	for (i = 0; i < GEN9_MOCS_SIZE; i++) {
402
		if (pre)
403
			old_v = vgpu_vreg_t(pre, offset);
404
		else
405
			old_v = gen9_render_mocs.control_table[ring_id][i];
406
		if (next)
407
			new_v = vgpu_vreg_t(next, offset);
408
		else
409
			new_v = gen9_render_mocs.control_table[ring_id][i];
410 411 412

		if (old_v != new_v)
			I915_WRITE_FW(offset, new_v);
Z
Zhi Wang 已提交
413 414 415 416 417 418

		offset.reg += 4;
	}

	if (ring_id == RCS) {
		l3_offset.reg = 0xb020;
419
		for (i = 0; i < GEN9_MOCS_SIZE / 2; i++) {
420
			if (pre)
421
				old_v = vgpu_vreg_t(pre, l3_offset);
422
			else
423
				old_v = gen9_render_mocs.l3cc_table[i];
424
			if (next)
425
				new_v = vgpu_vreg_t(next, l3_offset);
426
			else
427
				new_v = gen9_render_mocs.l3cc_table[i];
428 429 430

			if (old_v != new_v)
				I915_WRITE_FW(l3_offset, new_v);
431

Z
Zhi Wang 已提交
432 433 434 435 436
			l3_offset.reg += 4;
		}
	}
}

437 438
#define CTX_CONTEXT_CONTROL_VAL	0x03

439
bool is_inhibit_context(struct intel_context *ce)
440
{
441
	const u32 *reg_state = ce->lrc_reg_state;
442 443 444 445 446 447 448
	u32 inhibit_mask =
		_MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);

	return inhibit_mask ==
		(reg_state[CTX_CONTEXT_CONTROL_VAL] & inhibit_mask);
}

449 450 451 452
/* Switch ring mmio values (context). */
static void switch_mmio(struct intel_vgpu *pre,
			struct intel_vgpu *next,
			int ring_id)
Z
Zhi Wang 已提交
453
{
454 455
	struct drm_i915_private *dev_priv;
	struct intel_vgpu_submission *s;
456
	struct engine_mmio *mmio;
457
	u32 old_v, new_v;
Z
Zhi Wang 已提交
458

459
	dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv;
460 461 462
	if (IS_SKYLAKE(dev_priv)
		|| IS_KABYLAKE(dev_priv)
		|| IS_BROXTON(dev_priv))
463
		switch_mocs(pre, next, ring_id);
Z
Zhi Wang 已提交
464

465
	for (mmio = dev_priv->gvt->engine_mmio_list.mmio;
466
	     i915_mmio_reg_valid(mmio->reg); mmio++) {
Z
Zhi Wang 已提交
467 468
		if (mmio->ring_id != ring_id)
			continue;
469 470 471 472 473
		/*
		 * No need to do save or restore of the mmio which is in context
		 * state image on kabylake, it's initialized by lri command and
		 * save or restore with context together.
		 */
474 475
		if ((IS_KABYLAKE(dev_priv) || IS_BROXTON(dev_priv))
			&& mmio->in_context)
476 477
			continue;

478 479
		// save
		if (pre) {
480
			vgpu_vreg_t(pre, mmio->reg) = I915_READ_FW(mmio->reg);
481
			if (mmio->mask)
482
				vgpu_vreg_t(pre, mmio->reg) &=
483
						~(mmio->mask << 16);
484
			old_v = vgpu_vreg_t(pre, mmio->reg);
Z
Zhi Wang 已提交
485
		} else
486 487 488 489 490 491
			old_v = mmio->value = I915_READ_FW(mmio->reg);

		// restore
		if (next) {
			s = &next->submission;
			/*
492 493 494
			 * No need to restore the mmio which is in context state
			 * image if it's not inhibit context, it will restore
			 * itself.
495 496
			 */
			if (mmio->in_context &&
497
			    !is_inhibit_context(&s->shadow_ctx->__engine[ring_id]))
498 499 500
				continue;

			if (mmio->mask)
501
				new_v = vgpu_vreg_t(next, mmio->reg) |
502 503
							(mmio->mask << 16);
			else
504
				new_v = vgpu_vreg_t(next, mmio->reg);
505 506 507 508 509 510 511 512
		} else {
			if (mmio->in_context)
				continue;
			if (mmio->mask)
				new_v = mmio->value | (mmio->mask << 16);
			else
				new_v = mmio->value;
		}
513

514
		I915_WRITE_FW(mmio->reg, new_v);
Z
Zhi Wang 已提交
515

516 517 518
		trace_render_mmio(pre ? pre->id : 0,
				  next ? next->id : 0,
				  "switch",
519
				  i915_mmio_reg_offset(mmio->reg),
520
				  old_v, new_v);
Z
Zhi Wang 已提交
521
	}
522 523 524

	if (next)
		handle_tlb_pending_event(next, ring_id);
Z
Zhi Wang 已提交
525
}
526 527 528 529 530 531 532 533 534 535 536 537 538

/**
 * intel_gvt_switch_render_mmio - switch mmio context of specific engine
 * @pre: the last vGPU that own the engine
 * @next: the vGPU to switch to
 * @ring_id: specify the engine
 *
 * If pre is null indicates that host own the engine. If next is null
 * indicates that we are switching to host workload.
 */
void intel_gvt_switch_mmio(struct intel_vgpu *pre,
			   struct intel_vgpu *next, int ring_id)
{
539 540
	struct drm_i915_private *dev_priv;

541 542 543 544 545 546
	if (WARN_ON(!pre && !next))
		return;

	gvt_dbg_render("switch ring %d from %s to %s\n", ring_id,
		       pre ? "vGPU" : "host", next ? "vGPU" : "HOST");

547 548 549 550 551 552 553 554
	dev_priv = pre ? pre->gvt->dev_priv : next->gvt->dev_priv;

	/**
	 * We are using raw mmio access wrapper to improve the
	 * performace for batch mmio read/write, so we need
	 * handle forcewake mannually.
	 */
	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
555
	switch_mmio(pre, next, ring_id);
556
	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
557
}
558 559 560 561 562 563 564 565

/**
 * intel_gvt_init_engine_mmio_context - Initiate the engine mmio list
 * @gvt: GVT device
 *
 */
void intel_gvt_init_engine_mmio_context(struct intel_gvt *gvt)
{
566 567
	struct engine_mmio *mmio;

568 569 570
	if (IS_SKYLAKE(gvt->dev_priv) ||
		IS_KABYLAKE(gvt->dev_priv) ||
		IS_BROXTON(gvt->dev_priv))
571
		gvt->engine_mmio_list.mmio = gen9_engine_mmio_list;
572
	else
573 574 575 576
		gvt->engine_mmio_list.mmio = gen8_engine_mmio_list;

	for (mmio = gvt->engine_mmio_list.mmio;
	     i915_mmio_reg_valid(mmio->reg); mmio++) {
577
		if (mmio->in_context) {
578
			gvt->engine_mmio_list.ctx_mmio_count[mmio->ring_id]++;
579 580
			intel_gvt_mmio_set_in_ctx(gvt, mmio->reg.reg);
		}
581
	}
582
}