intel_uncore.c 50.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
/*
 * Copyright © 2013 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "i915_drv.h"
#include "intel_drv.h"
26
#include "i915_vgpu.h"
27

28
#include <linux/pm_runtime.h>
29
#include <linux/bsearch.h>
30

31
#define FORCEWAKE_ACK_TIMEOUT_MS 50
32

33
#define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__))
34

35 36 37 38 39 40 41
static const char * const forcewake_domain_names[] = {
	"render",
	"blitter",
	"media",
};

const char *
42
intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
43
{
44
	BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
45 46 47 48 49 50 51 52 53 54 55

	if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
		return forcewake_domain_names[id];

	WARN_ON(id);

	return "unknown";
}

static inline void
fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
56
{
57
	WARN_ON(!i915_mmio_reg_valid(d->reg_set));
58
	__raw_i915_write32(d->i915, d->reg_set, d->val_reset);
59 60
}

61 62
static inline void
fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
63
{
64 65 66 67 68
	d->wake_count++;
	hrtimer_start_range_ns(&d->timer,
			       ktime_set(0, NSEC_PER_MSEC),
			       NSEC_PER_MSEC,
			       HRTIMER_MODE_REL);
69 70
}

71 72
static inline void
fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d)
73
{
74 75
	if (wait_for_atomic((__raw_i915_read32(d->i915, d->reg_ack) &
			     FORCEWAKE_KERNEL) == 0,
76
			    FORCEWAKE_ACK_TIMEOUT_MS))
77 78 79
		DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
			  intel_uncore_forcewake_domain_to_str(d->id));
}
80

81 82 83 84 85
static inline void
fw_domain_get(const struct intel_uncore_forcewake_domain *d)
{
	__raw_i915_write32(d->i915, d->reg_set, d->val_set);
}
86

87 88 89 90 91
static inline void
fw_domain_wait_ack(const struct intel_uncore_forcewake_domain *d)
{
	if (wait_for_atomic((__raw_i915_read32(d->i915, d->reg_ack) &
			     FORCEWAKE_KERNEL),
92
			    FORCEWAKE_ACK_TIMEOUT_MS))
93 94 95
		DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
			  intel_uncore_forcewake_domain_to_str(d->id));
}
96

97 98 99 100
static inline void
fw_domain_put(const struct intel_uncore_forcewake_domain *d)
{
	__raw_i915_write32(d->i915, d->reg_set, d->val_clear);
101 102
}

103 104
static inline void
fw_domain_posting_read(const struct intel_uncore_forcewake_domain *d)
105
{
106
	/* something from same cacheline, but not from the set register */
107
	if (i915_mmio_reg_valid(d->reg_post))
108
		__raw_posting_read(d->i915, d->reg_post);
109 110
}

111
static void
112
fw_domains_get(struct drm_i915_private *dev_priv, enum forcewake_domains fw_domains)
113
{
114
	struct intel_uncore_forcewake_domain *d;
115

116
	for_each_fw_domain_masked(d, fw_domains, dev_priv) {
117 118 119
		fw_domain_wait_ack_clear(d);
		fw_domain_get(d);
	}
120 121 122

	for_each_fw_domain_masked(d, fw_domains, dev_priv)
		fw_domain_wait_ack(d);
123
}
124

125
static void
126
fw_domains_put(struct drm_i915_private *dev_priv, enum forcewake_domains fw_domains)
127 128
{
	struct intel_uncore_forcewake_domain *d;
129

130
	for_each_fw_domain_masked(d, fw_domains, dev_priv) {
131 132 133 134
		fw_domain_put(d);
		fw_domain_posting_read(d);
	}
}
135

136 137 138 139 140 141
static void
fw_domains_posting_read(struct drm_i915_private *dev_priv)
{
	struct intel_uncore_forcewake_domain *d;

	/* No need to do for all, just do for first found */
142
	for_each_fw_domain(d, dev_priv) {
143 144 145 146 147 148
		fw_domain_posting_read(d);
		break;
	}
}

static void
149
fw_domains_reset(struct drm_i915_private *dev_priv, enum forcewake_domains fw_domains)
150 151 152
{
	struct intel_uncore_forcewake_domain *d;

153 154
	if (dev_priv->uncore.fw_domains == 0)
		return;
155

156
	for_each_fw_domain_masked(d, fw_domains, dev_priv)
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
		fw_domain_reset(d);

	fw_domains_posting_read(dev_priv);
}

static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
{
	/* w/a for a sporadic read returning 0 by waiting for the GT
	 * thread to wake up.
	 */
	if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) &
				GEN6_GT_THREAD_STATUS_CORE_MASK) == 0, 500))
		DRM_ERROR("GT thread status wait timed out\n");
}

static void fw_domains_get_with_thread_status(struct drm_i915_private *dev_priv,
173
					      enum forcewake_domains fw_domains)
174 175
{
	fw_domains_get(dev_priv, fw_domains);
176

177
	/* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
178
	__gen6_gt_wait_for_thread_c0(dev_priv);
179 180 181 182 183
}

static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
{
	u32 gtfifodbg;
184 185

	gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
186 187
	if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg))
		__raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg);
188 189
}

190
static void fw_domains_put_with_fifo(struct drm_i915_private *dev_priv,
191
				     enum forcewake_domains fw_domains)
192
{
193
	fw_domains_put(dev_priv, fw_domains);
194 195 196
	gen6_gt_check_fifodbg(dev_priv);
}

197 198 199 200 201 202 203
static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv)
{
	u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL);

	return count & GT_FIFO_FREE_ENTRIES_MASK;
}

204 205 206 207
static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
{
	int ret = 0;

208 209
	/* On VLV, FIFO will be shared by both SW and HW.
	 * So, we need to read the FREE_ENTRIES everytime */
210
	if (IS_VALLEYVIEW(dev_priv))
211
		dev_priv->uncore.fifo_count = fifo_free_entries(dev_priv);
212

213 214
	if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
		int loop = 500;
215 216
		u32 fifo = fifo_free_entries(dev_priv);

217 218
		while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
			udelay(10);
219
			fifo = fifo_free_entries(dev_priv);
220 221 222 223 224 225 226 227 228 229
		}
		if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
			++ret;
		dev_priv->uncore.fifo_count = fifo;
	}
	dev_priv->uncore.fifo_count--;

	return ret;
}

230 231
static enum hrtimer_restart
intel_uncore_fw_release_timer(struct hrtimer *timer)
Z
Zhe Wang 已提交
232
{
233 234
	struct intel_uncore_forcewake_domain *domain =
	       container_of(timer, struct intel_uncore_forcewake_domain, timer);
235
	struct drm_i915_private *dev_priv = domain->i915;
236
	unsigned long irqflags;
Z
Zhe Wang 已提交
237

238
	assert_rpm_device_not_suspended(dev_priv);
Z
Zhe Wang 已提交
239

240
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
241 242 243
	if (WARN_ON(domain->wake_count == 0))
		domain->wake_count++;

244 245 246 247
	if (--domain->wake_count == 0) {
		dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask);
		dev_priv->uncore.fw_domains_active &= ~domain->mask;
	}
248

249
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
250 251

	return HRTIMER_NORESTART;
Z
Zhe Wang 已提交
252 253
}

254 255
void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
				  bool restore)
Z
Zhe Wang 已提交
256
{
257
	unsigned long irqflags;
258
	struct intel_uncore_forcewake_domain *domain;
259
	int retry_count = 100;
260
	enum forcewake_domains fw, active_domains;
Z
Zhe Wang 已提交
261

262 263 264 265 266 267
	/* Hold uncore.lock across reset to prevent any register access
	 * with forcewake not set correctly. Wait until all pending
	 * timers are run before holding.
	 */
	while (1) {
		active_domains = 0;
Z
Zhe Wang 已提交
268

269
		for_each_fw_domain(domain, dev_priv) {
270
			if (hrtimer_cancel(&domain->timer) == 0)
271
				continue;
Z
Zhe Wang 已提交
272

273
			intel_uncore_fw_release_timer(&domain->timer);
274
		}
275

276
		spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
277

278
		for_each_fw_domain(domain, dev_priv) {
279
			if (hrtimer_active(&domain->timer))
280
				active_domains |= domain->mask;
281
		}
282

283 284
		if (active_domains == 0)
			break;
285

286 287 288 289
		if (--retry_count == 0) {
			DRM_ERROR("Timed out waiting for forcewake timers to finish\n");
			break;
		}
290

291 292 293
		spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
		cond_resched();
	}
294

295 296
	WARN_ON(active_domains);

297
	fw = dev_priv->uncore.fw_domains_active;
298 299
	if (fw)
		dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
300

301
	fw_domains_reset(dev_priv, FORCEWAKE_ALL);
Z
Zhe Wang 已提交
302

303 304 305 306
	if (restore) { /* If reset with a user forcewake, try to restore */
		if (fw)
			dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);

307
		if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
308
			dev_priv->uncore.fifo_count =
309
				fifo_free_entries(dev_priv);
310 311
	}

312
	if (!restore)
313
		assert_forcewakes_inactive(dev_priv);
314

315
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
316 317
}

M
Mika Kuoppala 已提交
318 319 320 321 322 323 324 325 326 327 328 329
static u64 gen9_edram_size(struct drm_i915_private *dev_priv)
{
	const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
	const unsigned int sets[4] = { 1, 1, 2, 2 };
	const u32 cap = dev_priv->edram_cap;

	return EDRAM_NUM_BANKS(cap) *
		ways[EDRAM_WAYS_IDX(cap)] *
		sets[EDRAM_SETS_IDX(cap)] *
		1024 * 1024;
}

330
u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv)
331
{
332 333 334
	if (!HAS_EDRAM(dev_priv))
		return 0;

M
Mika Kuoppala 已提交
335 336
	/* The needed capability bits for size calculation
	 * are not there with pre gen9 so return 128MB always.
337
	 */
M
Mika Kuoppala 已提交
338 339
	if (INTEL_GEN(dev_priv) < 9)
		return 128 * 1024 * 1024;
340

M
Mika Kuoppala 已提交
341
	return gen9_edram_size(dev_priv);
342
}
343

344 345 346 347 348 349 350 351 352
static void intel_uncore_edram_detect(struct drm_i915_private *dev_priv)
{
	if (IS_HASWELL(dev_priv) ||
	    IS_BROADWELL(dev_priv) ||
	    INTEL_GEN(dev_priv) >= 9) {
		dev_priv->edram_cap = __raw_i915_read32(dev_priv,
							HSW_EDRAM_CAP);

		/* NB: We can't write IDICR yet because we do not have gt funcs
353
		 * set up */
354 355
	} else {
		dev_priv->edram_cap = 0;
356
	}
357 358 359 360

	if (HAS_EDRAM(dev_priv))
		DRM_INFO("Found %lluMB of eDRAM\n",
			 intel_uncore_edram_size(dev_priv) / (1024 * 1024));
361 362
}

363
static bool
364
fpga_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
365 366 367 368 369 370 371 372 373 374 375 376
{
	u32 dbg;

	dbg = __raw_i915_read32(dev_priv, FPGA_DBG);
	if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
		return false;

	__raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);

	return true;
}

377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402
static bool
vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
{
	u32 cer;

	cer = __raw_i915_read32(dev_priv, CLAIM_ER);
	if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
		return false;

	__raw_i915_write32(dev_priv, CLAIM_ER, CLAIM_ER_CLR);

	return true;
}

static bool
check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
{
	if (HAS_FPGA_DBG_UNCLAIMED(dev_priv))
		return fpga_check_for_unclaimed_mmio(dev_priv);

	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
		return vlv_check_for_unclaimed_mmio(dev_priv);

	return false;
}

403
static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
404 405
					  bool restore_forcewake)
{
406 407 408
	/* clear out unclaimed reg detection bit */
	if (check_for_unclaimed_mmio(dev_priv))
		DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
409

410
	/* clear out old GT FIFO errors */
411
	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
412 413 414
		__raw_i915_write32(dev_priv, GTFIFODBG,
				   __raw_i915_read32(dev_priv, GTFIFODBG));

415
	/* WaDisableShadowRegForCpd:chv */
416
	if (IS_CHERRYVIEW(dev_priv)) {
417 418 419 420 421 422
		__raw_i915_write32(dev_priv, GTFIFOCTL,
				   __raw_i915_read32(dev_priv, GTFIFOCTL) |
				   GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
				   GT_FIFO_CTL_RC6_POLICY_STALL);
	}

423
	intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
424 425
}

426 427
void intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
				 bool restore_forcewake)
428
{
429 430
	__intel_uncore_early_sanitize(dev_priv, restore_forcewake);
	i915_check_and_clear_faults(dev_priv);
431 432
}

433
void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
434
{
435
	i915.enable_rc6 = sanitize_rc6_option(dev_priv, i915.enable_rc6);
436

437
	/* BIOS often leaves RC6 enabled, but disable it for hw init */
438
	intel_sanitize_gt_powersave(dev_priv);
439 440
}

441 442 443 444 445 446 447
static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
					 enum forcewake_domains fw_domains)
{
	struct intel_uncore_forcewake_domain *domain;

	fw_domains &= dev_priv->uncore.fw_domains;

448
	for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
449
		if (domain->wake_count++)
450
			fw_domains &= ~domain->mask;
451 452
	}

453
	if (fw_domains) {
454
		dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
455 456
		dev_priv->uncore.fw_domains_active |= fw_domains;
	}
457 458
}

459 460 461 462 463 464 465 466 467 468 469 470
/**
 * intel_uncore_forcewake_get - grab forcewake domain references
 * @dev_priv: i915 device instance
 * @fw_domains: forcewake domains to get reference on
 *
 * This function can be used get GT's forcewake domain references.
 * Normal register access will handle the forcewake domains automatically.
 * However if some sequence requires the GT to not power down a particular
 * forcewake domains this function should be called at the beginning of the
 * sequence. And subsequently the reference should be dropped by symmetric
 * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
 * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
471
 */
472
void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
473
				enum forcewake_domains fw_domains)
474 475 476
{
	unsigned long irqflags;

477 478 479
	if (!dev_priv->uncore.funcs.force_wake_get)
		return;

480
	assert_rpm_wakelock_held(dev_priv);
481

482
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
483
	__intel_uncore_forcewake_get(dev_priv, fw_domains);
484 485 486
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
}

487
/**
488
 * intel_uncore_forcewake_get__locked - grab forcewake domain references
489
 * @dev_priv: i915 device instance
490
 * @fw_domains: forcewake domains to get reference on
491
 *
492 493
 * See intel_uncore_forcewake_get(). This variant places the onus
 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
494
 */
495 496 497 498 499 500 501 502 503 504 505 506 507
void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv,
					enum forcewake_domains fw_domains)
{
	assert_spin_locked(&dev_priv->uncore.lock);

	if (!dev_priv->uncore.funcs.force_wake_get)
		return;

	__intel_uncore_forcewake_get(dev_priv, fw_domains);
}

static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
					 enum forcewake_domains fw_domains)
508
{
509
	struct intel_uncore_forcewake_domain *domain;
510

511 512
	fw_domains &= dev_priv->uncore.fw_domains;

513
	for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
514 515 516 517 518 519
		if (WARN_ON(domain->wake_count == 0))
			continue;

		if (--domain->wake_count)
			continue;

520
		fw_domain_arm_timer(domain);
521
	}
522
}
523

524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541
/**
 * intel_uncore_forcewake_put - release a forcewake domain reference
 * @dev_priv: i915 device instance
 * @fw_domains: forcewake domains to put references
 *
 * This function drops the device-level forcewakes for specified
 * domains obtained by intel_uncore_forcewake_get().
 */
void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
				enum forcewake_domains fw_domains)
{
	unsigned long irqflags;

	if (!dev_priv->uncore.funcs.force_wake_put)
		return;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	__intel_uncore_forcewake_put(dev_priv, fw_domains);
542 543 544
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
}

545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563
/**
 * intel_uncore_forcewake_put__locked - grab forcewake domain references
 * @dev_priv: i915 device instance
 * @fw_domains: forcewake domains to get reference on
 *
 * See intel_uncore_forcewake_put(). This variant places the onus
 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 */
void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
					enum forcewake_domains fw_domains)
{
	assert_spin_locked(&dev_priv->uncore.lock);

	if (!dev_priv->uncore.funcs.force_wake_put)
		return;

	__intel_uncore_forcewake_put(dev_priv, fw_domains);
}

564
void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
565 566 567 568
{
	if (!dev_priv->uncore.funcs.force_wake_get)
		return;

569
	WARN_ON(dev_priv->uncore.fw_domains_active);
570 571
}

572
/* We give fast paths for the really cool registers */
573
#define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000)
574

575 576 577 578 579 580 581 582 583 584
#define __gen6_reg_read_fw_domains(offset) \
({ \
	enum forcewake_domains __fwd; \
	if (NEEDS_FORCE_WAKE(offset)) \
		__fwd = FORCEWAKE_RENDER; \
	else \
		__fwd = 0; \
	__fwd; \
})

585 586 587 588 589 590 591 592 593 594 595 596 597
static int fw_range_cmp(const void *key, const void *elt)
{
	const struct intel_forcewake_range *entry = elt;
	u32 offset = (u32)((unsigned long)key);

	if (offset < entry->start)
		return -1;
	else if (offset > entry->end)
		return 1;
	else
		return 0;
}

598
static enum forcewake_domains
599
find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
600
{
601 602
	const struct intel_forcewake_range *table, *entry;
	unsigned int num_entries;
603

604 605 606 607 608
	table = dev_priv->uncore.fw_domains_table;
	num_entries = dev_priv->uncore.fw_domains_table_entries;

	entry = bsearch((void *)(unsigned long)offset, (const void *)table,
			num_entries, sizeof(struct intel_forcewake_range),
609
			fw_range_cmp);
610

611
	return entry ? entry->domains : 0;
612 613
}

614
static void
615
intel_fw_table_check(struct drm_i915_private *dev_priv)
616
{
617 618
	const struct intel_forcewake_range *ranges;
	unsigned int num_ranges;
619 620 621 622 623 624
	s32 prev;
	unsigned int i;

	if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG))
		return;

625 626 627 628 629 630
	ranges = dev_priv->uncore.fw_domains_table;
	if (!ranges)
		return;

	num_ranges = dev_priv->uncore.fw_domains_table_entries;

631 632 633 634 635 636 637 638
	for (i = 0, prev = -1; i < num_ranges; i++, ranges++) {
		WARN_ON_ONCE(prev >= (s32)ranges->start);
		prev = ranges->start;
		WARN_ON_ONCE(prev >= (s32)ranges->end);
		prev = ranges->end;
	}
}

639 640
#define GEN_FW_RANGE(s, e, d) \
	{ .start = (s), .end = (e), .domains = (d) }
641

T
Tvrtko Ursulin 已提交
642 643 644 645 646
#define HAS_FWTABLE(dev_priv) \
	(IS_GEN9(dev_priv) || \
	 IS_CHERRYVIEW(dev_priv) || \
	 IS_VALLEYVIEW(dev_priv))

647
/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
648 649 650 651 652 653
static const struct intel_forcewake_range __vlv_fw_ranges[] = {
	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
	GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
	GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
654
	GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
655 656
	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
};
657

T
Tvrtko Ursulin 已提交
658
#define __fwtable_reg_read_fw_domains(offset) \
659 660
({ \
	enum forcewake_domains __fwd = 0; \
661
	if (NEEDS_FORCE_WAKE((offset))) \
662
		__fwd = find_fw_domain(dev_priv, offset); \
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695
	__fwd; \
})

static const i915_reg_t gen8_shadowed_regs[] = {
	GEN6_RPNSWREQ,
	GEN6_RC_VIDEO_FREQ,
	RING_TAIL(RENDER_RING_BASE),
	RING_TAIL(GEN6_BSD_RING_BASE),
	RING_TAIL(VEBOX_RING_BASE),
	RING_TAIL(BLT_RING_BASE),
	/* TODO: Other registers are not yet used */
};

static bool is_gen8_shadowed(u32 offset)
{
	int i;
	for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
		if (offset == gen8_shadowed_regs[i].reg)
			return true;

	return false;
}

#define __gen8_reg_write_fw_domains(offset) \
({ \
	enum forcewake_domains __fwd; \
	if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \
		__fwd = FORCEWAKE_RENDER; \
	else \
		__fwd = 0; \
	__fwd; \
})

696
/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
697 698
static const struct intel_forcewake_range __chv_fw_ranges[] = {
	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
699
	GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
700
	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
701
	GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
702
	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
703
	GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
704
	GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
705 706
	GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
707
	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
708 709
	GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
	GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
710 711 712 713 714
	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
	GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
	GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
	GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
};
715

716 717 718
#define __chv_reg_write_fw_domains(offset) \
({ \
	enum forcewake_domains __fwd = 0; \
719
	if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
720
		__fwd = find_fw_domain(dev_priv, offset); \
721 722 723
	__fwd; \
})

724
/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
725
static const struct intel_forcewake_range __gen9_fw_ranges[] = {
726
	GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
727 728
	GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
729
	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER),
730
	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
731
	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER),
732
	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
733
	GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER),
734
	GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
735
	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
736
	GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER),
737
	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
738
	GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER),
739
	GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
740
	GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER),
741
	GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
742
	GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER),
743
	GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
744
	GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
745
	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
746
	GEN_FW_RANGE(0xb480, 0xbfff, FORCEWAKE_BLITTER),
747
	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
748
	GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER),
749
	GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
750
	GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER),
751
	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
752
	GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER),
753
	GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
754
	GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER),
755
	GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
756
	GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER),
757 758
	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
};
759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781

static const i915_reg_t gen9_shadowed_regs[] = {
	RING_TAIL(RENDER_RING_BASE),
	RING_TAIL(GEN6_BSD_RING_BASE),
	RING_TAIL(VEBOX_RING_BASE),
	RING_TAIL(BLT_RING_BASE),
	GEN6_RPNSWREQ,
	GEN6_RC_VIDEO_FREQ,
	/* TODO: Other registers are not yet used */
};

static bool is_gen9_shadowed(u32 offset)
{
	int i;
	for (i = 0; i < ARRAY_SIZE(gen9_shadowed_regs); i++)
		if (offset == gen9_shadowed_regs[i].reg)
			return true;

	return false;
}

#define __gen9_reg_write_fw_domains(offset) \
({ \
782
	enum forcewake_domains __fwd = 0; \
783
	if (NEEDS_FORCE_WAKE((offset)) && !is_gen9_shadowed(offset)) \
784
		__fwd = find_fw_domain(dev_priv, offset); \
785 786 787
	__fwd; \
})

788 789 790 791 792 793
static void
ilk_dummy_write(struct drm_i915_private *dev_priv)
{
	/* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
	 * the chip from rc6 before touching it for real. MI_MODE is masked,
	 * hence harmless to write 0 into. */
794
	__raw_i915_write32(dev_priv, MI_MODE, 0);
795 796 797
}

static void
798 799 800 801
__unclaimed_reg_debug(struct drm_i915_private *dev_priv,
		      const i915_reg_t reg,
		      const bool read,
		      const bool before)
802
{
803 804 805
	if (WARN(check_for_unclaimed_mmio(dev_priv) && !before,
		 "Unclaimed %s register 0x%x\n",
		 read ? "read from" : "write to",
806
		 i915_mmio_reg_offset(reg)))
807
		i915.mmio_debug--; /* Only report the first N failures */
808 809
}

810 811 812 813 814 815 816 817 818 819 820 821
static inline void
unclaimed_reg_debug(struct drm_i915_private *dev_priv,
		    const i915_reg_t reg,
		    const bool read,
		    const bool before)
{
	if (likely(!i915.mmio_debug))
		return;

	__unclaimed_reg_debug(dev_priv, reg, read, before);
}

822
#define GEN2_READ_HEADER(x) \
B
Ben Widawsky 已提交
823
	u##x val = 0; \
824
	assert_rpm_wakelock_held(dev_priv);
B
Ben Widawsky 已提交
825

826
#define GEN2_READ_FOOTER \
B
Ben Widawsky 已提交
827 828 829
	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
	return val

830
#define __gen2_read(x) \
831
static u##x \
832
gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
833
	GEN2_READ_HEADER(x); \
834
	val = __raw_i915_read##x(dev_priv, reg); \
835
	GEN2_READ_FOOTER; \
836 837 838 839
}

#define __gen5_read(x) \
static u##x \
840
gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
841
	GEN2_READ_HEADER(x); \
842 843
	ilk_dummy_write(dev_priv); \
	val = __raw_i915_read##x(dev_priv, reg); \
844
	GEN2_READ_FOOTER; \
845 846
}

847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862
__gen5_read(8)
__gen5_read(16)
__gen5_read(32)
__gen5_read(64)
__gen2_read(8)
__gen2_read(16)
__gen2_read(32)
__gen2_read(64)

#undef __gen5_read
#undef __gen2_read

#undef GEN2_READ_FOOTER
#undef GEN2_READ_HEADER

#define GEN6_READ_HEADER(x) \
863
	u32 offset = i915_mmio_reg_offset(reg); \
864 865
	unsigned long irqflags; \
	u##x val = 0; \
866
	assert_rpm_wakelock_held(dev_priv); \
867 868
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
	unclaimed_reg_debug(dev_priv, reg, true, true)
869 870

#define GEN6_READ_FOOTER \
871
	unclaimed_reg_debug(dev_priv, reg, true, false); \
872 873 874 875
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
	return val

876 877
static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv,
					enum forcewake_domains fw_domains)
878 879 880
{
	struct intel_uncore_forcewake_domain *domain;

881 882 883 884 885 886 887 888 889 890
	for_each_fw_domain_masked(domain, fw_domains, dev_priv)
		fw_domain_arm_timer(domain);

	dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
	dev_priv->uncore.fw_domains_active |= fw_domains;
}

static inline void __force_wake_auto(struct drm_i915_private *dev_priv,
				     enum forcewake_domains fw_domains)
{
891 892 893
	if (WARN_ON(!fw_domains))
		return;

894 895 896
	/* Turn on all requested but inactive supported forcewake domains. */
	fw_domains &= dev_priv->uncore.fw_domains;
	fw_domains &= ~dev_priv->uncore.fw_domains_active;
897

898 899
	if (fw_domains)
		___force_wake_auto(dev_priv, fw_domains);
900 901
}

902 903
#define __gen6_read(x) \
static u##x \
904
gen6_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
905
	enum forcewake_domains fw_engine; \
906
	GEN6_READ_HEADER(x); \
907 908 909
	fw_engine = __gen6_reg_read_fw_domains(offset); \
	if (fw_engine) \
		__force_wake_auto(dev_priv, fw_engine); \
910
	val = __raw_i915_read##x(dev_priv, reg); \
911
	GEN6_READ_FOOTER; \
912 913
}

914
#define __fwtable_read(x) \
915
static u##x \
916
fwtable_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
917
	enum forcewake_domains fw_engine; \
918
	GEN6_READ_HEADER(x); \
T
Tvrtko Ursulin 已提交
919
	fw_engine = __fwtable_reg_read_fw_domains(offset); \
920
	if (fw_engine) \
921
		__force_wake_auto(dev_priv, fw_engine); \
922
	val = __raw_i915_read##x(dev_priv, reg); \
923
	GEN6_READ_FOOTER; \
924 925
}

926 927 928 929
__fwtable_read(8)
__fwtable_read(16)
__fwtable_read(32)
__fwtable_read(64)
930 931 932 933 934
__gen6_read(8)
__gen6_read(16)
__gen6_read(32)
__gen6_read(64)

935
#undef __fwtable_read
936
#undef __gen6_read
937 938
#undef GEN6_READ_FOOTER
#undef GEN6_READ_HEADER
B
Ben Widawsky 已提交
939

940 941 942
#define VGPU_READ_HEADER(x) \
	unsigned long irqflags; \
	u##x val = 0; \
943
	assert_rpm_device_not_suspended(dev_priv); \
944 945 946 947 948 949 950 951 952
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)

#define VGPU_READ_FOOTER \
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
	return val

#define __vgpu_read(x) \
static u##x \
953
vgpu_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
954 955 956 957 958 959 960 961 962 963 964 965 966 967
	VGPU_READ_HEADER(x); \
	val = __raw_i915_read##x(dev_priv, reg); \
	VGPU_READ_FOOTER; \
}

__vgpu_read(8)
__vgpu_read(16)
__vgpu_read(32)
__vgpu_read(64)

#undef __vgpu_read
#undef VGPU_READ_FOOTER
#undef VGPU_READ_HEADER

968
#define GEN2_WRITE_HEADER \
B
Ben Widawsky 已提交
969
	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
970
	assert_rpm_wakelock_held(dev_priv); \
971

972
#define GEN2_WRITE_FOOTER
V
Ville Syrjälä 已提交
973

974
#define __gen2_write(x) \
975
static void \
976
gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
977
	GEN2_WRITE_HEADER; \
978
	__raw_i915_write##x(dev_priv, reg, val); \
979
	GEN2_WRITE_FOOTER; \
980 981 982 983
}

#define __gen5_write(x) \
static void \
984
gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
985
	GEN2_WRITE_HEADER; \
986 987
	ilk_dummy_write(dev_priv); \
	__raw_i915_write##x(dev_priv, reg, val); \
988
	GEN2_WRITE_FOOTER; \
989 990
}

991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004
__gen5_write(8)
__gen5_write(16)
__gen5_write(32)
__gen2_write(8)
__gen2_write(16)
__gen2_write(32)

#undef __gen5_write
#undef __gen2_write

#undef GEN2_WRITE_FOOTER
#undef GEN2_WRITE_HEADER

#define GEN6_WRITE_HEADER \
1005
	u32 offset = i915_mmio_reg_offset(reg); \
1006 1007
	unsigned long irqflags; \
	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1008
	assert_rpm_wakelock_held(dev_priv); \
1009 1010
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
	unclaimed_reg_debug(dev_priv, reg, false, true)
1011 1012

#define GEN6_WRITE_FOOTER \
1013
	unclaimed_reg_debug(dev_priv, reg, false, false); \
1014 1015
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)

1016 1017
#define __gen6_write(x) \
static void \
1018
gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1019
	u32 __fifo_ret = 0; \
1020
	GEN6_WRITE_HEADER; \
1021
	if (NEEDS_FORCE_WAKE(offset)) { \
1022 1023 1024 1025 1026 1027
		__fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
	} \
	__raw_i915_write##x(dev_priv, reg, val); \
	if (unlikely(__fifo_ret)) { \
		gen6_gt_check_fifodbg(dev_priv); \
	} \
1028
	GEN6_WRITE_FOOTER; \
1029 1030
}

1031 1032
#define __gen8_write(x) \
static void \
1033
gen8_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1034
	enum forcewake_domains fw_engine; \
1035
	GEN6_WRITE_HEADER; \
1036 1037 1038
	fw_engine = __gen8_reg_write_fw_domains(offset); \
	if (fw_engine) \
		__force_wake_auto(dev_priv, fw_engine); \
1039
	__raw_i915_write##x(dev_priv, reg, val); \
1040
	GEN6_WRITE_FOOTER; \
1041 1042
}

1043 1044
#define __chv_write(x) \
static void \
1045
chv_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1046
	enum forcewake_domains fw_engine; \
1047
	GEN6_WRITE_HEADER; \
1048
	fw_engine = __chv_reg_write_fw_domains(offset); \
1049
	if (fw_engine) \
1050
		__force_wake_auto(dev_priv, fw_engine); \
1051
	__raw_i915_write##x(dev_priv, reg, val); \
1052
	GEN6_WRITE_FOOTER; \
1053 1054
}

1055 1056
#define __gen9_write(x) \
static void \
1057
gen9_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, \
1058
		bool trace) { \
1059
	enum forcewake_domains fw_engine; \
1060
	GEN6_WRITE_HEADER; \
1061
	fw_engine = __gen9_reg_write_fw_domains(offset); \
1062
	if (fw_engine) \
1063
		__force_wake_auto(dev_priv, fw_engine); \
1064
	__raw_i915_write##x(dev_priv, reg, val); \
1065
	GEN6_WRITE_FOOTER; \
1066 1067 1068 1069 1070
}

__gen9_write(8)
__gen9_write(16)
__gen9_write(32)
1071 1072 1073
__chv_write(8)
__chv_write(16)
__chv_write(32)
1074 1075 1076
__gen8_write(8)
__gen8_write(16)
__gen8_write(32)
1077 1078 1079 1080
__gen6_write(8)
__gen6_write(16)
__gen6_write(32)

1081
#undef __gen9_write
1082
#undef __chv_write
1083
#undef __gen8_write
1084
#undef __gen6_write
1085 1086
#undef GEN6_WRITE_FOOTER
#undef GEN6_WRITE_HEADER
1087

1088 1089 1090
#define VGPU_WRITE_HEADER \
	unsigned long irqflags; \
	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1091
	assert_rpm_device_not_suspended(dev_priv); \
1092 1093 1094 1095 1096 1097 1098
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)

#define VGPU_WRITE_FOOTER \
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)

#define __vgpu_write(x) \
static void vgpu_write##x(struct drm_i915_private *dev_priv, \
1099
			  i915_reg_t reg, u##x val, bool trace) { \
1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
	VGPU_WRITE_HEADER; \
	__raw_i915_write##x(dev_priv, reg, val); \
	VGPU_WRITE_FOOTER; \
}

__vgpu_write(8)
__vgpu_write(16)
__vgpu_write(32)

#undef __vgpu_write
#undef VGPU_WRITE_FOOTER
#undef VGPU_WRITE_HEADER

1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
#define ASSIGN_WRITE_MMIO_VFUNCS(x) \
do { \
	dev_priv->uncore.funcs.mmio_writeb = x##_write8; \
	dev_priv->uncore.funcs.mmio_writew = x##_write16; \
	dev_priv->uncore.funcs.mmio_writel = x##_write32; \
} while (0)

#define ASSIGN_READ_MMIO_VFUNCS(x) \
do { \
	dev_priv->uncore.funcs.mmio_readb = x##_read8; \
	dev_priv->uncore.funcs.mmio_readw = x##_read16; \
	dev_priv->uncore.funcs.mmio_readl = x##_read32; \
	dev_priv->uncore.funcs.mmio_readq = x##_read64; \
} while (0)

1128 1129

static void fw_domain_init(struct drm_i915_private *dev_priv,
1130
			   enum forcewake_domain_id domain_id,
1131 1132
			   i915_reg_t reg_set,
			   i915_reg_t reg_ack)
1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
{
	struct intel_uncore_forcewake_domain *d;

	if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT))
		return;

	d = &dev_priv->uncore.fw_domain[domain_id];

	WARN_ON(d->wake_count);

	d->wake_count = 0;
	d->reg_set = reg_set;
	d->reg_ack = reg_ack;

	if (IS_GEN6(dev_priv)) {
		d->val_reset = 0;
		d->val_set = FORCEWAKE_KERNEL;
		d->val_clear = 0;
	} else {
1152
		/* WaRsClearFWBitsAtReset:bdw,skl */
1153 1154 1155 1156 1157
		d->val_reset = _MASKED_BIT_DISABLE(0xffff);
		d->val_set = _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL);
		d->val_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL);
	}

1158
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1159 1160 1161 1162 1163 1164 1165
		d->reg_post = FORCEWAKE_ACK_VLV;
	else if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv) || IS_GEN8(dev_priv))
		d->reg_post = ECOBUS;

	d->i915 = dev_priv;
	d->id = domain_id;

1166 1167 1168 1169 1170 1171
	BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
	BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER));
	BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));

	d->mask = 1 << domain_id;

1172 1173
	hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	d->timer.function = intel_uncore_fw_release_timer;
1174 1175

	dev_priv->uncore.fw_domains |= (1 << domain_id);
1176 1177

	fw_domain_reset(d);
1178 1179
}

1180
static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
1181
{
1182
	if (INTEL_INFO(dev_priv)->gen <= 5)
1183 1184
		return;

1185
	if (IS_GEN9(dev_priv)) {
1186 1187 1188 1189 1190 1191 1192 1193 1194 1195
		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
			       FORCEWAKE_RENDER_GEN9,
			       FORCEWAKE_ACK_RENDER_GEN9);
		fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER,
			       FORCEWAKE_BLITTER_GEN9,
			       FORCEWAKE_ACK_BLITTER_GEN9);
		fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
			       FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
1196
	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1197
		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
1198
		if (!IS_CHERRYVIEW(dev_priv))
1199 1200 1201 1202
			dev_priv->uncore.funcs.force_wake_put =
				fw_domains_put_with_fifo;
		else
			dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1203 1204 1205 1206
		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
			       FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
		fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
			       FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
1207
	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
1208 1209
		dev_priv->uncore.funcs.force_wake_get =
			fw_domains_get_with_thread_status;
1210
		if (IS_HASWELL(dev_priv))
1211 1212 1213 1214
			dev_priv->uncore.funcs.force_wake_put =
				fw_domains_put_with_fifo;
		else
			dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1215 1216
		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
			       FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
1217
	} else if (IS_IVYBRIDGE(dev_priv)) {
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
		u32 ecobus;

		/* IVB configs may use multi-threaded forcewake */

		/* A small trick here - if the bios hasn't configured
		 * MT forcewake, and if the device is in RC6, then
		 * force_wake_mt_get will not wake the device and the
		 * ECOBUS read will return zero. Which will be
		 * (correctly) interpreted by the test below as MT
		 * forcewake being disabled.
		 */
1229 1230 1231 1232 1233
		dev_priv->uncore.funcs.force_wake_get =
			fw_domains_get_with_thread_status;
		dev_priv->uncore.funcs.force_wake_put =
			fw_domains_put_with_fifo;

1234 1235
		/* We need to init first for ECOBUS access and then
		 * determine later if we want to reinit, in case of MT access is
1236 1237 1238
		 * not working. In this stage we don't know which flavour this
		 * ivb is, so it is better to reset also the gen6 fw registers
		 * before the ecobus check.
1239
		 */
1240 1241 1242 1243

		__raw_i915_write32(dev_priv, FORCEWAKE, 0);
		__raw_posting_read(dev_priv, ECOBUS);

1244 1245
		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
			       FORCEWAKE_MT, FORCEWAKE_MT_ACK);
1246

1247
		spin_lock_irq(&dev_priv->uncore.lock);
1248
		fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_ALL);
1249
		ecobus = __raw_i915_read32(dev_priv, ECOBUS);
1250
		fw_domains_put_with_fifo(dev_priv, FORCEWAKE_ALL);
1251
		spin_unlock_irq(&dev_priv->uncore.lock);
1252

1253
		if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
1254 1255
			DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
			DRM_INFO("when using vblank-synced partial screen updates.\n");
1256 1257
			fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
				       FORCEWAKE, FORCEWAKE_ACK);
1258
		}
1259
	} else if (IS_GEN6(dev_priv)) {
1260
		dev_priv->uncore.funcs.force_wake_get =
1261
			fw_domains_get_with_thread_status;
1262
		dev_priv->uncore.funcs.force_wake_put =
1263 1264 1265
			fw_domains_put_with_fifo;
		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
			       FORCEWAKE, FORCEWAKE_ACK);
1266
	}
1267 1268 1269

	/* All future platforms are expected to require complex power gating */
	WARN_ON(dev_priv->uncore.fw_domains == 0);
1270 1271
}

1272 1273 1274 1275 1276 1277 1278
#define ASSIGN_FW_DOMAINS_TABLE(d) \
{ \
	dev_priv->uncore.fw_domains_table = \
			(struct intel_forcewake_range *)(d); \
	dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \
}

1279
void intel_uncore_init(struct drm_i915_private *dev_priv)
1280
{
1281
	i915_check_vgpu(dev_priv);
1282

1283
	intel_uncore_edram_detect(dev_priv);
1284 1285
	intel_uncore_fw_domains_init(dev_priv);
	__intel_uncore_early_sanitize(dev_priv, false);
1286

1287 1288
	dev_priv->uncore.unclaimed_mmio_check = 1;

1289
	switch (INTEL_INFO(dev_priv)->gen) {
1290
	default:
1291
	case 9:
1292
		ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
1293
		ASSIGN_WRITE_MMIO_VFUNCS(gen9);
1294
		ASSIGN_READ_MMIO_VFUNCS(fwtable);
1295 1296
		break;
	case 8:
1297
		if (IS_CHERRYVIEW(dev_priv)) {
1298
			ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
1299
			ASSIGN_WRITE_MMIO_VFUNCS(chv);
1300
			ASSIGN_READ_MMIO_VFUNCS(fwtable);
1301 1302

		} else {
1303 1304
			ASSIGN_WRITE_MMIO_VFUNCS(gen8);
			ASSIGN_READ_MMIO_VFUNCS(gen6);
1305
		}
1306
		break;
1307 1308
	case 7:
	case 6:
1309
		ASSIGN_WRITE_MMIO_VFUNCS(gen6);
1310

1311
		if (IS_VALLEYVIEW(dev_priv)) {
1312
			ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges);
1313
			ASSIGN_READ_MMIO_VFUNCS(fwtable);
1314
		} else {
1315
			ASSIGN_READ_MMIO_VFUNCS(gen6);
1316
		}
1317 1318
		break;
	case 5:
1319 1320
		ASSIGN_WRITE_MMIO_VFUNCS(gen5);
		ASSIGN_READ_MMIO_VFUNCS(gen5);
1321 1322 1323 1324
		break;
	case 4:
	case 3:
	case 2:
1325 1326
		ASSIGN_WRITE_MMIO_VFUNCS(gen2);
		ASSIGN_READ_MMIO_VFUNCS(gen2);
1327 1328
		break;
	}
1329

1330 1331
	intel_fw_table_check(dev_priv);

1332
	if (intel_vgpu_active(dev_priv)) {
1333 1334 1335 1336
		ASSIGN_WRITE_MMIO_VFUNCS(vgpu);
		ASSIGN_READ_MMIO_VFUNCS(vgpu);
	}

1337
	i915_check_and_clear_faults(dev_priv);
1338
}
1339 1340
#undef ASSIGN_WRITE_MMIO_VFUNCS
#undef ASSIGN_READ_MMIO_VFUNCS
1341

1342
void intel_uncore_fini(struct drm_i915_private *dev_priv)
1343 1344
{
	/* Paranoia: make sure we have disabled everything before we exit. */
1345 1346
	intel_uncore_sanitize(dev_priv);
	intel_uncore_forcewake_reset(dev_priv, false);
1347 1348
}

1349
#define GEN_RANGE(l, h) GENMASK((h) - 1, (l) - 1)
1350

1351
static const struct register_whitelist {
1352
	i915_reg_t offset_ldw, offset_udw;
1353
	uint32_t size;
1354 1355
	/* supported gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
	uint32_t gen_bitmask;
1356
} whitelist[] = {
1357 1358 1359
	{ .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE),
	  .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE),
	  .size = 8, .gen_bitmask = GEN_RANGE(4, 9) },
1360 1361 1362 1363 1364
};

int i915_reg_read_ioctl(struct drm_device *dev,
			void *data, struct drm_file *file)
{
1365
	struct drm_i915_private *dev_priv = to_i915(dev);
1366 1367
	struct drm_i915_reg_read *reg = data;
	struct register_whitelist const *entry = whitelist;
1368
	unsigned size;
1369
	i915_reg_t offset_ldw, offset_udw;
1370
	int i, ret = 0;
1371 1372

	for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
1373
		if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) &&
1374
		    (INTEL_INFO(dev)->gen_mask & entry->gen_bitmask))
1375 1376 1377 1378 1379 1380
			break;
	}

	if (i == ARRAY_SIZE(whitelist))
		return -EINVAL;

1381 1382 1383 1384
	/* We use the low bits to encode extra flags as the register should
	 * be naturally aligned (and those that are not so aligned merely
	 * limit the available flags for that register).
	 */
1385 1386
	offset_ldw = entry->offset_ldw;
	offset_udw = entry->offset_udw;
1387
	size = entry->size;
1388
	size |= reg->offset ^ i915_mmio_reg_offset(offset_ldw);
1389

1390 1391
	intel_runtime_pm_get(dev_priv);

1392 1393
	switch (size) {
	case 8 | 1:
1394
		reg->val = I915_READ64_2x32(offset_ldw, offset_udw);
1395
		break;
1396
	case 8:
1397
		reg->val = I915_READ64(offset_ldw);
1398 1399
		break;
	case 4:
1400
		reg->val = I915_READ(offset_ldw);
1401 1402
		break;
	case 2:
1403
		reg->val = I915_READ16(offset_ldw);
1404 1405
		break;
	case 1:
1406
		reg->val = I915_READ8(offset_ldw);
1407 1408
		break;
	default:
1409 1410
		ret = -EINVAL;
		goto out;
1411 1412
	}

1413 1414 1415
out:
	intel_runtime_pm_put(dev_priv);
	return ret;
1416 1417
}

1418
static int i915_reset_complete(struct pci_dev *pdev)
1419 1420
{
	u8 gdrst;
1421
	pci_read_config_byte(pdev, I915_GDRST, &gdrst);
V
Ville Syrjälä 已提交
1422
	return (gdrst & GRDOM_RESET_STATUS) == 0;
1423 1424
}

1425
static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1426
{
1427
	struct pci_dev *pdev = dev_priv->drm.pdev;
1428

V
Ville Syrjälä 已提交
1429
	/* assert reset for at least 20 usec */
1430
	pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
V
Ville Syrjälä 已提交
1431
	udelay(20);
1432
	pci_write_config_byte(pdev, I915_GDRST, 0);
1433

1434
	return wait_for(i915_reset_complete(pdev), 500);
V
Ville Syrjälä 已提交
1435 1436
}

1437
static int g4x_reset_complete(struct pci_dev *pdev)
V
Ville Syrjälä 已提交
1438 1439
{
	u8 gdrst;
1440
	pci_read_config_byte(pdev, I915_GDRST, &gdrst);
V
Ville Syrjälä 已提交
1441
	return (gdrst & GRDOM_RESET_ENABLE) == 0;
1442 1443
}

1444
static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1445
{
1446
	struct pci_dev *pdev = dev_priv->drm.pdev;
1447 1448
	pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
	return wait_for(g4x_reset_complete(pdev), 500);
1449 1450
}

1451
static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1452
{
1453
	struct pci_dev *pdev = dev_priv->drm.pdev;
1454 1455
	int ret;

1456
	pci_write_config_byte(pdev, I915_GDRST,
1457
			      GRDOM_RENDER | GRDOM_RESET_ENABLE);
1458
	ret =  wait_for(g4x_reset_complete(pdev), 500);
1459 1460 1461 1462 1463 1464 1465
	if (ret)
		return ret;

	/* WaVcpClkGateDisableForMediaReset:ctg,elk */
	I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
	POSTING_READ(VDECCLK_GATE_D);

1466
	pci_write_config_byte(pdev, I915_GDRST,
1467
			      GRDOM_MEDIA | GRDOM_RESET_ENABLE);
1468
	ret =  wait_for(g4x_reset_complete(pdev), 500);
1469 1470 1471 1472 1473 1474 1475
	if (ret)
		return ret;

	/* WaVcpClkGateDisableForMediaReset:ctg,elk */
	I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
	POSTING_READ(VDECCLK_GATE_D);

1476
	pci_write_config_byte(pdev, I915_GDRST, 0);
1477 1478 1479 1480

	return 0;
}

1481 1482
static int ironlake_do_reset(struct drm_i915_private *dev_priv,
			     unsigned engine_mask)
1483 1484 1485
{
	int ret;

1486
	I915_WRITE(ILK_GDSR,
1487
		   ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
1488 1489 1490
	ret = intel_wait_for_register(dev_priv,
				      ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
				      500);
1491 1492 1493
	if (ret)
		return ret;

1494
	I915_WRITE(ILK_GDSR,
1495
		   ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
1496 1497 1498
	ret = intel_wait_for_register(dev_priv,
				      ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
				      500);
1499 1500 1501
	if (ret)
		return ret;

1502
	I915_WRITE(ILK_GDSR, 0);
1503 1504

	return 0;
1505 1506
}

1507 1508 1509
/* Reset the hardware domains (GENX_GRDOM_*) specified by mask */
static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
				u32 hw_domain_mask)
1510 1511 1512 1513 1514
{
	/* GEN6_GDRST is not in the gt power well, no need to check
	 * for fifo space for the write or forcewake the chip for
	 * the read
	 */
1515
	__raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask);
1516

1517
	/* Spin waiting for the device to ack the reset requests */
1518 1519 1520
	return intel_wait_for_register_fw(dev_priv,
					  GEN6_GDRST, hw_domain_mask, 0,
					  500);
1521 1522 1523 1524
}

/**
 * gen6_reset_engines - reset individual engines
1525
 * @dev_priv: i915 device
1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
 * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset
 *
 * This function will reset the individual engines that are set in engine_mask.
 * If you provide ALL_ENGINES as mask, full global domain reset will be issued.
 *
 * Note: It is responsibility of the caller to handle the difference between
 * asking full domain reset versus reset for all available individual engines.
 *
 * Returns 0 on success, nonzero on error.
 */
1536 1537
static int gen6_reset_engines(struct drm_i915_private *dev_priv,
			      unsigned engine_mask)
1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552
{
	struct intel_engine_cs *engine;
	const u32 hw_engine_mask[I915_NUM_ENGINES] = {
		[RCS] = GEN6_GRDOM_RENDER,
		[BCS] = GEN6_GRDOM_BLT,
		[VCS] = GEN6_GRDOM_MEDIA,
		[VCS2] = GEN8_GRDOM_MEDIA2,
		[VECS] = GEN6_GRDOM_VECS,
	};
	u32 hw_mask;
	int ret;

	if (engine_mask == ALL_ENGINES) {
		hw_mask = GEN6_GRDOM_FULL;
	} else {
1553 1554
		unsigned int tmp;

1555
		hw_mask = 0;
1556
		for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1557 1558 1559 1560
			hw_mask |= hw_engine_mask[engine->id];
	}

	ret = gen6_hw_domain_reset(dev_priv, hw_mask);
1561

1562
	intel_uncore_forcewake_reset(dev_priv, true);
1563

1564 1565 1566
	return ret;
}

1567 1568 1569 1570 1571 1572 1573 1574 1575
/**
 * intel_wait_for_register_fw - wait until register matches expected state
 * @dev_priv: the i915 device
 * @reg: the register to read
 * @mask: mask to apply to register value
 * @value: expected value
 * @timeout_ms: timeout in millisecond
 *
 * This routine waits until the target register @reg contains the expected
1576 1577 1578 1579
 * @value after applying the @mask, i.e. it waits until ::
 *
 *     (I915_READ_FW(reg) & mask) == value
 *
1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611
 * Otherwise, the wait will timeout after @timeout_ms milliseconds.
 *
 * Note that this routine assumes the caller holds forcewake asserted, it is
 * not suitable for very long waits. See intel_wait_for_register() if you
 * wish to wait without holding forcewake for the duration (i.e. you expect
 * the wait to be slow).
 *
 * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
 */
int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
			       i915_reg_t reg,
			       const u32 mask,
			       const u32 value,
			       const unsigned long timeout_ms)
{
#define done ((I915_READ_FW(reg) & mask) == value)
	int ret = wait_for_us(done, 2);
	if (ret)
		ret = wait_for(done, timeout_ms);
	return ret;
#undef done
}

/**
 * intel_wait_for_register - wait until register matches expected state
 * @dev_priv: the i915 device
 * @reg: the register to read
 * @mask: mask to apply to register value
 * @value: expected value
 * @timeout_ms: timeout in millisecond
 *
 * This routine waits until the target register @reg contains the expected
1612 1613 1614 1615
 * @value after applying the @mask, i.e. it waits until ::
 *
 *     (I915_READ(reg) & mask) == value
 *
1616 1617 1618 1619 1620 1621 1622 1623 1624
 * Otherwise, the wait will timeout after @timeout_ms milliseconds.
 *
 * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
 */
int intel_wait_for_register(struct drm_i915_private *dev_priv,
			    i915_reg_t reg,
			    const u32 mask,
			    const u32 value,
			    const unsigned long timeout_ms)
1625
{
1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638

	unsigned fw =
		intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
	int ret;

	intel_uncore_forcewake_get(dev_priv, fw);
	ret = wait_for_us((I915_READ_FW(reg) & mask) == value, 2);
	intel_uncore_forcewake_put(dev_priv, fw);
	if (ret)
		ret = wait_for((I915_READ_NOTRACE(reg) & mask) == value,
			       timeout_ms);

	return ret;
1639 1640 1641 1642
}

static int gen8_request_engine_reset(struct intel_engine_cs *engine)
{
1643
	struct drm_i915_private *dev_priv = engine->i915;
1644 1645 1646 1647 1648
	int ret;

	I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
		      _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET));

1649 1650 1651 1652 1653
	ret = intel_wait_for_register_fw(dev_priv,
					 RING_RESET_CTL(engine->mmio_base),
					 RESET_CTL_READY_TO_RESET,
					 RESET_CTL_READY_TO_RESET,
					 700);
1654 1655 1656 1657 1658 1659 1660 1661
	if (ret)
		DRM_ERROR("%s: reset request timeout\n", engine->name);

	return ret;
}

static void gen8_unrequest_engine_reset(struct intel_engine_cs *engine)
{
1662
	struct drm_i915_private *dev_priv = engine->i915;
1663 1664 1665

	I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
		      _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
1666 1667
}

1668 1669
static int gen8_reset_engines(struct drm_i915_private *dev_priv,
			      unsigned engine_mask)
1670 1671
{
	struct intel_engine_cs *engine;
1672
	unsigned int tmp;
1673

1674
	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1675
		if (gen8_request_engine_reset(engine))
1676 1677
			goto not_ready;

1678
	return gen6_reset_engines(dev_priv, engine_mask);
1679 1680

not_ready:
1681
	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1682
		gen8_unrequest_engine_reset(engine);
1683 1684 1685 1686

	return -EIO;
}

1687 1688 1689
typedef int (*reset_func)(struct drm_i915_private *, unsigned engine_mask);

static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv)
1690
{
1691 1692 1693
	if (!i915.reset)
		return NULL;

1694
	if (INTEL_INFO(dev_priv)->gen >= 8)
1695
		return gen8_reset_engines;
1696
	else if (INTEL_INFO(dev_priv)->gen >= 6)
1697
		return gen6_reset_engines;
1698
	else if (IS_GEN5(dev_priv))
1699
		return ironlake_do_reset;
1700
	else if (IS_G4X(dev_priv))
1701
		return g4x_do_reset;
1702
	else if (IS_G33(dev_priv))
1703
		return g33_do_reset;
1704
	else if (INTEL_INFO(dev_priv)->gen >= 3)
1705
		return i915_do_reset;
1706
	else
1707 1708 1709
		return NULL;
}

1710
int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1711
{
1712
	reset_func reset;
1713
	int ret;
1714

1715
	reset = intel_get_gpu_reset(dev_priv);
1716
	if (reset == NULL)
1717
		return -ENODEV;
1718

1719 1720 1721 1722
	/* If the power well sleeps during the reset, the reset
	 * request may be dropped and never completes (causing -EIO).
	 */
	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1723
	ret = reset(dev_priv, engine_mask);
1724 1725 1726
	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);

	return ret;
1727 1728
}

1729
bool intel_has_gpu_reset(struct drm_i915_private *dev_priv)
1730
{
1731
	return intel_get_gpu_reset(dev_priv) != NULL;
1732 1733
}

1734 1735 1736 1737 1738
int intel_guc_reset(struct drm_i915_private *dev_priv)
{
	int ret;
	unsigned long irqflags;

1739
	if (!HAS_GUC(dev_priv))
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
		return -EINVAL;

	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);

	ret = gen6_hw_domain_reset(dev_priv, GEN9_GRDOM_GUC);

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);

	return ret;
}

1753
bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv)
1754
{
1755
	return check_for_unclaimed_mmio(dev_priv);
1756
}
1757

1758
bool
1759 1760 1761 1762
intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv)
{
	if (unlikely(i915.mmio_debug ||
		     dev_priv->uncore.unclaimed_mmio_check <= 0))
1763
		return false;
1764 1765 1766 1767 1768 1769 1770

	if (unlikely(intel_uncore_unclaimed_mmio(dev_priv))) {
		DRM_DEBUG("Unclaimed register detected, "
			  "enabling oneshot unclaimed register reporting. "
			  "Please use i915.mmio_debug=N for more information.\n");
		i915.mmio_debug++;
		dev_priv->uncore.unclaimed_mmio_check--;
1771
		return true;
1772
	}
1773 1774

	return false;
1775
}
1776 1777 1778 1779 1780

static enum forcewake_domains
intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
				i915_reg_t reg)
{
T
Tvrtko Ursulin 已提交
1781
	u32 offset = i915_mmio_reg_offset(reg);
1782 1783
	enum forcewake_domains fw_domains;

T
Tvrtko Ursulin 已提交
1784 1785 1786 1787 1788 1789 1790
	if (HAS_FWTABLE(dev_priv)) {
		fw_domains = __fwtable_reg_read_fw_domains(offset);
	} else if (INTEL_GEN(dev_priv) >= 6) {
		fw_domains = __gen6_reg_read_fw_domains(offset);
	} else {
		WARN_ON(!IS_GEN(dev_priv, 2, 5));
		fw_domains = 0;
1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803
	}

	WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);

	return fw_domains;
}

static enum forcewake_domains
intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
				 i915_reg_t reg)
{
	enum forcewake_domains fw_domains;

1804
	switch (INTEL_GEN(dev_priv)) {
1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853
	case 9:
		fw_domains = __gen9_reg_write_fw_domains(i915_mmio_reg_offset(reg));
		break;
	case 8:
		if (IS_CHERRYVIEW(dev_priv))
			fw_domains = __chv_reg_write_fw_domains(i915_mmio_reg_offset(reg));
		else
			fw_domains = __gen8_reg_write_fw_domains(i915_mmio_reg_offset(reg));
		break;
	case 7:
	case 6:
		fw_domains = FORCEWAKE_RENDER;
		break;
	default:
		MISSING_CASE(INTEL_INFO(dev_priv)->gen);
	case 5:
	case 4:
	case 3:
	case 2:
		return 0;
	}

	WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);

	return fw_domains;
}

/**
 * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
 * 				    a register
 * @dev_priv: pointer to struct drm_i915_private
 * @reg: register in question
 * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
 *
 * Returns a set of forcewake domains required to be taken with for example
 * intel_uncore_forcewake_get for the specified register to be accessible in the
 * specified mode (read, write or read/write) with raw mmio accessors.
 *
 * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
 * callers to do FIFO management on their own or risk losing writes.
 */
enum forcewake_domains
intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
			       i915_reg_t reg, unsigned int op)
{
	enum forcewake_domains fw_domains = 0;

	WARN_ON(!op);

T
Tvrtko Ursulin 已提交
1854 1855 1856
	if (intel_vgpu_active(dev_priv))
		return 0;

1857 1858 1859 1860 1861 1862 1863 1864
	if (op & FW_REG_READ)
		fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg);

	if (op & FW_REG_WRITE)
		fw_domains |= intel_uncore_forcewake_for_write(dev_priv, reg);

	return fw_domains;
}