intel_uncore.c 55.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25
/*
 * Copyright © 2013 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include "i915_drv.h"
#include "intel_drv.h"
26
#include "i915_vgpu.h"
27

28
#include <asm/iosf_mbi.h>
29 30
#include <linux/pm_runtime.h>

31
#define FORCEWAKE_ACK_TIMEOUT_MS 50
32

33
#define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__))
34

35 36 37 38 39 40 41
static const char * const forcewake_domain_names[] = {
	"render",
	"blitter",
	"media",
};

const char *
42
intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
43
{
44
	BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
45 46 47 48 49 50 51 52 53 54 55

	if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
		return forcewake_domain_names[id];

	WARN_ON(id);

	return "unknown";
}

static inline void
fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
56
{
57
	WARN_ON(!i915_mmio_reg_valid(d->reg_set));
58
	__raw_i915_write32(d->i915, d->reg_set, d->val_reset);
59 60
}

61 62
static inline void
fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
63
{
64 65
	d->wake_count++;
	hrtimer_start_range_ns(&d->timer,
T
Thomas Gleixner 已提交
66
			       NSEC_PER_MSEC,
67 68
			       NSEC_PER_MSEC,
			       HRTIMER_MODE_REL);
69 70
}

71 72
static inline void
fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d)
73
{
74 75
	if (wait_for_atomic((__raw_i915_read32(d->i915, d->reg_ack) &
			     FORCEWAKE_KERNEL) == 0,
76
			    FORCEWAKE_ACK_TIMEOUT_MS))
77 78 79
		DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
			  intel_uncore_forcewake_domain_to_str(d->id));
}
80

81 82 83 84 85
static inline void
fw_domain_get(const struct intel_uncore_forcewake_domain *d)
{
	__raw_i915_write32(d->i915, d->reg_set, d->val_set);
}
86

87 88 89 90 91
static inline void
fw_domain_wait_ack(const struct intel_uncore_forcewake_domain *d)
{
	if (wait_for_atomic((__raw_i915_read32(d->i915, d->reg_ack) &
			     FORCEWAKE_KERNEL),
92
			    FORCEWAKE_ACK_TIMEOUT_MS))
93 94 95
		DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
			  intel_uncore_forcewake_domain_to_str(d->id));
}
96

97 98 99 100
static inline void
fw_domain_put(const struct intel_uncore_forcewake_domain *d)
{
	__raw_i915_write32(d->i915, d->reg_set, d->val_clear);
101 102
}

103 104
static inline void
fw_domain_posting_read(const struct intel_uncore_forcewake_domain *d)
105
{
106
	/* something from same cacheline, but not from the set register */
107
	if (i915_mmio_reg_valid(d->reg_post))
108
		__raw_posting_read(d->i915, d->reg_post);
109 110
}

111
static void
112
fw_domains_get(struct drm_i915_private *dev_priv, enum forcewake_domains fw_domains)
113
{
114
	struct intel_uncore_forcewake_domain *d;
115

116
	for_each_fw_domain_masked(d, fw_domains, dev_priv) {
117 118 119
		fw_domain_wait_ack_clear(d);
		fw_domain_get(d);
	}
120 121 122

	for_each_fw_domain_masked(d, fw_domains, dev_priv)
		fw_domain_wait_ack(d);
123
}
124

125
static void
126
fw_domains_put(struct drm_i915_private *dev_priv, enum forcewake_domains fw_domains)
127 128
{
	struct intel_uncore_forcewake_domain *d;
129

130
	for_each_fw_domain_masked(d, fw_domains, dev_priv) {
131 132 133 134
		fw_domain_put(d);
		fw_domain_posting_read(d);
	}
}
135

136 137 138 139 140 141
static void
fw_domains_posting_read(struct drm_i915_private *dev_priv)
{
	struct intel_uncore_forcewake_domain *d;

	/* No need to do for all, just do for first found */
142
	for_each_fw_domain(d, dev_priv) {
143 144 145 146 147 148
		fw_domain_posting_read(d);
		break;
	}
}

static void
149
fw_domains_reset(struct drm_i915_private *dev_priv, enum forcewake_domains fw_domains)
150 151 152
{
	struct intel_uncore_forcewake_domain *d;

153 154
	if (dev_priv->uncore.fw_domains == 0)
		return;
155

156
	for_each_fw_domain_masked(d, fw_domains, dev_priv)
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
		fw_domain_reset(d);

	fw_domains_posting_read(dev_priv);
}

static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
{
	/* w/a for a sporadic read returning 0 by waiting for the GT
	 * thread to wake up.
	 */
	if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) &
				GEN6_GT_THREAD_STATUS_CORE_MASK) == 0, 500))
		DRM_ERROR("GT thread status wait timed out\n");
}

static void fw_domains_get_with_thread_status(struct drm_i915_private *dev_priv,
173
					      enum forcewake_domains fw_domains)
174 175
{
	fw_domains_get(dev_priv, fw_domains);
176

177
	/* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
178
	__gen6_gt_wait_for_thread_c0(dev_priv);
179 180 181 182 183
}

static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
{
	u32 gtfifodbg;
184 185

	gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
186 187
	if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg))
		__raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg);
188 189
}

190
static void fw_domains_put_with_fifo(struct drm_i915_private *dev_priv,
191
				     enum forcewake_domains fw_domains)
192
{
193
	fw_domains_put(dev_priv, fw_domains);
194 195 196
	gen6_gt_check_fifodbg(dev_priv);
}

197 198 199 200 201 202 203
static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv)
{
	u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL);

	return count & GT_FIFO_FREE_ENTRIES_MASK;
}

204 205 206 207
static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
{
	int ret = 0;

208 209
	/* On VLV, FIFO will be shared by both SW and HW.
	 * So, we need to read the FREE_ENTRIES everytime */
210
	if (IS_VALLEYVIEW(dev_priv))
211
		dev_priv->uncore.fifo_count = fifo_free_entries(dev_priv);
212

213 214
	if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) {
		int loop = 500;
215 216
		u32 fifo = fifo_free_entries(dev_priv);

217 218
		while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) {
			udelay(10);
219
			fifo = fifo_free_entries(dev_priv);
220 221 222 223 224 225 226 227 228 229
		}
		if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES))
			++ret;
		dev_priv->uncore.fifo_count = fifo;
	}
	dev_priv->uncore.fifo_count--;

	return ret;
}

230 231
static enum hrtimer_restart
intel_uncore_fw_release_timer(struct hrtimer *timer)
Z
Zhe Wang 已提交
232
{
233 234
	struct intel_uncore_forcewake_domain *domain =
	       container_of(timer, struct intel_uncore_forcewake_domain, timer);
235
	struct drm_i915_private *dev_priv = domain->i915;
236
	unsigned long irqflags;
Z
Zhe Wang 已提交
237

238
	assert_rpm_device_not_suspended(dev_priv);
Z
Zhe Wang 已提交
239

240
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
241 242 243
	if (WARN_ON(domain->wake_count == 0))
		domain->wake_count++;

244 245 246 247
	if (--domain->wake_count == 0) {
		dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask);
		dev_priv->uncore.fw_domains_active &= ~domain->mask;
	}
248

249
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
250 251

	return HRTIMER_NORESTART;
Z
Zhe Wang 已提交
252 253
}

254 255
static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
					 bool restore)
Z
Zhe Wang 已提交
256
{
257
	unsigned long irqflags;
258
	struct intel_uncore_forcewake_domain *domain;
259
	int retry_count = 100;
260
	enum forcewake_domains fw, active_domains;
Z
Zhe Wang 已提交
261

262 263 264 265 266 267
	/* Hold uncore.lock across reset to prevent any register access
	 * with forcewake not set correctly. Wait until all pending
	 * timers are run before holding.
	 */
	while (1) {
		active_domains = 0;
Z
Zhe Wang 已提交
268

269
		for_each_fw_domain(domain, dev_priv) {
270
			if (hrtimer_cancel(&domain->timer) == 0)
271
				continue;
Z
Zhe Wang 已提交
272

273
			intel_uncore_fw_release_timer(&domain->timer);
274
		}
275

276
		spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
277

278
		for_each_fw_domain(domain, dev_priv) {
279
			if (hrtimer_active(&domain->timer))
280
				active_domains |= domain->mask;
281
		}
282

283 284
		if (active_domains == 0)
			break;
285

286 287 288 289
		if (--retry_count == 0) {
			DRM_ERROR("Timed out waiting for forcewake timers to finish\n");
			break;
		}
290

291 292 293
		spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
		cond_resched();
	}
294

295 296
	WARN_ON(active_domains);

297
	fw = dev_priv->uncore.fw_domains_active;
298 299
	if (fw)
		dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
300

301
	fw_domains_reset(dev_priv, FORCEWAKE_ALL);
Z
Zhe Wang 已提交
302

303 304 305 306
	if (restore) { /* If reset with a user forcewake, try to restore */
		if (fw)
			dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);

307
		if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
308
			dev_priv->uncore.fifo_count =
309
				fifo_free_entries(dev_priv);
310 311
	}

312
	if (!restore)
313
		assert_forcewakes_inactive(dev_priv);
314

315
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
316 317
}

M
Mika Kuoppala 已提交
318 319 320 321 322 323 324 325 326 327 328 329
static u64 gen9_edram_size(struct drm_i915_private *dev_priv)
{
	const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
	const unsigned int sets[4] = { 1, 1, 2, 2 };
	const u32 cap = dev_priv->edram_cap;

	return EDRAM_NUM_BANKS(cap) *
		ways[EDRAM_WAYS_IDX(cap)] *
		sets[EDRAM_SETS_IDX(cap)] *
		1024 * 1024;
}

330
u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv)
331
{
332 333 334
	if (!HAS_EDRAM(dev_priv))
		return 0;

M
Mika Kuoppala 已提交
335 336
	/* The needed capability bits for size calculation
	 * are not there with pre gen9 so return 128MB always.
337
	 */
M
Mika Kuoppala 已提交
338 339
	if (INTEL_GEN(dev_priv) < 9)
		return 128 * 1024 * 1024;
340

M
Mika Kuoppala 已提交
341
	return gen9_edram_size(dev_priv);
342
}
343

344 345 346 347 348 349 350 351 352
static void intel_uncore_edram_detect(struct drm_i915_private *dev_priv)
{
	if (IS_HASWELL(dev_priv) ||
	    IS_BROADWELL(dev_priv) ||
	    INTEL_GEN(dev_priv) >= 9) {
		dev_priv->edram_cap = __raw_i915_read32(dev_priv,
							HSW_EDRAM_CAP);

		/* NB: We can't write IDICR yet because we do not have gt funcs
353
		 * set up */
354 355
	} else {
		dev_priv->edram_cap = 0;
356
	}
357 358 359 360

	if (HAS_EDRAM(dev_priv))
		DRM_INFO("Found %lluMB of eDRAM\n",
			 intel_uncore_edram_size(dev_priv) / (1024 * 1024));
361 362
}

363
static bool
364
fpga_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
365 366 367 368 369 370 371 372 373 374 375 376
{
	u32 dbg;

	dbg = __raw_i915_read32(dev_priv, FPGA_DBG);
	if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
		return false;

	__raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);

	return true;
}

377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402
static bool
vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
{
	u32 cer;

	cer = __raw_i915_read32(dev_priv, CLAIM_ER);
	if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
		return false;

	__raw_i915_write32(dev_priv, CLAIM_ER, CLAIM_ER_CLR);

	return true;
}

static bool
check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
{
	if (HAS_FPGA_DBG_UNCLAIMED(dev_priv))
		return fpga_check_for_unclaimed_mmio(dev_priv);

	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
		return vlv_check_for_unclaimed_mmio(dev_priv);

	return false;
}

403
static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
404 405
					  bool restore_forcewake)
{
406 407
	struct intel_device_info *info = mkwrite_device_info(dev_priv);

408 409 410
	/* clear out unclaimed reg detection bit */
	if (check_for_unclaimed_mmio(dev_priv))
		DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
411

412
	/* clear out old GT FIFO errors */
413
	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
414 415 416
		__raw_i915_write32(dev_priv, GTFIFODBG,
				   __raw_i915_read32(dev_priv, GTFIFODBG));

417
	/* WaDisableShadowRegForCpd:chv */
418
	if (IS_CHERRYVIEW(dev_priv)) {
419 420 421 422 423 424
		__raw_i915_write32(dev_priv, GTFIFOCTL,
				   __raw_i915_read32(dev_priv, GTFIFOCTL) |
				   GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
				   GT_FIFO_CTL_RC6_POLICY_STALL);
	}

425
	if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST))
426 427
		info->has_decoupled_mmio = false;

428
	intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
429 430
}

431
void intel_uncore_suspend(struct drm_i915_private *dev_priv)
432
{
433 434
	iosf_mbi_unregister_pmic_bus_access_notifier(
		&dev_priv->uncore.pmic_bus_access_nb);
435 436 437 438 439 440
	intel_uncore_forcewake_reset(dev_priv, false);
}

void intel_uncore_resume_early(struct drm_i915_private *dev_priv)
{
	__intel_uncore_early_sanitize(dev_priv, true);
441 442
	iosf_mbi_register_pmic_bus_access_notifier(
		&dev_priv->uncore.pmic_bus_access_nb);
443
	i915_check_and_clear_faults(dev_priv);
444 445
}

446
void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
447
{
448
	i915.enable_rc6 = sanitize_rc6_option(dev_priv, i915.enable_rc6);
449

450
	/* BIOS often leaves RC6 enabled, but disable it for hw init */
451
	intel_sanitize_gt_powersave(dev_priv);
452 453
}

454 455 456 457 458 459 460
static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
					 enum forcewake_domains fw_domains)
{
	struct intel_uncore_forcewake_domain *domain;

	fw_domains &= dev_priv->uncore.fw_domains;

461
	for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
462
		if (domain->wake_count++)
463
			fw_domains &= ~domain->mask;
464 465
	}

466
	if (fw_domains) {
467
		dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
468 469
		dev_priv->uncore.fw_domains_active |= fw_domains;
	}
470 471
}

472 473 474 475 476 477 478 479 480 481 482 483
/**
 * intel_uncore_forcewake_get - grab forcewake domain references
 * @dev_priv: i915 device instance
 * @fw_domains: forcewake domains to get reference on
 *
 * This function can be used get GT's forcewake domain references.
 * Normal register access will handle the forcewake domains automatically.
 * However if some sequence requires the GT to not power down a particular
 * forcewake domains this function should be called at the beginning of the
 * sequence. And subsequently the reference should be dropped by symmetric
 * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
 * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
484
 */
485
void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
486
				enum forcewake_domains fw_domains)
487 488 489
{
	unsigned long irqflags;

490 491 492
	if (!dev_priv->uncore.funcs.force_wake_get)
		return;

493
	assert_rpm_wakelock_held(dev_priv);
494

495
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
496
	__intel_uncore_forcewake_get(dev_priv, fw_domains);
497 498 499
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
}

500
/**
501
 * intel_uncore_forcewake_get__locked - grab forcewake domain references
502
 * @dev_priv: i915 device instance
503
 * @fw_domains: forcewake domains to get reference on
504
 *
505 506
 * See intel_uncore_forcewake_get(). This variant places the onus
 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
507
 */
508 509 510 511 512 513 514 515 516 517 518 519 520
void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv,
					enum forcewake_domains fw_domains)
{
	assert_spin_locked(&dev_priv->uncore.lock);

	if (!dev_priv->uncore.funcs.force_wake_get)
		return;

	__intel_uncore_forcewake_get(dev_priv, fw_domains);
}

static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
					 enum forcewake_domains fw_domains)
521
{
522
	struct intel_uncore_forcewake_domain *domain;
523

524 525
	fw_domains &= dev_priv->uncore.fw_domains;

526
	for_each_fw_domain_masked(domain, fw_domains, dev_priv) {
527 528 529 530 531 532
		if (WARN_ON(domain->wake_count == 0))
			continue;

		if (--domain->wake_count)
			continue;

533
		fw_domain_arm_timer(domain);
534
	}
535
}
536

537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
/**
 * intel_uncore_forcewake_put - release a forcewake domain reference
 * @dev_priv: i915 device instance
 * @fw_domains: forcewake domains to put references
 *
 * This function drops the device-level forcewakes for specified
 * domains obtained by intel_uncore_forcewake_get().
 */
void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
				enum forcewake_domains fw_domains)
{
	unsigned long irqflags;

	if (!dev_priv->uncore.funcs.force_wake_put)
		return;

	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
	__intel_uncore_forcewake_put(dev_priv, fw_domains);
555 556 557
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
}

558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
/**
 * intel_uncore_forcewake_put__locked - grab forcewake domain references
 * @dev_priv: i915 device instance
 * @fw_domains: forcewake domains to get reference on
 *
 * See intel_uncore_forcewake_put(). This variant places the onus
 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
 */
void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
					enum forcewake_domains fw_domains)
{
	assert_spin_locked(&dev_priv->uncore.lock);

	if (!dev_priv->uncore.funcs.force_wake_put)
		return;

	__intel_uncore_forcewake_put(dev_priv, fw_domains);
}

577
void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
578 579 580 581
{
	if (!dev_priv->uncore.funcs.force_wake_get)
		return;

582
	WARN_ON(dev_priv->uncore.fw_domains_active);
583 584
}

585
/* We give fast paths for the really cool registers */
586
#define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000)
587

588 589 590 591 592 593 594 595 596 597
#define __gen6_reg_read_fw_domains(offset) \
({ \
	enum forcewake_domains __fwd; \
	if (NEEDS_FORCE_WAKE(offset)) \
		__fwd = FORCEWAKE_RENDER; \
	else \
		__fwd = 0; \
	__fwd; \
})

T
Tvrtko Ursulin 已提交
598
static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
599 600 601 602 603 604 605 606 607
{
	if (offset < entry->start)
		return -1;
	else if (offset > entry->end)
		return 1;
	else
		return 0;
}

T
Tvrtko Ursulin 已提交
608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
/* Copied and "macroized" from lib/bsearch.c */
#define BSEARCH(key, base, num, cmp) ({                                 \
	unsigned int start__ = 0, end__ = (num);                        \
	typeof(base) result__ = NULL;                                   \
	while (start__ < end__) {                                       \
		unsigned int mid__ = start__ + (end__ - start__) / 2;   \
		int ret__ = (cmp)((key), (base) + mid__);               \
		if (ret__ < 0) {                                        \
			end__ = mid__;                                  \
		} else if (ret__ > 0) {                                 \
			start__ = mid__ + 1;                            \
		} else {                                                \
			result__ = (base) + mid__;                      \
			break;                                          \
		}                                                       \
	}                                                               \
	result__;                                                       \
})

627
static enum forcewake_domains
628
find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
629
{
T
Tvrtko Ursulin 已提交
630
	const struct intel_forcewake_range *entry;
631

T
Tvrtko Ursulin 已提交
632 633 634
	entry = BSEARCH(offset,
			dev_priv->uncore.fw_domains_table,
			dev_priv->uncore.fw_domains_table_entries,
635
			fw_range_cmp);
636

637 638 639 640 641 642 643 644
	if (!entry)
		return 0;

	WARN(entry->domains & ~dev_priv->uncore.fw_domains,
	     "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
	     entry->domains & ~dev_priv->uncore.fw_domains, offset);

	return entry->domains;
645 646
}

647
static void
648
intel_fw_table_check(struct drm_i915_private *dev_priv)
649
{
650 651
	const struct intel_forcewake_range *ranges;
	unsigned int num_ranges;
652 653 654 655 656 657
	s32 prev;
	unsigned int i;

	if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG))
		return;

658 659 660 661 662 663
	ranges = dev_priv->uncore.fw_domains_table;
	if (!ranges)
		return;

	num_ranges = dev_priv->uncore.fw_domains_table_entries;

664
	for (i = 0, prev = -1; i < num_ranges; i++, ranges++) {
665 666
		WARN_ON_ONCE(IS_GEN9(dev_priv) &&
			     (prev + 1) != (s32)ranges->start);
667 668 669 670 671 672 673
		WARN_ON_ONCE(prev >= (s32)ranges->start);
		prev = ranges->start;
		WARN_ON_ONCE(prev >= (s32)ranges->end);
		prev = ranges->end;
	}
}

674 675
#define GEN_FW_RANGE(s, e, d) \
	{ .start = (s), .end = (e), .domains = (d) }
676

T
Tvrtko Ursulin 已提交
677 678 679 680 681
#define HAS_FWTABLE(dev_priv) \
	(IS_GEN9(dev_priv) || \
	 IS_CHERRYVIEW(dev_priv) || \
	 IS_VALLEYVIEW(dev_priv))

682
/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
683 684 685 686 687 688
static const struct intel_forcewake_range __vlv_fw_ranges[] = {
	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
	GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
	GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
689
	GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
690 691
	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
};
692

T
Tvrtko Ursulin 已提交
693
#define __fwtable_reg_read_fw_domains(offset) \
694 695
({ \
	enum forcewake_domains __fwd = 0; \
696
	if (NEEDS_FORCE_WAKE((offset))) \
697
		__fwd = find_fw_domain(dev_priv, offset); \
698 699 700
	__fwd; \
})

701
/* *Must* be sorted by offset! See intel_shadow_table_check(). */
702
static const i915_reg_t gen8_shadowed_regs[] = {
703 704 705 706 707 708
	RING_TAIL(RENDER_RING_BASE),	/* 0x2000 (base) */
	GEN6_RPNSWREQ,			/* 0xA008 */
	GEN6_RC_VIDEO_FREQ,		/* 0xA00C */
	RING_TAIL(GEN6_BSD_RING_BASE),	/* 0x12000 (base) */
	RING_TAIL(VEBOX_RING_BASE),	/* 0x1a000 (base) */
	RING_TAIL(BLT_RING_BASE),	/* 0x22000 (base) */
709 710 711
	/* TODO: Other registers are not yet used */
};

712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
static void intel_shadow_table_check(void)
{
	const i915_reg_t *reg = gen8_shadowed_regs;
	s32 prev;
	u32 offset;
	unsigned int i;

	if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG))
		return;

	for (i = 0, prev = -1; i < ARRAY_SIZE(gen8_shadowed_regs); i++, reg++) {
		offset = i915_mmio_reg_offset(*reg);
		WARN_ON_ONCE(prev >= (s32)offset);
		prev = offset;
	}
}

T
Tvrtko Ursulin 已提交
729
static int mmio_reg_cmp(u32 key, const i915_reg_t *reg)
730
{
T
Tvrtko Ursulin 已提交
731
	u32 offset = i915_mmio_reg_offset(*reg);
732

T
Tvrtko Ursulin 已提交
733
	if (key < offset)
734
		return -1;
T
Tvrtko Ursulin 已提交
735
	else if (key > offset)
736 737 738 739 740
		return 1;
	else
		return 0;
}

741 742
static bool is_gen8_shadowed(u32 offset)
{
T
Tvrtko Ursulin 已提交
743
	const i915_reg_t *regs = gen8_shadowed_regs;
744

T
Tvrtko Ursulin 已提交
745 746
	return BSEARCH(offset, regs, ARRAY_SIZE(gen8_shadowed_regs),
		       mmio_reg_cmp);
747 748 749 750 751 752 753 754 755 756 757 758
}

#define __gen8_reg_write_fw_domains(offset) \
({ \
	enum forcewake_domains __fwd; \
	if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \
		__fwd = FORCEWAKE_RENDER; \
	else \
		__fwd = 0; \
	__fwd; \
})

759
/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
760 761
static const struct intel_forcewake_range __chv_fw_ranges[] = {
	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
762
	GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
763
	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
764
	GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
765
	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
766
	GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
767
	GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
768 769
	GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
770
	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
771 772
	GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
	GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
773 774 775 776 777
	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
	GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
	GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
	GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
};
778

779
#define __fwtable_reg_write_fw_domains(offset) \
780 781
({ \
	enum forcewake_domains __fwd = 0; \
782
	if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
783
		__fwd = find_fw_domain(dev_priv, offset); \
784 785 786
	__fwd; \
})

787
/* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
788
static const struct intel_forcewake_range __gen9_fw_ranges[] = {
789
	GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
790 791
	GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
792
	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER),
793
	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
794
	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER),
795
	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
796
	GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER),
797
	GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
798
	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
799
	GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER),
800
	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
801
	GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER),
802
	GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
803
	GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER),
804
	GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
805
	GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER),
806
	GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
807
	GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
808
	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
809
	GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER),
810
	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
811
	GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER),
812
	GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
813
	GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER),
814
	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
815
	GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER),
816
	GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
817
	GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER),
818
	GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
819
	GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER),
820 821
	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
};
822

823 824 825 826 827 828
static void
ilk_dummy_write(struct drm_i915_private *dev_priv)
{
	/* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
	 * the chip from rc6 before touching it for real. MI_MODE is masked,
	 * hence harmless to write 0 into. */
829
	__raw_i915_write32(dev_priv, MI_MODE, 0);
830 831 832
}

static void
833 834 835 836
__unclaimed_reg_debug(struct drm_i915_private *dev_priv,
		      const i915_reg_t reg,
		      const bool read,
		      const bool before)
837
{
838 839 840
	if (WARN(check_for_unclaimed_mmio(dev_priv) && !before,
		 "Unclaimed %s register 0x%x\n",
		 read ? "read from" : "write to",
841
		 i915_mmio_reg_offset(reg)))
842
		i915.mmio_debug--; /* Only report the first N failures */
843 844
}

845 846 847 848 849 850 851 852 853 854 855 856
static inline void
unclaimed_reg_debug(struct drm_i915_private *dev_priv,
		    const i915_reg_t reg,
		    const bool read,
		    const bool before)
{
	if (likely(!i915.mmio_debug))
		return;

	__unclaimed_reg_debug(dev_priv, reg, read, before);
}

857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916
static const enum decoupled_power_domain fw2dpd_domain[] = {
	GEN9_DECOUPLED_PD_RENDER,
	GEN9_DECOUPLED_PD_BLITTER,
	GEN9_DECOUPLED_PD_ALL,
	GEN9_DECOUPLED_PD_MEDIA,
	GEN9_DECOUPLED_PD_ALL,
	GEN9_DECOUPLED_PD_ALL,
	GEN9_DECOUPLED_PD_ALL
};

/*
 * Decoupled MMIO access for only 1 DWORD
 */
static void __gen9_decoupled_mmio_access(struct drm_i915_private *dev_priv,
					 u32 reg,
					 enum forcewake_domains fw_domain,
					 enum decoupled_ops operation)
{
	enum decoupled_power_domain dp_domain;
	u32 ctrl_reg_data = 0;

	dp_domain = fw2dpd_domain[fw_domain - 1];

	ctrl_reg_data |= reg;
	ctrl_reg_data |= (operation << GEN9_DECOUPLED_OP_SHIFT);
	ctrl_reg_data |= (dp_domain << GEN9_DECOUPLED_PD_SHIFT);
	ctrl_reg_data |= GEN9_DECOUPLED_DW1_GO;
	__raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW1, ctrl_reg_data);

	if (wait_for_atomic((__raw_i915_read32(dev_priv,
			    GEN9_DECOUPLED_REG0_DW1) &
			    GEN9_DECOUPLED_DW1_GO) == 0,
			    FORCEWAKE_ACK_TIMEOUT_MS))
		DRM_ERROR("Decoupled MMIO wait timed out\n");
}

static inline u32
__gen9_decoupled_mmio_read32(struct drm_i915_private *dev_priv,
			     u32 reg,
			     enum forcewake_domains fw_domain)
{
	__gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
				     GEN9_DECOUPLED_OP_READ);

	return __raw_i915_read32(dev_priv, GEN9_DECOUPLED_REG0_DW0);
}

static inline void
__gen9_decoupled_mmio_write(struct drm_i915_private *dev_priv,
			    u32 reg, u32 data,
			    enum forcewake_domains fw_domain)
{

	__raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW0, data);

	__gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
				     GEN9_DECOUPLED_OP_WRITE);
}


917
#define GEN2_READ_HEADER(x) \
B
Ben Widawsky 已提交
918
	u##x val = 0; \
919
	assert_rpm_wakelock_held(dev_priv);
B
Ben Widawsky 已提交
920

921
#define GEN2_READ_FOOTER \
B
Ben Widawsky 已提交
922 923 924
	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
	return val

925
#define __gen2_read(x) \
926
static u##x \
927
gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
928
	GEN2_READ_HEADER(x); \
929
	val = __raw_i915_read##x(dev_priv, reg); \
930
	GEN2_READ_FOOTER; \
931 932 933 934
}

#define __gen5_read(x) \
static u##x \
935
gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
936
	GEN2_READ_HEADER(x); \
937 938
	ilk_dummy_write(dev_priv); \
	val = __raw_i915_read##x(dev_priv, reg); \
939
	GEN2_READ_FOOTER; \
940 941
}

942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957
__gen5_read(8)
__gen5_read(16)
__gen5_read(32)
__gen5_read(64)
__gen2_read(8)
__gen2_read(16)
__gen2_read(32)
__gen2_read(64)

#undef __gen5_read
#undef __gen2_read

#undef GEN2_READ_FOOTER
#undef GEN2_READ_HEADER

#define GEN6_READ_HEADER(x) \
958
	u32 offset = i915_mmio_reg_offset(reg); \
959 960
	unsigned long irqflags; \
	u##x val = 0; \
961
	assert_rpm_wakelock_held(dev_priv); \
962 963
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
	unclaimed_reg_debug(dev_priv, reg, true, true)
964 965

#define GEN6_READ_FOOTER \
966
	unclaimed_reg_debug(dev_priv, reg, true, false); \
967 968 969 970
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
	return val

971 972
static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv,
					enum forcewake_domains fw_domains)
973 974 975
{
	struct intel_uncore_forcewake_domain *domain;

976 977 978 979 980 981 982 983 984 985
	for_each_fw_domain_masked(domain, fw_domains, dev_priv)
		fw_domain_arm_timer(domain);

	dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
	dev_priv->uncore.fw_domains_active |= fw_domains;
}

static inline void __force_wake_auto(struct drm_i915_private *dev_priv,
				     enum forcewake_domains fw_domains)
{
986 987 988
	if (WARN_ON(!fw_domains))
		return;

989 990 991
	/* Turn on all requested but inactive supported forcewake domains. */
	fw_domains &= dev_priv->uncore.fw_domains;
	fw_domains &= ~dev_priv->uncore.fw_domains_active;
992

993 994
	if (fw_domains)
		___force_wake_auto(dev_priv, fw_domains);
995 996
}

997 998
#define __gen6_read(x) \
static u##x \
999
gen6_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1000
	enum forcewake_domains fw_engine; \
1001
	GEN6_READ_HEADER(x); \
1002 1003 1004
	fw_engine = __gen6_reg_read_fw_domains(offset); \
	if (fw_engine) \
		__force_wake_auto(dev_priv, fw_engine); \
1005
	val = __raw_i915_read##x(dev_priv, reg); \
1006
	GEN6_READ_FOOTER; \
1007 1008
}

1009
#define __fwtable_read(x) \
1010
static u##x \
1011
fwtable_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1012
	enum forcewake_domains fw_engine; \
1013
	GEN6_READ_HEADER(x); \
T
Tvrtko Ursulin 已提交
1014
	fw_engine = __fwtable_reg_read_fw_domains(offset); \
1015
	if (fw_engine) \
1016
		__force_wake_auto(dev_priv, fw_engine); \
1017
	val = __raw_i915_read##x(dev_priv, reg); \
1018
	GEN6_READ_FOOTER; \
1019 1020
}

1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
#define __gen9_decoupled_read(x) \
static u##x \
gen9_decoupled_read##x(struct drm_i915_private *dev_priv, \
		       i915_reg_t reg, bool trace) { \
	enum forcewake_domains fw_engine; \
	GEN6_READ_HEADER(x); \
	fw_engine = __fwtable_reg_read_fw_domains(offset); \
	if (fw_engine & ~dev_priv->uncore.fw_domains_active) { \
		unsigned i; \
		u32 *ptr_data = (u32 *) &val; \
		for (i = 0; i < x/32; i++, offset += sizeof(u32), ptr_data++) \
			*ptr_data = __gen9_decoupled_mmio_read32(dev_priv, \
								 offset, \
								 fw_engine); \
	} else { \
		val = __raw_i915_read##x(dev_priv, reg); \
	} \
	GEN6_READ_FOOTER; \
}

__gen9_decoupled_read(32)
__gen9_decoupled_read(64)
1043 1044 1045 1046
__fwtable_read(8)
__fwtable_read(16)
__fwtable_read(32)
__fwtable_read(64)
1047 1048 1049 1050 1051
__gen6_read(8)
__gen6_read(16)
__gen6_read(32)
__gen6_read(64)

1052
#undef __fwtable_read
1053
#undef __gen6_read
1054 1055
#undef GEN6_READ_FOOTER
#undef GEN6_READ_HEADER
B
Ben Widawsky 已提交
1056

1057 1058 1059
#define VGPU_READ_HEADER(x) \
	unsigned long irqflags; \
	u##x val = 0; \
1060
	assert_rpm_device_not_suspended(dev_priv); \
1061 1062 1063 1064 1065 1066 1067 1068 1069
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)

#define VGPU_READ_FOOTER \
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
	return val

#define __vgpu_read(x) \
static u##x \
1070
vgpu_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
	VGPU_READ_HEADER(x); \
	val = __raw_i915_read##x(dev_priv, reg); \
	VGPU_READ_FOOTER; \
}

__vgpu_read(8)
__vgpu_read(16)
__vgpu_read(32)
__vgpu_read(64)

#undef __vgpu_read
#undef VGPU_READ_FOOTER
#undef VGPU_READ_HEADER

1085
#define GEN2_WRITE_HEADER \
B
Ben Widawsky 已提交
1086
	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1087
	assert_rpm_wakelock_held(dev_priv); \
1088

1089
#define GEN2_WRITE_FOOTER
V
Ville Syrjälä 已提交
1090

1091
#define __gen2_write(x) \
1092
static void \
1093
gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1094
	GEN2_WRITE_HEADER; \
1095
	__raw_i915_write##x(dev_priv, reg, val); \
1096
	GEN2_WRITE_FOOTER; \
1097 1098 1099 1100
}

#define __gen5_write(x) \
static void \
1101
gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1102
	GEN2_WRITE_HEADER; \
1103 1104
	ilk_dummy_write(dev_priv); \
	__raw_i915_write##x(dev_priv, reg, val); \
1105
	GEN2_WRITE_FOOTER; \
1106 1107
}

1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
__gen5_write(8)
__gen5_write(16)
__gen5_write(32)
__gen2_write(8)
__gen2_write(16)
__gen2_write(32)

#undef __gen5_write
#undef __gen2_write

#undef GEN2_WRITE_FOOTER
#undef GEN2_WRITE_HEADER

#define GEN6_WRITE_HEADER \
1122
	u32 offset = i915_mmio_reg_offset(reg); \
1123 1124
	unsigned long irqflags; \
	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1125
	assert_rpm_wakelock_held(dev_priv); \
1126 1127
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
	unclaimed_reg_debug(dev_priv, reg, false, true)
1128 1129

#define GEN6_WRITE_FOOTER \
1130
	unclaimed_reg_debug(dev_priv, reg, false, false); \
1131 1132
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)

1133 1134
#define __gen6_write(x) \
static void \
1135
gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1136
	u32 __fifo_ret = 0; \
1137
	GEN6_WRITE_HEADER; \
1138
	if (NEEDS_FORCE_WAKE(offset)) { \
1139 1140 1141 1142 1143 1144
		__fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
	} \
	__raw_i915_write##x(dev_priv, reg, val); \
	if (unlikely(__fifo_ret)) { \
		gen6_gt_check_fifodbg(dev_priv); \
	} \
1145
	GEN6_WRITE_FOOTER; \
1146 1147
}

1148 1149
#define __gen8_write(x) \
static void \
1150
gen8_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1151
	enum forcewake_domains fw_engine; \
1152
	GEN6_WRITE_HEADER; \
1153 1154 1155
	fw_engine = __gen8_reg_write_fw_domains(offset); \
	if (fw_engine) \
		__force_wake_auto(dev_priv, fw_engine); \
1156
	__raw_i915_write##x(dev_priv, reg, val); \
1157
	GEN6_WRITE_FOOTER; \
1158 1159
}

1160
#define __fwtable_write(x) \
1161
static void \
1162
fwtable_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1163
	enum forcewake_domains fw_engine; \
1164
	GEN6_WRITE_HEADER; \
1165
	fw_engine = __fwtable_reg_write_fw_domains(offset); \
1166
	if (fw_engine) \
1167
		__force_wake_auto(dev_priv, fw_engine); \
1168
	__raw_i915_write##x(dev_priv, reg, val); \
1169
	GEN6_WRITE_FOOTER; \
1170 1171
}

1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190
#define __gen9_decoupled_write(x) \
static void \
gen9_decoupled_write##x(struct drm_i915_private *dev_priv, \
			i915_reg_t reg, u##x val, \
		bool trace) { \
	enum forcewake_domains fw_engine; \
	GEN6_WRITE_HEADER; \
	fw_engine = __fwtable_reg_write_fw_domains(offset); \
	if (fw_engine & ~dev_priv->uncore.fw_domains_active) \
		__gen9_decoupled_mmio_write(dev_priv, \
					    offset, \
					    val, \
					    fw_engine); \
	else \
		__raw_i915_write##x(dev_priv, reg, val); \
	GEN6_WRITE_FOOTER; \
}

__gen9_decoupled_write(32)
1191 1192 1193
__fwtable_write(8)
__fwtable_write(16)
__fwtable_write(32)
1194 1195 1196
__gen8_write(8)
__gen8_write(16)
__gen8_write(32)
1197 1198 1199 1200
__gen6_write(8)
__gen6_write(16)
__gen6_write(32)

1201
#undef __fwtable_write
1202
#undef __gen8_write
1203
#undef __gen6_write
1204 1205
#undef GEN6_WRITE_FOOTER
#undef GEN6_WRITE_HEADER
1206

1207 1208 1209
#define VGPU_WRITE_HEADER \
	unsigned long irqflags; \
	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1210
	assert_rpm_device_not_suspended(dev_priv); \
1211 1212 1213 1214 1215 1216 1217
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)

#define VGPU_WRITE_FOOTER \
	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)

#define __vgpu_write(x) \
static void vgpu_write##x(struct drm_i915_private *dev_priv, \
1218
			  i915_reg_t reg, u##x val, bool trace) { \
1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231
	VGPU_WRITE_HEADER; \
	__raw_i915_write##x(dev_priv, reg, val); \
	VGPU_WRITE_FOOTER; \
}

__vgpu_write(8)
__vgpu_write(16)
__vgpu_write(32)

#undef __vgpu_write
#undef VGPU_WRITE_FOOTER
#undef VGPU_WRITE_HEADER

1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246
#define ASSIGN_WRITE_MMIO_VFUNCS(x) \
do { \
	dev_priv->uncore.funcs.mmio_writeb = x##_write8; \
	dev_priv->uncore.funcs.mmio_writew = x##_write16; \
	dev_priv->uncore.funcs.mmio_writel = x##_write32; \
} while (0)

#define ASSIGN_READ_MMIO_VFUNCS(x) \
do { \
	dev_priv->uncore.funcs.mmio_readb = x##_read8; \
	dev_priv->uncore.funcs.mmio_readw = x##_read16; \
	dev_priv->uncore.funcs.mmio_readl = x##_read32; \
	dev_priv->uncore.funcs.mmio_readq = x##_read64; \
} while (0)

1247 1248

static void fw_domain_init(struct drm_i915_private *dev_priv,
1249
			   enum forcewake_domain_id domain_id,
1250 1251
			   i915_reg_t reg_set,
			   i915_reg_t reg_ack)
1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
{
	struct intel_uncore_forcewake_domain *d;

	if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT))
		return;

	d = &dev_priv->uncore.fw_domain[domain_id];

	WARN_ON(d->wake_count);

	d->wake_count = 0;
	d->reg_set = reg_set;
	d->reg_ack = reg_ack;

	if (IS_GEN6(dev_priv)) {
		d->val_reset = 0;
		d->val_set = FORCEWAKE_KERNEL;
		d->val_clear = 0;
	} else {
1271
		/* WaRsClearFWBitsAtReset:bdw,skl */
1272 1273 1274 1275 1276
		d->val_reset = _MASKED_BIT_DISABLE(0xffff);
		d->val_set = _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL);
		d->val_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL);
	}

1277
	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
1278 1279 1280 1281 1282 1283 1284
		d->reg_post = FORCEWAKE_ACK_VLV;
	else if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv) || IS_GEN8(dev_priv))
		d->reg_post = ECOBUS;

	d->i915 = dev_priv;
	d->id = domain_id;

1285 1286 1287 1288 1289 1290
	BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
	BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER));
	BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));

	d->mask = 1 << domain_id;

1291 1292
	hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	d->timer.function = intel_uncore_fw_release_timer;
1293 1294

	dev_priv->uncore.fw_domains |= (1 << domain_id);
1295 1296

	fw_domain_reset(d);
1297 1298
}

1299
static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
1300
{
1301
	if (INTEL_INFO(dev_priv)->gen <= 5)
1302 1303
		return;

1304
	if (IS_GEN9(dev_priv)) {
1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
			       FORCEWAKE_RENDER_GEN9,
			       FORCEWAKE_ACK_RENDER_GEN9);
		fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER,
			       FORCEWAKE_BLITTER_GEN9,
			       FORCEWAKE_ACK_BLITTER_GEN9);
		fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
			       FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
1315
	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
1316
		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
1317
		if (!IS_CHERRYVIEW(dev_priv))
1318 1319 1320 1321
			dev_priv->uncore.funcs.force_wake_put =
				fw_domains_put_with_fifo;
		else
			dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1322 1323 1324 1325
		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
			       FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
		fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
			       FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
1326
	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
1327 1328
		dev_priv->uncore.funcs.force_wake_get =
			fw_domains_get_with_thread_status;
1329
		if (IS_HASWELL(dev_priv))
1330 1331 1332 1333
			dev_priv->uncore.funcs.force_wake_put =
				fw_domains_put_with_fifo;
		else
			dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
1334 1335
		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
			       FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
1336
	} else if (IS_IVYBRIDGE(dev_priv)) {
1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
		u32 ecobus;

		/* IVB configs may use multi-threaded forcewake */

		/* A small trick here - if the bios hasn't configured
		 * MT forcewake, and if the device is in RC6, then
		 * force_wake_mt_get will not wake the device and the
		 * ECOBUS read will return zero. Which will be
		 * (correctly) interpreted by the test below as MT
		 * forcewake being disabled.
		 */
1348 1349 1350 1351 1352
		dev_priv->uncore.funcs.force_wake_get =
			fw_domains_get_with_thread_status;
		dev_priv->uncore.funcs.force_wake_put =
			fw_domains_put_with_fifo;

1353 1354
		/* We need to init first for ECOBUS access and then
		 * determine later if we want to reinit, in case of MT access is
1355 1356 1357
		 * not working. In this stage we don't know which flavour this
		 * ivb is, so it is better to reset also the gen6 fw registers
		 * before the ecobus check.
1358
		 */
1359 1360 1361 1362

		__raw_i915_write32(dev_priv, FORCEWAKE, 0);
		__raw_posting_read(dev_priv, ECOBUS);

1363 1364
		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
			       FORCEWAKE_MT, FORCEWAKE_MT_ACK);
1365

1366
		spin_lock_irq(&dev_priv->uncore.lock);
1367
		fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_ALL);
1368
		ecobus = __raw_i915_read32(dev_priv, ECOBUS);
1369
		fw_domains_put_with_fifo(dev_priv, FORCEWAKE_ALL);
1370
		spin_unlock_irq(&dev_priv->uncore.lock);
1371

1372
		if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
1373 1374
			DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
			DRM_INFO("when using vblank-synced partial screen updates.\n");
1375 1376
			fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
				       FORCEWAKE, FORCEWAKE_ACK);
1377
		}
1378
	} else if (IS_GEN6(dev_priv)) {
1379
		dev_priv->uncore.funcs.force_wake_get =
1380
			fw_domains_get_with_thread_status;
1381
		dev_priv->uncore.funcs.force_wake_put =
1382 1383 1384
			fw_domains_put_with_fifo;
		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
			       FORCEWAKE, FORCEWAKE_ACK);
1385
	}
1386 1387 1388

	/* All future platforms are expected to require complex power gating */
	WARN_ON(dev_priv->uncore.fw_domains == 0);
1389 1390
}

1391 1392 1393 1394 1395 1396 1397
#define ASSIGN_FW_DOMAINS_TABLE(d) \
{ \
	dev_priv->uncore.fw_domains_table = \
			(struct intel_forcewake_range *)(d); \
	dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \
}

1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423
static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
					 unsigned long action, void *data)
{
	struct drm_i915_private *dev_priv = container_of(nb,
			struct drm_i915_private, uncore.pmic_bus_access_nb);

	switch (action) {
	case MBI_PMIC_BUS_ACCESS_BEGIN:
		/*
		 * forcewake all now to make sure that we don't need to do a
		 * forcewake later which on systems where this notifier gets
		 * called requires the punit to access to the shared pmic i2c
		 * bus, which will be busy after this notification, leading to:
		 * "render: timed out waiting for forcewake ack request."
		 * errors.
		 */
		intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
		break;
	case MBI_PMIC_BUS_ACCESS_END:
		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
		break;
	}

	return NOTIFY_OK;
}

1424
void intel_uncore_init(struct drm_i915_private *dev_priv)
1425
{
1426
	i915_check_vgpu(dev_priv);
1427

1428
	intel_uncore_edram_detect(dev_priv);
1429 1430
	intel_uncore_fw_domains_init(dev_priv);
	__intel_uncore_early_sanitize(dev_priv, false);
1431

1432
	dev_priv->uncore.unclaimed_mmio_check = 1;
1433 1434
	dev_priv->uncore.pmic_bus_access_nb.notifier_call =
		i915_pmic_bus_access_notifier;
1435

1436
	switch (INTEL_INFO(dev_priv)->gen) {
1437
	default:
1438
	case 9:
1439
		ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
1440
		ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
1441
		ASSIGN_READ_MMIO_VFUNCS(fwtable);
1442 1443 1444 1445 1446 1447 1448 1449
		if (HAS_DECOUPLED_MMIO(dev_priv)) {
			dev_priv->uncore.funcs.mmio_readl =
						gen9_decoupled_read32;
			dev_priv->uncore.funcs.mmio_readq =
						gen9_decoupled_read64;
			dev_priv->uncore.funcs.mmio_writel =
						gen9_decoupled_write32;
		}
1450 1451
		break;
	case 8:
1452
		if (IS_CHERRYVIEW(dev_priv)) {
1453
			ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
1454
			ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
1455
			ASSIGN_READ_MMIO_VFUNCS(fwtable);
1456 1457

		} else {
1458 1459
			ASSIGN_WRITE_MMIO_VFUNCS(gen8);
			ASSIGN_READ_MMIO_VFUNCS(gen6);
1460
		}
1461
		break;
1462 1463
	case 7:
	case 6:
1464
		ASSIGN_WRITE_MMIO_VFUNCS(gen6);
1465

1466
		if (IS_VALLEYVIEW(dev_priv)) {
1467
			ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges);
1468
			ASSIGN_READ_MMIO_VFUNCS(fwtable);
1469
		} else {
1470
			ASSIGN_READ_MMIO_VFUNCS(gen6);
1471
		}
1472 1473
		break;
	case 5:
1474 1475
		ASSIGN_WRITE_MMIO_VFUNCS(gen5);
		ASSIGN_READ_MMIO_VFUNCS(gen5);
1476 1477 1478 1479
		break;
	case 4:
	case 3:
	case 2:
1480 1481
		ASSIGN_WRITE_MMIO_VFUNCS(gen2);
		ASSIGN_READ_MMIO_VFUNCS(gen2);
1482 1483
		break;
	}
1484

1485
	intel_fw_table_check(dev_priv);
1486 1487
	if (INTEL_GEN(dev_priv) >= 8)
		intel_shadow_table_check();
1488

1489
	if (intel_vgpu_active(dev_priv)) {
1490 1491 1492 1493
		ASSIGN_WRITE_MMIO_VFUNCS(vgpu);
		ASSIGN_READ_MMIO_VFUNCS(vgpu);
	}

1494 1495 1496
	iosf_mbi_register_pmic_bus_access_notifier(
		&dev_priv->uncore.pmic_bus_access_nb);

1497
	i915_check_and_clear_faults(dev_priv);
1498
}
1499 1500
#undef ASSIGN_WRITE_MMIO_VFUNCS
#undef ASSIGN_READ_MMIO_VFUNCS
1501

1502
void intel_uncore_fini(struct drm_i915_private *dev_priv)
1503
{
1504 1505 1506
	iosf_mbi_unregister_pmic_bus_access_notifier(
		&dev_priv->uncore.pmic_bus_access_nb);

1507
	/* Paranoia: make sure we have disabled everything before we exit. */
1508 1509
	intel_uncore_sanitize(dev_priv);
	intel_uncore_forcewake_reset(dev_priv, false);
1510 1511
}

1512
#define GEN_RANGE(l, h) GENMASK((h) - 1, (l) - 1)
1513

1514
static const struct register_whitelist {
1515
	i915_reg_t offset_ldw, offset_udw;
1516
	uint32_t size;
1517 1518
	/* supported gens, 0x10 for 4, 0x30 for 4 and 5, etc. */
	uint32_t gen_bitmask;
1519
} whitelist[] = {
1520 1521 1522
	{ .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE),
	  .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE),
	  .size = 8, .gen_bitmask = GEN_RANGE(4, 9) },
1523 1524 1525 1526 1527
};

int i915_reg_read_ioctl(struct drm_device *dev,
			void *data, struct drm_file *file)
{
1528
	struct drm_i915_private *dev_priv = to_i915(dev);
1529 1530
	struct drm_i915_reg_read *reg = data;
	struct register_whitelist const *entry = whitelist;
1531
	unsigned size;
1532
	i915_reg_t offset_ldw, offset_udw;
1533
	int i, ret = 0;
1534 1535

	for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
1536
		if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) &&
1537
		    (INTEL_INFO(dev_priv)->gen_mask & entry->gen_bitmask))
1538 1539 1540 1541 1542 1543
			break;
	}

	if (i == ARRAY_SIZE(whitelist))
		return -EINVAL;

1544 1545 1546 1547
	/* We use the low bits to encode extra flags as the register should
	 * be naturally aligned (and those that are not so aligned merely
	 * limit the available flags for that register).
	 */
1548 1549
	offset_ldw = entry->offset_ldw;
	offset_udw = entry->offset_udw;
1550
	size = entry->size;
1551
	size |= reg->offset ^ i915_mmio_reg_offset(offset_ldw);
1552

1553 1554
	intel_runtime_pm_get(dev_priv);

1555 1556
	switch (size) {
	case 8 | 1:
1557
		reg->val = I915_READ64_2x32(offset_ldw, offset_udw);
1558
		break;
1559
	case 8:
1560
		reg->val = I915_READ64(offset_ldw);
1561 1562
		break;
	case 4:
1563
		reg->val = I915_READ(offset_ldw);
1564 1565
		break;
	case 2:
1566
		reg->val = I915_READ16(offset_ldw);
1567 1568
		break;
	case 1:
1569
		reg->val = I915_READ8(offset_ldw);
1570 1571
		break;
	default:
1572 1573
		ret = -EINVAL;
		goto out;
1574 1575
	}

1576 1577 1578
out:
	intel_runtime_pm_put(dev_priv);
	return ret;
1579 1580
}

1581
static int i915_reset_complete(struct pci_dev *pdev)
1582 1583
{
	u8 gdrst;
1584
	pci_read_config_byte(pdev, I915_GDRST, &gdrst);
V
Ville Syrjälä 已提交
1585
	return (gdrst & GRDOM_RESET_STATUS) == 0;
1586 1587
}

1588
static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1589
{
1590
	struct pci_dev *pdev = dev_priv->drm.pdev;
1591

V
Ville Syrjälä 已提交
1592
	/* assert reset for at least 20 usec */
1593
	pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
V
Ville Syrjälä 已提交
1594
	udelay(20);
1595
	pci_write_config_byte(pdev, I915_GDRST, 0);
1596

1597
	return wait_for(i915_reset_complete(pdev), 500);
V
Ville Syrjälä 已提交
1598 1599
}

1600
static int g4x_reset_complete(struct pci_dev *pdev)
V
Ville Syrjälä 已提交
1601 1602
{
	u8 gdrst;
1603
	pci_read_config_byte(pdev, I915_GDRST, &gdrst);
V
Ville Syrjälä 已提交
1604
	return (gdrst & GRDOM_RESET_ENABLE) == 0;
1605 1606
}

1607
static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1608
{
1609
	struct pci_dev *pdev = dev_priv->drm.pdev;
1610 1611
	pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
	return wait_for(g4x_reset_complete(pdev), 500);
1612 1613
}

1614
static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1615
{
1616
	struct pci_dev *pdev = dev_priv->drm.pdev;
1617 1618
	int ret;

1619
	pci_write_config_byte(pdev, I915_GDRST,
1620
			      GRDOM_RENDER | GRDOM_RESET_ENABLE);
1621
	ret =  wait_for(g4x_reset_complete(pdev), 500);
1622 1623 1624 1625 1626 1627 1628
	if (ret)
		return ret;

	/* WaVcpClkGateDisableForMediaReset:ctg,elk */
	I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
	POSTING_READ(VDECCLK_GATE_D);

1629
	pci_write_config_byte(pdev, I915_GDRST,
1630
			      GRDOM_MEDIA | GRDOM_RESET_ENABLE);
1631
	ret =  wait_for(g4x_reset_complete(pdev), 500);
1632 1633 1634 1635 1636 1637 1638
	if (ret)
		return ret;

	/* WaVcpClkGateDisableForMediaReset:ctg,elk */
	I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
	POSTING_READ(VDECCLK_GATE_D);

1639
	pci_write_config_byte(pdev, I915_GDRST, 0);
1640 1641 1642 1643

	return 0;
}

1644 1645
static int ironlake_do_reset(struct drm_i915_private *dev_priv,
			     unsigned engine_mask)
1646 1647 1648
{
	int ret;

1649
	I915_WRITE(ILK_GDSR,
1650
		   ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
1651 1652 1653
	ret = intel_wait_for_register(dev_priv,
				      ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
				      500);
1654 1655 1656
	if (ret)
		return ret;

1657
	I915_WRITE(ILK_GDSR,
1658
		   ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
1659 1660 1661
	ret = intel_wait_for_register(dev_priv,
				      ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
				      500);
1662 1663 1664
	if (ret)
		return ret;

1665
	I915_WRITE(ILK_GDSR, 0);
1666 1667

	return 0;
1668 1669
}

1670 1671 1672
/* Reset the hardware domains (GENX_GRDOM_*) specified by mask */
static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
				u32 hw_domain_mask)
1673 1674 1675 1676 1677
{
	/* GEN6_GDRST is not in the gt power well, no need to check
	 * for fifo space for the write or forcewake the chip for
	 * the read
	 */
1678
	__raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask);
1679

1680
	/* Spin waiting for the device to ack the reset requests */
1681 1682 1683
	return intel_wait_for_register_fw(dev_priv,
					  GEN6_GDRST, hw_domain_mask, 0,
					  500);
1684 1685 1686 1687
}

/**
 * gen6_reset_engines - reset individual engines
1688
 * @dev_priv: i915 device
1689 1690 1691 1692 1693 1694 1695 1696 1697 1698
 * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset
 *
 * This function will reset the individual engines that are set in engine_mask.
 * If you provide ALL_ENGINES as mask, full global domain reset will be issued.
 *
 * Note: It is responsibility of the caller to handle the difference between
 * asking full domain reset versus reset for all available individual engines.
 *
 * Returns 0 on success, nonzero on error.
 */
1699 1700
static int gen6_reset_engines(struct drm_i915_private *dev_priv,
			      unsigned engine_mask)
1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715
{
	struct intel_engine_cs *engine;
	const u32 hw_engine_mask[I915_NUM_ENGINES] = {
		[RCS] = GEN6_GRDOM_RENDER,
		[BCS] = GEN6_GRDOM_BLT,
		[VCS] = GEN6_GRDOM_MEDIA,
		[VCS2] = GEN8_GRDOM_MEDIA2,
		[VECS] = GEN6_GRDOM_VECS,
	};
	u32 hw_mask;
	int ret;

	if (engine_mask == ALL_ENGINES) {
		hw_mask = GEN6_GRDOM_FULL;
	} else {
1716 1717
		unsigned int tmp;

1718
		hw_mask = 0;
1719
		for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1720 1721 1722 1723
			hw_mask |= hw_engine_mask[engine->id];
	}

	ret = gen6_hw_domain_reset(dev_priv, hw_mask);
1724

1725
	intel_uncore_forcewake_reset(dev_priv, true);
1726

1727 1728 1729
	return ret;
}

1730 1731 1732 1733 1734 1735 1736 1737 1738
/**
 * intel_wait_for_register_fw - wait until register matches expected state
 * @dev_priv: the i915 device
 * @reg: the register to read
 * @mask: mask to apply to register value
 * @value: expected value
 * @timeout_ms: timeout in millisecond
 *
 * This routine waits until the target register @reg contains the expected
1739 1740 1741 1742
 * @value after applying the @mask, i.e. it waits until ::
 *
 *     (I915_READ_FW(reg) & mask) == value
 *
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774
 * Otherwise, the wait will timeout after @timeout_ms milliseconds.
 *
 * Note that this routine assumes the caller holds forcewake asserted, it is
 * not suitable for very long waits. See intel_wait_for_register() if you
 * wish to wait without holding forcewake for the duration (i.e. you expect
 * the wait to be slow).
 *
 * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
 */
int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
			       i915_reg_t reg,
			       const u32 mask,
			       const u32 value,
			       const unsigned long timeout_ms)
{
#define done ((I915_READ_FW(reg) & mask) == value)
	int ret = wait_for_us(done, 2);
	if (ret)
		ret = wait_for(done, timeout_ms);
	return ret;
#undef done
}

/**
 * intel_wait_for_register - wait until register matches expected state
 * @dev_priv: the i915 device
 * @reg: the register to read
 * @mask: mask to apply to register value
 * @value: expected value
 * @timeout_ms: timeout in millisecond
 *
 * This routine waits until the target register @reg contains the expected
1775 1776 1777 1778
 * @value after applying the @mask, i.e. it waits until ::
 *
 *     (I915_READ(reg) & mask) == value
 *
1779 1780 1781 1782 1783 1784 1785 1786 1787
 * Otherwise, the wait will timeout after @timeout_ms milliseconds.
 *
 * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
 */
int intel_wait_for_register(struct drm_i915_private *dev_priv,
			    i915_reg_t reg,
			    const u32 mask,
			    const u32 value,
			    const unsigned long timeout_ms)
1788
{
1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801

	unsigned fw =
		intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
	int ret;

	intel_uncore_forcewake_get(dev_priv, fw);
	ret = wait_for_us((I915_READ_FW(reg) & mask) == value, 2);
	intel_uncore_forcewake_put(dev_priv, fw);
	if (ret)
		ret = wait_for((I915_READ_NOTRACE(reg) & mask) == value,
			       timeout_ms);

	return ret;
1802 1803 1804 1805
}

static int gen8_request_engine_reset(struct intel_engine_cs *engine)
{
1806
	struct drm_i915_private *dev_priv = engine->i915;
1807 1808 1809 1810 1811
	int ret;

	I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
		      _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET));

1812 1813 1814 1815 1816
	ret = intel_wait_for_register_fw(dev_priv,
					 RING_RESET_CTL(engine->mmio_base),
					 RESET_CTL_READY_TO_RESET,
					 RESET_CTL_READY_TO_RESET,
					 700);
1817 1818 1819 1820 1821 1822 1823 1824
	if (ret)
		DRM_ERROR("%s: reset request timeout\n", engine->name);

	return ret;
}

static void gen8_unrequest_engine_reset(struct intel_engine_cs *engine)
{
1825
	struct drm_i915_private *dev_priv = engine->i915;
1826 1827 1828

	I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
		      _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
1829 1830
}

1831 1832
static int gen8_reset_engines(struct drm_i915_private *dev_priv,
			      unsigned engine_mask)
1833 1834
{
	struct intel_engine_cs *engine;
1835
	unsigned int tmp;
1836

1837
	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1838
		if (gen8_request_engine_reset(engine))
1839 1840
			goto not_ready;

1841
	return gen6_reset_engines(dev_priv, engine_mask);
1842 1843

not_ready:
1844
	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1845
		gen8_unrequest_engine_reset(engine);
1846 1847 1848 1849

	return -EIO;
}

1850 1851 1852
typedef int (*reset_func)(struct drm_i915_private *, unsigned engine_mask);

static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv)
1853
{
1854 1855 1856
	if (!i915.reset)
		return NULL;

1857
	if (INTEL_INFO(dev_priv)->gen >= 8)
1858
		return gen8_reset_engines;
1859
	else if (INTEL_INFO(dev_priv)->gen >= 6)
1860
		return gen6_reset_engines;
1861
	else if (IS_GEN5(dev_priv))
1862
		return ironlake_do_reset;
1863
	else if (IS_G4X(dev_priv))
1864
		return g4x_do_reset;
1865
	else if (IS_G33(dev_priv) || IS_PINEVIEW(dev_priv))
1866
		return g33_do_reset;
1867
	else if (INTEL_INFO(dev_priv)->gen >= 3)
1868
		return i915_do_reset;
1869
	else
1870 1871 1872
		return NULL;
}

1873
int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
1874
{
1875
	reset_func reset;
1876
	int ret;
1877

1878
	reset = intel_get_gpu_reset(dev_priv);
1879
	if (reset == NULL)
1880
		return -ENODEV;
1881

1882 1883 1884 1885
	/* If the power well sleeps during the reset, the reset
	 * request may be dropped and never completes (causing -EIO).
	 */
	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1886
	ret = reset(dev_priv, engine_mask);
1887 1888 1889
	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);

	return ret;
1890 1891
}

1892
bool intel_has_gpu_reset(struct drm_i915_private *dev_priv)
1893
{
1894
	return intel_get_gpu_reset(dev_priv) != NULL;
1895 1896
}

1897 1898 1899 1900 1901
int intel_guc_reset(struct drm_i915_private *dev_priv)
{
	int ret;
	unsigned long irqflags;

1902
	if (!HAS_GUC(dev_priv))
1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915
		return -EINVAL;

	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);

	ret = gen6_hw_domain_reset(dev_priv, GEN9_GRDOM_GUC);

	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);

	return ret;
}

1916
bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv)
1917
{
1918
	return check_for_unclaimed_mmio(dev_priv);
1919
}
1920

1921
bool
1922 1923 1924 1925
intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv)
{
	if (unlikely(i915.mmio_debug ||
		     dev_priv->uncore.unclaimed_mmio_check <= 0))
1926
		return false;
1927 1928 1929 1930 1931 1932 1933

	if (unlikely(intel_uncore_unclaimed_mmio(dev_priv))) {
		DRM_DEBUG("Unclaimed register detected, "
			  "enabling oneshot unclaimed register reporting. "
			  "Please use i915.mmio_debug=N for more information.\n");
		i915.mmio_debug++;
		dev_priv->uncore.unclaimed_mmio_check--;
1934
		return true;
1935
	}
1936 1937

	return false;
1938
}
1939 1940 1941 1942 1943

static enum forcewake_domains
intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
				i915_reg_t reg)
{
T
Tvrtko Ursulin 已提交
1944
	u32 offset = i915_mmio_reg_offset(reg);
1945 1946
	enum forcewake_domains fw_domains;

T
Tvrtko Ursulin 已提交
1947 1948 1949 1950 1951 1952 1953
	if (HAS_FWTABLE(dev_priv)) {
		fw_domains = __fwtable_reg_read_fw_domains(offset);
	} else if (INTEL_GEN(dev_priv) >= 6) {
		fw_domains = __gen6_reg_read_fw_domains(offset);
	} else {
		WARN_ON(!IS_GEN(dev_priv, 2, 5));
		fw_domains = 0;
1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964
	}

	WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);

	return fw_domains;
}

static enum forcewake_domains
intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
				 i915_reg_t reg)
{
1965
	u32 offset = i915_mmio_reg_offset(reg);
1966 1967
	enum forcewake_domains fw_domains;

1968 1969 1970 1971 1972
	if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) {
		fw_domains = __fwtable_reg_write_fw_domains(offset);
	} else if (IS_GEN8(dev_priv)) {
		fw_domains = __gen8_reg_write_fw_domains(offset);
	} else if (IS_GEN(dev_priv, 6, 7)) {
1973
		fw_domains = FORCEWAKE_RENDER;
1974 1975 1976
	} else {
		WARN_ON(!IS_GEN(dev_priv, 2, 5));
		fw_domains = 0;
1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005
	}

	WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);

	return fw_domains;
}

/**
 * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
 * 				    a register
 * @dev_priv: pointer to struct drm_i915_private
 * @reg: register in question
 * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
 *
 * Returns a set of forcewake domains required to be taken with for example
 * intel_uncore_forcewake_get for the specified register to be accessible in the
 * specified mode (read, write or read/write) with raw mmio accessors.
 *
 * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
 * callers to do FIFO management on their own or risk losing writes.
 */
enum forcewake_domains
intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
			       i915_reg_t reg, unsigned int op)
{
	enum forcewake_domains fw_domains = 0;

	WARN_ON(!op);

T
Tvrtko Ursulin 已提交
2006 2007 2008
	if (intel_vgpu_active(dev_priv))
		return 0;

2009 2010 2011 2012 2013 2014 2015 2016
	if (op & FW_REG_READ)
		fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg);

	if (op & FW_REG_WRITE)
		fw_domains |= intel_uncore_forcewake_for_write(dev_priv, reg);

	return fw_domains;
}