i915_pmu.c 26.8 KB
Newer Older
1
/*
2
 * SPDX-License-Identifier: MIT
3
 *
4
 * Copyright © 2017-2018 Intel Corporation
5 6
 */

7
#include <linux/irq.h>
8
#include <linux/pm_runtime.h>
9 10

#include "gt/intel_engine.h"
11
#include "gt/intel_engine_pm.h"
12
#include "gt/intel_engine_user.h"
13
#include "gt/intel_gt_pm.h"
14

15
#include "i915_drv.h"
16 17
#include "i915_pmu.h"
#include "intel_pm.h"
18 19 20 21 22 23 24 25 26 27 28 29

/* Frequency for the sampling timer for events which need it. */
#define FREQUENCY 200
#define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY)

#define ENGINE_SAMPLE_MASK \
	(BIT(I915_SAMPLE_BUSY) | \
	 BIT(I915_SAMPLE_WAIT) | \
	 BIT(I915_SAMPLE_SEMA))

#define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS)

30
static cpumask_t i915_pmu_cpumask;
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79

static u8 engine_config_sample(u64 config)
{
	return config & I915_PMU_SAMPLE_MASK;
}

static u8 engine_event_sample(struct perf_event *event)
{
	return engine_config_sample(event->attr.config);
}

static u8 engine_event_class(struct perf_event *event)
{
	return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff;
}

static u8 engine_event_instance(struct perf_event *event)
{
	return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff;
}

static bool is_engine_config(u64 config)
{
	return config < __I915_PMU_OTHER(0);
}

static unsigned int config_enabled_bit(u64 config)
{
	if (is_engine_config(config))
		return engine_config_sample(config);
	else
		return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0));
}

static u64 config_enabled_mask(u64 config)
{
	return BIT_ULL(config_enabled_bit(config));
}

static bool is_engine_event(struct perf_event *event)
{
	return is_engine_config(event->attr.config);
}

static unsigned int event_enabled_bit(struct perf_event *event)
{
	return config_enabled_bit(event->attr.config);
}

80
static bool pmu_needs_timer(struct i915_pmu *pmu, bool gpu_active)
81
{
82
	struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu);
83 84 85 86 87 88 89
	u64 enable;

	/*
	 * Only some counters need the sampling timer.
	 *
	 * We start with a bitmask of all currently enabled events.
	 */
90
	enable = pmu->enable;
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105

	/*
	 * Mask out all the ones which do not need the timer, or in
	 * other words keep all the ones that could need the timer.
	 */
	enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) |
		  config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) |
		  ENGINE_SAMPLE_MASK;

	/*
	 * When the GPU is idle per-engine counters do not need to be
	 * running so clear those bits out.
	 */
	if (!gpu_active)
		enable &= ~ENGINE_SAMPLE_MASK;
106 107 108 109
	/*
	 * Also there is software busyness tracking available we do not
	 * need the timer for I915_SAMPLE_BUSY counter.
	 */
110
	else if (i915->caps.scheduler & I915_SCHEDULER_CAP_ENGINE_BUSY_STATS)
111
		enable &= ~BIT(I915_SAMPLE_BUSY);
112 113 114 115 116 117 118 119 120

	/*
	 * If some bits remain it means we need the sampling timer running.
	 */
	return enable;
}

void i915_pmu_gt_parked(struct drm_i915_private *i915)
{
121 122 123
	struct i915_pmu *pmu = &i915->pmu;

	if (!pmu->base.event_init)
124 125
		return;

126
	spin_lock_irq(&pmu->lock);
127 128 129 130
	/*
	 * Signal sampling timer to stop if only engine events are enabled and
	 * GPU went idle.
	 */
131 132
	pmu->timer_enabled = pmu_needs_timer(pmu, false);
	spin_unlock_irq(&pmu->lock);
133 134
}

135
static void __i915_pmu_maybe_start_timer(struct i915_pmu *pmu)
136
{
137 138 139 140
	if (!pmu->timer_enabled && pmu_needs_timer(pmu, true)) {
		pmu->timer_enabled = true;
		pmu->timer_last = ktime_get();
		hrtimer_start_range_ns(&pmu->timer,
141 142 143 144 145 146 147
				       ns_to_ktime(PERIOD), 0,
				       HRTIMER_MODE_REL_PINNED);
	}
}

void i915_pmu_gt_unparked(struct drm_i915_private *i915)
{
148 149 150
	struct i915_pmu *pmu = &i915->pmu;

	if (!pmu->base.event_init)
151 152
		return;

153
	spin_lock_irq(&pmu->lock);
154 155 156
	/*
	 * Re-enable sampling timer when GPU goes active.
	 */
157 158
	__i915_pmu_maybe_start_timer(pmu);
	spin_unlock_irq(&pmu->lock);
159 160
}

161
static void
162
add_sample(struct i915_pmu_sample *sample, u32 val)
163
{
164
	sample->cur += val;
165 166
}

167
static void
168
engines_sample(struct intel_gt *gt, unsigned int period_ns)
169
{
170
	struct drm_i915_private *i915 = gt->i915;
171 172 173
	struct intel_engine_cs *engine;
	enum intel_engine_id id;

174
	if ((i915->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
175 176
		return;

177
	for_each_engine(engine, i915, id) {
178
		struct intel_engine_pmu *pmu = &engine->pmu;
179
		unsigned long flags;
180
		bool busy;
181 182
		u32 val;

183 184 185 186 187
		if (!intel_engine_pm_get_if_awake(engine))
			continue;

		spin_lock_irqsave(&engine->uncore->lock, flags);

188
		val = ENGINE_READ_FW(engine, RING_CTL);
189
		if (val == 0) /* powerwell off => engine idle */
190
			goto skip;
191

192
		if (val & RING_WAIT)
193
			add_sample(&pmu->sample[I915_SAMPLE_WAIT], period_ns);
194
		if (val & RING_WAIT_SEMAPHORE)
195 196
			add_sample(&pmu->sample[I915_SAMPLE_SEMA], period_ns);

197 198 199 200
		/* No need to sample when busy stats are supported. */
		if (intel_engine_supports_stats(engine))
			goto skip;

201 202 203 204 205 206 207 208 209
		/*
		 * While waiting on a semaphore or event, MI_MODE reports the
		 * ring as idle. However, previously using the seqno, and with
		 * execlists sampling, we account for the ring waiting as the
		 * engine being busy. Therefore, we record the sample as being
		 * busy if either waiting or !idle.
		 */
		busy = val & (RING_WAIT_SEMAPHORE | RING_WAIT);
		if (!busy) {
210
			val = ENGINE_READ_FW(engine, RING_MI_MODE);
211 212 213 214
			busy = !(val & MODE_IDLE);
		}
		if (busy)
			add_sample(&pmu->sample[I915_SAMPLE_BUSY], period_ns);
215

216 217 218 219
skip:
		spin_unlock_irqrestore(&engine->uncore->lock, flags);
		intel_engine_pm_put(engine);
	}
220 221
}

222 223 224 225 226 227 228
static void
add_sample_mult(struct i915_pmu_sample *sample, u32 val, u32 mul)
{
	sample->cur += mul_u32_u32(val, mul);
}

static void
229
frequency_sample(struct intel_gt *gt, unsigned int period_ns)
230
{
231 232 233 234 235
	struct drm_i915_private *i915 = gt->i915;
	struct intel_uncore *uncore = gt->uncore;
	struct i915_pmu *pmu = &i915->pmu;

	if (pmu->enable & config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) {
236 237
		u32 val;

238
		val = i915->gt_pm.rps.cur_freq;
239 240 241 242
		if (intel_gt_pm_get_if_awake(gt)) {
			val = intel_uncore_read_notrace(uncore, GEN6_RPSTAT1);
			val = intel_get_cagf(i915, val);
			intel_gt_pm_put(gt);
243 244
		}

245 246
		add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_ACT],
				intel_gpu_freq(i915, val),
247
				period_ns / 1000);
248 249
	}

250 251 252
	if (pmu->enable & config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) {
		add_sample_mult(&pmu->sample[__I915_SAMPLE_FREQ_REQ],
				intel_gpu_freq(i915, i915->gt_pm.rps.cur_freq),
253
				period_ns / 1000);
254 255 256 257 258 259 260
	}
}

static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer)
{
	struct drm_i915_private *i915 =
		container_of(hrtimer, struct drm_i915_private, pmu.timer);
261
	struct i915_pmu *pmu = &i915->pmu;
262
	struct intel_gt *gt = &i915->gt;
263 264
	unsigned int period_ns;
	ktime_t now;
265

266
	if (!READ_ONCE(pmu->timer_enabled))
267 268
		return HRTIMER_NORESTART;

269
	now = ktime_get();
270 271
	period_ns = ktime_to_ns(ktime_sub(now, pmu->timer_last));
	pmu->timer_last = now;
272 273 274 275 276 277 278

	/*
	 * Strictly speaking the passed in period may not be 100% accurate for
	 * all internal calculation, since some amount of time can be spent on
	 * grabbing the forcewake. However the potential error from timer call-
	 * back delay greatly dominates this so we keep it simple.
	 */
279 280
	engines_sample(gt, period_ns);
	frequency_sample(gt, period_ns);
281 282

	hrtimer_forward(hrtimer, now, ns_to_ktime(PERIOD));
283 284 285 286

	return HRTIMER_RESTART;
}

287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
static u64 count_interrupts(struct drm_i915_private *i915)
{
	/* open-coded kstat_irqs() */
	struct irq_desc *desc = irq_to_desc(i915->drm.pdev->irq);
	u64 sum = 0;
	int cpu;

	if (!desc || !desc->kstat_irqs)
		return 0;

	for_each_possible_cpu(cpu)
		sum += *per_cpu_ptr(desc->kstat_irqs, cpu);

	return sum;
}

303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319
static void engine_event_destroy(struct perf_event *event)
{
	struct drm_i915_private *i915 =
		container_of(event->pmu, typeof(*i915), pmu.base);
	struct intel_engine_cs *engine;

	engine = intel_engine_lookup_user(i915,
					  engine_event_class(event),
					  engine_event_instance(event));
	if (WARN_ON_ONCE(!engine))
		return;

	if (engine_event_sample(event) == I915_SAMPLE_BUSY &&
	    intel_engine_supports_stats(engine))
		intel_disable_engine_stats(engine);
}

320 321 322
static void i915_pmu_event_destroy(struct perf_event *event)
{
	WARN_ON(event->parent);
323 324 325

	if (is_engine_event(event))
		engine_event_destroy(event);
326 327
}

328 329 330
static int
engine_event_status(struct intel_engine_cs *engine,
		    enum drm_i915_pmu_engine_sample sample)
331
{
332
	switch (sample) {
333 334 335 336
	case I915_SAMPLE_BUSY:
	case I915_SAMPLE_WAIT:
		break;
	case I915_SAMPLE_SEMA:
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
		if (INTEL_GEN(engine->i915) < 6)
			return -ENODEV;
		break;
	default:
		return -ENOENT;
	}

	return 0;
}

static int
config_status(struct drm_i915_private *i915, u64 config)
{
	switch (config) {
	case I915_PMU_ACTUAL_FREQUENCY:
		if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
			/* Requires a mutex for sampling! */
			return -ENODEV;
		/* Fall-through. */
	case I915_PMU_REQUESTED_FREQUENCY:
357 358 359
		if (INTEL_GEN(i915) < 6)
			return -ENODEV;
		break;
360 361 362 363 364 365
	case I915_PMU_INTERRUPTS:
		break;
	case I915_PMU_RC6_RESIDENCY:
		if (!HAS_RC6(i915))
			return -ENODEV;
		break;
366 367 368 369 370 371 372
	default:
		return -ENOENT;
	}

	return 0;
}

373 374 375 376 377
static int engine_event_init(struct perf_event *event)
{
	struct drm_i915_private *i915 =
		container_of(event->pmu, typeof(*i915), pmu.base);
	struct intel_engine_cs *engine;
378 379
	u8 sample;
	int ret;
380 381 382 383 384 385

	engine = intel_engine_lookup_user(i915, engine_event_class(event),
					  engine_event_instance(event));
	if (!engine)
		return -ENODEV;

386 387 388 389 390 391 392 393 394
	sample = engine_event_sample(event);
	ret = engine_event_status(engine, sample);
	if (ret)
		return ret;

	if (sample == I915_SAMPLE_BUSY && intel_engine_supports_stats(engine))
		ret = intel_enable_engine_stats(engine);

	return ret;
395 396
}

397 398 399 400
static int i915_pmu_event_init(struct perf_event *event)
{
	struct drm_i915_private *i915 =
		container_of(event->pmu, typeof(*i915), pmu.base);
401
	int ret;
402 403 404 405 406 407 408 409 410 411 412 413 414 415

	if (event->attr.type != event->pmu->type)
		return -ENOENT;

	/* unsupported modes and filters */
	if (event->attr.sample_period) /* no sampling */
		return -EINVAL;

	if (has_branch_stack(event))
		return -EOPNOTSUPP;

	if (event->cpu < 0)
		return -EINVAL;

416 417
	/* only allow running on one cpu at a time */
	if (!cpumask_test_cpu(event->cpu, &i915_pmu_cpumask))
418
		return -EINVAL;
419

420
	if (is_engine_event(event))
421
		ret = engine_event_init(event);
422 423
	else
		ret = config_status(i915, event->attr.config);
424 425 426 427 428 429 430 431 432
	if (ret)
		return ret;

	if (!event->parent)
		event->destroy = i915_pmu_event_destroy;

	return 0;
}

433
static u64 __get_rc6(struct intel_gt *gt)
434
{
435
	struct drm_i915_private *i915 = gt->i915;
436 437
	u64 val;

438 439 440 441
	val = intel_rc6_residency_ns(i915,
				     IS_VALLEYVIEW(i915) ?
				     VLV_GT_RENDER_RC6 :
				     GEN6_GT_GFX_RC6);
442

443 444 445 446 447 448 449 450
	if (HAS_RC6p(i915))
		val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6p);

	if (HAS_RC6pp(i915))
		val += intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6pp);

	return val;
}
451

452
static u64 get_rc6(struct intel_gt *gt)
453 454
{
#if IS_ENABLED(CONFIG_PM)
455
	struct drm_i915_private *i915 = gt->i915;
456
	struct intel_runtime_pm *rpm = &i915->runtime_pm;
457
	struct i915_pmu *pmu = &i915->pmu;
C
Chris Wilson 已提交
458
	intel_wakeref_t wakeref;
459 460
	unsigned long flags;
	u64 val;
461

462
	wakeref = intel_runtime_pm_get_if_in_use(rpm);
C
Chris Wilson 已提交
463
	if (wakeref) {
464
		val = __get_rc6(gt);
465
		intel_runtime_pm_put(rpm, wakeref);
466 467 468 469 470 471 472

		/*
		 * If we are coming back from being runtime suspended we must
		 * be careful not to report a larger value than returned
		 * previously.
		 */

473
		spin_lock_irqsave(&pmu->lock, flags);
474

475 476 477
		if (val >= pmu->sample[__I915_SAMPLE_RC6_ESTIMATED].cur) {
			pmu->sample[__I915_SAMPLE_RC6_ESTIMATED].cur = 0;
			pmu->sample[__I915_SAMPLE_RC6].cur = val;
478
		} else {
479
			val = pmu->sample[__I915_SAMPLE_RC6_ESTIMATED].cur;
480 481
		}

482
		spin_unlock_irqrestore(&pmu->lock, flags);
483
	} else {
484
		struct device *kdev = rpm->kdev;
485 486 487 488 489 490 491 492

		/*
		 * We are runtime suspended.
		 *
		 * Report the delta from when the device was suspended to now,
		 * on top of the last known real value, as the approximated RC6
		 * counter value.
		 */
493
		spin_lock_irqsave(&pmu->lock, flags);
494

495 496 497 498 499 500 501 502 503 504 505
		/*
		 * After the above branch intel_runtime_pm_get_if_in_use failed
		 * to get the runtime PM reference we cannot assume we are in
		 * runtime suspend since we can either: a) race with coming out
		 * of it before we took the power.lock, or b) there are other
		 * states than suspended which can bring us here.
		 *
		 * We need to double-check that we are indeed currently runtime
		 * suspended and if not we cannot do better than report the last
		 * known RC6 value.
		 */
506 507
		if (pm_runtime_status_suspended(kdev)) {
			val = pm_runtime_suspended_time(kdev);
508

509 510
			if (!pmu->sample[__I915_SAMPLE_RC6_ESTIMATED].cur)
				pmu->suspended_time_last = val;
511

512 513
			val -= pmu->suspended_time_last;
			val += pmu->sample[__I915_SAMPLE_RC6].cur;
514

515 516 517
			pmu->sample[__I915_SAMPLE_RC6_ESTIMATED].cur = val;
		} else if (pmu->sample[__I915_SAMPLE_RC6_ESTIMATED].cur) {
			val = pmu->sample[__I915_SAMPLE_RC6_ESTIMATED].cur;
518
		} else {
519
			val = pmu->sample[__I915_SAMPLE_RC6].cur;
520
		}
521

522
		spin_unlock_irqrestore(&pmu->lock, flags);
523 524 525
	}

	return val;
526
#else
527
	return __get_rc6(gt);
528
#endif
529 530
}

531
static u64 __i915_pmu_event_read(struct perf_event *event)
532 533 534
{
	struct drm_i915_private *i915 =
		container_of(event->pmu, typeof(*i915), pmu.base);
535
	struct i915_pmu *pmu = &i915->pmu;
536 537 538 539 540 541 542 543 544 545 546 547
	u64 val = 0;

	if (is_engine_event(event)) {
		u8 sample = engine_event_sample(event);
		struct intel_engine_cs *engine;

		engine = intel_engine_lookup_user(i915,
						  engine_event_class(event),
						  engine_event_instance(event));

		if (WARN_ON_ONCE(!engine)) {
			/* Do nothing */
548
		} else if (sample == I915_SAMPLE_BUSY &&
549
			   intel_engine_supports_stats(engine)) {
550
			val = ktime_to_ns(intel_engine_get_busy_time(engine));
551 552 553 554 555 556 557
		} else {
			val = engine->pmu.sample[sample].cur;
		}
	} else {
		switch (event->attr.config) {
		case I915_PMU_ACTUAL_FREQUENCY:
			val =
558
			   div_u64(pmu->sample[__I915_SAMPLE_FREQ_ACT].cur,
559
				   USEC_PER_SEC /* to MHz */);
560 561 562
			break;
		case I915_PMU_REQUESTED_FREQUENCY:
			val =
563
			   div_u64(pmu->sample[__I915_SAMPLE_FREQ_REQ].cur,
564
				   USEC_PER_SEC /* to MHz */);
565
			break;
566 567 568
		case I915_PMU_INTERRUPTS:
			val = count_interrupts(i915);
			break;
569
		case I915_PMU_RC6_RESIDENCY:
570
			val = get_rc6(&i915->gt);
571
			break;
572 573 574 575 576 577 578 579 580 581 582 583 584
		}
	}

	return val;
}

static void i915_pmu_event_read(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;
	u64 prev, new;

again:
	prev = local64_read(&hwc->prev_count);
585
	new = __i915_pmu_event_read(event);
586 587 588 589 590 591 592 593 594 595 596 597

	if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev)
		goto again;

	local64_add(new - prev, &event->count);
}

static void i915_pmu_enable(struct perf_event *event)
{
	struct drm_i915_private *i915 =
		container_of(event->pmu, typeof(*i915), pmu.base);
	unsigned int bit = event_enabled_bit(event);
598
	struct i915_pmu *pmu = &i915->pmu;
599 600
	unsigned long flags;

601
	spin_lock_irqsave(&pmu->lock, flags);
602 603 604 605 606

	/*
	 * Update the bitmask of enabled events and increment
	 * the event reference counter.
	 */
607 608 609 610 611
	BUILD_BUG_ON(ARRAY_SIZE(pmu->enable_count) != I915_PMU_MASK_BITS);
	GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count));
	GEM_BUG_ON(pmu->enable_count[bit] == ~0);
	pmu->enable |= BIT_ULL(bit);
	pmu->enable_count[bit]++;
612

613 614 615
	/*
	 * Start the sampling timer if needed and not already enabled.
	 */
616
	__i915_pmu_maybe_start_timer(pmu);
617

618 619 620 621 622 623 624 625 626 627 628 629
	/*
	 * For per-engine events the bitmask and reference counting
	 * is stored per engine.
	 */
	if (is_engine_event(event)) {
		u8 sample = engine_event_sample(event);
		struct intel_engine_cs *engine;

		engine = intel_engine_lookup_user(i915,
						  engine_event_class(event),
						  engine_event_instance(event));

630 631 632 633 634 635
		BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.enable_count) !=
			     I915_ENGINE_SAMPLE_COUNT);
		BUILD_BUG_ON(ARRAY_SIZE(engine->pmu.sample) !=
			     I915_ENGINE_SAMPLE_COUNT);
		GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
		GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
636
		GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0);
637 638

		engine->pmu.enable |= BIT(sample);
639
		engine->pmu.enable_count[sample]++;
640 641
	}

642
	spin_unlock_irqrestore(&pmu->lock, flags);
643

644 645 646 647 648
	/*
	 * Store the current counter value so we can report the correct delta
	 * for all listeners. Even when the event was already enabled and has
	 * an existing non-zero value.
	 */
649
	local64_set(&event->hw.prev_count, __i915_pmu_event_read(event));
650 651 652 653 654 655 656
}

static void i915_pmu_disable(struct perf_event *event)
{
	struct drm_i915_private *i915 =
		container_of(event->pmu, typeof(*i915), pmu.base);
	unsigned int bit = event_enabled_bit(event);
657
	struct i915_pmu *pmu = &i915->pmu;
658 659
	unsigned long flags;

660
	spin_lock_irqsave(&pmu->lock, flags);
661 662 663 664 665 666 667 668

	if (is_engine_event(event)) {
		u8 sample = engine_event_sample(event);
		struct intel_engine_cs *engine;

		engine = intel_engine_lookup_user(i915,
						  engine_event_class(event),
						  engine_event_instance(event));
669 670 671

		GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.enable_count));
		GEM_BUG_ON(sample >= ARRAY_SIZE(engine->pmu.sample));
672
		GEM_BUG_ON(engine->pmu.enable_count[sample] == 0);
673

674 675 676 677
		/*
		 * Decrement the reference count and clear the enabled
		 * bitmask when the last listener on an event goes away.
		 */
678
		if (--engine->pmu.enable_count[sample] == 0)
679 680 681
			engine->pmu.enable &= ~BIT(sample);
	}

682 683
	GEM_BUG_ON(bit >= ARRAY_SIZE(pmu->enable_count));
	GEM_BUG_ON(pmu->enable_count[bit] == 0);
684 685 686 687
	/*
	 * Decrement the reference count and clear the enabled
	 * bitmask when the last listener on an event goes away.
	 */
688 689 690
	if (--pmu->enable_count[bit] == 0) {
		pmu->enable &= ~BIT_ULL(bit);
		pmu->timer_enabled &= pmu_needs_timer(pmu, true);
691
	}
692

693
	spin_unlock_irqrestore(&pmu->lock, flags);
694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727
}

static void i915_pmu_event_start(struct perf_event *event, int flags)
{
	i915_pmu_enable(event);
	event->hw.state = 0;
}

static void i915_pmu_event_stop(struct perf_event *event, int flags)
{
	if (flags & PERF_EF_UPDATE)
		i915_pmu_event_read(event);
	i915_pmu_disable(event);
	event->hw.state = PERF_HES_STOPPED;
}

static int i915_pmu_event_add(struct perf_event *event, int flags)
{
	if (flags & PERF_EF_START)
		i915_pmu_event_start(event, flags);

	return 0;
}

static void i915_pmu_event_del(struct perf_event *event, int flags)
{
	i915_pmu_event_stop(event, PERF_EF_UPDATE);
}

static int i915_pmu_event_event_idx(struct perf_event *event)
{
	return 0;
}

728 729 730 731 732
struct i915_str_attribute {
	struct device_attribute attr;
	const char *str;
};

733 734 735
static ssize_t i915_pmu_format_show(struct device *dev,
				    struct device_attribute *attr, char *buf)
{
736
	struct i915_str_attribute *eattr;
737

738 739
	eattr = container_of(attr, struct i915_str_attribute, attr);
	return sprintf(buf, "%s\n", eattr->str);
740 741 742
}

#define I915_PMU_FORMAT_ATTR(_name, _config) \
743
	(&((struct i915_str_attribute[]) { \
744
		{ .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \
745
		  .str = _config, } \
746 747 748 749 750 751 752 753 754 755 756 757
	})[0].attr.attr)

static struct attribute *i915_pmu_format_attrs[] = {
	I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"),
	NULL,
};

static const struct attribute_group i915_pmu_format_attr_group = {
	.name = "format",
	.attrs = i915_pmu_format_attrs,
};

758 759 760 761 762
struct i915_ext_attribute {
	struct device_attribute attr;
	unsigned long val;
};

763 764 765
static ssize_t i915_pmu_event_show(struct device *dev,
				   struct device_attribute *attr, char *buf)
{
766
	struct i915_ext_attribute *eattr;
767

768 769
	eattr = container_of(attr, struct i915_ext_attribute, attr);
	return sprintf(buf, "config=0x%lx\n", eattr->val);
770 771
}

772
static struct attribute_group i915_pmu_events_attr_group = {
773
	.name = "events",
774
	/* Patch in attrs at runtime. */
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
};

static ssize_t
i915_pmu_get_attr_cpumask(struct device *dev,
			  struct device_attribute *attr,
			  char *buf)
{
	return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask);
}

static DEVICE_ATTR(cpumask, 0444, i915_pmu_get_attr_cpumask, NULL);

static struct attribute *i915_cpumask_attrs[] = {
	&dev_attr_cpumask.attr,
	NULL,
};

792
static const struct attribute_group i915_pmu_cpumask_attr_group = {
793 794 795 796 797 798 799 800 801 802
	.attrs = i915_cpumask_attrs,
};

static const struct attribute_group *i915_pmu_attr_groups[] = {
	&i915_pmu_format_attr_group,
	&i915_pmu_events_attr_group,
	&i915_pmu_cpumask_attr_group,
	NULL
};

803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818
#define __event(__config, __name, __unit) \
{ \
	.config = (__config), \
	.name = (__name), \
	.unit = (__unit), \
}

#define __engine_event(__sample, __name) \
{ \
	.sample = (__sample), \
	.name = (__name), \
}

static struct i915_ext_attribute *
add_i915_attr(struct i915_ext_attribute *attr, const char *name, u64 config)
{
819
	sysfs_attr_init(&attr->attr.attr);
820 821 822 823 824 825 826 827 828 829 830 831
	attr->attr.attr.name = name;
	attr->attr.attr.mode = 0444;
	attr->attr.show = i915_pmu_event_show;
	attr->val = config;

	return ++attr;
}

static struct perf_pmu_events_attr *
add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name,
	     const char *str)
{
832
	sysfs_attr_init(&attr->attr.attr);
833 834 835 836 837 838 839 840 841
	attr->attr.attr.name = name;
	attr->attr.attr.mode = 0444;
	attr->attr.show = perf_event_sysfs_show;
	attr->event_str = str;

	return ++attr;
}

static struct attribute **
842
create_event_attributes(struct i915_pmu *pmu)
843
{
844
	struct drm_i915_private *i915 = container_of(pmu, typeof(*i915), pmu);
845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875
	static const struct {
		u64 config;
		const char *name;
		const char *unit;
	} events[] = {
		__event(I915_PMU_ACTUAL_FREQUENCY, "actual-frequency", "MHz"),
		__event(I915_PMU_REQUESTED_FREQUENCY, "requested-frequency", "MHz"),
		__event(I915_PMU_INTERRUPTS, "interrupts", NULL),
		__event(I915_PMU_RC6_RESIDENCY, "rc6-residency", "ns"),
	};
	static const struct {
		enum drm_i915_pmu_engine_sample sample;
		char *name;
	} engine_events[] = {
		__engine_event(I915_SAMPLE_BUSY, "busy"),
		__engine_event(I915_SAMPLE_SEMA, "sema"),
		__engine_event(I915_SAMPLE_WAIT, "wait"),
	};
	unsigned int count = 0;
	struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter;
	struct i915_ext_attribute *i915_attr = NULL, *i915_iter;
	struct attribute **attr = NULL, **attr_iter;
	struct intel_engine_cs *engine;
	unsigned int i;

	/* Count how many counters we will be exposing. */
	for (i = 0; i < ARRAY_SIZE(events); i++) {
		if (!config_status(i915, events[i].config))
			count++;
	}

876
	for_each_uabi_engine(engine, i915) {
877 878 879 880 881 882 883 884
		for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
			if (!engine_event_status(engine,
						 engine_events[i].sample))
				count++;
		}
	}

	/* Allocate attribute objects and table. */
885
	i915_attr = kcalloc(count, sizeof(*i915_attr), GFP_KERNEL);
886 887 888
	if (!i915_attr)
		goto err_alloc;

889
	pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL);
890 891 892 893
	if (!pmu_attr)
		goto err_alloc;

	/* Max one pointer of each attribute type plus a termination entry. */
894
	attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL);
895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926
	if (!attr)
		goto err_alloc;

	i915_iter = i915_attr;
	pmu_iter = pmu_attr;
	attr_iter = attr;

	/* Initialize supported non-engine counters. */
	for (i = 0; i < ARRAY_SIZE(events); i++) {
		char *str;

		if (config_status(i915, events[i].config))
			continue;

		str = kstrdup(events[i].name, GFP_KERNEL);
		if (!str)
			goto err;

		*attr_iter++ = &i915_iter->attr.attr;
		i915_iter = add_i915_attr(i915_iter, str, events[i].config);

		if (events[i].unit) {
			str = kasprintf(GFP_KERNEL, "%s.unit", events[i].name);
			if (!str)
				goto err;

			*attr_iter++ = &pmu_iter->attr.attr;
			pmu_iter = add_pmu_attr(pmu_iter, str, events[i].unit);
		}
	}

	/* Initialize supported engine counters. */
927
	for_each_uabi_engine(engine, i915) {
928 929 930 931 932 933 934 935 936 937 938 939 940 941 942
		for (i = 0; i < ARRAY_SIZE(engine_events); i++) {
			char *str;

			if (engine_event_status(engine,
						engine_events[i].sample))
				continue;

			str = kasprintf(GFP_KERNEL, "%s-%s",
					engine->name, engine_events[i].name);
			if (!str)
				goto err;

			*attr_iter++ = &i915_iter->attr.attr;
			i915_iter =
				add_i915_attr(i915_iter, str,
943
					      __I915_PMU_ENGINE(engine->uabi_class,
944
								engine->uabi_instance,
945 946 947 948 949 950 951 952 953 954 955 956
								engine_events[i].sample));

			str = kasprintf(GFP_KERNEL, "%s-%s.unit",
					engine->name, engine_events[i].name);
			if (!str)
				goto err;

			*attr_iter++ = &pmu_iter->attr.attr;
			pmu_iter = add_pmu_attr(pmu_iter, str, "ns");
		}
	}

957 958
	pmu->i915_attr = i915_attr;
	pmu->pmu_attr = pmu_attr;
959 960 961 962 963 964 965 966 967 968 969 970 971 972 973

	return attr;

err:;
	for (attr_iter = attr; *attr_iter; attr_iter++)
		kfree((*attr_iter)->name);

err_alloc:
	kfree(attr);
	kfree(i915_attr);
	kfree(pmu_attr);

	return NULL;
}

974
static void free_event_attributes(struct i915_pmu *pmu)
975 976 977 978 979 980 981
{
	struct attribute **attr_iter = i915_pmu_events_attr_group.attrs;

	for (; *attr_iter; attr_iter++)
		kfree((*attr_iter)->name);

	kfree(i915_pmu_events_attr_group.attrs);
982 983
	kfree(pmu->i915_attr);
	kfree(pmu->pmu_attr);
984 985

	i915_pmu_events_attr_group.attrs = NULL;
986 987
	pmu->i915_attr = NULL;
	pmu->pmu_attr = NULL;
988 989
}

990 991 992 993 994 995 996
static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
{
	struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);

	GEM_BUG_ON(!pmu->base.event_init);

	/* Select the first online CPU as a designated reader. */
997
	if (!cpumask_weight(&i915_pmu_cpumask))
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
		cpumask_set_cpu(cpu, &i915_pmu_cpumask);

	return 0;
}

static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node)
{
	struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
	unsigned int target;

	GEM_BUG_ON(!pmu->base.event_init);

	if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) {
		target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
		/* Migrate events if there is a valid target */
		if (target < nr_cpu_ids) {
			cpumask_set_cpu(target, &i915_pmu_cpumask);
			perf_pmu_migrate_context(&pmu->base, cpu, target);
		}
	}

	return 0;
}

static enum cpuhp_state cpuhp_slot = CPUHP_INVALID;

1024
static int i915_pmu_register_cpuhp_state(struct i915_pmu *pmu)
1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036
{
	enum cpuhp_state slot;
	int ret;

	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
				      "perf/x86/intel/i915:online",
				      i915_pmu_cpu_online,
				      i915_pmu_cpu_offline);
	if (ret < 0)
		return ret;

	slot = ret;
1037
	ret = cpuhp_state_add_instance(slot, &pmu->node);
1038 1039 1040 1041 1042 1043 1044 1045 1046
	if (ret) {
		cpuhp_remove_multi_state(slot);
		return ret;
	}

	cpuhp_slot = slot;
	return 0;
}

1047
static void i915_pmu_unregister_cpuhp_state(struct i915_pmu *pmu)
1048 1049
{
	WARN_ON(cpuhp_slot == CPUHP_INVALID);
1050
	WARN_ON(cpuhp_state_remove_instance(cpuhp_slot, &pmu->node));
1051 1052 1053 1054 1055
	cpuhp_remove_multi_state(cpuhp_slot);
}

void i915_pmu_register(struct drm_i915_private *i915)
{
1056
	struct i915_pmu *pmu = &i915->pmu;
1057 1058 1059
	int ret;

	if (INTEL_GEN(i915) <= 2) {
1060
		dev_info(i915->drm.dev, "PMU not supported for this GPU.");
1061 1062 1063
		return;
	}

1064
	i915_pmu_events_attr_group.attrs = create_event_attributes(pmu);
1065 1066 1067 1068 1069
	if (!i915_pmu_events_attr_group.attrs) {
		ret = -ENOMEM;
		goto err;
	}

1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
	pmu->base.attr_groups	= i915_pmu_attr_groups;
	pmu->base.task_ctx_nr	= perf_invalid_context;
	pmu->base.event_init	= i915_pmu_event_init;
	pmu->base.add		= i915_pmu_event_add;
	pmu->base.del		= i915_pmu_event_del;
	pmu->base.start		= i915_pmu_event_start;
	pmu->base.stop		= i915_pmu_event_stop;
	pmu->base.read		= i915_pmu_event_read;
	pmu->base.event_idx	= i915_pmu_event_event_idx;

	spin_lock_init(&pmu->lock);
	hrtimer_init(&pmu->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	pmu->timer.function = i915_sample;

	ret = perf_pmu_register(&pmu->base, "i915", -1);
1085 1086 1087
	if (ret)
		goto err;

1088
	ret = i915_pmu_register_cpuhp_state(pmu);
1089 1090 1091 1092 1093 1094
	if (ret)
		goto err_unreg;

	return;

err_unreg:
1095
	perf_pmu_unregister(&pmu->base);
1096
err:
1097 1098
	pmu->base.event_init = NULL;
	free_event_attributes(pmu);
1099 1100 1101 1102 1103
	DRM_NOTE("Failed to register PMU! (err=%d)\n", ret);
}

void i915_pmu_unregister(struct drm_i915_private *i915)
{
1104 1105 1106
	struct i915_pmu *pmu = &i915->pmu;

	if (!pmu->base.event_init)
1107 1108
		return;

1109
	WARN_ON(pmu->enable);
1110

1111
	hrtimer_cancel(&pmu->timer);
1112

1113
	i915_pmu_unregister_cpuhp_state(pmu);
1114

1115 1116 1117
	perf_pmu_unregister(&pmu->base);
	pmu->base.event_init = NULL;
	free_event_attributes(pmu);
1118
}