coresight-etm4x.c 28.7 KB
Newer Older
1 2 3
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2014, The Linux Foundation. All rights reserved.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
 */

#include <linux/kernel.h>
#include <linux/moduleparam.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/device.h>
#include <linux/io.h>
#include <linux/err.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/smp.h>
#include <linux/sysfs.h>
#include <linux/stat.h>
#include <linux/clk.h>
#include <linux/cpu.h>
#include <linux/coresight.h>
22
#include <linux/coresight-pmu.h>
23 24 25 26
#include <linux/pm_wakeup.h>
#include <linux/amba/bus.h>
#include <linux/seq_file.h>
#include <linux/uaccess.h>
27
#include <linux/perf_event.h>
28 29
#include <linux/pm_runtime.h>
#include <asm/sections.h>
30
#include <asm/local.h>
31 32

#include "coresight-etm4x.h"
33
#include "coresight-etm-perf.h"
34 35 36 37 38 39 40

static int boot_enable;
module_param_named(boot_enable, boot_enable, int, S_IRUGO);

/* The number of ETMv4 currently registered */
static int etm4_count;
static struct etmv4_drvdata *etmdrvdata[NR_CPUS];
41 42 43
static void etm4_set_default_config(struct etmv4_config *config);
static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
				  struct perf_event *event);
44

45 46
static enum cpuhp_state hp_online;

47
static void etm4_os_unlock(struct etmv4_drvdata *drvdata)
48 49 50
{
	/* Writing any value to ETMOSLAR unlocks the trace registers */
	writel_relaxed(0x0, drvdata->base + TRCOSLAR);
51
	drvdata->os_unlock = true;
52 53 54 55 56
	isb();
}

static bool etm4_arch_supported(u8 arch)
{
57 58
	/* Mask out the minor version number */
	switch (arch & 0xf0) {
59 60 61 62 63 64 65 66
	case ETM_ARCH_V4:
		break;
	default:
		return false;
	}
	return true;
}

67 68 69 70 71 72 73
static int etm4_cpu_id(struct coresight_device *csdev)
{
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

	return drvdata->cpu;
}

74 75 76 77
static int etm4_trace_id(struct coresight_device *csdev)
{
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

78
	return drvdata->trcid;
79 80 81 82 83 84
}

static void etm4_enable_hw(void *info)
{
	int i;
	struct etmv4_drvdata *drvdata = info;
85
	struct etmv4_config *config = &drvdata->config;
86 87 88 89 90 91 92 93 94 95 96

	CS_UNLOCK(drvdata->base);

	etm4_os_unlock(drvdata);

	/* Disable the trace unit before programming trace registers */
	writel_relaxed(0, drvdata->base + TRCPRGCTLR);

	/* wait for TRCSTATR.IDLE to go up */
	if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1))
		dev_err(drvdata->dev,
97
			"timeout while waiting for Idle Trace Status\n");
98

99 100
	writel_relaxed(config->pe_sel, drvdata->base + TRCPROCSELR);
	writel_relaxed(config->cfg, drvdata->base + TRCCONFIGR);
101 102
	/* nothing specific implemented */
	writel_relaxed(0x0, drvdata->base + TRCAUXCTLR);
103 104 105 106 107 108 109
	writel_relaxed(config->eventctrl0, drvdata->base + TRCEVENTCTL0R);
	writel_relaxed(config->eventctrl1, drvdata->base + TRCEVENTCTL1R);
	writel_relaxed(config->stall_ctrl, drvdata->base + TRCSTALLCTLR);
	writel_relaxed(config->ts_ctrl, drvdata->base + TRCTSCTLR);
	writel_relaxed(config->syncfreq, drvdata->base + TRCSYNCPR);
	writel_relaxed(config->ccctlr, drvdata->base + TRCCCCTLR);
	writel_relaxed(config->bb_ctrl, drvdata->base + TRCBBCTLR);
110
	writel_relaxed(drvdata->trcid, drvdata->base + TRCTRACEIDR);
111 112 113
	writel_relaxed(config->vinst_ctrl, drvdata->base + TRCVICTLR);
	writel_relaxed(config->viiectlr, drvdata->base + TRCVIIECTLR);
	writel_relaxed(config->vissctlr,
114
		       drvdata->base + TRCVISSCTLR);
115
	writel_relaxed(config->vipcssctlr,
116 117
		       drvdata->base + TRCVIPCSSCTLR);
	for (i = 0; i < drvdata->nrseqstate - 1; i++)
118
		writel_relaxed(config->seq_ctrl[i],
119
			       drvdata->base + TRCSEQEVRn(i));
120 121 122
	writel_relaxed(config->seq_rst, drvdata->base + TRCSEQRSTEVR);
	writel_relaxed(config->seq_state, drvdata->base + TRCSEQSTR);
	writel_relaxed(config->ext_inp, drvdata->base + TRCEXTINSELR);
123
	for (i = 0; i < drvdata->nr_cntr; i++) {
124
		writel_relaxed(config->cntrldvr[i],
125
			       drvdata->base + TRCCNTRLDVRn(i));
126
		writel_relaxed(config->cntr_ctrl[i],
127
			       drvdata->base + TRCCNTCTLRn(i));
128
		writel_relaxed(config->cntr_val[i],
129 130
			       drvdata->base + TRCCNTVRn(i));
	}
131 132

	/* Resource selector pair 0 is always implemented and reserved */
133 134
	for (i = 0; i < drvdata->nr_resource * 2; i++)
		writel_relaxed(config->res_ctrl[i],
135 136 137
			       drvdata->base + TRCRSCTLRn(i));

	for (i = 0; i < drvdata->nr_ss_cmp; i++) {
138
		writel_relaxed(config->ss_ctrl[i],
139
			       drvdata->base + TRCSSCCRn(i));
140
		writel_relaxed(config->ss_status[i],
141
			       drvdata->base + TRCSSCSRn(i));
142
		writel_relaxed(config->ss_pe_cmp[i],
143 144 145
			       drvdata->base + TRCSSPCICRn(i));
	}
	for (i = 0; i < drvdata->nr_addr_cmp; i++) {
146
		writeq_relaxed(config->addr_val[i],
147
			       drvdata->base + TRCACVRn(i));
148
		writeq_relaxed(config->addr_acc[i],
149 150 151
			       drvdata->base + TRCACATRn(i));
	}
	for (i = 0; i < drvdata->numcidc; i++)
152
		writeq_relaxed(config->ctxid_pid[i],
153
			       drvdata->base + TRCCIDCVRn(i));
154 155
	writel_relaxed(config->ctxid_mask0, drvdata->base + TRCCIDCCTLR0);
	writel_relaxed(config->ctxid_mask1, drvdata->base + TRCCIDCCTLR1);
156 157

	for (i = 0; i < drvdata->numvmidc; i++)
158
		writeq_relaxed(config->vmid_val[i],
159
			       drvdata->base + TRCVMIDCVRn(i));
160 161
	writel_relaxed(config->vmid_mask0, drvdata->base + TRCVMIDCCTLR0);
	writel_relaxed(config->vmid_mask1, drvdata->base + TRCVMIDCCTLR1);
162

163 164 165 166 167 168 169
	/*
	 * Request to keep the trace unit powered and also
	 * emulation of powerdown
	 */
	writel_relaxed(readl_relaxed(drvdata->base + TRCPDCR) | TRCPDCR_PU,
		       drvdata->base + TRCPDCR);

170 171 172 173 174 175
	/* Enable the trace unit */
	writel_relaxed(1, drvdata->base + TRCPRGCTLR);

	/* wait for TRCSTATR.IDLE to go back down to '0' */
	if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 0))
		dev_err(drvdata->dev,
176
			"timeout while waiting for Idle Trace Status\n");
177 178 179 180 181 182
	/*
	 * As recommended by section 4.3.7 ("Synchronization when using the
	 * memory-mapped interface") of ARM IHI 0064D
	 */
	dsb(sy);
	isb();
183 184 185 186 187 188

	CS_LOCK(drvdata->base);

	dev_dbg(drvdata->dev, "cpu: %d enable smp call done\n", drvdata->cpu);
}

189
static int etm4_parse_event_config(struct etmv4_drvdata *drvdata,
190
				   struct perf_event *event)
191
{
192
	int ret = 0;
193
	struct etmv4_config *config = &drvdata->config;
194
	struct perf_event_attr *attr = &event->attr;
195

196 197 198 199
	if (!attr) {
		ret = -EINVAL;
		goto out;
	}
200 201 202 203 204 205 206 207 208 209 210

	/* Clear configuration from previous run */
	memset(config, 0, sizeof(struct etmv4_config));

	if (attr->exclude_kernel)
		config->mode = ETM_MODE_EXCL_KERN;

	if (attr->exclude_user)
		config->mode = ETM_MODE_EXCL_USER;

	/* Always start from the default config */
211 212 213 214 215 216
	etm4_set_default_config(config);

	/* Configure filters specified on the perf cmd line, if any. */
	ret = etm4_set_event_filters(drvdata, event);
	if (ret)
		goto out;
217 218

	/* Go from generic option to ETMv4 specifics */
219 220 221 222 223
	if (attr->config & BIT(ETM_OPT_CYCACC)) {
		config->cfg |= BIT(4);
		/* TRM: Must program this for cycacc to work */
		config->ccctlr = ETM_CYC_THRESHOLD_DEFAULT;
	}
224
	if (attr->config & BIT(ETM_OPT_TS))
225 226
		/* bit[11], Global timestamp tracing bit */
		config->cfg |= BIT(11);
227 228 229 230
	/* return stack - enable if selected and supported */
	if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack)
		/* bit[12], Return stack enable bit */
		config->cfg |= BIT(12);
231

232 233
out:
	return ret;
234 235 236
}

static int etm4_enable_perf(struct coresight_device *csdev,
237
			    struct perf_event *event)
238
{
239
	int ret = 0;
240 241
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

242 243 244 245
	if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) {
		ret = -EINVAL;
		goto out;
	}
246 247

	/* Configure the tracer based on the session's specifics */
248 249 250
	ret = etm4_parse_event_config(drvdata, event);
	if (ret)
		goto out;
251 252 253
	/* And enable it */
	etm4_enable_hw(drvdata);

254 255
out:
	return ret;
256 257
}

258
static int etm4_enable_sysfs(struct coresight_device *csdev)
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
{
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
	int ret;

	spin_lock(&drvdata->spinlock);

	/*
	 * Executing etm4_enable_hw on the cpu whose ETM is being enabled
	 * ensures that register writes occur when cpu is powered.
	 */
	ret = smp_call_function_single(drvdata->cpu,
				       etm4_enable_hw, drvdata, 1);
	if (ret)
		goto err;

274
	drvdata->sticky_enable = true;
275 276 277 278
	spin_unlock(&drvdata->spinlock);

	dev_info(drvdata->dev, "ETM tracing enabled\n");
	return 0;
279

280 281 282 283 284
err:
	spin_unlock(&drvdata->spinlock);
	return ret;
}

285
static int etm4_enable(struct coresight_device *csdev,
286
		       struct perf_event *event, u32 mode)
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
{
	int ret;
	u32 val;
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

	val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode);

	/* Someone is already using the tracer */
	if (val)
		return -EBUSY;

	switch (mode) {
	case CS_MODE_SYSFS:
		ret = etm4_enable_sysfs(csdev);
		break;
302
	case CS_MODE_PERF:
303
		ret = etm4_enable_perf(csdev, event);
304
		break;
305 306 307 308 309 310 311 312 313 314 315
	default:
		ret = -EINVAL;
	}

	/* The tracer didn't start */
	if (ret)
		local_set(&drvdata->mode, CS_MODE_DISABLED);

	return ret;
}

316 317 318 319 320 321 322
static void etm4_disable_hw(void *info)
{
	u32 control;
	struct etmv4_drvdata *drvdata = info;

	CS_UNLOCK(drvdata->base);

323 324 325 326 327
	/* power can be removed from the trace unit now */
	control = readl_relaxed(drvdata->base + TRCPDCR);
	control &= ~TRCPDCR_PU;
	writel_relaxed(control, drvdata->base + TRCPDCR);

328 329 330 331 332
	control = readl_relaxed(drvdata->base + TRCPRGCTLR);

	/* EN, bit[0] Trace unit enable bit */
	control &= ~0x1;

333 334 335 336 337 338
	/*
	 * Make sure everything completes before disabling, as recommended
	 * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register,
	 * SSTATUS") of ARM IHI 0064D
	 */
	dsb(sy);
339 340 341 342 343 344 345 346
	isb();
	writel_relaxed(control, drvdata->base + TRCPRGCTLR);

	CS_LOCK(drvdata->base);

	dev_dbg(drvdata->dev, "cpu: %d disable smp call done\n", drvdata->cpu);
}

347 348
static int etm4_disable_perf(struct coresight_device *csdev,
			     struct perf_event *event)
349
{
350 351
	u32 control;
	struct etm_filters *filters = event->hw.addr_filters;
352 353 354 355 356 357
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

	if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id()))
		return -EINVAL;

	etm4_disable_hw(drvdata);
358 359 360 361 362 363 364 365 366 367 368

	/*
	 * Check if the start/stop logic was active when the unit was stopped.
	 * That way we can re-enable the start/stop logic when the process is
	 * scheduled again.  Configuration of the start/stop logic happens in
	 * function etm4_set_event_filters().
	 */
	control = readl_relaxed(drvdata->base + TRCVICTLR);
	/* TRCVICTLR::SSSTATUS, bit[9] */
	filters->ssstatus = (control & BIT(9));

369 370 371
	return 0;
}

372
static void etm4_disable_sysfs(struct coresight_device *csdev)
373 374 375 376 377 378 379 380 381
{
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

	/*
	 * Taking hotplug lock here protects from clocks getting disabled
	 * with tracing being left on (crash scenario) if user disable occurs
	 * after cpu online mask indicates the cpu is offline but before the
	 * DYING hotplug callback is serviced by the ETM driver.
	 */
382
	cpus_read_lock();
383 384 385 386 387 388 389 390 391
	spin_lock(&drvdata->spinlock);

	/*
	 * Executing etm4_disable_hw on the cpu whose ETM is being disabled
	 * ensures that register writes occur when cpu is powered.
	 */
	smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1);

	spin_unlock(&drvdata->spinlock);
392
	cpus_read_unlock();
393 394 395 396

	dev_info(drvdata->dev, "ETM tracing disabled\n");
}

397 398
static void etm4_disable(struct coresight_device *csdev,
			 struct perf_event *event)
399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
{
	u32 mode;
	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);

	/*
	 * For as long as the tracer isn't disabled another entity can't
	 * change its status.  As such we can read the status here without
	 * fearing it will change under us.
	 */
	mode = local_read(&drvdata->mode);

	switch (mode) {
	case CS_MODE_DISABLED:
		break;
	case CS_MODE_SYSFS:
		etm4_disable_sysfs(csdev);
		break;
416
	case CS_MODE_PERF:
417
		etm4_disable_perf(csdev, event);
418
		break;
419 420 421 422 423 424
	}

	if (mode)
		local_set(&drvdata->mode, CS_MODE_DISABLED);
}

425
static const struct coresight_ops_source etm4_source_ops = {
426
	.cpu_id		= etm4_cpu_id,
427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445
	.trace_id	= etm4_trace_id,
	.enable		= etm4_enable,
	.disable	= etm4_disable,
};

static const struct coresight_ops etm4_cs_ops = {
	.source_ops	= &etm4_source_ops,
};

static void etm4_init_arch_data(void *info)
{
	u32 etmidr0;
	u32 etmidr1;
	u32 etmidr2;
	u32 etmidr3;
	u32 etmidr4;
	u32 etmidr5;
	struct etmv4_drvdata *drvdata = info;

446 447 448
	/* Make sure all registers are accessible */
	etm4_os_unlock(drvdata);

449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
	CS_UNLOCK(drvdata->base);

	/* find all capabilities of the tracing unit */
	etmidr0 = readl_relaxed(drvdata->base + TRCIDR0);

	/* INSTP0, bits[2:1] P0 tracing support field */
	if (BMVAL(etmidr0, 1, 1) && BMVAL(etmidr0, 2, 2))
		drvdata->instrp0 = true;
	else
		drvdata->instrp0 = false;

	/* TRCBB, bit[5] Branch broadcast tracing support bit */
	if (BMVAL(etmidr0, 5, 5))
		drvdata->trcbb = true;
	else
		drvdata->trcbb = false;

	/* TRCCOND, bit[6] Conditional instruction tracing support bit */
	if (BMVAL(etmidr0, 6, 6))
		drvdata->trccond = true;
	else
		drvdata->trccond = false;

	/* TRCCCI, bit[7] Cycle counting instruction bit */
	if (BMVAL(etmidr0, 7, 7))
		drvdata->trccci = true;
	else
		drvdata->trccci = false;

	/* RETSTACK, bit[9] Return stack bit */
	if (BMVAL(etmidr0, 9, 9))
		drvdata->retstack = true;
	else
		drvdata->retstack = false;

	/* NUMEVENT, bits[11:10] Number of events field */
	drvdata->nr_event = BMVAL(etmidr0, 10, 11);
	/* QSUPP, bits[16:15] Q element support field */
	drvdata->q_support = BMVAL(etmidr0, 15, 16);
	/* TSSIZE, bits[28:24] Global timestamp size field */
	drvdata->ts_size = BMVAL(etmidr0, 24, 28);

	/* base architecture of trace unit */
	etmidr1 = readl_relaxed(drvdata->base + TRCIDR1);
	/*
	 * TRCARCHMIN, bits[7:4] architecture the minor version number
	 * TRCARCHMAJ, bits[11:8] architecture major versin number
	 */
	drvdata->arch = BMVAL(etmidr1, 4, 11);

	/* maximum size of resources */
	etmidr2 = readl_relaxed(drvdata->base + TRCIDR2);
	/* CIDSIZE, bits[9:5] Indicates the Context ID size */
	drvdata->ctxid_size = BMVAL(etmidr2, 5, 9);
	/* VMIDSIZE, bits[14:10] Indicates the VMID size */
	drvdata->vmid_size = BMVAL(etmidr2, 10, 14);
	/* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */
	drvdata->ccsize = BMVAL(etmidr2, 25, 28);

	etmidr3 = readl_relaxed(drvdata->base + TRCIDR3);
	/* CCITMIN, bits[11:0] minimum threshold value that can be programmed */
	drvdata->ccitmin = BMVAL(etmidr3, 0, 11);
	/* EXLEVEL_S, bits[19:16] Secure state instruction tracing */
	drvdata->s_ex_level = BMVAL(etmidr3, 16, 19);
	/* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */
	drvdata->ns_ex_level = BMVAL(etmidr3, 20, 23);

	/*
	 * TRCERR, bit[24] whether a trace unit can trace a
	 * system error exception.
	 */
	if (BMVAL(etmidr3, 24, 24))
		drvdata->trc_error = true;
	else
		drvdata->trc_error = false;

	/* SYNCPR, bit[25] implementation has a fixed synchronization period? */
	if (BMVAL(etmidr3, 25, 25))
		drvdata->syncpr = true;
	else
		drvdata->syncpr = false;

	/* STALLCTL, bit[26] is stall control implemented? */
	if (BMVAL(etmidr3, 26, 26))
		drvdata->stallctl = true;
	else
		drvdata->stallctl = false;

	/* SYSSTALL, bit[27] implementation can support stall control? */
	if (BMVAL(etmidr3, 27, 27))
		drvdata->sysstall = true;
	else
		drvdata->sysstall = false;

	/* NUMPROC, bits[30:28] the number of PEs available for tracing */
	drvdata->nr_pe = BMVAL(etmidr3, 28, 30);

	/* NOOVERFLOW, bit[31] is trace overflow prevention supported */
	if (BMVAL(etmidr3, 31, 31))
		drvdata->nooverflow = true;
	else
		drvdata->nooverflow = false;

	/* number of resources trace unit supports */
	etmidr4 = readl_relaxed(drvdata->base + TRCIDR4);
	/* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */
	drvdata->nr_addr_cmp = BMVAL(etmidr4, 0, 3);
	/* NUMPC, bits[15:12] number of PE comparator inputs for tracing */
	drvdata->nr_pe_cmp = BMVAL(etmidr4, 12, 15);
558 559 560 561 562 563 564
	/*
	 * NUMRSPAIR, bits[19:16]
	 * The number of resource pairs conveyed by the HW starts at 0, i.e a
	 * value of 0x0 indicate 1 resource pair, 0x1 indicate two and so on.
	 * As such add 1 to the value of NUMRSPAIR for a better representation.
	 */
	drvdata->nr_resource = BMVAL(etmidr4, 16, 19) + 1;
565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
	/*
	 * NUMSSCC, bits[23:20] the number of single-shot
	 * comparator control for tracing
	 */
	drvdata->nr_ss_cmp = BMVAL(etmidr4, 20, 23);
	/* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */
	drvdata->numcidc = BMVAL(etmidr4, 24, 27);
	/* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */
	drvdata->numvmidc = BMVAL(etmidr4, 28, 31);

	etmidr5 = readl_relaxed(drvdata->base + TRCIDR5);
	/* NUMEXTIN, bits[8:0] number of external inputs implemented */
	drvdata->nr_ext_inp = BMVAL(etmidr5, 0, 8);
	/* TRACEIDSIZE, bits[21:16] indicates the trace ID width */
	drvdata->trcid_size = BMVAL(etmidr5, 16, 21);
	/* ATBTRIG, bit[22] implementation can support ATB triggers? */
	if (BMVAL(etmidr5, 22, 22))
		drvdata->atbtrig = true;
	else
		drvdata->atbtrig = false;
	/*
	 * LPOVERRIDE, bit[23] implementation supports
	 * low-power state override
	 */
	if (BMVAL(etmidr5, 23, 23))
		drvdata->lpoverride = true;
	else
		drvdata->lpoverride = false;
	/* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */
	drvdata->nrseqstate = BMVAL(etmidr5, 25, 27);
	/* NUMCNTR, bits[30:28] number of counters available for tracing */
	drvdata->nr_cntr = BMVAL(etmidr5, 28, 30);
	CS_LOCK(drvdata->base);
}

600
static void etm4_set_default_config(struct etmv4_config *config)
601 602
{
	/* disable all events tracing */
603 604
	config->eventctrl0 = 0x0;
	config->eventctrl1 = 0x0;
605 606

	/* disable stalling */
607
	config->stall_ctrl = 0x0;
608

609 610 611
	/* enable trace synchronization every 4096 bytes, if available */
	config->syncfreq = 0xC;

612
	/* disable timestamp event */
613
	config->ts_ctrl = 0x0;
614

615 616
	/* TRCVICTLR::EVENT = 0x01, select the always on logic */
	config->vinst_ctrl |= BIT(0);
617
}
618

619
static u64 etm4_get_access_type(struct etmv4_config *config)
620
{
621
	u64 access_type = 0;
622

623 624 625 626 627 628 629 630 631 632
	/*
	 * EXLEVEL_NS, bits[15:12]
	 * The Exception levels are:
	 *   Bit[12] Exception level 0 - Application
	 *   Bit[13] Exception level 1 - OS
	 *   Bit[14] Exception level 2 - Hypervisor
	 *   Bit[15] Never implemented
	 *
	 * Always stay away from hypervisor mode.
	 */
633
	access_type = ETM_EXLEVEL_NS_HYP;
634

635 636 637 638 639 640
	if (config->mode & ETM_MODE_EXCL_KERN)
		access_type |= ETM_EXLEVEL_NS_OS;

	if (config->mode & ETM_MODE_EXCL_USER)
		access_type |= ETM_EXLEVEL_NS_APP;

641
	/*
642 643 644 645 646 647 648
	 * EXLEVEL_S, bits[11:8], don't trace anything happening
	 * in secure state.
	 */
	access_type |= (ETM_EXLEVEL_S_APP	|
			ETM_EXLEVEL_S_OS	|
			ETM_EXLEVEL_S_HYP);

649 650 651 652 653 654 655 656
	return access_type;
}

static void etm4_set_comparator_filter(struct etmv4_config *config,
				       u64 start, u64 stop, int comparator)
{
	u64 access_type = etm4_get_access_type(config);

657
	/* First half of default address comparator */
658 659 660
	config->addr_val[comparator] = start;
	config->addr_acc[comparator] = access_type;
	config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE;
661 662

	/* Second half of default address comparator */
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
	config->addr_val[comparator + 1] = stop;
	config->addr_acc[comparator + 1] = access_type;
	config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE;

	/*
	 * Configure the ViewInst function to include this address range
	 * comparator.
	 *
	 * @comparator is divided by two since it is the index in the
	 * etmv4_config::addr_val array but register TRCVIIECTLR deals with
	 * address range comparator _pairs_.
	 *
	 * Therefore:
	 *	index 0 -> compatator pair 0
	 *	index 2 -> comparator pair 1
	 *	index 4 -> comparator pair 2
	 *	...
	 *	index 14 -> comparator pair 7
	 */
	config->viiectlr |= BIT(comparator / 2);
}

685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705
static void etm4_set_start_stop_filter(struct etmv4_config *config,
				       u64 address, int comparator,
				       enum etm_addr_type type)
{
	int shift;
	u64 access_type = etm4_get_access_type(config);

	/* Configure the comparator */
	config->addr_val[comparator] = address;
	config->addr_acc[comparator] = access_type;
	config->addr_type[comparator] = type;

	/*
	 * Configure ViewInst Start-Stop control register.
	 * Addresses configured to start tracing go from bit 0 to n-1,
	 * while those configured to stop tracing from 16 to 16 + n-1.
	 */
	shift = (type == ETM_ADDR_TYPE_START ? 0 : 16);
	config->vissctlr |= BIT(shift + comparator);
}

706 707 708
static void etm4_set_default_filter(struct etmv4_config *config)
{
	u64 start, stop;
709 710

	/*
711 712
	 * Configure address range comparator '0' to encompass all
	 * possible addresses.
713
	 */
714 715 716 717 718
	start = 0x0;
	stop = ~0x0;

	etm4_set_comparator_filter(config, start, stop,
				   ETM_DEFAULT_ADDR_COMP);
719

720 721 722 723 724 725 726
	/*
	 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
	 * in the started state
	 */
	config->vinst_ctrl |= BIT(9);

	/* No start-stop filtering for ViewInst */
727
	config->vissctlr = 0x0;
728 729
}

730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
static void etm4_set_default(struct etmv4_config *config)
{
	if (WARN_ON_ONCE(!config))
		return;

	/*
	 * Make default initialisation trace everything
	 *
	 * Select the "always true" resource selector on the
	 * "Enablign Event" line and configure address range comparator
	 * '0' to trace all the possible address range.  From there
	 * configure the "include/exclude" engine to include address
	 * range comparator '0'.
	 */
	etm4_set_default_config(config);
	etm4_set_default_filter(config);
}

748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769
static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type)
{
	int nr_comparator, index = 0;
	struct etmv4_config *config = &drvdata->config;

	/*
	 * nr_addr_cmp holds the number of comparator _pair_, so time 2
	 * for the total number of comparators.
	 */
	nr_comparator = drvdata->nr_addr_cmp * 2;

	/* Go through the tally of comparators looking for a free one. */
	while (index < nr_comparator) {
		switch (type) {
		case ETM_ADDR_TYPE_RANGE:
			if (config->addr_type[index] == ETM_ADDR_TYPE_NONE &&
			    config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE)
				return index;

			/* Address range comparators go in pairs */
			index += 2;
			break;
770 771 772 773 774 775 776 777
		case ETM_ADDR_TYPE_START:
		case ETM_ADDR_TYPE_STOP:
			if (config->addr_type[index] == ETM_ADDR_TYPE_NONE)
				return index;

			/* Start/stop address can have odd indexes */
			index += 1;
			break;
778 779 780 781 782 783 784 785 786 787 788 789 790
		default:
			return -EINVAL;
		}
	}

	/* If we are here all the comparators have been used. */
	return -ENOSPC;
}

static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
				  struct perf_event *event)
{
	int i, comparator, ret = 0;
791
	u64 address;
792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833
	struct etmv4_config *config = &drvdata->config;
	struct etm_filters *filters = event->hw.addr_filters;

	if (!filters)
		goto default_filter;

	/* Sync events with what Perf got */
	perf_event_addr_filters_sync(event);

	/*
	 * If there are no filters to deal with simply go ahead with
	 * the default filter, i.e the entire address range.
	 */
	if (!filters->nr_filters)
		goto default_filter;

	for (i = 0; i < filters->nr_filters; i++) {
		struct etm_filter *filter = &filters->etm_filter[i];
		enum etm_addr_type type = filter->type;

		/* See if a comparator is free. */
		comparator = etm4_get_next_comparator(drvdata, type);
		if (comparator < 0) {
			ret = comparator;
			goto out;
		}

		switch (type) {
		case ETM_ADDR_TYPE_RANGE:
			etm4_set_comparator_filter(config,
						   filter->start_addr,
						   filter->stop_addr,
						   comparator);
			/*
			 * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
			 * in the started state
			 */
			config->vinst_ctrl |= BIT(9);

			/* No start-stop filtering for ViewInst */
			config->vissctlr = 0x0;
			break;
834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
		case ETM_ADDR_TYPE_START:
		case ETM_ADDR_TYPE_STOP:
			/* Get the right start or stop address */
			address = (type == ETM_ADDR_TYPE_START ?
				   filter->start_addr :
				   filter->stop_addr);

			/* Configure comparator */
			etm4_set_start_stop_filter(config, address,
						   comparator, type);

			/*
			 * If filters::ssstatus == 1, trace acquisition was
			 * started but the process was yanked away before the
			 * the stop address was hit.  As such the start/stop
			 * logic needs to be re-started so that tracing can
			 * resume where it left.
			 *
			 * The start/stop logic status when a process is
			 * scheduled out is checked in function
			 * etm4_disable_perf().
			 */
			if (filters->ssstatus)
				config->vinst_ctrl |= BIT(9);

			/* No include/exclude filtering for ViewInst */
			config->viiectlr = 0x0;
			break;
862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877
		default:
			ret = -EINVAL;
			goto out;
		}
	}

	goto out;


default_filter:
	etm4_set_default_filter(config);

out:
	return ret;
}

878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912
void etm4_config_trace_mode(struct etmv4_config *config)
{
	u32 addr_acc, mode;

	mode = config->mode;
	mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER);

	/* excluding kernel AND user space doesn't make sense */
	WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER));

	/* nothing to do if neither flags are set */
	if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER))
		return;

	addr_acc = config->addr_acc[ETM_DEFAULT_ADDR_COMP];
	/* clear default config */
	addr_acc &= ~(ETM_EXLEVEL_NS_APP | ETM_EXLEVEL_NS_OS);

	/*
	 * EXLEVEL_NS, bits[15:12]
	 * The Exception levels are:
	 *   Bit[12] Exception level 0 - Application
	 *   Bit[13] Exception level 1 - OS
	 *   Bit[14] Exception level 2 - Hypervisor
	 *   Bit[15] Never implemented
	 */
	if (mode & ETM_MODE_EXCL_KERN)
		addr_acc |= ETM_EXLEVEL_NS_OS;
	else
		addr_acc |= ETM_EXLEVEL_NS_APP;

	config->addr_acc[ETM_DEFAULT_ADDR_COMP] = addr_acc;
	config->addr_acc[ETM_DEFAULT_ADDR_COMP + 1] = addr_acc;
}

913
static int etm4_online_cpu(unsigned int cpu)
914 915
{
	if (!etmdrvdata[cpu])
916
		return 0;
917

918 919 920 921
	if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable)
		coresight_enable(etmdrvdata[cpu]->csdev);
	return 0;
}
922

923 924 925 926 927 928 929 930 931
static int etm4_starting_cpu(unsigned int cpu)
{
	if (!etmdrvdata[cpu])
		return 0;

	spin_lock(&etmdrvdata[cpu]->spinlock);
	if (!etmdrvdata[cpu]->os_unlock) {
		etm4_os_unlock(etmdrvdata[cpu]);
		etmdrvdata[cpu]->os_unlock = true;
932
	}
933 934 935 936 937

	if (local_read(&etmdrvdata[cpu]->mode))
		etm4_enable_hw(etmdrvdata[cpu]);
	spin_unlock(&etmdrvdata[cpu]->spinlock);
	return 0;
938 939
}

940 941 942 943 944 945 946 947 948 949 950
static int etm4_dying_cpu(unsigned int cpu)
{
	if (!etmdrvdata[cpu])
		return 0;

	spin_lock(&etmdrvdata[cpu]->spinlock);
	if (local_read(&etmdrvdata[cpu]->mode))
		etm4_disable_hw(etmdrvdata[cpu]);
	spin_unlock(&etmdrvdata[cpu]->spinlock);
	return 0;
}
951

952 953 954 955 956
static void etm4_init_trace_id(struct etmv4_drvdata *drvdata)
{
	drvdata->trcid = coresight_get_trace_id(drvdata->cpu);
}

957 958 959 960 961 962 963 964
static int etm4_probe(struct amba_device *adev, const struct amba_id *id)
{
	int ret;
	void __iomem *base;
	struct device *dev = &adev->dev;
	struct coresight_platform_data *pdata = NULL;
	struct etmv4_drvdata *drvdata;
	struct resource *res = &adev->res;
965
	struct coresight_desc desc = { 0 };
966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
	struct device_node *np = adev->dev.of_node;

	drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
	if (!drvdata)
		return -ENOMEM;

	if (np) {
		pdata = of_get_coresight_platform_data(dev, np);
		if (IS_ERR(pdata))
			return PTR_ERR(pdata);
		adev->dev.platform_data = pdata;
	}

	drvdata->dev = &adev->dev;
	dev_set_drvdata(dev, drvdata);

	/* Validity for the resource is already checked by the AMBA core */
	base = devm_ioremap_resource(dev, res);
	if (IS_ERR(base))
		return PTR_ERR(base);

	drvdata->base = base;

	spin_lock_init(&drvdata->spinlock);

	drvdata->cpu = pdata ? pdata->cpu : 0;

993
	cpus_read_lock();
994 995 996 997 998 999
	etmdrvdata[drvdata->cpu] = drvdata;

	if (smp_call_function_single(drvdata->cpu,
				etm4_init_arch_data,  drvdata, 1))
		dev_err(dev, "ETM arch init failed\n");

1000
	if (!etm4_count++) {
1001 1002 1003 1004 1005 1006
		cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ARM_CORESIGHT_STARTING,
						     "arm/coresight4:starting",
						     etm4_starting_cpu, etm4_dying_cpu);
		ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
							   "arm/coresight4:online",
							   etm4_online_cpu, NULL);
1007 1008 1009 1010
		if (ret < 0)
			goto err_arch_supported;
		hp_online = ret;
	}
1011

1012
	cpus_read_unlock();
1013 1014 1015 1016 1017

	if (etm4_arch_supported(drvdata->arch) == false) {
		ret = -EINVAL;
		goto err_arch_supported;
	}
1018 1019 1020

	etm4_init_trace_id(drvdata);
	etm4_set_default(&drvdata->config);
1021

1022 1023 1024 1025 1026 1027 1028
	desc.type = CORESIGHT_DEV_TYPE_SOURCE;
	desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
	desc.ops = &etm4_cs_ops;
	desc.pdata = pdata;
	desc.dev = dev;
	desc.groups = coresight_etmv4_groups;
	drvdata->csdev = coresight_register(&desc);
1029 1030
	if (IS_ERR(drvdata->csdev)) {
		ret = PTR_ERR(drvdata->csdev);
1031
		goto err_arch_supported;
1032 1033
	}

1034 1035 1036 1037 1038 1039 1040
	ret = etm_perf_symlink(drvdata->csdev, true);
	if (ret) {
		coresight_unregister(drvdata->csdev);
		goto err_arch_supported;
	}

	pm_runtime_put(&adev->dev);
1041 1042
	dev_info(dev, "CPU%d: ETM v%d.%d initialized\n",
		 drvdata->cpu, drvdata->arch >> 4, drvdata->arch & 0xf);
1043 1044 1045 1046 1047 1048 1049 1050 1051

	if (boot_enable) {
		coresight_enable(drvdata->csdev);
		drvdata->boot_enable = true;
	}

	return 0;

err_arch_supported:
1052
	if (--etm4_count == 0) {
1053
		cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
1054 1055 1056
		if (hp_online)
			cpuhp_remove_state_nocalls(hp_online);
	}
1057 1058 1059
	return ret;
}

1060 1061 1062 1063 1064 1065
#define ETM4x_AMBA_ID(pid)			\
	{					\
		.id	= pid,			\
		.mask	= 0x000fffff,		\
	}

1066
static const struct amba_id etm4_ids[] = {
1067 1068 1069 1070 1071 1072
	ETM4x_AMBA_ID(0x000bb95d),		/* Cortex-A53 */
	ETM4x_AMBA_ID(0x000bb95e),		/* Cortex-A57 */
	ETM4x_AMBA_ID(0x000bb95a),		/* Cortex-A72 */
	ETM4x_AMBA_ID(0x000bb959),		/* Cortex-A73 */
	ETM4x_AMBA_ID(0x000bb9da),		/* Cortex-A35 */
	{},
1073 1074 1075 1076 1077
};

static struct amba_driver etm4x_driver = {
	.drv = {
		.name   = "coresight-etm4x",
1078
		.suppress_bind_attrs = true,
1079 1080 1081 1082
	},
	.probe		= etm4_probe,
	.id_table	= etm4_ids,
};
1083
builtin_amba_driver(etm4x_driver);