diff --git a/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x b/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x index 881f0cd99ce4ed7c894ec4bc35c59bf8dc7f1ecc..8e53a32f815056f97220b2092249128c12a848d0 100644 --- a/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x +++ b/Documentation/ABI/testing/sysfs-bus-coresight-devices-etm4x @@ -371,6 +371,14 @@ Contact: Mathieu Poirier Description: (Read) Print the content of the Device ID Register (0xFC8). The value is taken directly from the HW. +What: /sys/bus/coresight/devices/etm/mgmt/trcdevarch +Date: January 2021 +KernelVersion: 5.12 +Contact: Mathieu Poirier +Description: (Read) Print the content of the Device Architecture Register + (offset 0xFBC). The value is taken directly read + from the HW. + What: /sys/bus/coresight/devices/etm/mgmt/trcdevtype Date: April 2015 KernelVersion: 4.01 diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 085b1109d8796610554a3fa04e56a62f1dbf832f..dd1ace7acbf0dd49fd28b81203751b4fbbb952b5 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -7244,7 +7244,7 @@ CONFIG_IO_STRICT_DEVMEM=y # CONFIG_ARM64_RELOC_TEST is not set CONFIG_CORESIGHT=m CONFIG_CORESIGHT_LINKS_AND_SINKS=m -# CONFIG_CORESIGHT_LINK_AND_SINK_TMC is not set +CONFIG_CORESIGHT_LINK_AND_SINK_TMC=m # CONFIG_CORESIGHT_SINK_TPIU is not set # CONFIG_CORESIGHT_SINK_ETBV10 is not set CONFIG_CORESIGHT_SOURCE_ETM4X=m @@ -7252,7 +7252,7 @@ CONFIG_ETM4X_IMPDEF_FEATURE=y # CONFIG_CORESIGHT_STM is not set # CONFIG_CORESIGHT_CPU_DEBUG is not set # CONFIG_CORESIGHT_CTI is not set -# CONFIG_CORESIGHT_TRBE is not set +CONFIG_CORESIGHT_TRBE=m CONFIG_ULTRASOC_SMB=m # end of arm64 Debugging diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index 2a13b1c2c3904adecd2501d7423e8111fe561089..f150a862dadd27027d33646581980d1c46001dd0 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -23,6 +23,7 @@ #define dsb(opt) asm volatile("dsb " #opt : : : "memory") #define psb_csync() asm volatile("hint #17" : : : "memory") +#define tsb_csync() asm volatile("hint #18" : : : "memory") #define csdb() asm volatile("hint #20" : : : "memory") /* diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 098247c4c6685aa12d37f978dd41a92f4aada6f4..8907c408ff72944e7eb603c3eb80820e8c5b0ecd 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -192,6 +192,7 @@ #define SYS_GCR_EL1 sys_reg(3, 0, 1, 0, 6) #define SYS_ZCR_EL1 sys_reg(3, 0, 1, 2, 0) +#define SYS_TRFCR_EL1 sys_reg(3, 0, 1, 2, 1) #define SYS_TTBR0_EL1 sys_reg(3, 0, 2, 0, 0) #define SYS_TTBR1_EL1 sys_reg(3, 0, 2, 0, 1) @@ -333,6 +334,55 @@ /*** End of Statistical Profiling Extension ***/ +/* + * TRBE Registers + */ +#define SYS_TRBLIMITR_EL1 sys_reg(3, 0, 9, 11, 0) +#define SYS_TRBPTR_EL1 sys_reg(3, 0, 9, 11, 1) +#define SYS_TRBBASER_EL1 sys_reg(3, 0, 9, 11, 2) +#define SYS_TRBSR_EL1 sys_reg(3, 0, 9, 11, 3) +#define SYS_TRBMAR_EL1 sys_reg(3, 0, 9, 11, 4) +#define SYS_TRBTRG_EL1 sys_reg(3, 0, 9, 11, 6) +#define SYS_TRBIDR_EL1 sys_reg(3, 0, 9, 11, 7) + +#define TRBLIMITR_LIMIT_MASK GENMASK_ULL(51, 0) +#define TRBLIMITR_LIMIT_SHIFT 12 +#define TRBLIMITR_NVM BIT(5) +#define TRBLIMITR_TRIG_MODE_MASK GENMASK(1, 0) +#define TRBLIMITR_TRIG_MODE_SHIFT 3 +#define TRBLIMITR_FILL_MODE_MASK GENMASK(1, 0) +#define TRBLIMITR_FILL_MODE_SHIFT 1 +#define TRBLIMITR_ENABLE BIT(0) +#define TRBPTR_PTR_MASK GENMASK_ULL(63, 0) +#define TRBPTR_PTR_SHIFT 0 +#define TRBBASER_BASE_MASK GENMASK_ULL(51, 0) +#define TRBBASER_BASE_SHIFT 12 +#define TRBSR_EC_MASK GENMASK(5, 0) +#define TRBSR_EC_SHIFT 26 +#define TRBSR_IRQ BIT(22) +#define TRBSR_TRG BIT(21) +#define TRBSR_WRAP BIT(20) +#define TRBSR_ABORT BIT(18) +#define TRBSR_STOP BIT(17) +#define TRBSR_MSS_MASK GENMASK(15, 0) +#define TRBSR_MSS_SHIFT 0 +#define TRBSR_BSC_MASK GENMASK(5, 0) +#define TRBSR_BSC_SHIFT 0 +#define TRBSR_FSC_MASK GENMASK(5, 0) +#define TRBSR_FSC_SHIFT 0 +#define TRBMAR_SHARE_MASK GENMASK(1, 0) +#define TRBMAR_SHARE_SHIFT 8 +#define TRBMAR_OUTER_MASK GENMASK(3, 0) +#define TRBMAR_OUTER_SHIFT 4 +#define TRBMAR_INNER_MASK GENMASK(3, 0) +#define TRBMAR_INNER_SHIFT 0 +#define TRBTRG_TRG_MASK GENMASK(31, 0) +#define TRBTRG_TRG_SHIFT 0 +#define TRBIDR_FLAG BIT(5) +#define TRBIDR_PROG BIT(4) +#define TRBIDR_ALIGN_MASK GENMASK(3, 0) +#define TRBIDR_ALIGN_SHIFT 0 + #define SYS_PMINTENSET_EL1 sys_reg(3, 0, 9, 14, 1) #define SYS_PMINTENCLR_EL1 sys_reg(3, 0, 9, 14, 2) @@ -478,6 +528,7 @@ #define SYS_PMCCFILTR_EL0 sys_reg(3, 3, 14, 15, 7) #define SYS_ZCR_EL2 sys_reg(3, 4, 1, 2, 0) +#define SYS_TRFCR_EL2 sys_reg(3, 4, 1, 2, 1) #define SYS_DACR32_EL2 sys_reg(3, 4, 3, 0, 0) #define SYS_SPSR_EL2 sys_reg(3, 4, 4, 0, 0) #define SYS_ELR_EL2 sys_reg(3, 4, 4, 0, 1) @@ -855,6 +906,8 @@ #define ID_AA64MMFR2_CNP_SHIFT 0 /* id_aa64dfr0 */ +#define ID_AA64DFR0_TRBE_SHIFT 44 +#define ID_AA64DFR0_TRACE_FILT_SHIFT 40 #define ID_AA64DFR0_DOUBLELOCK_SHIFT 36 #define ID_AA64DFR0_PMSVER_SHIFT 32 #define ID_AA64DFR0_CTX_CMPS_SHIFT 28 @@ -1032,6 +1085,14 @@ /* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */ #define SYS_MPIDR_SAFE_VAL (BIT(31)) +#define TRFCR_ELx_TS_SHIFT 5 +#define TRFCR_ELx_TS_VIRTUAL ((0x1UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_ELx_TS_GUEST_PHYSICAL ((0x2UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_ELx_TS_PHYSICAL ((0x3UL) << TRFCR_ELx_TS_SHIFT) +#define TRFCR_EL2_CX BIT(3) +#define TRFCR_ELx_ExTRE BIT(1) +#define TRFCR_ELx_E0TRE BIT(0) + #ifdef __ASSEMBLY__ .irp num,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 6c1bc564b91fbe1cd8d989a71a8d948cdcf81021..33050c41a3177795c7a7997b2c4f4acb8fbf6bec 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1573,6 +1573,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_GCR_EL1), undef_access }, { SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility }, + { SYS_DESC(SYS_TRFCR_EL1), undef_access }, { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 }, { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 }, { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 }, diff --git a/drivers/hwtracing/coresight/Kconfig b/drivers/hwtracing/coresight/Kconfig index 17c1fa4fcf70f049276cf9528255ebc85d347bdd..017551d9b89e50eaa63afa14347d66aca9bbe562 100644 --- a/drivers/hwtracing/coresight/Kconfig +++ b/drivers/hwtracing/coresight/Kconfig @@ -97,15 +97,15 @@ config CORESIGHT_SOURCE_ETM3X module will be called coresight-etm3x. config CORESIGHT_SOURCE_ETM4X - tristate "CoreSight Embedded Trace Macrocell 4.x driver" + tristate "CoreSight ETMv4.x / ETE driver" depends on ARM64 select CORESIGHT_LINKS_AND_SINKS select PID_IN_CONTEXTIDR help - This driver provides support for the ETM4.x tracer module, tracing the - instructions that a processor is executing. This is primarily useful - for instruction level tracing. Depending on the implemented version - data tracing may also be available. + This driver provides support for the CoreSight Embedded Trace Macrocell + version 4.x and the Embedded Trace Extensions (ETE). Both are CPU tracer + modules, tracing the instructions that a processor is executing. This is + primarily useful for instruction level tracing. To compile this driver as a module, choose M here: the module will be called coresight-etm4x. @@ -174,6 +174,20 @@ config CORESIGHT_CTI_INTEGRATION_REGS registers are not used in normal operation and can leave devices in an inconsistent state. +config CORESIGHT_TRBE + tristate "Trace Buffer Extension (TRBE) driver" + depends on ARM64 && CORESIGHT_SOURCE_ETM4X + help + This driver provides support for percpu Trace Buffer Extension (TRBE). + TRBE always needs to be used along with it's corresponding percpu ETE + component. ETE generates trace data which is then captured with TRBE. + Unlike traditional sink devices, TRBE is a CPU feature accessible via + system registers. But it's explicit dependency with trace unit (ETE) + requires it to be plugged in as a coresight sink device. + + To compile this driver as a module, choose M here: the module will be + called coresight-trbe. + config ULTRASOC_SMB tristate "Ultrasoc system memory buffer drivers" depends on ARM64 && CORESIGHT_LINKS_AND_SINKS diff --git a/drivers/hwtracing/coresight/Makefile b/drivers/hwtracing/coresight/Makefile index 28acf9600f8339172069c6188a09c9b20b2c1f2b..c2d6456953ad7410781629926d3086a55dd85769 100644 --- a/drivers/hwtracing/coresight/Makefile +++ b/drivers/hwtracing/coresight/Makefile @@ -21,6 +21,7 @@ obj-$(CONFIG_CORESIGHT_STM) += coresight-stm.o obj-$(CONFIG_CORESIGHT_CPU_DEBUG) += coresight-cpu-debug.o obj-$(CONFIG_CORESIGHT_CATU) += coresight-catu.o obj-$(CONFIG_CORESIGHT_CTI) += coresight-cti.o +obj-$(CONFIG_CORESIGHT_TRBE) += coresight-trbe.o coresight-cti-y := coresight-cti-core.o coresight-cti-platform.o \ coresight-cti-sysfs.o obj-$(CONFIG_ULTRASOC_SMB) += ultrasoc-smb.o diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c index 8e19e8cdcce5e735f946bf510a5dab2b45ed3e6d..e0740c6dbd54235aa8c2cacf7f21a13facea07fb 100644 --- a/drivers/hwtracing/coresight/coresight-catu.c +++ b/drivers/hwtracing/coresight/coresight-catu.c @@ -401,8 +401,9 @@ static const struct attribute_group *catu_groups[] = { static inline int catu_wait_for_ready(struct catu_drvdata *drvdata) { - return coresight_timeout(drvdata->base, - CATU_STATUS, CATU_STATUS_READY, 1); + struct csdev_access *csa = &drvdata->csdev->access; + + return coresight_timeout(csa, CATU_STATUS, CATU_STATUS_READY, 1); } static int catu_enable_hw(struct catu_drvdata *drvdata, void *data) @@ -411,6 +412,7 @@ static int catu_enable_hw(struct catu_drvdata *drvdata, void *data) u32 control, mode; struct etr_buf *etr_buf = data; struct device *dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; if (catu_wait_for_ready(drvdata)) dev_warn(dev, "Timeout while waiting for READY\n"); @@ -421,7 +423,7 @@ static int catu_enable_hw(struct catu_drvdata *drvdata, void *data) return -EBUSY; } - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) return rc; @@ -465,9 +467,10 @@ static int catu_disable_hw(struct catu_drvdata *drvdata) { int rc = 0; struct device *dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; catu_write_control(drvdata, 0); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); if (catu_wait_for_ready(drvdata)) { dev_info(dev, "Timeout while waiting for READY\n"); rc = -EAGAIN; @@ -551,6 +554,7 @@ static int catu_probe(struct amba_device *adev, const struct amba_id *id) dev->platform_data = pdata; drvdata->base = base; + catu_desc.access = CSDEV_ACCESS_IOMEM(base); catu_desc.pdata = pdata; catu_desc.dev = dev; catu_desc.groups = catu_groups; diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 5ddc8103503b516ba868f12a5d99f7c395f0b85e..f6b33c681b42912a81fd1aa3cf7d910e68988cf1 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -23,6 +23,7 @@ #include "coresight-priv.h" static DEFINE_MUTEX(coresight_mutex); +DEFINE_PER_CPU(struct coresight_device *, csdev_sink); /** * struct coresight_node - elements of a path, from source to sink @@ -70,6 +71,18 @@ void coresight_remove_cti_ops(void) } EXPORT_SYMBOL_GPL(coresight_remove_cti_ops); +void coresight_set_percpu_sink(int cpu, struct coresight_device *csdev) +{ + per_cpu(csdev_sink, cpu) = csdev; +} +EXPORT_SYMBOL_GPL(coresight_set_percpu_sink); + +struct coresight_device *coresight_get_percpu_sink(int cpu) +{ + return per_cpu(csdev_sink, cpu); +} +EXPORT_SYMBOL_GPL(coresight_get_percpu_sink); + static int coresight_id_match(struct device *dev, void *data) { int trace_id, i_trace_id; @@ -145,30 +158,32 @@ static int coresight_find_link_outport(struct coresight_device *csdev, return -ENODEV; } -static inline u32 coresight_read_claim_tags(void __iomem *base) +static inline u32 coresight_read_claim_tags(struct coresight_device *csdev) { - return readl_relaxed(base + CORESIGHT_CLAIMCLR); + return csdev_access_relaxed_read32(&csdev->access, CORESIGHT_CLAIMCLR); } -static inline bool coresight_is_claimed_self_hosted(void __iomem *base) +static inline bool coresight_is_claimed_self_hosted(struct coresight_device *csdev) { - return coresight_read_claim_tags(base) == CORESIGHT_CLAIM_SELF_HOSTED; + return coresight_read_claim_tags(csdev) == CORESIGHT_CLAIM_SELF_HOSTED; } -static inline bool coresight_is_claimed_any(void __iomem *base) +static inline bool coresight_is_claimed_any(struct coresight_device *csdev) { - return coresight_read_claim_tags(base) != 0; + return coresight_read_claim_tags(csdev) != 0; } -static inline void coresight_set_claim_tags(void __iomem *base) +static inline void coresight_set_claim_tags(struct coresight_device *csdev) { - writel_relaxed(CORESIGHT_CLAIM_SELF_HOSTED, base + CORESIGHT_CLAIMSET); + csdev_access_relaxed_write32(&csdev->access, CORESIGHT_CLAIM_SELF_HOSTED, + CORESIGHT_CLAIMSET); isb(); } -static inline void coresight_clear_claim_tags(void __iomem *base) +static inline void coresight_clear_claim_tags(struct coresight_device *csdev) { - writel_relaxed(CORESIGHT_CLAIM_SELF_HOSTED, base + CORESIGHT_CLAIMCLR); + csdev_access_relaxed_write32(&csdev->access, CORESIGHT_CLAIM_SELF_HOSTED, + CORESIGHT_CLAIMCLR); isb(); } @@ -182,27 +197,33 @@ static inline void coresight_clear_claim_tags(void __iomem *base) * Called with CS_UNLOCKed for the component. * Returns : 0 on success */ -int coresight_claim_device_unlocked(void __iomem *base) +int coresight_claim_device_unlocked(struct coresight_device *csdev) { - if (coresight_is_claimed_any(base)) + if (WARN_ON(!csdev)) + return -EINVAL; + + if (coresight_is_claimed_any(csdev)) return -EBUSY; - coresight_set_claim_tags(base); - if (coresight_is_claimed_self_hosted(base)) + coresight_set_claim_tags(csdev); + if (coresight_is_claimed_self_hosted(csdev)) return 0; /* There was a race setting the tags, clean up and fail */ - coresight_clear_claim_tags(base); + coresight_clear_claim_tags(csdev); return -EBUSY; } EXPORT_SYMBOL_GPL(coresight_claim_device_unlocked); -int coresight_claim_device(void __iomem *base) +int coresight_claim_device(struct coresight_device *csdev) { int rc; - CS_UNLOCK(base); - rc = coresight_claim_device_unlocked(base); - CS_LOCK(base); + if (WARN_ON(!csdev)) + return -EINVAL; + + CS_UNLOCK(csdev->access.base); + rc = coresight_claim_device_unlocked(csdev); + CS_LOCK(csdev->access.base); return rc; } @@ -212,11 +233,14 @@ EXPORT_SYMBOL_GPL(coresight_claim_device); * coresight_disclaim_device_unlocked : Clear the claim tags for the device. * Called with CS_UNLOCKed for the component. */ -void coresight_disclaim_device_unlocked(void __iomem *base) +void coresight_disclaim_device_unlocked(struct coresight_device *csdev) { - if (coresight_is_claimed_self_hosted(base)) - coresight_clear_claim_tags(base); + if (WARN_ON(!csdev)) + return; + + if (coresight_is_claimed_self_hosted(csdev)) + coresight_clear_claim_tags(csdev); else /* * The external agent may have not honoured our claim @@ -227,11 +251,14 @@ void coresight_disclaim_device_unlocked(void __iomem *base) } EXPORT_SYMBOL_GPL(coresight_disclaim_device_unlocked); -void coresight_disclaim_device(void __iomem *base) +void coresight_disclaim_device(struct coresight_device *csdev) { - CS_UNLOCK(base); - coresight_disclaim_device_unlocked(base); - CS_LOCK(base); + if (WARN_ON(!csdev)) + return; + + CS_UNLOCK(csdev->access.base); + coresight_disclaim_device_unlocked(csdev); + CS_LOCK(csdev->access.base); } EXPORT_SYMBOL_GPL(coresight_disclaim_device); @@ -764,6 +791,14 @@ static int _coresight_build_path(struct coresight_device *csdev, if (csdev == sink) goto out; + if (coresight_is_percpu_source(csdev) && coresight_is_percpu_sink(sink) && + sink == per_cpu(csdev_sink, source_ops(csdev)->cpu_id(csdev))) { + if (_coresight_build_path(sink, sink, path) == 0) { + found = true; + goto out; + } + } + /* Not a sink - recursively explore each port found on this element */ for (i = 0; i < csdev->pdata->nr_outport; i++) { struct coresight_device *child_dev; @@ -979,8 +1014,12 @@ coresight_find_default_sink(struct coresight_device *csdev) int depth = 0; /* look for a default sink if we have not found for this device */ - if (!csdev->def_sink) - csdev->def_sink = coresight_find_sink(csdev, &depth); + if (!csdev->def_sink) { + if (coresight_is_percpu_source(csdev)) + csdev->def_sink = per_cpu(csdev_sink, source_ops(csdev)->cpu_id(csdev)); + if (!csdev->def_sink) + csdev->def_sink = coresight_find_sink(csdev, &depth); + } return csdev->def_sink; } @@ -1413,23 +1452,24 @@ static void coresight_remove_conns(struct coresight_device *csdev) } /** - * coresight_timeout - loop until a bit has changed to a specific state. - * @addr: base address of the area of interest. - * @offset: address of a register, starting from @addr. + * coresight_timeout - loop until a bit has changed to a specific register + * state. + * @csa: coresight device access for the device + * @offset: Offset of the register from the base of the device. * @position: the position of the bit of interest. * @value: the value the bit should have. * * Return: 0 as soon as the bit has taken the desired state or -EAGAIN if * TIMEOUT_US has elapsed, which ever happens first. */ - -int coresight_timeout(void __iomem *addr, u32 offset, int position, int value) +int coresight_timeout(struct csdev_access *csa, u32 offset, + int position, int value) { int i; u32 val; for (i = TIMEOUT_US; i > 0; i--) { - val = __raw_readl(addr + offset); + val = csdev_access_read32(csa, offset); /* waiting on the bit to go from 0 to 1 */ if (value) { if (val & BIT(position)) @@ -1453,6 +1493,48 @@ int coresight_timeout(void __iomem *addr, u32 offset, int position, int value) } EXPORT_SYMBOL_GPL(coresight_timeout); +u32 coresight_relaxed_read32(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_relaxed_read32(&csdev->access, offset); +} + +u32 coresight_read32(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_read32(&csdev->access, offset); +} + +void coresight_relaxed_write32(struct coresight_device *csdev, + u32 val, u32 offset) +{ + csdev_access_relaxed_write32(&csdev->access, val, offset); +} + +void coresight_write32(struct coresight_device *csdev, u32 val, u32 offset) +{ + csdev_access_write32(&csdev->access, val, offset); +} + +u64 coresight_relaxed_read64(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_relaxed_read64(&csdev->access, offset); +} + +u64 coresight_read64(struct coresight_device *csdev, u32 offset) +{ + return csdev_access_read64(&csdev->access, offset); +} + +void coresight_relaxed_write64(struct coresight_device *csdev, + u64 val, u32 offset) +{ + csdev_access_relaxed_write64(&csdev->access, val, offset); +} + +void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset) +{ + csdev_access_write64(&csdev->access, val, offset); +} + /* * coresight_release_platform_data: Release references to the devices connected * to the output port of this device. @@ -1519,6 +1601,7 @@ struct coresight_device *coresight_register(struct coresight_desc *desc) csdev->type = desc->type; csdev->subtype = desc->subtype; csdev->ops = desc->ops; + csdev->access = desc->access; csdev->orphan = false; csdev->dev.type = &coresight_dev_type[desc->type]; diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index 0276700c246d59351c4792b09c19b5f793f1a2bc..8988b2ed2ea6f5d3f5fed1606d824d6bf6f1781f 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -102,7 +102,7 @@ static int cti_enable_hw(struct cti_drvdata *drvdata) goto cti_state_unchanged; /* claim the device */ - rc = coresight_claim_device(drvdata->base); + rc = coresight_claim_device(drvdata->csdev); if (rc) goto cti_err_not_enabled; @@ -136,7 +136,7 @@ static void cti_cpuhp_enable_hw(struct cti_drvdata *drvdata) goto cti_hp_not_enabled; /* try to claim the device */ - if (coresight_claim_device(drvdata->base)) + if (coresight_claim_device(drvdata->csdev)) goto cti_hp_not_enabled; cti_write_all_hw_regs(drvdata); @@ -154,6 +154,7 @@ static int cti_disable_hw(struct cti_drvdata *drvdata) { struct cti_config *config = &drvdata->config; struct device *dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; spin_lock(&drvdata->spinlock); @@ -171,7 +172,7 @@ static int cti_disable_hw(struct cti_drvdata *drvdata) writel_relaxed(0, drvdata->base + CTICONTROL); config->hw_enabled = false; - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); spin_unlock(&drvdata->spinlock); pm_runtime_put(dev->parent); @@ -655,6 +656,7 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, void *v) { struct cti_drvdata *drvdata; + struct coresight_device *csdev; unsigned int cpu = smp_processor_id(); int notify_res = NOTIFY_OK; @@ -662,6 +664,7 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, return NOTIFY_OK; drvdata = cti_cpu_drvdata[cpu]; + csdev = drvdata->csdev; if (WARN_ON_ONCE(drvdata->ctidev.cpu != cpu)) return NOTIFY_BAD; @@ -673,13 +676,13 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, /* CTI regs all static - we have a copy & nothing to save */ drvdata->config.hw_powered = false; if (drvdata->config.hw_enabled) - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(csdev); break; case CPU_PM_ENTER_FAILED: drvdata->config.hw_powered = true; if (drvdata->config.hw_enabled) { - if (coresight_claim_device(drvdata->base)) + if (coresight_claim_device(csdev)) drvdata->config.hw_enabled = false; } break; @@ -692,7 +695,7 @@ static int cti_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, /* check enable reference count to enable HW */ if (atomic_read(&drvdata->config.enable_req_count)) { /* check we can claim the device as we re-power */ - if (coresight_claim_device(drvdata->base)) + if (coresight_claim_device(csdev)) goto cti_notify_exit; drvdata->config.hw_enabled = true; @@ -736,7 +739,7 @@ static int cti_dying_cpu(unsigned int cpu) spin_lock(&drvdata->spinlock); drvdata->config.hw_powered = false; if (drvdata->config.hw_enabled) - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); spin_unlock(&drvdata->spinlock); return 0; } @@ -868,6 +871,7 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + cti_desc.access = CSDEV_ACCESS_IOMEM(base); dev_set_drvdata(dev, drvdata); diff --git a/drivers/hwtracing/coresight/coresight-etb10.c b/drivers/hwtracing/coresight/coresight-etb10.c index 51c801c05e5c30cac3e4f1b84bb79e01a5da4eb5..f775cbee12b88d7c65ac0c46d71a595fb7cda6d3 100644 --- a/drivers/hwtracing/coresight/coresight-etb10.c +++ b/drivers/hwtracing/coresight/coresight-etb10.c @@ -132,7 +132,7 @@ static void __etb_enable_hw(struct etb_drvdata *drvdata) static int etb_enable_hw(struct etb_drvdata *drvdata) { - int rc = coresight_claim_device(drvdata->base); + int rc = coresight_claim_device(drvdata->csdev); if (rc) return rc; @@ -252,6 +252,7 @@ static void __etb_disable_hw(struct etb_drvdata *drvdata) { u32 ffcr; struct device *dev = &drvdata->csdev->dev; + struct csdev_access *csa = &drvdata->csdev->access; CS_UNLOCK(drvdata->base); @@ -263,7 +264,7 @@ static void __etb_disable_hw(struct etb_drvdata *drvdata) ffcr |= ETB_FFCR_FON_MAN; writel_relaxed(ffcr, drvdata->base + ETB_FFCR); - if (coresight_timeout(drvdata->base, ETB_FFCR, ETB_FFCR_BIT, 0)) { + if (coresight_timeout(csa, ETB_FFCR, ETB_FFCR_BIT, 0)) { dev_err(dev, "timeout while waiting for completion of Manual Flush\n"); } @@ -271,7 +272,7 @@ static void __etb_disable_hw(struct etb_drvdata *drvdata) /* disable trace capture */ writel_relaxed(0x0, drvdata->base + ETB_CTL_REG); - if (coresight_timeout(drvdata->base, ETB_FFSR, ETB_FFSR_BIT, 1)) { + if (coresight_timeout(csa, ETB_FFSR, ETB_FFSR_BIT, 1)) { dev_err(dev, "timeout while waiting for Formatter to Stop\n"); } @@ -344,7 +345,7 @@ static void etb_disable_hw(struct etb_drvdata *drvdata) { __etb_disable_hw(drvdata); etb_dump_hw(drvdata); - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); } static int etb_disable(struct coresight_device *csdev) @@ -757,6 +758,7 @@ static int etb_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); spin_lock_init(&drvdata->spinlock); diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index bdc34ca449f7145ba06755738e4e97029310564f..34a920fbb08d45b6ca9b9d9686d63488c65839f9 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -24,7 +24,26 @@ static struct pmu etm_pmu; static bool etm_perf_up; -static DEFINE_PER_CPU(struct perf_output_handle, ctx_handle); +/* + * An ETM context for a running event includes the perf aux handle + * and aux_data. For ETM, the aux_data (etm_event_data), consists of + * the trace path and the sink configuration. The event data is accessible + * via perf_get_aux(handle). However, a sink could "end" a perf output + * handle via the IRQ handler. And if the "sink" encounters a failure + * to "begin" another session (e.g due to lack of space in the buffer), + * the handle will be cleared. Thus, the event_data may not be accessible + * from the handle when we get to the etm_event_stop(), which is required + * for stopping the trace path. The event_data is guaranteed to stay alive + * until "free_aux()", which cannot happen as long as the event is active on + * the ETM. Thus the event_data for the session must be part of the ETM context + * to make sure we can disable the trace path. + */ +struct etm_ctxt { + struct perf_output_handle handle; + struct etm_event_data *event_data; +}; + +static DEFINE_PER_CPU(struct etm_ctxt, etm_ctxt); static DEFINE_PER_CPU(struct coresight_device *, csdev_src); /* ETMv3.5/PTM's ETMCR is 'config' */ @@ -304,13 +323,18 @@ static void etm_event_start(struct perf_event *event, int flags) { int cpu = smp_processor_id(); struct etm_event_data *event_data; - struct perf_output_handle *handle = this_cpu_ptr(&ctx_handle); + struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt); + struct perf_output_handle *handle = &ctxt->handle; struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); struct list_head *path; if (!csdev) goto fail; + /* Have we messed up our tracking ? */ + if (WARN_ON(ctxt->event_data)) + goto fail; + /* * Deal with the ring buffer API and get a handle on the * session's information. @@ -325,9 +349,14 @@ static void etm_event_start(struct perf_event *event, int flags) * sink from this ETM. We can't do much in this case if * the sink was specified or hinted to the driver. For * now, simply don't record anything on this ETM. + * + * As such we pretend that everything is fine, and let + * it continue without actually tracing. The event could + * continue tracing when it moves to a CPU where it is + * reachable to a sink. */ if (!cpumask_test_cpu(cpu, &event_data->mask)) - goto fail_end_stop; + goto out; path = etm_event_cpu_path(event_data, cpu); /* We need a sink, no need to continue without one */ @@ -339,24 +368,32 @@ static void etm_event_start(struct perf_event *event, int flags) if (coresight_enable_path(path, CS_MODE_PERF, handle)) goto fail_end_stop; - /* Tell the perf core the event is alive */ - event->hw.state = 0; - /* Finally enable the tracer */ if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF)) goto fail_disable_path; out: + /* Tell the perf core the event is alive */ + event->hw.state = 0; + /* Save the event_data for this ETM */ + ctxt->event_data = event_data; return; fail_disable_path: coresight_disable_path(path); fail_end_stop: - perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); - perf_aux_output_end(handle, 0); + /* + * Check if the handle is still associated with the event, + * to handle cases where if the sink failed to start the + * trace and TRUNCATED the handle already. + */ + if (READ_ONCE(handle->event)) { + perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + perf_aux_output_end(handle, 0); + } fail: event->hw.state = PERF_HES_STOPPED; - goto out; + return; } static void etm_event_stop(struct perf_event *event, int mode) @@ -364,13 +401,43 @@ static void etm_event_stop(struct perf_event *event, int mode) int cpu = smp_processor_id(); unsigned long size; struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); - struct perf_output_handle *handle = this_cpu_ptr(&ctx_handle); - struct etm_event_data *event_data = perf_get_aux(handle); + struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt); + struct perf_output_handle *handle = &ctxt->handle; + struct etm_event_data *event_data; struct list_head *path; + /* + * If we still have access to the event_data via handle, + * confirm that we haven't messed up the tracking. + */ + if (handle->event && + WARN_ON(perf_get_aux(handle) != ctxt->event_data)) + return; + + event_data = ctxt->event_data; + /* Clear the event_data as this ETM is stopping the trace. */ + ctxt->event_data = NULL; + if (event->hw.state == PERF_HES_STOPPED) return; + /* We must have a valid event_data for a running event */ + if (WARN_ON(!event_data)) + return; + + /* + * Check if this ETM was allowed to trace, as decided at + * etm_setup_aux(). If it wasn't allowed to trace, then + * nothing needs to be torn down other than outputting a + * zero sized record. + */ + if (handle->event && (mode & PERF_EF_UPDATE) && + !cpumask_test_cpu(cpu, &event_data->mask)) { + event->hw.state = PERF_HES_STOPPED; + perf_aux_output_end(handle, 0); + return; + } + if (!csdev) return; @@ -388,7 +455,13 @@ static void etm_event_stop(struct perf_event *event, int mode) /* tell the core */ event->hw.state = PERF_HES_STOPPED; - if (mode & PERF_EF_UPDATE) { + /* + * If the handle is not bound to an event anymore + * (e.g, the sink driver was unable to restart the + * handle due to lack of buffer space), we don't + * have to do anything here. + */ + if (handle->event && (mode & PERF_EF_UPDATE)) { if (WARN_ON_ONCE(handle->event != event)) return; @@ -398,7 +471,21 @@ static void etm_event_stop(struct perf_event *event, int mode) size = sink_ops(sink)->update_buffer(sink, handle, event_data->snk_config); - perf_aux_output_end(handle, size); + /* + * Make sure the handle is still valid as the + * sink could have closed it from an IRQ. + * The sink driver must handle the race with + * update_buffer() and IRQ. Thus either we + * should get a valid handle and valid size + * (which may be 0). + * + * But we should never get a non-zero size with + * an invalid handle. + */ + if (READ_ONCE(handle->event)) + perf_aux_output_end(handle, size); + else + WARN_ON(size); } /* Disabling the path make its elements available to other sessions */ diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c index 683a69e88efda89ab5e100d92a048a47c323181c..cf64ce73a7412136a358ade02ab0cc7bacaddbf9 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c @@ -358,10 +358,11 @@ static int etm_enable_hw(struct etm_drvdata *drvdata) int i, rc; u32 etmcr; struct etm_config *config = &drvdata->config; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) goto done; @@ -566,6 +567,7 @@ static void etm_disable_hw(void *info) int i; struct etm_drvdata *drvdata = info; struct etm_config *config = &drvdata->config; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); etm_set_prog(drvdata); @@ -577,7 +579,7 @@ static void etm_disable_hw(void *info) config->cntr_val[i] = etm_readl(drvdata, ETMCNTVRn(i)); etm_set_pwrdwn(drvdata); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); @@ -602,7 +604,7 @@ static void etm_disable_perf(struct coresight_device *csdev) * power down the tracer. */ etm_set_pwrdwn(drvdata); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } @@ -839,6 +841,7 @@ static int etm_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); spin_lock_init(&drvdata->spinlock); diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 881da29cbd5abf3fe8b0ada7c2c47cd8665a7e37..9d0895cde9d6587b66513f903be5ba111b9f398b 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -27,9 +27,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -37,6 +39,7 @@ #include "coresight-etm4x.h" #include "coresight-etm-perf.h" +#include "coresight-self-hosted-trace.h" static int boot_enable; module_param(boot_enable, int, 0444); @@ -59,32 +62,143 @@ static u64 etm4_get_access_type(struct etmv4_config *config); static enum cpuhp_state hp_online; -static void etm4_os_unlock(struct etmv4_drvdata *drvdata) +struct etm4_init_arg { + struct etmv4_drvdata *drvdata; + struct csdev_access *csa; +}; + +u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit) { - /* Writing 0 to TRCOSLAR unlocks the trace registers */ - writel_relaxed(0x0, drvdata->base + TRCOSLAR); - drvdata->os_unlock = true; - isb(); + u64 res = 0; + + switch (offset) { + ETM4x_READ_SYSREG_CASES(res) + default : + pr_warn_ratelimited("etm4x: trying to read unsupported register @%x\n", + offset); + } + + if (!_relaxed) + __iormb(res); /* Imitate the !relaxed I/O helpers */ + + return res; } -static void etm4_os_lock(struct etmv4_drvdata *drvdata) +void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) { - /* Writing 0x1 to TRCOSLAR locks the trace registers */ - writel_relaxed(0x1, drvdata->base + TRCOSLAR); - drvdata->os_unlock = false; - isb(); + if (!_relaxed) + __iowmb(); /* Imitate the !relaxed I/O helpers */ + if (!_64bit) + val &= GENMASK(31, 0); + + switch (offset) { + ETM4x_WRITE_SYSREG_CASES(val) + default : + pr_warn_ratelimited("etm4x: trying to write to unsupported register @%x\n", + offset); + } +} + +static u64 ete_sysreg_read(u32 offset, bool _relaxed, bool _64bit) +{ + u64 res = 0; + + switch (offset) { + ETE_READ_CASES(res) + default : + pr_warn_ratelimited("ete: trying to read unsupported register @%x\n", + offset); + } + + if (!_relaxed) + __iormb(res); /* Imitate the !relaxed I/O helpers */ + + return res; } -static bool etm4_arch_supported(u8 arch) +static void ete_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit) { - /* Mask out the minor version number */ - switch (arch & 0xf0) { - case ETM_ARCH_V4: + if (!_relaxed) + __iowmb(); /* Imitate the !relaxed I/O helpers */ + if (!_64bit) + val &= GENMASK(31, 0); + + switch (offset) { + ETE_WRITE_CASES(val) + default : + pr_warn_ratelimited("ete: trying to write to unsupported register @%x\n", + offset); + } +} + +static void etm_detect_os_lock(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + u32 oslsr = etm4x_relaxed_read32(csa, TRCOSLSR); + + drvdata->os_lock_model = ETM_OSLSR_OSLM(oslsr); +} + +static void etm_write_os_lock(struct etmv4_drvdata *drvdata, + struct csdev_access *csa, u32 val) +{ + val = !!val; + + switch (drvdata->os_lock_model) { + case ETM_OSLOCK_PRESENT: + etm4x_relaxed_write32(csa, val, TRCOSLAR); + break; + case ETM_OSLOCK_PE: + write_sysreg_s(val, SYS_OSLAR_EL1); break; default: - return false; + pr_warn_once("CPU%d: Unsupported Trace OSLock model: %x\n", + smp_processor_id(), drvdata->os_lock_model); + fallthrough; + case ETM_OSLOCK_NI: + return; } - return true; + isb(); +} + +static inline void etm4_os_unlock_csa(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + WARN_ON(drvdata->cpu != smp_processor_id()); + + /* Writing 0 to OS Lock unlocks the trace unit registers */ + etm_write_os_lock(drvdata, csa, 0x0); + drvdata->os_unlock = true; +} + +static void etm4_os_unlock(struct etmv4_drvdata *drvdata) +{ + if (!WARN_ON(!drvdata->csdev)) + etm4_os_unlock_csa(drvdata, &drvdata->csdev->access); +} + +static void etm4_os_lock(struct etmv4_drvdata *drvdata) +{ + if (WARN_ON(!drvdata->csdev)) + return; + /* Writing 0x1 to OS Lock locks the trace registers */ + etm_write_os_lock(drvdata, &drvdata->csdev->access, 0x1); + drvdata->os_unlock = false; +} + +static void etm4_cs_lock(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + /* Software Lock is only accessible via memory mapped interface */ + if (csa->io_mem) + CS_LOCK(csa->base); +} + +static void etm4_cs_unlock(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + if (csa->io_mem) + CS_UNLOCK(csa->base); } static int etm4_cpu_id(struct coresight_device *csdev) @@ -106,6 +220,45 @@ struct etm4_enable_arg { int rc; }; +/* + * etm4x_prohibit_trace - Prohibit the CPU from tracing at all ELs. + * When the CPU supports FEAT_TRF, we could move the ETM to a trace + * prohibited state by filtering the Exception levels via TRFCR_EL1. + */ +static void etm4x_prohibit_trace(struct etmv4_drvdata *drvdata) +{ + /* If the CPU doesn't support FEAT_TRF, nothing to do */ + if (!drvdata->trfcr) + return; + cpu_prohibit_trace(); +} + +/* + * etm4x_allow_trace - Allow CPU tracing in the respective ELs, + * as configured by the drvdata->config.mode for the current + * session. Even though we have TRCVICTLR bits to filter the + * trace in the ELs, it doesn't prevent the ETM from generating + * a packet (e.g, TraceInfo) that might contain the addresses from + * the excluded levels. Thus we use the additional controls provided + * via the Trace Filtering controls (FEAT_TRF) to make sure no trace + * is generated for the excluded ELs. + */ +static void etm4x_allow_trace(struct etmv4_drvdata *drvdata) +{ + u64 trfcr = drvdata->trfcr; + + /* If the CPU doesn't support FEAT_TRF, nothing to do */ + if (!trfcr) + return; + + if (drvdata->config.mode & ETM_MODE_EXCL_KERN) + trfcr &= ~TRFCR_ELx_ExTRE; + if (drvdata->config.mode & ETM_MODE_EXCL_USER) + trfcr &= ~TRFCR_ELx_E0TRE; + + write_trfcr(trfcr); +} + #ifdef CONFIG_ETM4X_IMPDEF_FEATURE #define HISI_HIP08_AMBA_ID 0x000b6d01 @@ -146,12 +299,12 @@ static void etm4_hisi_config_core_commit(void *info) write_sysreg_s(val, HISI_HIP08_CORE_COMMIT_REG); } -static void etm4_hisi_config_auxctrlr(void *info) +static void etm4_hisi_config_set_auxctrlr(void *info) { - struct etmv4_drvdata *drvdata = info; + struct csdev_access *csa = info; /* Switch the ETM to idle state */ - writel_relaxed(HISI_HIP08_AUXCTRL_CHICKEN_BIT, drvdata->base + TRCAUXCTLR); + etm4x_relaxed_write32(csa, HISI_HIP08_AUXCTRL_CHICKEN_BIT, TRCAUXCTLR); } static struct etm4_arch_features etm4_features[] = { @@ -159,7 +312,7 @@ static struct etm4_arch_features etm4_features[] = { .arch_callback = etm4_hisi_config_core_commit, }, [ETM4_IMPDEF_HISI_SET_AUXCTRLR] = { - .arch_callback = etm4_hisi_config_auxctrlr, + .arch_callback = etm4_hisi_config_set_auxctrlr, }, {}, }; @@ -177,7 +330,7 @@ static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata) ftr->arch_callback(&enable); if (bit == ETM4_IMPDEF_HISI_SET_AUXCTRLR && ftr->arch_callback) - ftr->arch_callback(drvdata); + ftr->arch_callback(&drvdata->csdev->access); } } @@ -206,7 +359,9 @@ static void etm4_check_arch_features(struct etmv4_drvdata *drvdata, #else static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata) { - writel_relaxed(0x0, drvdata->base + TRCAUXCTLR); + struct csdev_access *csa = &drvdata->csdev->access; + + etm4x_relaxed_write32(csa, 0x0, TRCAUXCTLR); } static void etm4_disable_arch_specific(struct etmv4_drvdata *drvdata) @@ -223,57 +378,64 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) { int i, rc; struct etmv4_config *config = &drvdata->config; - struct device *etm_dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; + struct device *etm_dev = &csdev->dev; + struct csdev_access *csa = &csdev->access; - CS_UNLOCK(drvdata->base); + + etm4_cs_unlock(drvdata, csa); etm4_enable_arch_specific(drvdata); etm4_os_unlock(drvdata); - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) goto done; /* Disable the trace unit before programming trace registers */ - writel_relaxed(0, drvdata->base + TRCPRGCTLR); + etm4x_relaxed_write32(csa, 0, TRCPRGCTLR); + + /* + * If we use system instructions, we need to synchronize the + * write to the TRCPRGCTLR, before accessing the TRCSTATR. + * See ARM IHI0064F, section + * "4.3.7 Synchronization of register updates" + */ + if (!csa->io_mem) + isb(); /* wait for TRCSTATR.IDLE to go up */ - if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) dev_err(etm_dev, "timeout while waiting for Idle Trace Status\n"); if (drvdata->nr_pe) - writel_relaxed(config->pe_sel, drvdata->base + TRCPROCSELR); - writel_relaxed(config->cfg, drvdata->base + TRCCONFIGR); + etm4x_relaxed_write32(csa, config->pe_sel, TRCPROCSELR); + etm4x_relaxed_write32(csa, config->cfg, TRCCONFIGR); /* nothing specific implemented */ - writel_relaxed(config->eventctrl0, drvdata->base + TRCEVENTCTL0R); - writel_relaxed(config->eventctrl1, drvdata->base + TRCEVENTCTL1R); + etm4x_relaxed_write32(csa, config->eventctrl0, TRCEVENTCTL0R); + etm4x_relaxed_write32(csa, config->eventctrl1, TRCEVENTCTL1R); if (drvdata->stallctl) - writel_relaxed(config->stall_ctrl, drvdata->base + TRCSTALLCTLR); - writel_relaxed(config->ts_ctrl, drvdata->base + TRCTSCTLR); - writel_relaxed(config->syncfreq, drvdata->base + TRCSYNCPR); - writel_relaxed(config->ccctlr, drvdata->base + TRCCCCTLR); - writel_relaxed(config->bb_ctrl, drvdata->base + TRCBBCTLR); - writel_relaxed(drvdata->trcid, drvdata->base + TRCTRACEIDR); - writel_relaxed(config->vinst_ctrl, drvdata->base + TRCVICTLR); - writel_relaxed(config->viiectlr, drvdata->base + TRCVIIECTLR); - writel_relaxed(config->vissctlr, - drvdata->base + TRCVISSCTLR); + etm4x_relaxed_write32(csa, config->stall_ctrl, TRCSTALLCTLR); + etm4x_relaxed_write32(csa, config->ts_ctrl, TRCTSCTLR); + etm4x_relaxed_write32(csa, config->syncfreq, TRCSYNCPR); + etm4x_relaxed_write32(csa, config->ccctlr, TRCCCCTLR); + etm4x_relaxed_write32(csa, config->bb_ctrl, TRCBBCTLR); + etm4x_relaxed_write32(csa, drvdata->trcid, TRCTRACEIDR); + etm4x_relaxed_write32(csa, config->vinst_ctrl, TRCVICTLR); + etm4x_relaxed_write32(csa, config->viiectlr, TRCVIIECTLR); + etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR); if (drvdata->nr_pe_cmp) - writel_relaxed(config->vipcssctlr, - drvdata->base + TRCVIPCSSCTLR); + etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR); for (i = 0; i < drvdata->nrseqstate - 1; i++) - writel_relaxed(config->seq_ctrl[i], - drvdata->base + TRCSEQEVRn(i)); - writel_relaxed(config->seq_rst, drvdata->base + TRCSEQRSTEVR); - writel_relaxed(config->seq_state, drvdata->base + TRCSEQSTR); - writel_relaxed(config->ext_inp, drvdata->base + TRCEXTINSELR); + etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i)); + etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR); + etm4x_relaxed_write32(csa, config->seq_state, TRCSEQSTR); + etm4x_relaxed_write32(csa, config->ext_inp, TRCEXTINSELR); + for (i = 0; i < drvdata->nr_cntr; i++) { - writel_relaxed(config->cntrldvr[i], - drvdata->base + TRCCNTRLDVRn(i)); - writel_relaxed(config->cntr_ctrl[i], - drvdata->base + TRCCNTCTLRn(i)); - writel_relaxed(config->cntr_val[i], - drvdata->base + TRCCNTVRn(i)); + etm4x_relaxed_write32(csa, config->cntrldvr[i], TRCCNTRLDVRn(i)); + etm4x_relaxed_write32(csa, config->cntr_ctrl[i], TRCCNTCTLRn(i)); + etm4x_relaxed_write32(csa, config->cntr_val[i], TRCCNTVRn(i)); } /* @@ -281,54 +443,59 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) * such start at 2. */ for (i = 2; i < drvdata->nr_resource * 2; i++) - writel_relaxed(config->res_ctrl[i], - drvdata->base + TRCRSCTLRn(i)); + etm4x_relaxed_write32(csa, config->res_ctrl[i], TRCRSCTLRn(i)); for (i = 0; i < drvdata->nr_ss_cmp; i++) { /* always clear status bit on restart if using single-shot */ if (config->ss_ctrl[i] || config->ss_pe_cmp[i]) config->ss_status[i] &= ~BIT(31); - writel_relaxed(config->ss_ctrl[i], - drvdata->base + TRCSSCCRn(i)); - writel_relaxed(config->ss_status[i], - drvdata->base + TRCSSCSRn(i)); - writel_relaxed(config->ss_pe_cmp[i], - drvdata->base + TRCSSPCICRn(i)); + etm4x_relaxed_write32(csa, config->ss_ctrl[i], TRCSSCCRn(i)); + etm4x_relaxed_write32(csa, config->ss_status[i], TRCSSCSRn(i)); + etm4x_relaxed_write32(csa, config->ss_pe_cmp[i], TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp; i++) { - writeq_relaxed(config->addr_val[i], - drvdata->base + TRCACVRn(i)); - writeq_relaxed(config->addr_acc[i], - drvdata->base + TRCACATRn(i)); + etm4x_relaxed_write64(csa, config->addr_val[i], TRCACVRn(i)); + etm4x_relaxed_write64(csa, config->addr_acc[i], TRCACATRn(i)); } for (i = 0; i < drvdata->numcidc; i++) - writeq_relaxed(config->ctxid_pid[i], - drvdata->base + TRCCIDCVRn(i)); - writel_relaxed(config->ctxid_mask0, drvdata->base + TRCCIDCCTLR0); + etm4x_relaxed_write64(csa, config->ctxid_pid[i], TRCCIDCVRn(i)); + etm4x_relaxed_write32(csa, config->ctxid_mask0, TRCCIDCCTLR0); if (drvdata->numcidc > 4) - writel_relaxed(config->ctxid_mask1, drvdata->base + TRCCIDCCTLR1); + etm4x_relaxed_write32(csa, config->ctxid_mask1, TRCCIDCCTLR1); for (i = 0; i < drvdata->numvmidc; i++) - writeq_relaxed(config->vmid_val[i], - drvdata->base + TRCVMIDCVRn(i)); - writel_relaxed(config->vmid_mask0, drvdata->base + TRCVMIDCCTLR0); + etm4x_relaxed_write64(csa, config->vmid_val[i], TRCVMIDCVRn(i)); + etm4x_relaxed_write32(csa, config->vmid_mask0, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - writel_relaxed(config->vmid_mask1, drvdata->base + TRCVMIDCCTLR1); + etm4x_relaxed_write32(csa, config->vmid_mask1, TRCVMIDCCTLR1); if (!drvdata->skip_power_up) { + u32 trcpdcr = etm4x_relaxed_read32(csa, TRCPDCR); + /* * Request to keep the trace unit powered and also * emulation of powerdown */ - writel_relaxed(readl_relaxed(drvdata->base + TRCPDCR) | - TRCPDCR_PU, drvdata->base + TRCPDCR); + etm4x_relaxed_write32(csa, trcpdcr | TRCPDCR_PU, TRCPDCR); } + /* + * ETE mandates that the TRCRSR is written to before + * enabling it. + */ + if (etm4x_is_ete(drvdata)) + etm4x_relaxed_write32(csa, TRCRSR_TA, TRCRSR); + + etm4x_allow_trace(drvdata); /* Enable the trace unit */ - writel_relaxed(1, drvdata->base + TRCPRGCTLR); + etm4x_relaxed_write32(csa, 1, TRCPRGCTLR); + + /* Synchronize the register updates for sysreg access */ + if (!csa->io_mem) + isb(); /* wait for TRCSTATR.IDLE to go back down to '0' */ - if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 0)) + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 0)) dev_err(etm_dev, "timeout while waiting for Idle Trace Status\n"); @@ -340,7 +507,7 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) isb(); done: - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n", drvdata->cpu, rc); @@ -592,24 +759,31 @@ static void etm4_disable_hw(void *info) u32 control; struct etmv4_drvdata *drvdata = info; struct etmv4_config *config = &drvdata->config; - struct device *etm_dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; + struct device *etm_dev = &csdev->dev; + struct csdev_access *csa = &csdev->access; int i; - CS_UNLOCK(drvdata->base); + etm4_cs_unlock(drvdata, csa); etm4_disable_arch_specific(drvdata); if (!drvdata->skip_power_up) { /* power can be removed from the trace unit now */ - control = readl_relaxed(drvdata->base + TRCPDCR); + control = etm4x_relaxed_read32(csa, TRCPDCR); control &= ~TRCPDCR_PU; - writel_relaxed(control, drvdata->base + TRCPDCR); + etm4x_relaxed_write32(csa, control, TRCPDCR); } - control = readl_relaxed(drvdata->base + TRCPRGCTLR); + control = etm4x_relaxed_read32(csa, TRCPRGCTLR); /* EN, bit[0] Trace unit enable bit */ control &= ~0x1; + /* + * If the CPU supports v8.4 Trace filter Control, + * set the ETM to trace prohibited region. + */ + etm4x_prohibit_trace(drvdata); /* * Make sure everything completes before disabling, as recommended * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register, @@ -617,29 +791,28 @@ static void etm4_disable_hw(void *info) */ dsb(sy); isb(); - writel_relaxed(control, drvdata->base + TRCPRGCTLR); + /* Trace synchronization barrier, is a nop if not supported */ + tsb_csync(); + etm4x_relaxed_write32(csa, control, TRCPRGCTLR); /* wait for TRCSTATR.PMSTABLE to go to '1' */ - if (coresight_timeout(drvdata->base, TRCSTATR, - TRCSTATR_PMSTABLE_BIT, 1)) + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) dev_err(etm_dev, "timeout while waiting for PM stable Trace Status\n"); - /* read the status of the single shot comparators */ for (i = 0; i < drvdata->nr_ss_cmp; i++) { config->ss_status[i] = - readl_relaxed(drvdata->base + TRCSSCSRn(i)); + etm4x_relaxed_read32(csa, TRCSSCSRn(i)); } /* read back the current counter values */ for (i = 0; i < drvdata->nr_cntr; i++) { config->cntr_val[i] = - readl_relaxed(drvdata->base + TRCCNTVRn(i)); + etm4x_relaxed_read32(csa, TRCCNTVRn(i)); } - coresight_disclaim_device_unlocked(drvdata->base); - - CS_LOCK(drvdata->base); + coresight_disclaim_device_unlocked(csdev); + etm4_cs_lock(drvdata, csa); dev_dbg(&drvdata->csdev->dev, "cpu: %d disable smp call done\n", drvdata->cpu); @@ -663,7 +836,7 @@ static int etm4_disable_perf(struct coresight_device *csdev, * scheduled again. Configuration of the start/stop logic happens in * function etm4_set_event_filters(). */ - control = readl_relaxed(drvdata->base + TRCVICTLR); + control = etm4x_relaxed_read32(&csdev->access, TRCVICTLR); /* TRCVICTLR::SSSTATUS, bit[9] */ filters->ssstatus = (control & BIT(9)); @@ -734,24 +907,149 @@ static const struct coresight_ops etm4_cs_ops = { .source_ops = &etm4_source_ops, }; +static inline bool cpu_supports_sysreg_trace(void) +{ + u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); + + return ((dfr0 >> ID_AA64DFR0_TRACEVER_SHIFT) & 0xfUL) > 0; +} + +static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + u32 devarch; + + if (!cpu_supports_sysreg_trace()) + return false; + + /* + * ETMs implementing sysreg access must implement TRCDEVARCH. + */ + devarch = read_etm4x_sysreg_const_offset(TRCDEVARCH); + switch (devarch & ETM_DEVARCH_ID_MASK) { + case ETM_DEVARCH_ETMv4x_ARCH: + *csa = (struct csdev_access) { + .io_mem = false, + .read = etm4x_sysreg_read, + .write = etm4x_sysreg_write, + }; + break; + case ETM_DEVARCH_ETE_ARCH: + *csa = (struct csdev_access) { + .io_mem = false, + .read = ete_sysreg_read, + .write = ete_sysreg_write, + }; + break; + default: + return false; + } + + drvdata->arch = etm_devarch_to_arch(devarch); + return true; +} + +static bool etm4_init_iomem_access(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + u32 devarch = readl_relaxed(drvdata->base + TRCDEVARCH); + u32 idr1 = readl_relaxed(drvdata->base + TRCIDR1); + + /* + * All ETMs must implement TRCDEVARCH to indicate that + * the component is an ETMv4. To support any broken + * implementations we fall back to TRCIDR1 check, which + * is not really reliable. + */ + if ((devarch & ETM_DEVARCH_ID_MASK) == ETM_DEVARCH_ETMv4x_ARCH) { + drvdata->arch = etm_devarch_to_arch(devarch); + } else { + pr_warn("CPU%d: ETM4x incompatible TRCDEVARCH: %x, falling back to TRCIDR1\n", + smp_processor_id(), devarch); + + if (ETM_TRCIDR1_ARCH_MAJOR(idr1) != ETM_TRCIDR1_ARCH_ETMv4) + return false; + drvdata->arch = etm_trcidr_to_arch(idr1); + } + + *csa = CSDEV_ACCESS_IOMEM(drvdata->base); + return true; +} + +static bool etm4_init_csdev_access(struct etmv4_drvdata *drvdata, + struct csdev_access *csa) +{ + /* + * Always choose the memory mapped io, if there is + * a memory map to prevent sysreg access on broken + * systems. + */ + if (drvdata->base) + return etm4_init_iomem_access(drvdata, csa); + + if (etm4_init_sysreg_access(drvdata, csa)) + return true; + + return false; +} + +static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata) +{ + u64 dfr0 = read_sysreg(id_aa64dfr0_el1); + u64 trfcr; + + drvdata->trfcr = 0; + if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRACE_FILT_SHIFT)) + return; + + /* + * If the CPU supports v8.4 SelfHosted Tracing, enable + * tracing at the kernel EL and EL0, forcing to use the + * virtual time as the timestamp. + */ + trfcr = (TRFCR_ELx_TS_VIRTUAL | + TRFCR_ELx_ExTRE | + TRFCR_ELx_E0TRE); + + /* If we are running at EL2, allow tracing the CONTEXTIDR_EL2. */ + if (is_kernel_in_hyp_mode()) + trfcr |= TRFCR_EL2_CX; + + drvdata->trfcr = trfcr; +} + static void etm4_init_arch_data(void *info) { u32 etmidr0; - u32 etmidr1; u32 etmidr2; u32 etmidr3; u32 etmidr4; u32 etmidr5; - struct etmv4_drvdata *drvdata = info; + struct etm4_init_arg *init_arg = info; + struct etmv4_drvdata *drvdata; + struct csdev_access *csa; int i; - /* Make sure all registers are accessible */ - etm4_os_unlock(drvdata); + drvdata = init_arg->drvdata; + csa = init_arg->csa; + + /* + * If we are unable to detect the access mechanism, + * or unable to detect the trace unit type, fail + * early. + */ + if (!etm4_init_csdev_access(drvdata, csa)) + return; + + /* Detect the support for OS Lock before we actually use it */ + etm_detect_os_lock(drvdata, csa); - CS_UNLOCK(drvdata->base); + /* Make sure all registers are accessible */ + etm4_os_unlock_csa(drvdata, csa); + etm4_cs_unlock(drvdata, csa); /* find all capabilities of the tracing unit */ - etmidr0 = readl_relaxed(drvdata->base + TRCIDR0); + etmidr0 = etm4x_relaxed_read32(csa, TRCIDR0); /* INSTP0, bits[2:1] P0 tracing support field */ if (BMVAL(etmidr0, 1, 1) && BMVAL(etmidr0, 2, 2)) @@ -790,17 +1088,8 @@ static void etm4_init_arch_data(void *info) /* TSSIZE, bits[28:24] Global timestamp size field */ drvdata->ts_size = BMVAL(etmidr0, 24, 28); - /* base architecture of trace unit */ - etmidr1 = readl_relaxed(drvdata->base + TRCIDR1); - /* - * TRCARCHMIN, bits[7:4] architecture the minor version number - * TRCARCHMAJ, bits[11:8] architecture major versin number - */ - drvdata->arch = BMVAL(etmidr1, 4, 11); - drvdata->config.arch = drvdata->arch; - /* maximum size of resources */ - etmidr2 = readl_relaxed(drvdata->base + TRCIDR2); + etmidr2 = etm4x_relaxed_read32(csa, TRCIDR2); /* CIDSIZE, bits[9:5] Indicates the Context ID size */ drvdata->ctxid_size = BMVAL(etmidr2, 5, 9); /* VMIDSIZE, bits[14:10] Indicates the VMID size */ @@ -808,7 +1097,7 @@ static void etm4_init_arch_data(void *info) /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */ drvdata->ccsize = BMVAL(etmidr2, 25, 28); - etmidr3 = readl_relaxed(drvdata->base + TRCIDR3); + etmidr3 = etm4x_relaxed_read32(csa, TRCIDR3); /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */ drvdata->ccitmin = BMVAL(etmidr3, 0, 11); /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */ @@ -853,7 +1142,7 @@ static void etm4_init_arch_data(void *info) drvdata->nooverflow = false; /* number of resources trace unit supports */ - etmidr4 = readl_relaxed(drvdata->base + TRCIDR4); + etmidr4 = etm4x_relaxed_read32(csa, TRCIDR4); /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */ drvdata->nr_addr_cmp = BMVAL(etmidr4, 0, 3); /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */ @@ -869,7 +1158,7 @@ static void etm4_init_arch_data(void *info) * Otherwise for values 0x1 and above the number is N + 1 as per v4.2. */ drvdata->nr_resource = BMVAL(etmidr4, 16, 19); - if ((drvdata->arch < ETM4X_ARCH_4V3) || (drvdata->nr_resource > 0)) + if ((drvdata->arch < ETM_ARCH_V4_3) || (drvdata->nr_resource > 0)) drvdata->nr_resource += 1; /* * NUMSSCC, bits[23:20] the number of single-shot @@ -879,14 +1168,14 @@ static void etm4_init_arch_data(void *info) drvdata->nr_ss_cmp = BMVAL(etmidr4, 20, 23); for (i = 0; i < drvdata->nr_ss_cmp; i++) { drvdata->config.ss_status[i] = - readl_relaxed(drvdata->base + TRCSSCSRn(i)); + etm4x_relaxed_read32(csa, TRCSSCSRn(i)); } /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */ drvdata->numcidc = BMVAL(etmidr4, 24, 27); /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */ drvdata->numvmidc = BMVAL(etmidr4, 28, 31); - etmidr5 = readl_relaxed(drvdata->base + TRCIDR5); + etmidr5 = etm4x_relaxed_read32(csa, TRCIDR5); /* NUMEXTIN, bits[8:0] number of external inputs implemented */ drvdata->nr_ext_inp = BMVAL(etmidr5, 0, 8); /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */ @@ -908,7 +1197,8 @@ static void etm4_init_arch_data(void *info) drvdata->nrseqstate = BMVAL(etmidr5, 25, 27); /* NUMCNTR, bits[30:28] number of counters available for tracing */ drvdata->nr_cntr = BMVAL(etmidr5, 28, 30); - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); + cpu_detect_trace_filtering(drvdata); } /* Set ELx trace filter access in the TRCVICTLR register */ @@ -1268,11 +1558,19 @@ static void etm4_init_trace_id(struct etmv4_drvdata *drvdata) drvdata->trcid = coresight_get_trace_id(drvdata->cpu); } -static int etm4_cpu_save(struct etmv4_drvdata *drvdata) +static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) { int i, ret = 0; struct etmv4_save_state *state; - struct device *etm_dev = &drvdata->csdev->dev; + struct coresight_device *csdev = drvdata->csdev; + struct csdev_access *csa; + struct device *etm_dev; + + if (WARN_ON(!csdev)) + return -ENODEV; + + etm_dev = &csdev->dev; + csa = &csdev->access; /* * As recommended by 3.4.1 ("The procedure when powering down the PE") @@ -1281,14 +1579,12 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) dsb(sy); isb(); - CS_UNLOCK(drvdata->base); - + etm4_cs_unlock(drvdata, csa); /* Lock the OS lock to disable trace and external debugger access */ etm4_os_lock(drvdata); /* wait for TRCSTATR.PMSTABLE to go up */ - if (coresight_timeout(drvdata->base, TRCSTATR, - TRCSTATR_PMSTABLE_BIT, 1)) { + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_PMSTABLE_BIT, 1)) { dev_err(etm_dev, "timeout while waiting for PM Stable Status\n"); etm4_os_unlock(drvdata); @@ -1298,56 +1594,57 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) state = drvdata->save_state; - state->trcprgctlr = readl(drvdata->base + TRCPRGCTLR); + state->trcprgctlr = etm4x_read32(csa, TRCPRGCTLR); if (drvdata->nr_pe) - state->trcprocselr = readl(drvdata->base + TRCPROCSELR); - state->trcconfigr = readl(drvdata->base + TRCCONFIGR); - state->trcauxctlr = readl(drvdata->base + TRCAUXCTLR); - state->trceventctl0r = readl(drvdata->base + TRCEVENTCTL0R); - state->trceventctl1r = readl(drvdata->base + TRCEVENTCTL1R); + state->trcprocselr = etm4x_read32(csa, TRCPROCSELR); + state->trcconfigr = etm4x_read32(csa, TRCCONFIGR); + state->trcauxctlr = etm4x_read32(csa, TRCAUXCTLR); + state->trceventctl0r = etm4x_read32(csa, TRCEVENTCTL0R); + state->trceventctl1r = etm4x_read32(csa, TRCEVENTCTL1R); if (drvdata->stallctl) - state->trcstallctlr = readl(drvdata->base + TRCSTALLCTLR); - state->trctsctlr = readl(drvdata->base + TRCTSCTLR); - state->trcsyncpr = readl(drvdata->base + TRCSYNCPR); - state->trcccctlr = readl(drvdata->base + TRCCCCTLR); - state->trcbbctlr = readl(drvdata->base + TRCBBCTLR); - state->trctraceidr = readl(drvdata->base + TRCTRACEIDR); - state->trcqctlr = readl(drvdata->base + TRCQCTLR); - - state->trcvictlr = readl(drvdata->base + TRCVICTLR); - state->trcviiectlr = readl(drvdata->base + TRCVIIECTLR); - state->trcvissctlr = readl(drvdata->base + TRCVISSCTLR); + state->trcstallctlr = etm4x_read32(csa, TRCSTALLCTLR); + state->trctsctlr = etm4x_read32(csa, TRCTSCTLR); + state->trcsyncpr = etm4x_read32(csa, TRCSYNCPR); + state->trcccctlr = etm4x_read32(csa, TRCCCCTLR); + state->trcbbctlr = etm4x_read32(csa, TRCBBCTLR); + state->trctraceidr = etm4x_read32(csa, TRCTRACEIDR); + state->trcqctlr = etm4x_read32(csa, TRCQCTLR); + + state->trcvictlr = etm4x_read32(csa, TRCVICTLR); + state->trcviiectlr = etm4x_read32(csa, TRCVIIECTLR); + state->trcvissctlr = etm4x_read32(csa, TRCVISSCTLR); if (drvdata->nr_pe_cmp) - state->trcvipcssctlr = readl(drvdata->base + TRCVIPCSSCTLR); - state->trcvdctlr = readl(drvdata->base + TRCVDCTLR); - state->trcvdsacctlr = readl(drvdata->base + TRCVDSACCTLR); - state->trcvdarcctlr = readl(drvdata->base + TRCVDARCCTLR); + state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR); + state->trcvdctlr = etm4x_read32(csa, TRCVDCTLR); + state->trcvdsacctlr = etm4x_read32(csa, TRCVDSACCTLR); + state->trcvdarcctlr = etm4x_read32(csa, TRCVDARCCTLR); for (i = 0; i < drvdata->nrseqstate - 1; i++) - state->trcseqevr[i] = readl(drvdata->base + TRCSEQEVRn(i)); + state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i)); + + state->trcseqrstevr = etm4x_read32(csa, TRCSEQRSTEVR); - state->trcseqrstevr = readl(drvdata->base + TRCSEQRSTEVR); - state->trcseqstr = readl(drvdata->base + TRCSEQSTR); - state->trcextinselr = readl(drvdata->base + TRCEXTINSELR); + state->trcseqstr = etm4x_read32(csa, TRCSEQSTR); + state->trcextinselr = etm4x_read32(csa, TRCEXTINSELR); for (i = 0; i < drvdata->nr_cntr; i++) { - state->trccntrldvr[i] = readl(drvdata->base + TRCCNTRLDVRn(i)); - state->trccntctlr[i] = readl(drvdata->base + TRCCNTCTLRn(i)); - state->trccntvr[i] = readl(drvdata->base + TRCCNTVRn(i)); + state->trccntrldvr[i] = etm4x_read32(csa, TRCCNTRLDVRn(i)); + state->trccntctlr[i] = etm4x_read32(csa, TRCCNTCTLRn(i)); + state->trccntvr[i] = etm4x_read32(csa, TRCCNTVRn(i)); } for (i = 0; i < drvdata->nr_resource * 2; i++) - state->trcrsctlr[i] = readl(drvdata->base + TRCRSCTLRn(i)); + state->trcrsctlr[i] = etm4x_read32(csa, TRCRSCTLRn(i)); for (i = 0; i < drvdata->nr_ss_cmp; i++) { - state->trcssccr[i] = readl(drvdata->base + TRCSSCCRn(i)); - state->trcsscsr[i] = readl(drvdata->base + TRCSSCSRn(i)); - state->trcsspcicr[i] = readl(drvdata->base + TRCSSPCICRn(i)); + state->trcssccr[i] = etm4x_read32(csa, TRCSSCCRn(i)); + state->trcsscsr[i] = etm4x_read32(csa, TRCSSCSRn(i)); + state->trcsspcicr[i] = etm4x_read32(csa, TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { - state->trcacvr[i] = readq(drvdata->base + TRCACVRn(i)); - state->trcacatr[i] = readq(drvdata->base + TRCACATRn(i)); + state->trcacvr[i] = etm4x_read64(csa, TRCACVRn(i)); + state->trcacatr[i] = etm4x_read64(csa, TRCACATRn(i)); } /* @@ -1358,26 +1655,26 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) */ for (i = 0; i < drvdata->numcidc; i++) - state->trccidcvr[i] = readq(drvdata->base + TRCCIDCVRn(i)); + state->trccidcvr[i] = etm4x_read64(csa, TRCCIDCVRn(i)); for (i = 0; i < drvdata->numvmidc; i++) - state->trcvmidcvr[i] = readq(drvdata->base + TRCVMIDCVRn(i)); + state->trcvmidcvr[i] = etm4x_read64(csa, TRCVMIDCVRn(i)); - state->trccidcctlr0 = readl(drvdata->base + TRCCIDCCTLR0); + state->trccidcctlr0 = etm4x_read32(csa, TRCCIDCCTLR0); if (drvdata->numcidc > 4) - state->trccidcctlr1 = readl(drvdata->base + TRCCIDCCTLR1); + state->trccidcctlr1 = etm4x_read32(csa, TRCCIDCCTLR1); - state->trcvmidcctlr0 = readl(drvdata->base + TRCVMIDCCTLR0); + state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - state->trcvmidcctlr1 = readl(drvdata->base + TRCVMIDCCTLR1); + state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1); - state->trcclaimset = readl(drvdata->base + TRCCLAIMCLR); + state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR); if (!drvdata->skip_power_up) - state->trcpdcr = readl(drvdata->base + TRCPDCR); + state->trcpdcr = etm4x_read32(csa, TRCPDCR); /* wait for TRCSTATR.IDLE to go up */ - if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) { + if (coresight_timeout(csa, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) { dev_err(etm_dev, "timeout while waiting for Idle Trace Status\n"); etm4_os_unlock(drvdata); @@ -1393,104 +1690,109 @@ static int etm4_cpu_save(struct etmv4_drvdata *drvdata) * despite requesting software to save/restore state. */ if (!drvdata->skip_power_up) - writel_relaxed((state->trcpdcr & ~TRCPDCR_PU), - drvdata->base + TRCPDCR); + etm4x_relaxed_write32(csa, (state->trcpdcr & ~TRCPDCR_PU), + TRCPDCR); out: - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); return ret; } -static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) +static int etm4_cpu_save(struct etmv4_drvdata *drvdata) +{ + int ret = 0; + + /* Save the TRFCR irrespective of whether the ETM is ON */ + if (drvdata->trfcr) + drvdata->save_trfcr = read_trfcr(); + /* + * Save and restore the ETM Trace registers only if + * the ETM is active. + */ + if (local_read(&drvdata->mode) && drvdata->save_state) + ret = __etm4_cpu_save(drvdata); + return ret; +} + +static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) { int i; struct etmv4_save_state *state = drvdata->save_state; + struct csdev_access tmp_csa = CSDEV_ACCESS_IOMEM(drvdata->base); + struct csdev_access *csa = &tmp_csa; - CS_UNLOCK(drvdata->base); + etm4_cs_unlock(drvdata, csa); + etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); - writel_relaxed(state->trcclaimset, drvdata->base + TRCCLAIMSET); - - writel_relaxed(state->trcprgctlr, drvdata->base + TRCPRGCTLR); + etm4x_relaxed_write32(csa, state->trcprgctlr, TRCPRGCTLR); if (drvdata->nr_pe) - writel_relaxed(state->trcprocselr, drvdata->base + TRCPROCSELR); - writel_relaxed(state->trcconfigr, drvdata->base + TRCCONFIGR); - writel_relaxed(state->trcauxctlr, drvdata->base + TRCAUXCTLR); - writel_relaxed(state->trceventctl0r, drvdata->base + TRCEVENTCTL0R); - writel_relaxed(state->trceventctl1r, drvdata->base + TRCEVENTCTL1R); + etm4x_relaxed_write32(csa, state->trcprocselr, TRCPROCSELR); + etm4x_relaxed_write32(csa, state->trcconfigr, TRCCONFIGR); + etm4x_relaxed_write32(csa, state->trcauxctlr, TRCAUXCTLR); + etm4x_relaxed_write32(csa, state->trceventctl0r, TRCEVENTCTL0R); + etm4x_relaxed_write32(csa, state->trceventctl1r, TRCEVENTCTL1R); if (drvdata->stallctl) - writel_relaxed(state->trcstallctlr, drvdata->base + TRCSTALLCTLR); - writel_relaxed(state->trctsctlr, drvdata->base + TRCTSCTLR); - writel_relaxed(state->trcsyncpr, drvdata->base + TRCSYNCPR); - writel_relaxed(state->trcccctlr, drvdata->base + TRCCCCTLR); - writel_relaxed(state->trcbbctlr, drvdata->base + TRCBBCTLR); - writel_relaxed(state->trctraceidr, drvdata->base + TRCTRACEIDR); - writel_relaxed(state->trcqctlr, drvdata->base + TRCQCTLR); - - writel_relaxed(state->trcvictlr, drvdata->base + TRCVICTLR); - writel_relaxed(state->trcviiectlr, drvdata->base + TRCVIIECTLR); - writel_relaxed(state->trcvissctlr, drvdata->base + TRCVISSCTLR); + etm4x_relaxed_write32(csa, state->trcstallctlr, TRCSTALLCTLR); + etm4x_relaxed_write32(csa, state->trctsctlr, TRCTSCTLR); + etm4x_relaxed_write32(csa, state->trcsyncpr, TRCSYNCPR); + etm4x_relaxed_write32(csa, state->trcccctlr, TRCCCCTLR); + etm4x_relaxed_write32(csa, state->trcbbctlr, TRCBBCTLR); + etm4x_relaxed_write32(csa, state->trctraceidr, TRCTRACEIDR); + etm4x_relaxed_write32(csa, state->trcqctlr, TRCQCTLR); + + etm4x_relaxed_write32(csa, state->trcvictlr, TRCVICTLR); + etm4x_relaxed_write32(csa, state->trcviiectlr, TRCVIIECTLR); + etm4x_relaxed_write32(csa, state->trcvissctlr, TRCVISSCTLR); if (drvdata->nr_pe_cmp) - writel_relaxed(state->trcvipcssctlr, drvdata->base + TRCVIPCSSCTLR); - writel_relaxed(state->trcvdctlr, drvdata->base + TRCVDCTLR); - writel_relaxed(state->trcvdsacctlr, drvdata->base + TRCVDSACCTLR); - writel_relaxed(state->trcvdarcctlr, drvdata->base + TRCVDARCCTLR); + etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR); + etm4x_relaxed_write32(csa, state->trcvdctlr, TRCVDCTLR); + etm4x_relaxed_write32(csa, state->trcvdsacctlr, TRCVDSACCTLR); + etm4x_relaxed_write32(csa, state->trcvdarcctlr, TRCVDARCCTLR); for (i = 0; i < drvdata->nrseqstate - 1; i++) - writel_relaxed(state->trcseqevr[i], - drvdata->base + TRCSEQEVRn(i)); + etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i)); - writel_relaxed(state->trcseqrstevr, drvdata->base + TRCSEQRSTEVR); - writel_relaxed(state->trcseqstr, drvdata->base + TRCSEQSTR); - writel_relaxed(state->trcextinselr, drvdata->base + TRCEXTINSELR); + etm4x_relaxed_write32(csa, state->trcseqrstevr, TRCSEQRSTEVR); + etm4x_relaxed_write32(csa, state->trcseqstr, TRCSEQSTR); + etm4x_relaxed_write32(csa, state->trcextinselr, TRCEXTINSELR); for (i = 0; i < drvdata->nr_cntr; i++) { - writel_relaxed(state->trccntrldvr[i], - drvdata->base + TRCCNTRLDVRn(i)); - writel_relaxed(state->trccntctlr[i], - drvdata->base + TRCCNTCTLRn(i)); - writel_relaxed(state->trccntvr[i], - drvdata->base + TRCCNTVRn(i)); + etm4x_relaxed_write32(csa, state->trccntrldvr[i], TRCCNTRLDVRn(i)); + etm4x_relaxed_write32(csa, state->trccntctlr[i], TRCCNTCTLRn(i)); + etm4x_relaxed_write32(csa, state->trccntvr[i], TRCCNTVRn(i)); } for (i = 0; i < drvdata->nr_resource * 2; i++) - writel_relaxed(state->trcrsctlr[i], - drvdata->base + TRCRSCTLRn(i)); + etm4x_relaxed_write32(csa, state->trcrsctlr[i], TRCRSCTLRn(i)); for (i = 0; i < drvdata->nr_ss_cmp; i++) { - writel_relaxed(state->trcssccr[i], - drvdata->base + TRCSSCCRn(i)); - writel_relaxed(state->trcsscsr[i], - drvdata->base + TRCSSCSRn(i)); - writel_relaxed(state->trcsspcicr[i], - drvdata->base + TRCSSPCICRn(i)); + etm4x_relaxed_write32(csa, state->trcssccr[i], TRCSSCCRn(i)); + etm4x_relaxed_write32(csa, state->trcsscsr[i], TRCSSCSRn(i)); + etm4x_relaxed_write32(csa, state->trcsspcicr[i], TRCSSPCICRn(i)); } for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) { - writeq_relaxed(state->trcacvr[i], - drvdata->base + TRCACVRn(i)); - writeq_relaxed(state->trcacatr[i], - drvdata->base + TRCACATRn(i)); + etm4x_relaxed_write64(csa, state->trcacvr[i], TRCACVRn(i)); + etm4x_relaxed_write64(csa, state->trcacatr[i], TRCACATRn(i)); } for (i = 0; i < drvdata->numcidc; i++) - writeq_relaxed(state->trccidcvr[i], - drvdata->base + TRCCIDCVRn(i)); + etm4x_relaxed_write64(csa, state->trccidcvr[i], TRCCIDCVRn(i)); for (i = 0; i < drvdata->numvmidc; i++) - writeq_relaxed(state->trcvmidcvr[i], - drvdata->base + TRCVMIDCVRn(i)); + etm4x_relaxed_write64(csa, state->trcvmidcvr[i], TRCVMIDCVRn(i)); - writel_relaxed(state->trccidcctlr0, drvdata->base + TRCCIDCCTLR0); + etm4x_relaxed_write32(csa, state->trccidcctlr0, TRCCIDCCTLR0); if (drvdata->numcidc > 4) - writel_relaxed(state->trccidcctlr1, drvdata->base + TRCCIDCCTLR1); + etm4x_relaxed_write32(csa, state->trccidcctlr1, TRCCIDCCTLR1); - writel_relaxed(state->trcvmidcctlr0, drvdata->base + TRCVMIDCCTLR0); + etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0); if (drvdata->numvmidc > 4) - writel_relaxed(state->trcvmidcctlr1, drvdata->base + TRCVMIDCCTLR1); + etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1); - writel_relaxed(state->trcclaimset, drvdata->base + TRCCLAIMSET); + etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET); if (!drvdata->skip_power_up) - writel_relaxed(state->trcpdcr, drvdata->base + TRCPDCR); + etm4x_relaxed_write32(csa, state->trcpdcr, TRCPDCR); drvdata->state_needs_restore = false; @@ -1503,7 +1805,15 @@ static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) /* Unlock the OS lock to re-enable trace and external debug access */ etm4_os_unlock(drvdata); - CS_LOCK(drvdata->base); + etm4_cs_lock(drvdata, csa); +} + +static void etm4_cpu_restore(struct etmv4_drvdata *drvdata) +{ + if (drvdata->trfcr) + write_trfcr(drvdata->save_trfcr); + if (drvdata->state_needs_restore) + __etm4_cpu_restore(drvdata); } static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, @@ -1517,23 +1827,17 @@ static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd, drvdata = etmdrvdata[cpu]; - if (!drvdata->save_state) - return NOTIFY_OK; - if (WARN_ON_ONCE(drvdata->cpu != cpu)) return NOTIFY_BAD; switch (cmd) { case CPU_PM_ENTER: - /* save the state if self-hosted coresight is in use */ - if (local_read(&drvdata->mode)) - if (etm4_cpu_save(drvdata)) - return NOTIFY_BAD; + if (etm4_cpu_save(drvdata)) + return NOTIFY_BAD; break; case CPU_PM_EXIT: case CPU_PM_ENTER_FAILED: - if (drvdata->state_needs_restore) - etm4_cpu_restore(drvdata); + etm4_cpu_restore(drvdata); break; default: return NOTIFY_DONE; @@ -1590,15 +1894,15 @@ static void etm4_pm_clear(void) } } -static int etm4_probe(struct amba_device *adev, const struct amba_id *id) +static int etm4_probe(struct device *dev, void __iomem *base, u32 etm_pid) { int ret; - void __iomem *base; - struct device *dev = &adev->dev; struct coresight_platform_data *pdata = NULL; struct etmv4_drvdata *drvdata; - struct resource *res = &adev->res; struct coresight_desc desc = { 0 }; + struct etm4_init_arg init_arg = { 0 }; + u8 major, minor; + char *type_name; drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) @@ -1617,14 +1921,6 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) return -ENOMEM; } - if (fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up")) - drvdata->skip_power_up = true; - - /* Validity for the resource is already checked by the AMBA core */ - base = devm_ioremap_resource(dev, res); - if (IS_ERR(base)) - return PTR_ERR(base); - drvdata->base = base; spin_lock_init(&drvdata->spinlock); @@ -1633,17 +1929,37 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) if (drvdata->cpu < 0) return drvdata->cpu; - desc.name = devm_kasprintf(dev, GFP_KERNEL, "etm%d", drvdata->cpu); - if (!desc.name) - return -ENOMEM; + init_arg.drvdata = drvdata; + init_arg.csa = &desc.access; if (smp_call_function_single(drvdata->cpu, - etm4_init_arch_data, drvdata, 1)) + etm4_init_arch_data, &init_arg, 1)) dev_err(dev, "ETM arch init failed\n"); - if (etm4_arch_supported(drvdata->arch) == false) + if (!drvdata->arch) return -EINVAL; + /* TRCPDCR is not accessible with system instructions. */ + if (!desc.access.io_mem || + fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up")) + drvdata->skip_power_up = true; + + major = ETM_ARCH_MAJOR_VERSION(drvdata->arch); + minor = ETM_ARCH_MINOR_VERSION(drvdata->arch); + + if (etm4x_is_ete(drvdata)) { + type_name = "ete"; + /* ETE v1 has major version == 0b101. Adjust this for logging.*/ + major -= 4; + } else { + type_name = "etm"; + } + + desc.name = devm_kasprintf(dev, GFP_KERNEL, + "%s%d", type_name, drvdata->cpu); + if (!desc.name) + return -ENOMEM; + etm4_init_trace_id(drvdata); etm4_set_default(&drvdata->config); @@ -1651,7 +1967,7 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) if (IS_ERR(pdata)) return PTR_ERR(pdata); - adev->dev.platform_data = pdata; + dev->platform_data = pdata; desc.type = CORESIGHT_DEV_TYPE_SOURCE; desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC; @@ -1671,25 +1987,62 @@ static int etm4_probe(struct amba_device *adev, const struct amba_id *id) etmdrvdata[drvdata->cpu] = drvdata; - pm_runtime_put(&adev->dev); - dev_info(&drvdata->csdev->dev, "CPU%d: ETM v%d.%d initialized\n", - drvdata->cpu, drvdata->arch >> 4, drvdata->arch & 0xf); + dev_info(&drvdata->csdev->dev, "CPU%d: %s v%d.%d initialized\n", + drvdata->cpu, type_name, major, minor); if (boot_enable) { coresight_enable(drvdata->csdev); drvdata->boot_enable = true; } - etm4_check_arch_features(drvdata, id->id); + etm4_check_arch_features(drvdata, etm_pid); return 0; } +static int etm4_probe_amba(struct amba_device *adev, const struct amba_id *id) +{ + void __iomem *base; + struct device *dev = &adev->dev; + struct resource *res = &adev->res; + int ret; + + /* Validity for the resource is already checked by the AMBA core */ + base = devm_ioremap_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + ret = etm4_probe(dev, base, id->id); + if (!ret) + pm_runtime_put(&adev->dev); + + return ret; +} + +static int etm4_probe_platform_dev(struct platform_device *pdev) +{ + int ret; + + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + /* + * System register based devices could match the + * HW by reading appropriate registers on the HW + * and thus we could skip the PID. + */ + ret = etm4_probe(&pdev->dev, NULL, 0); + + pm_runtime_put(&pdev->dev); + return ret; +} + static struct amba_cs_uci_id uci_id_etm4[] = { { /* ETMv4 UCI data */ - .devarch = 0x47704a13, - .devarch_mask = 0xfff0ffff, + .devarch = ETM_DEVARCH_ETMv4x_ARCH, + .devarch_mask = ETM_DEVARCH_ID_MASK, .devtype = 0x00000013, } }; @@ -1701,15 +2054,12 @@ static void clear_etmdrvdata(void *info) etmdrvdata[cpu] = NULL; } -static void etm4_remove(struct amba_device *adev) +static int __exit etm4_remove_dev(struct etmv4_drvdata *drvdata) { - struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev); - etm_perf_symlink(drvdata->csdev, false); - /* - * Taking hotplug lock here to avoid racing between etm4_remove and - * CPU hotplug call backs. + * Taking hotplug lock here to avoid racing between etm4_remove_dev() + * and CPU hotplug call backs. */ cpus_read_lock(); /* @@ -1724,6 +2074,27 @@ static void etm4_remove(struct amba_device *adev) cpus_read_unlock(); coresight_unregister(drvdata->csdev); + + return 0; +} + +static void __exit etm4_remove_amba(struct amba_device *adev) +{ + struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev); + + if (drvdata) + etm4_remove_dev(drvdata); +} + +static int __exit etm4_remove_platform_dev(struct platform_device *pdev) +{ + int ret = 0; + struct etmv4_drvdata *drvdata = dev_get_drvdata(&pdev->dev); + + if (drvdata) + ret = etm4_remove_dev(drvdata); + pm_runtime_disable(&pdev->dev); + return ret; } static const struct amba_id etm4_ids[] = { @@ -1747,17 +2118,33 @@ static const struct amba_id etm4_ids[] = { MODULE_DEVICE_TABLE(amba, etm4_ids); -static struct amba_driver etm4x_driver = { +static struct amba_driver etm4x_amba_driver = { .drv = { .name = "coresight-etm4x", .owner = THIS_MODULE, .suppress_bind_attrs = true, }, - .probe = etm4_probe, - .remove = etm4_remove, + .probe = etm4_probe_amba, + .remove = etm4_remove_amba, .id_table = etm4_ids, }; +static const struct of_device_id etm4_sysreg_match[] = { + { .compatible = "arm,coresight-etm4x-sysreg" }, + { .compatible = "arm,embedded-trace-extension" }, + {} +}; + +static struct platform_driver etm4_platform_driver = { + .probe = etm4_probe_platform_dev, + .remove = etm4_remove_platform_dev, + .driver = { + .name = "coresight-etm4x", + .of_match_table = etm4_sysreg_match, + .suppress_bind_attrs = true, + }, +}; + static int __init etm4x_init(void) { int ret; @@ -1768,18 +2155,28 @@ static int __init etm4x_init(void) if (ret) return ret; - ret = amba_driver_register(&etm4x_driver); + ret = amba_driver_register(&etm4x_amba_driver); if (ret) { - pr_err("Error registering etm4x driver\n"); - etm4_pm_clear(); + pr_err("Error registering etm4x AMBA driver\n"); + goto clear_pm; } + ret = platform_driver_register(&etm4_platform_driver); + if (!ret) + return 0; + + pr_err("Error registering etm4x platform driver\n"); + amba_driver_unregister(&etm4x_amba_driver); + +clear_pm: + etm4_pm_clear(); return ret; } static void __exit etm4x_exit(void) { - amba_driver_unregister(&etm4x_driver); + amba_driver_unregister(&etm4x_amba_driver); + platform_driver_unregister(&etm4_platform_driver); etm4_pm_clear(); } diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index 42cc38c89f3ba54e8fc46eb66b1b570e295ebd1d..4ce3927024aef282db4b9b9cdd14384b9c2a51d7 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -2323,7 +2323,8 @@ static struct attribute *coresight_etmv4_attrs[] = { }; struct etmv4_reg { - void __iomem *addr; + struct coresight_device *csdev; + u32 offset; u32 data; }; @@ -2331,15 +2332,16 @@ static void do_smp_cross_read(void *data) { struct etmv4_reg *reg = data; - reg->data = readl_relaxed(reg->addr); + reg->data = etm4x_relaxed_read32(®->csdev->access, reg->offset); } -static u32 etmv4_cross_read(const struct device *dev, u32 offset) +static u32 etmv4_cross_read(const struct etmv4_drvdata *drvdata, u32 offset) { - struct etmv4_drvdata *drvdata = dev_get_drvdata(dev); struct etmv4_reg reg; - reg.addr = drvdata->base + offset; + reg.offset = offset; + reg.csdev = drvdata->csdev; + /* * smp cross call ensures the CPU will be powered up before * accessing the ETMv4 trace core registers @@ -2348,72 +2350,133 @@ static u32 etmv4_cross_read(const struct device *dev, u32 offset) return reg.data; } -#define coresight_etm4x_reg(name, offset) \ - coresight_simple_reg32(struct etmv4_drvdata, name, offset) +static inline u32 coresight_etm4x_attr_to_offset(struct device_attribute *attr) +{ + struct dev_ext_attribute *eattr; + + eattr = container_of(attr, struct dev_ext_attribute, attr); + return (u32)(unsigned long)eattr->var; +} + +static ssize_t coresight_etm4x_reg_show(struct device *dev, + struct device_attribute *d_attr, + char *buf) +{ + u32 val, offset; + struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); + + offset = coresight_etm4x_attr_to_offset(d_attr); + + pm_runtime_get_sync(dev->parent); + val = etmv4_cross_read(drvdata, offset); + pm_runtime_put_sync(dev->parent); + + return scnprintf(buf, PAGE_SIZE, "0x%x\n", val); +} + +static inline bool +etm4x_register_implemented(struct etmv4_drvdata *drvdata, u32 offset) +{ + switch (offset) { + ETM_COMMON_SYSREG_LIST_CASES + /* + * Common registers to ETE & ETM4x accessible via system + * instructions are always implemented. + */ + return true; + + ETM4x_ONLY_SYSREG_LIST_CASES + /* + * We only support etm4x and ete. So if the device is not + * ETE, it must be ETMv4x. + */ + return !etm4x_is_ete(drvdata); + + ETM4x_MMAP_LIST_CASES + /* + * Registers accessible only via memory-mapped registers + * must not be accessed via system instructions. + * We cannot access the drvdata->csdev here, as this + * function is called during the device creation, via + * coresight_register() and the csdev is not initialized + * until that is done. So rely on the drvdata->base to + * detect if we have a memory mapped access. + * Also ETE doesn't implement memory mapped access, thus + * it is sufficient to check that we are using mmio. + */ + return !!drvdata->base; + + ETE_ONLY_SYSREG_LIST_CASES + return etm4x_is_ete(drvdata); + } + + return false; +} + +/* + * Hide the ETM4x registers that may not be available on the + * hardware. + * There are certain management registers unavailable via system + * instructions. Make those sysfs attributes hidden on such + * systems. + */ +static umode_t +coresight_etm4x_attr_reg_implemented(struct kobject *kobj, + struct attribute *attr, int unused) +{ + struct device *dev = kobj_to_dev(kobj); + struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); + struct device_attribute *d_attr; + u32 offset; -#define coresight_etm4x_cross_read(name, offset) \ - coresight_simple_func(struct etmv4_drvdata, etmv4_cross_read, \ - name, offset) + d_attr = container_of(attr, struct device_attribute, attr); + offset = coresight_etm4x_attr_to_offset(d_attr); -coresight_etm4x_reg(trcpdcr, TRCPDCR); -coresight_etm4x_reg(trcpdsr, TRCPDSR); -coresight_etm4x_reg(trclsr, TRCLSR); -coresight_etm4x_reg(trcauthstatus, TRCAUTHSTATUS); -coresight_etm4x_reg(trcdevid, TRCDEVID); -coresight_etm4x_reg(trcdevtype, TRCDEVTYPE); -coresight_etm4x_reg(trcpidr0, TRCPIDR0); -coresight_etm4x_reg(trcpidr1, TRCPIDR1); -coresight_etm4x_reg(trcpidr2, TRCPIDR2); -coresight_etm4x_reg(trcpidr3, TRCPIDR3); -coresight_etm4x_cross_read(trcoslsr, TRCOSLSR); -coresight_etm4x_cross_read(trcconfig, TRCCONFIGR); -coresight_etm4x_cross_read(trctraceid, TRCTRACEIDR); + if (etm4x_register_implemented(drvdata, offset)) + return attr->mode; + return 0; +} + +#define coresight_etm4x_reg(name, offset) \ + &((struct dev_ext_attribute[]) { \ + { \ + __ATTR(name, 0444, coresight_etm4x_reg_show, NULL), \ + (void *)(unsigned long)offset \ + } \ + })[0].attr.attr static struct attribute *coresight_etmv4_mgmt_attrs[] = { - &dev_attr_trcoslsr.attr, - &dev_attr_trcpdcr.attr, - &dev_attr_trcpdsr.attr, - &dev_attr_trclsr.attr, - &dev_attr_trcconfig.attr, - &dev_attr_trctraceid.attr, - &dev_attr_trcauthstatus.attr, - &dev_attr_trcdevid.attr, - &dev_attr_trcdevtype.attr, - &dev_attr_trcpidr0.attr, - &dev_attr_trcpidr1.attr, - &dev_attr_trcpidr2.attr, - &dev_attr_trcpidr3.attr, + coresight_etm4x_reg(trcpdcr, TRCPDCR), + coresight_etm4x_reg(trcpdsr, TRCPDSR), + coresight_etm4x_reg(trclsr, TRCLSR), + coresight_etm4x_reg(trcauthstatus, TRCAUTHSTATUS), + coresight_etm4x_reg(trcdevid, TRCDEVID), + coresight_etm4x_reg(trcdevtype, TRCDEVTYPE), + coresight_etm4x_reg(trcpidr0, TRCPIDR0), + coresight_etm4x_reg(trcpidr1, TRCPIDR1), + coresight_etm4x_reg(trcpidr2, TRCPIDR2), + coresight_etm4x_reg(trcpidr3, TRCPIDR3), + coresight_etm4x_reg(trcoslsr, TRCOSLSR), + coresight_etm4x_reg(trcconfig, TRCCONFIGR), + coresight_etm4x_reg(trctraceid, TRCTRACEIDR), + coresight_etm4x_reg(trcdevarch, TRCDEVARCH), NULL, }; -coresight_etm4x_cross_read(trcidr0, TRCIDR0); -coresight_etm4x_cross_read(trcidr1, TRCIDR1); -coresight_etm4x_cross_read(trcidr2, TRCIDR2); -coresight_etm4x_cross_read(trcidr3, TRCIDR3); -coresight_etm4x_cross_read(trcidr4, TRCIDR4); -coresight_etm4x_cross_read(trcidr5, TRCIDR5); -/* trcidr[6,7] are reserved */ -coresight_etm4x_cross_read(trcidr8, TRCIDR8); -coresight_etm4x_cross_read(trcidr9, TRCIDR9); -coresight_etm4x_cross_read(trcidr10, TRCIDR10); -coresight_etm4x_cross_read(trcidr11, TRCIDR11); -coresight_etm4x_cross_read(trcidr12, TRCIDR12); -coresight_etm4x_cross_read(trcidr13, TRCIDR13); - static struct attribute *coresight_etmv4_trcidr_attrs[] = { - &dev_attr_trcidr0.attr, - &dev_attr_trcidr1.attr, - &dev_attr_trcidr2.attr, - &dev_attr_trcidr3.attr, - &dev_attr_trcidr4.attr, - &dev_attr_trcidr5.attr, + coresight_etm4x_reg(trcidr0, TRCIDR0), + coresight_etm4x_reg(trcidr1, TRCIDR1), + coresight_etm4x_reg(trcidr2, TRCIDR2), + coresight_etm4x_reg(trcidr3, TRCIDR3), + coresight_etm4x_reg(trcidr4, TRCIDR4), + coresight_etm4x_reg(trcidr5, TRCIDR5), /* trcidr[6,7] are reserved */ - &dev_attr_trcidr8.attr, - &dev_attr_trcidr9.attr, - &dev_attr_trcidr10.attr, - &dev_attr_trcidr11.attr, - &dev_attr_trcidr12.attr, - &dev_attr_trcidr13.attr, + coresight_etm4x_reg(trcidr8, TRCIDR8), + coresight_etm4x_reg(trcidr9, TRCIDR9), + coresight_etm4x_reg(trcidr10, TRCIDR10), + coresight_etm4x_reg(trcidr11, TRCIDR11), + coresight_etm4x_reg(trcidr12, TRCIDR12), + coresight_etm4x_reg(trcidr13, TRCIDR13), NULL, }; @@ -2422,6 +2485,7 @@ static const struct attribute_group coresight_etmv4_group = { }; static const struct attribute_group coresight_etmv4_mgmt_group = { + .is_visible = coresight_etm4x_attr_reg_implemented, .attrs = coresight_etmv4_mgmt_attrs, .name = "mgmt", }; diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index ef9d7365c2da8c07f223eac7603358ac85957ad2..894041f69842430d833ded3baf6f72cf26cbb1d7 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -7,6 +7,7 @@ #define _CORESIGHT_CORESIGHT_ETM_H #include +#include #include #include #include "coresight-priv.h" @@ -29,6 +30,7 @@ #define TRCAUXCTLR 0x018 #define TRCEVENTCTL0R 0x020 #define TRCEVENTCTL1R 0x024 +#define TRCRSR 0x028 #define TRCSTALLCTLR 0x02C #define TRCTSCTLR 0x030 #define TRCSYNCPR 0x034 @@ -49,6 +51,7 @@ #define TRCSEQRSTEVR 0x118 #define TRCSEQSTR 0x11C #define TRCEXTINSELR 0x120 +#define TRCEXTINSELRn(n) (0x120 + (n * 4)) /* n = 0-3 */ #define TRCCNTRLDVRn(n) (0x140 + (n * 4)) #define TRCCNTCTLRn(n) (0x150 + (n * 4)) #define TRCCNTVRn(n) (0x160 + (n * 4)) @@ -121,6 +124,367 @@ #define TRCCIDR2 0xFF8 #define TRCCIDR3 0xFFC +#define TRCRSR_TA BIT(12) + +/* + * System instructions to access ETM registers. + * See ETMv4.4 spec ARM IHI0064F section 4.3.6 System instructions + */ +#define ETM4x_OFFSET_TO_REG(x) ((x) >> 2) + +#define ETM4x_CRn(n) (((n) >> 7) & 0x7) +#define ETM4x_Op2(n) (((n) >> 4) & 0x7) +#define ETM4x_CRm(n) ((n) & 0xf) + +#include +#define ETM4x_REG_NUM_TO_SYSREG(n) \ + sys_reg(2, 1, ETM4x_CRn(n), ETM4x_CRm(n), ETM4x_Op2(n)) + +#define READ_ETM4x_REG(reg) \ + read_sysreg_s(ETM4x_REG_NUM_TO_SYSREG((reg))) +#define WRITE_ETM4x_REG(val, reg) \ + write_sysreg_s(val, ETM4x_REG_NUM_TO_SYSREG((reg))) + +#define read_etm4x_sysreg_const_offset(offset) \ + READ_ETM4x_REG(ETM4x_OFFSET_TO_REG(offset)) + +#define write_etm4x_sysreg_const_offset(val, offset) \ + WRITE_ETM4x_REG(val, ETM4x_OFFSET_TO_REG(offset)) + +#define CASE_READ(res, x) \ + case (x): { (res) = read_etm4x_sysreg_const_offset((x)); break; } + +#define CASE_WRITE(val, x) \ + case (x): { write_etm4x_sysreg_const_offset((val), (x)); break; } + +#define CASE_NOP(__unused, x) \ + case (x): + +#define ETE_ONLY_SYSREG_LIST(op, val) \ + CASE_##op((val), TRCRSR) \ + CASE_##op((val), TRCEXTINSELRn(1)) \ + CASE_##op((val), TRCEXTINSELRn(2)) \ + CASE_##op((val), TRCEXTINSELRn(3)) + +/* List of registers accessible via System instructions */ +#define ETM4x_ONLY_SYSREG_LIST(op, val) \ + CASE_##op((val), TRCPROCSELR) \ + CASE_##op((val), TRCVDCTLR) \ + CASE_##op((val), TRCVDSACCTLR) \ + CASE_##op((val), TRCVDARCCTLR) \ + CASE_##op((val), TRCOSLAR) + +#define ETM_COMMON_SYSREG_LIST(op, val) \ + CASE_##op((val), TRCPRGCTLR) \ + CASE_##op((val), TRCSTATR) \ + CASE_##op((val), TRCCONFIGR) \ + CASE_##op((val), TRCAUXCTLR) \ + CASE_##op((val), TRCEVENTCTL0R) \ + CASE_##op((val), TRCEVENTCTL1R) \ + CASE_##op((val), TRCSTALLCTLR) \ + CASE_##op((val), TRCTSCTLR) \ + CASE_##op((val), TRCSYNCPR) \ + CASE_##op((val), TRCCCCTLR) \ + CASE_##op((val), TRCBBCTLR) \ + CASE_##op((val), TRCTRACEIDR) \ + CASE_##op((val), TRCQCTLR) \ + CASE_##op((val), TRCVICTLR) \ + CASE_##op((val), TRCVIIECTLR) \ + CASE_##op((val), TRCVISSCTLR) \ + CASE_##op((val), TRCVIPCSSCTLR) \ + CASE_##op((val), TRCSEQEVRn(0)) \ + CASE_##op((val), TRCSEQEVRn(1)) \ + CASE_##op((val), TRCSEQEVRn(2)) \ + CASE_##op((val), TRCSEQRSTEVR) \ + CASE_##op((val), TRCSEQSTR) \ + CASE_##op((val), TRCEXTINSELR) \ + CASE_##op((val), TRCCNTRLDVRn(0)) \ + CASE_##op((val), TRCCNTRLDVRn(1)) \ + CASE_##op((val), TRCCNTRLDVRn(2)) \ + CASE_##op((val), TRCCNTRLDVRn(3)) \ + CASE_##op((val), TRCCNTCTLRn(0)) \ + CASE_##op((val), TRCCNTCTLRn(1)) \ + CASE_##op((val), TRCCNTCTLRn(2)) \ + CASE_##op((val), TRCCNTCTLRn(3)) \ + CASE_##op((val), TRCCNTVRn(0)) \ + CASE_##op((val), TRCCNTVRn(1)) \ + CASE_##op((val), TRCCNTVRn(2)) \ + CASE_##op((val), TRCCNTVRn(3)) \ + CASE_##op((val), TRCIDR8) \ + CASE_##op((val), TRCIDR9) \ + CASE_##op((val), TRCIDR10) \ + CASE_##op((val), TRCIDR11) \ + CASE_##op((val), TRCIDR12) \ + CASE_##op((val), TRCIDR13) \ + CASE_##op((val), TRCIMSPECn(0)) \ + CASE_##op((val), TRCIMSPECn(1)) \ + CASE_##op((val), TRCIMSPECn(2)) \ + CASE_##op((val), TRCIMSPECn(3)) \ + CASE_##op((val), TRCIMSPECn(4)) \ + CASE_##op((val), TRCIMSPECn(5)) \ + CASE_##op((val), TRCIMSPECn(6)) \ + CASE_##op((val), TRCIMSPECn(7)) \ + CASE_##op((val), TRCIDR0) \ + CASE_##op((val), TRCIDR1) \ + CASE_##op((val), TRCIDR2) \ + CASE_##op((val), TRCIDR3) \ + CASE_##op((val), TRCIDR4) \ + CASE_##op((val), TRCIDR5) \ + CASE_##op((val), TRCIDR6) \ + CASE_##op((val), TRCIDR7) \ + CASE_##op((val), TRCRSCTLRn(2)) \ + CASE_##op((val), TRCRSCTLRn(3)) \ + CASE_##op((val), TRCRSCTLRn(4)) \ + CASE_##op((val), TRCRSCTLRn(5)) \ + CASE_##op((val), TRCRSCTLRn(6)) \ + CASE_##op((val), TRCRSCTLRn(7)) \ + CASE_##op((val), TRCRSCTLRn(8)) \ + CASE_##op((val), TRCRSCTLRn(9)) \ + CASE_##op((val), TRCRSCTLRn(10)) \ + CASE_##op((val), TRCRSCTLRn(11)) \ + CASE_##op((val), TRCRSCTLRn(12)) \ + CASE_##op((val), TRCRSCTLRn(13)) \ + CASE_##op((val), TRCRSCTLRn(14)) \ + CASE_##op((val), TRCRSCTLRn(15)) \ + CASE_##op((val), TRCRSCTLRn(16)) \ + CASE_##op((val), TRCRSCTLRn(17)) \ + CASE_##op((val), TRCRSCTLRn(18)) \ + CASE_##op((val), TRCRSCTLRn(19)) \ + CASE_##op((val), TRCRSCTLRn(20)) \ + CASE_##op((val), TRCRSCTLRn(21)) \ + CASE_##op((val), TRCRSCTLRn(22)) \ + CASE_##op((val), TRCRSCTLRn(23)) \ + CASE_##op((val), TRCRSCTLRn(24)) \ + CASE_##op((val), TRCRSCTLRn(25)) \ + CASE_##op((val), TRCRSCTLRn(26)) \ + CASE_##op((val), TRCRSCTLRn(27)) \ + CASE_##op((val), TRCRSCTLRn(28)) \ + CASE_##op((val), TRCRSCTLRn(29)) \ + CASE_##op((val), TRCRSCTLRn(30)) \ + CASE_##op((val), TRCRSCTLRn(31)) \ + CASE_##op((val), TRCSSCCRn(0)) \ + CASE_##op((val), TRCSSCCRn(1)) \ + CASE_##op((val), TRCSSCCRn(2)) \ + CASE_##op((val), TRCSSCCRn(3)) \ + CASE_##op((val), TRCSSCCRn(4)) \ + CASE_##op((val), TRCSSCCRn(5)) \ + CASE_##op((val), TRCSSCCRn(6)) \ + CASE_##op((val), TRCSSCCRn(7)) \ + CASE_##op((val), TRCSSCSRn(0)) \ + CASE_##op((val), TRCSSCSRn(1)) \ + CASE_##op((val), TRCSSCSRn(2)) \ + CASE_##op((val), TRCSSCSRn(3)) \ + CASE_##op((val), TRCSSCSRn(4)) \ + CASE_##op((val), TRCSSCSRn(5)) \ + CASE_##op((val), TRCSSCSRn(6)) \ + CASE_##op((val), TRCSSCSRn(7)) \ + CASE_##op((val), TRCSSPCICRn(0)) \ + CASE_##op((val), TRCSSPCICRn(1)) \ + CASE_##op((val), TRCSSPCICRn(2)) \ + CASE_##op((val), TRCSSPCICRn(3)) \ + CASE_##op((val), TRCSSPCICRn(4)) \ + CASE_##op((val), TRCSSPCICRn(5)) \ + CASE_##op((val), TRCSSPCICRn(6)) \ + CASE_##op((val), TRCSSPCICRn(7)) \ + CASE_##op((val), TRCOSLSR) \ + CASE_##op((val), TRCACVRn(0)) \ + CASE_##op((val), TRCACVRn(1)) \ + CASE_##op((val), TRCACVRn(2)) \ + CASE_##op((val), TRCACVRn(3)) \ + CASE_##op((val), TRCACVRn(4)) \ + CASE_##op((val), TRCACVRn(5)) \ + CASE_##op((val), TRCACVRn(6)) \ + CASE_##op((val), TRCACVRn(7)) \ + CASE_##op((val), TRCACVRn(8)) \ + CASE_##op((val), TRCACVRn(9)) \ + CASE_##op((val), TRCACVRn(10)) \ + CASE_##op((val), TRCACVRn(11)) \ + CASE_##op((val), TRCACVRn(12)) \ + CASE_##op((val), TRCACVRn(13)) \ + CASE_##op((val), TRCACVRn(14)) \ + CASE_##op((val), TRCACVRn(15)) \ + CASE_##op((val), TRCACATRn(0)) \ + CASE_##op((val), TRCACATRn(1)) \ + CASE_##op((val), TRCACATRn(2)) \ + CASE_##op((val), TRCACATRn(3)) \ + CASE_##op((val), TRCACATRn(4)) \ + CASE_##op((val), TRCACATRn(5)) \ + CASE_##op((val), TRCACATRn(6)) \ + CASE_##op((val), TRCACATRn(7)) \ + CASE_##op((val), TRCACATRn(8)) \ + CASE_##op((val), TRCACATRn(9)) \ + CASE_##op((val), TRCACATRn(10)) \ + CASE_##op((val), TRCACATRn(11)) \ + CASE_##op((val), TRCACATRn(12)) \ + CASE_##op((val), TRCACATRn(13)) \ + CASE_##op((val), TRCACATRn(14)) \ + CASE_##op((val), TRCACATRn(15)) \ + CASE_##op((val), TRCDVCVRn(0)) \ + CASE_##op((val), TRCDVCVRn(1)) \ + CASE_##op((val), TRCDVCVRn(2)) \ + CASE_##op((val), TRCDVCVRn(3)) \ + CASE_##op((val), TRCDVCVRn(4)) \ + CASE_##op((val), TRCDVCVRn(5)) \ + CASE_##op((val), TRCDVCVRn(6)) \ + CASE_##op((val), TRCDVCVRn(7)) \ + CASE_##op((val), TRCDVCMRn(0)) \ + CASE_##op((val), TRCDVCMRn(1)) \ + CASE_##op((val), TRCDVCMRn(2)) \ + CASE_##op((val), TRCDVCMRn(3)) \ + CASE_##op((val), TRCDVCMRn(4)) \ + CASE_##op((val), TRCDVCMRn(5)) \ + CASE_##op((val), TRCDVCMRn(6)) \ + CASE_##op((val), TRCDVCMRn(7)) \ + CASE_##op((val), TRCCIDCVRn(0)) \ + CASE_##op((val), TRCCIDCVRn(1)) \ + CASE_##op((val), TRCCIDCVRn(2)) \ + CASE_##op((val), TRCCIDCVRn(3)) \ + CASE_##op((val), TRCCIDCVRn(4)) \ + CASE_##op((val), TRCCIDCVRn(5)) \ + CASE_##op((val), TRCCIDCVRn(6)) \ + CASE_##op((val), TRCCIDCVRn(7)) \ + CASE_##op((val), TRCVMIDCVRn(0)) \ + CASE_##op((val), TRCVMIDCVRn(1)) \ + CASE_##op((val), TRCVMIDCVRn(2)) \ + CASE_##op((val), TRCVMIDCVRn(3)) \ + CASE_##op((val), TRCVMIDCVRn(4)) \ + CASE_##op((val), TRCVMIDCVRn(5)) \ + CASE_##op((val), TRCVMIDCVRn(6)) \ + CASE_##op((val), TRCVMIDCVRn(7)) \ + CASE_##op((val), TRCCIDCCTLR0) \ + CASE_##op((val), TRCCIDCCTLR1) \ + CASE_##op((val), TRCVMIDCCTLR0) \ + CASE_##op((val), TRCVMIDCCTLR1) \ + CASE_##op((val), TRCCLAIMSET) \ + CASE_##op((val), TRCCLAIMCLR) \ + CASE_##op((val), TRCAUTHSTATUS) \ + CASE_##op((val), TRCDEVARCH) \ + CASE_##op((val), TRCDEVID) + +/* List of registers only accessible via memory-mapped interface */ +#define ETM_MMAP_LIST(op, val) \ + CASE_##op((val), TRCDEVTYPE) \ + CASE_##op((val), TRCPDCR) \ + CASE_##op((val), TRCPDSR) \ + CASE_##op((val), TRCDEVAFF0) \ + CASE_##op((val), TRCDEVAFF1) \ + CASE_##op((val), TRCLAR) \ + CASE_##op((val), TRCLSR) \ + CASE_##op((val), TRCITCTRL) \ + CASE_##op((val), TRCPIDR4) \ + CASE_##op((val), TRCPIDR0) \ + CASE_##op((val), TRCPIDR1) \ + CASE_##op((val), TRCPIDR2) \ + CASE_##op((val), TRCPIDR3) + +#define ETM4x_READ_SYSREG_CASES(res) \ + ETM_COMMON_SYSREG_LIST(READ, (res)) \ + ETM4x_ONLY_SYSREG_LIST(READ, (res)) + +#define ETM4x_WRITE_SYSREG_CASES(val) \ + ETM_COMMON_SYSREG_LIST(WRITE, (val)) \ + ETM4x_ONLY_SYSREG_LIST(WRITE, (val)) + +#define ETM_COMMON_SYSREG_LIST_CASES \ + ETM_COMMON_SYSREG_LIST(NOP, __unused) + +#define ETM4x_ONLY_SYSREG_LIST_CASES \ + ETM4x_ONLY_SYSREG_LIST(NOP, __unused) + +#define ETM4x_SYSREG_LIST_CASES \ + ETM_COMMON_SYSREG_LIST_CASES \ + ETM4x_ONLY_SYSREG_LIST(NOP, __unused) + +#define ETM4x_MMAP_LIST_CASES ETM_MMAP_LIST(NOP, __unused) + +/* ETE only supports system register access */ +#define ETE_READ_CASES(res) \ + ETM_COMMON_SYSREG_LIST(READ, (res)) \ + ETE_ONLY_SYSREG_LIST(READ, (res)) + +#define ETE_WRITE_CASES(val) \ + ETM_COMMON_SYSREG_LIST(WRITE, (val)) \ + ETE_ONLY_SYSREG_LIST(WRITE, (val)) + +#define ETE_ONLY_SYSREG_LIST_CASES \ + ETE_ONLY_SYSREG_LIST(NOP, __unused) + +#define read_etm4x_sysreg_offset(offset, _64bit) \ + ({ \ + u64 __val; \ + \ + if (__is_constexpr((offset))) \ + __val = read_etm4x_sysreg_const_offset((offset)); \ + else \ + __val = etm4x_sysreg_read((offset), true, (_64bit)); \ + __val; \ + }) + +#define write_etm4x_sysreg_offset(val, offset, _64bit) \ + do { \ + if (__builtin_constant_p((offset))) \ + write_etm4x_sysreg_const_offset((val), \ + (offset)); \ + else \ + etm4x_sysreg_write((val), (offset), true, \ + (_64bit)); \ + } while (0) + + +#define etm4x_relaxed_read32(csa, offset) \ + ((u32)((csa)->io_mem ? \ + readl_relaxed((csa)->base + (offset)) : \ + read_etm4x_sysreg_offset((offset), false))) + +#define etm4x_relaxed_read64(csa, offset) \ + ((u64)((csa)->io_mem ? \ + readq_relaxed((csa)->base + (offset)) : \ + read_etm4x_sysreg_offset((offset), true))) + +#define etm4x_read32(csa, offset) \ + ({ \ + u32 __val = etm4x_relaxed_read32((csa), (offset)); \ + __iormb(__val); \ + __val; \ + }) + +#define etm4x_read64(csa, offset) \ + ({ \ + u64 __val = etm4x_relaxed_read64((csa), (offset)); \ + __iormb(__val); \ + __val; \ + }) + +#define etm4x_relaxed_write32(csa, val, offset) \ + do { \ + if ((csa)->io_mem) \ + writel_relaxed((val), (csa)->base + (offset)); \ + else \ + write_etm4x_sysreg_offset((val), (offset), \ + false); \ + } while (0) + +#define etm4x_relaxed_write64(csa, val, offset) \ + do { \ + if ((csa)->io_mem) \ + writeq_relaxed((val), (csa)->base + (offset)); \ + else \ + write_etm4x_sysreg_offset((val), (offset), \ + true); \ + } while (0) + +#define etm4x_write32(csa, val, offset) \ + do { \ + __iowmb(); \ + etm4x_relaxed_write32((csa), (val), (offset)); \ + } while (0) + +#define etm4x_write64(csa, val, offset) \ + do { \ + __iowmb(); \ + etm4x_relaxed_write64((csa), (val), (offset)); \ + } while (0) + /* ETMv4 resources */ #define ETM_MAX_NR_PE 8 #define ETMv4_MAX_CNTR 4 @@ -137,7 +501,6 @@ #define ETM_MAX_RES_SEL 32 #define ETM_MAX_SS_CMP 8 -#define ETM_ARCH_V4 0x40 #define ETMv4_SYNC_MASK 0x1F #define ETM_CYC_THRESHOLD_MASK 0xFFF #define ETM_CYC_THRESHOLD_DEFAULT 0x100 @@ -175,6 +538,65 @@ ETM_MODE_EXCL_KERN | \ ETM_MODE_EXCL_USER) +/* + * TRCOSLSR.OSLM advertises the OS Lock model. + * OSLM[2:0] = TRCOSLSR[4:3,0] + * + * 0b000 - Trace OS Lock is not implemented. + * 0b010 - Trace OS Lock is implemented. + * 0b100 - Trace OS Lock is not implemented, unit is controlled by PE OS Lock. + */ +#define ETM_OSLOCK_NI 0b000 +#define ETM_OSLOCK_PRESENT 0b010 +#define ETM_OSLOCK_PE 0b100 + +#define ETM_OSLSR_OSLM(oslsr) ((((oslsr) & GENMASK(4, 3)) >> 2) | (oslsr & 0x1)) + +/* + * TRCDEVARCH Bit field definitions + * Bits[31:21] - ARCHITECT = Always Arm Ltd. + * * Bits[31:28] = 0x4 + * * Bits[27:21] = 0b0111011 + * Bit[20] - PRESENT, Indicates the presence of this register. + * + * Bit[19:16] - REVISION, Revision of the architecture. + * + * Bit[15:0] - ARCHID, Identifies this component as an ETM + * * Bits[15:12] - architecture version of ETM + * * = 4 for ETMv4 + * * Bits[11:0] = 0xA13, architecture part number for ETM. + */ +#define ETM_DEVARCH_ARCHITECT_MASK GENMASK(31, 21) +#define ETM_DEVARCH_ARCHITECT_ARM ((0x4 << 28) | (0b0111011 << 21)) +#define ETM_DEVARCH_PRESENT BIT(20) +#define ETM_DEVARCH_REVISION_SHIFT 16 +#define ETM_DEVARCH_REVISION_MASK GENMASK(19, 16) +#define ETM_DEVARCH_REVISION(x) \ + (((x) & ETM_DEVARCH_REVISION_MASK) >> ETM_DEVARCH_REVISION_SHIFT) +#define ETM_DEVARCH_ARCHID_MASK GENMASK(15, 0) +#define ETM_DEVARCH_ARCHID_ARCH_VER_SHIFT 12 +#define ETM_DEVARCH_ARCHID_ARCH_VER_MASK GENMASK(15, 12) +#define ETM_DEVARCH_ARCHID_ARCH_VER(x) \ + (((x) & ETM_DEVARCH_ARCHID_ARCH_VER_MASK) >> ETM_DEVARCH_ARCHID_ARCH_VER_SHIFT) + +#define ETM_DEVARCH_MAKE_ARCHID_ARCH_VER(ver) \ + (((ver) << ETM_DEVARCH_ARCHID_ARCH_VER_SHIFT) & ETM_DEVARCH_ARCHID_ARCH_VER_MASK) + +#define ETM_DEVARCH_ARCHID_ARCH_PART(x) ((x) & 0xfffUL) + +#define ETM_DEVARCH_MAKE_ARCHID(major) \ + ((ETM_DEVARCH_MAKE_ARCHID_ARCH_VER(major)) | ETM_DEVARCH_ARCHID_ARCH_PART(0xA13)) + +#define ETM_DEVARCH_ARCHID_ETMv4x ETM_DEVARCH_MAKE_ARCHID(0x4) +#define ETM_DEVARCH_ARCHID_ETE ETM_DEVARCH_MAKE_ARCHID(0x5) + +#define ETM_DEVARCH_ID_MASK \ + (ETM_DEVARCH_ARCHITECT_MASK | ETM_DEVARCH_ARCHID_MASK | ETM_DEVARCH_PRESENT) +#define ETM_DEVARCH_ETMv4x_ARCH \ + (ETM_DEVARCH_ARCHITECT_ARM | ETM_DEVARCH_ARCHID_ETMv4x | ETM_DEVARCH_PRESENT) +#define ETM_DEVARCH_ETE_ARCH \ + (ETM_DEVARCH_ARCHITECT_ARM | ETM_DEVARCH_ARCHID_ETE | ETM_DEVARCH_PRESENT) + #define TRCSTATR_IDLE_BIT 0 #define TRCSTATR_PMSTABLE_BIT 1 #define ETM_DEFAULT_ADDR_COMP 0 @@ -201,8 +623,65 @@ /* NS MON (EL3) mode never implemented */ #define ETM_EXLEVEL_NS_VICTLR_MASK GENMASK(22, 20) +#define ETM_TRCIDR1_ARCH_MAJOR_SHIFT 8 +#define ETM_TRCIDR1_ARCH_MAJOR_MASK (0xfU << ETM_TRCIDR1_ARCH_MAJOR_SHIFT) +#define ETM_TRCIDR1_ARCH_MAJOR(x) \ + (((x) & ETM_TRCIDR1_ARCH_MAJOR_MASK) >> ETM_TRCIDR1_ARCH_MAJOR_SHIFT) +#define ETM_TRCIDR1_ARCH_MINOR_SHIFT 4 +#define ETM_TRCIDR1_ARCH_MINOR_MASK (0xfU << ETM_TRCIDR1_ARCH_MINOR_SHIFT) +#define ETM_TRCIDR1_ARCH_MINOR(x) \ + (((x) & ETM_TRCIDR1_ARCH_MINOR_MASK) >> ETM_TRCIDR1_ARCH_MINOR_SHIFT) +#define ETM_TRCIDR1_ARCH_SHIFT ETM_TRCIDR1_ARCH_MINOR_SHIFT +#define ETM_TRCIDR1_ARCH_MASK \ + (ETM_TRCIDR1_ARCH_MAJOR_MASK | ETM_TRCIDR1_ARCH_MINOR_MASK) + +#define ETM_TRCIDR1_ARCH_ETMv4 0x4 + +/* + * Driver representation of the ETM architecture. + * The version of an ETM component can be detected from + * + * TRCDEVARCH - CoreSight architected register + * - Bits[15:12] - Major version + * - Bits[19:16] - Minor version + * TRCIDR1 - ETM architected register + * - Bits[11:8] - Major version + * - Bits[7:4] - Minor version + * We must rely on TRCDEVARCH for the version information, + * however we don't want to break the support for potential + * old implementations which might not implement it. Thus + * we fall back to TRCIDR1 if TRCDEVARCH is not implemented + * for memory mapped components. + * Now to make certain decisions easier based on the version + * we use an internal representation of the version in the + * driver, as follows : + * + * ETM_ARCH_VERSION[7:0], where : + * Bits[7:4] - Major version + * Bits[3:0] - Minro version + */ +#define ETM_ARCH_VERSION(major, minor) \ + ((((major) & 0xfU) << 4) | (((minor) & 0xfU))) +#define ETM_ARCH_MAJOR_VERSION(arch) (((arch) >> 4) & 0xfU) +#define ETM_ARCH_MINOR_VERSION(arch) ((arch) & 0xfU) + +#define ETM_ARCH_V4 ETM_ARCH_VERSION(4, 0) +#define ETM_ARCH_ETE ETM_ARCH_VERSION(5, 0) /* Interpretation of resource numbers change at ETM v4.3 architecture */ #define ETM4X_ARCH_4V3 0x43 +#define ETM_ARCH_V4_3 ETM_ARCH_VERSION(4, 3) + +static inline u8 etm_devarch_to_arch(u32 devarch) +{ + return ETM_ARCH_VERSION(ETM_DEVARCH_ARCHID_ARCH_VER(devarch), + ETM_DEVARCH_REVISION(devarch)); +} + +static inline u8 etm_trcidr_to_arch(u32 trcidr1) +{ + return ETM_ARCH_VERSION(ETM_TRCIDR1_ARCH_MAJOR(trcidr1), + ETM_TRCIDR1_ARCH_MINOR(trcidr1)); +} enum etm_impdef_type { ETM4_IMPDEF_HISI_CORE_COMMIT, @@ -370,7 +849,7 @@ struct etmv4_save_state { * @spinlock: Only one at a time pls. * @mode: This tracer's mode, i.e sysFS, Perf or disabled. * @cpu: The cpu this component is affined to. - * @arch: ETM version number. + * @arch: ETM architecture version. * @nr_pe: The number of processing entity available for tracing. * @nr_pe_cmp: The number of processing entity comparator inputs that are * available for tracing. @@ -417,7 +896,12 @@ struct etmv4_save_state { * @nooverflow: Indicate if overflow prevention is supported. * @atbtrig: If the implementation can support ATB triggers * @lpoverride: If the implementation can support low-power state over. + * @trfcr: If the CPU supports FEAT_TRF, value of the TRFCR_ELx that + * allows tracing at all ELs. We don't want to compute this + * at runtime, due to the additional setting of TRFCR_CX when + * in EL2. Otherwise, 0. * @config: structure holding configuration parameters. + * @save_trfcr: Saved TRFCR_EL1 register during a CPU PM event. * @save_state: State to be preserved across power loss * @state_needs_restore: True when there is context to restore after PM exit * @skip_power_up: Indicates if an implementation can skip powering up @@ -452,6 +936,7 @@ struct etmv4_drvdata { u8 s_ex_level; u8 ns_ex_level; u8 q_support; + u8 os_lock_model; bool sticky_enable; bool boot_enable; bool os_unlock; @@ -467,7 +952,9 @@ struct etmv4_drvdata { bool nooverflow; bool atbtrig; bool lpoverride; + u32 trfcr; struct etmv4_config config; + u64 save_trfcr; struct etmv4_save_state *save_state; bool state_needs_restore; bool skip_power_up; @@ -492,4 +979,12 @@ enum etm_addr_ctxtype { extern const struct attribute_group *coresight_etmv4_groups[]; void etm4_config_trace_mode(struct etmv4_config *config); + +u64 etm4x_sysreg_read(u32 offset, bool _relaxed, bool _64bit); +void etm4x_sysreg_write(u64 val, u32 offset, bool _relaxed, bool _64bit); + +static inline bool etm4x_is_ete(struct etmv4_drvdata *drvdata) +{ + return drvdata->arch >= ETM_ARCH_ETE; +} #endif diff --git a/drivers/hwtracing/coresight/coresight-funnel.c b/drivers/hwtracing/coresight/coresight-funnel.c index b2fb853776d791721034ee9dff4f3134f0f74bf9..5c073af15b8c25f827cfe3b03f85f8313f6fa9b1 100644 --- a/drivers/hwtracing/coresight/coresight-funnel.c +++ b/drivers/hwtracing/coresight/coresight-funnel.c @@ -52,13 +52,14 @@ static int dynamic_funnel_enable_hw(struct funnel_drvdata *drvdata, int port) { u32 functl; int rc = 0; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); functl = readl_relaxed(drvdata->base + FUNNEL_FUNCTL); /* Claim the device only when we enable the first slave */ if (!(functl & FUNNEL_ENSx_MASK)) { - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (rc) goto done; } @@ -101,6 +102,7 @@ static void dynamic_funnel_disable_hw(struct funnel_drvdata *drvdata, int inport) { u32 functl; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); @@ -110,7 +112,7 @@ static void dynamic_funnel_disable_hw(struct funnel_drvdata *drvdata, /* Disclaim the device if none of the slaves are now active */ if (!(functl & FUNNEL_ENSx_MASK)) - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } @@ -242,6 +244,7 @@ static int funnel_probe(struct device *dev, struct resource *res) } drvdata->base = base; desc.groups = coresight_funnel_groups; + desc.access = CSDEV_ACCESS_IOMEM(base); } dev_set_drvdata(dev, drvdata); diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index f5f654ea29946dd53d842492ba279fb6e18172c7..ff1dd2092ac5bbe9a476637be33099b45922cb36 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -232,4 +232,7 @@ coresight_find_csdev_by_fwnode(struct fwnode_handle *r_fwnode); void coresight_set_assoc_ectdev_mutex(struct coresight_device *csdev, struct coresight_device *ect_csdev); +void coresight_set_percpu_sink(int cpu, struct coresight_device *csdev); +struct coresight_device *coresight_get_percpu_sink(int cpu); + #endif diff --git a/drivers/hwtracing/coresight/coresight-replicator.c b/drivers/hwtracing/coresight/coresight-replicator.c index da2bfeeabc1b48c3c79f4006aa66aba009798235..dfff2801d7615327d47792b918e0201f3c9306d5 100644 --- a/drivers/hwtracing/coresight/coresight-replicator.c +++ b/drivers/hwtracing/coresight/coresight-replicator.c @@ -45,12 +45,14 @@ struct replicator_drvdata { static void dynamic_replicator_reset(struct replicator_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + CS_UNLOCK(drvdata->base); - if (!coresight_claim_device_unlocked(drvdata->base)) { + if (!coresight_claim_device_unlocked(csdev)) { writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER0); writel_relaxed(0xff, drvdata->base + REPLICATOR_IDFILTER1); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); } CS_LOCK(drvdata->base); @@ -70,6 +72,7 @@ static int dynamic_replicator_enable(struct replicator_drvdata *drvdata, { int rc = 0; u32 id0val, id1val; + struct coresight_device *csdev = drvdata->csdev; CS_UNLOCK(drvdata->base); @@ -84,7 +87,7 @@ static int dynamic_replicator_enable(struct replicator_drvdata *drvdata, id0val = id1val = 0xff; if (id0val == 0xff && id1val == 0xff) - rc = coresight_claim_device_unlocked(drvdata->base); + rc = coresight_claim_device_unlocked(csdev); if (!rc) { switch (outport) { @@ -140,6 +143,7 @@ static void dynamic_replicator_disable(struct replicator_drvdata *drvdata, int inport, int outport) { u32 reg; + struct coresight_device *csdev = drvdata->csdev; switch (outport) { case 0: @@ -160,7 +164,7 @@ static void dynamic_replicator_disable(struct replicator_drvdata *drvdata, if ((readl_relaxed(drvdata->base + REPLICATOR_IDFILTER0) == 0xff) && (readl_relaxed(drvdata->base + REPLICATOR_IDFILTER1) == 0xff)) - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } @@ -254,6 +258,7 @@ static int replicator_probe(struct device *dev, struct resource *res) } drvdata->base = base; desc.groups = replicator_groups; + desc.access = CSDEV_ACCESS_IOMEM(base); } if (fwnode_property_present(dev_fwnode(dev), diff --git a/drivers/hwtracing/coresight/coresight-self-hosted-trace.h b/drivers/hwtracing/coresight/coresight-self-hosted-trace.h new file mode 100644 index 0000000000000000000000000000000000000000..53840a2c41f2c7051e7d4acabd895f6934275ce7 --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-self-hosted-trace.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Arm v8 Self-Hosted trace support. + * + * Copyright (C) 2021 ARM Ltd. + */ + +#ifndef __CORESIGHT_SELF_HOSTED_TRACE_H +#define __CORESIGHT_SELF_HOSTED_TRACE_H + +#include + +static inline u64 read_trfcr(void) +{ + return read_sysreg_s(SYS_TRFCR_EL1); +} + +static inline void write_trfcr(u64 val) +{ + write_sysreg_s(val, SYS_TRFCR_EL1); + isb(); +} + +static inline u64 cpu_prohibit_trace(void) +{ + u64 trfcr = read_trfcr(); + + /* Prohibit tracing at EL0 & the kernel EL */ + write_trfcr(trfcr & ~(TRFCR_ELx_ExTRE | TRFCR_ELx_E0TRE)); + /* Return the original value of the TRFCR */ + return trfcr; +} +#endif /* __CORESIGHT_SELF_HOSTED_TRACE_H */ diff --git a/drivers/hwtracing/coresight/coresight-stm.c b/drivers/hwtracing/coresight/coresight-stm.c index 0ecca9f93f3a1b9c8d814b76bd62619f2ca583db..ed9a47ce769c42fa32deb9a2a1a2bdc3068681ea 100644 --- a/drivers/hwtracing/coresight/coresight-stm.c +++ b/drivers/hwtracing/coresight/coresight-stm.c @@ -258,6 +258,7 @@ static void stm_disable(struct coresight_device *csdev, struct perf_event *event) { struct stm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct csdev_access *csa = &csdev->access; /* * For as long as the tracer isn't disabled another entity can't @@ -270,7 +271,7 @@ static void stm_disable(struct coresight_device *csdev, spin_unlock(&drvdata->spinlock); /* Wait until the engine has completely stopped */ - coresight_timeout(drvdata->base, STMTCSR, STMTCSR_BUSY_BIT, 0); + coresight_timeout(csa, STMTCSR, STMTCSR_BUSY_BIT, 0); pm_runtime_put(csdev->dev.parent); @@ -884,6 +885,7 @@ static int stm_probe(struct amba_device *adev, const struct amba_id *id) if (IS_ERR(base)) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); ret = stm_get_stimulus_area(dev, &ch_res); if (ret) diff --git a/drivers/hwtracing/coresight/coresight-tmc-core.c b/drivers/hwtracing/coresight/coresight-tmc-core.c index e29b3914fc0ff14fbdeba40c2ad296d8fcfd3df4..74c6323d4d6ab9582e4e01350f24d53f3fdc4dce 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-core.c +++ b/drivers/hwtracing/coresight/coresight-tmc-core.c @@ -33,16 +33,20 @@ DEFINE_CORESIGHT_DEVLIST(etr_devs, "tmc_etr"); void tmc_wait_for_tmcready(struct tmc_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + struct csdev_access *csa = &csdev->access; + /* Ensure formatter, unformatter and hardware fifo are empty */ - if (coresight_timeout(drvdata->base, - TMC_STS, TMC_STS_TMCREADY_BIT, 1)) { - dev_err(&drvdata->csdev->dev, + if (coresight_timeout(csa, TMC_STS, TMC_STS_TMCREADY_BIT, 1)) { + dev_err(&csdev->dev, "timeout while waiting for TMC to be Ready\n"); } } void tmc_flush_and_stop(struct tmc_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + struct csdev_access *csa = &csdev->access; u32 ffcr; ffcr = readl_relaxed(drvdata->base + TMC_FFCR); @@ -51,9 +55,8 @@ void tmc_flush_and_stop(struct tmc_drvdata *drvdata) ffcr |= BIT(TMC_FFCR_FLUSHMAN_BIT); writel_relaxed(ffcr, drvdata->base + TMC_FFCR); /* Ensure flush completes */ - if (coresight_timeout(drvdata->base, - TMC_FFCR, TMC_FFCR_FLUSHMAN_BIT, 0)) { - dev_err(&drvdata->csdev->dev, + if (coresight_timeout(csa, TMC_FFCR, TMC_FFCR_FLUSHMAN_BIT, 0)) { + dev_err(&csdev->dev, "timeout while waiting for completion of Manual Flush\n"); } @@ -456,6 +459,7 @@ static int tmc_probe(struct amba_device *adev, const struct amba_id *id) } drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); spin_lock_init(&drvdata->spinlock); diff --git a/drivers/hwtracing/coresight/coresight-tmc-etf.c b/drivers/hwtracing/coresight/coresight-tmc-etf.c index 8978f3410bee5c05711fc700e52a747d08de284d..cd0fb7bfba6849cb1efb1a09b8c8ada7af09cee7 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etf.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etf.c @@ -37,7 +37,7 @@ static void __tmc_etb_enable_hw(struct tmc_drvdata *drvdata) static int tmc_etb_enable_hw(struct tmc_drvdata *drvdata) { - int rc = coresight_claim_device(drvdata->base); + int rc = coresight_claim_device(drvdata->csdev); if (rc) return rc; @@ -88,7 +88,7 @@ static void __tmc_etb_disable_hw(struct tmc_drvdata *drvdata) static void tmc_etb_disable_hw(struct tmc_drvdata *drvdata) { __tmc_etb_disable_hw(drvdata); - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); } static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) @@ -109,7 +109,7 @@ static void __tmc_etf_enable_hw(struct tmc_drvdata *drvdata) static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) { - int rc = coresight_claim_device(drvdata->base); + int rc = coresight_claim_device(drvdata->csdev); if (rc) return rc; @@ -120,11 +120,13 @@ static int tmc_etf_enable_hw(struct tmc_drvdata *drvdata) static void tmc_etf_disable_hw(struct tmc_drvdata *drvdata) { + struct coresight_device *csdev = drvdata->csdev; + CS_UNLOCK(drvdata->base); tmc_flush_and_stop(drvdata); tmc_disable_hw(drvdata); - coresight_disclaim_device_unlocked(drvdata->base); + coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); } diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c index 3309b1344ffc019c16d864a97cfed58de1c8bdb2..c631b16d367d500c7f14d2d59bfafc086b44b96e 100644 --- a/drivers/hwtracing/coresight/coresight-tmc-etr.c +++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c @@ -609,8 +609,9 @@ static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata, if (!flat_buf) return -ENOMEM; - flat_buf->vaddr = dma_alloc_coherent(real_dev, etr_buf->size, - &flat_buf->daddr, GFP_KERNEL); + flat_buf->vaddr = dma_alloc_noncoherent(real_dev, etr_buf->size, + &flat_buf->daddr, + DMA_FROM_DEVICE, GFP_KERNEL); if (!flat_buf->vaddr) { kfree(flat_buf); return -ENOMEM; @@ -631,14 +632,18 @@ static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf) if (flat_buf && flat_buf->daddr) { struct device *real_dev = flat_buf->dev->parent; - dma_free_coherent(real_dev, flat_buf->size, - flat_buf->vaddr, flat_buf->daddr); + dma_free_noncoherent(real_dev, etr_buf->size, + flat_buf->vaddr, flat_buf->daddr, + DMA_FROM_DEVICE); } kfree(flat_buf); } static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp) { + struct etr_flat_buf *flat_buf = etr_buf->private; + struct device *real_dev = flat_buf->dev->parent; + /* * Adjust the buffer to point to the beginning of the trace data * and update the available trace data. @@ -648,6 +653,19 @@ static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp) etr_buf->len = etr_buf->size; else etr_buf->len = rwp - rrp; + + /* + * The driver always starts tracing at the beginning of the buffer, + * the only reason why we would get a wrap around is when the buffer + * is full. Sync the entire buffer in one go for this case. + */ + if (etr_buf->offset + etr_buf->len > etr_buf->size) + dma_sync_single_for_cpu(real_dev, flat_buf->daddr, + etr_buf->size, DMA_FROM_DEVICE); + else + dma_sync_single_for_cpu(real_dev, + flat_buf->daddr + etr_buf->offset, + etr_buf->len, DMA_FROM_DEVICE); } static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf, @@ -1040,7 +1058,7 @@ static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata, rc = tmc_etr_enable_catu(drvdata, etr_buf); if (rc) return rc; - rc = coresight_claim_device(drvdata->base); + rc = coresight_claim_device(drvdata->csdev); if (!rc) { drvdata->etr_buf = etr_buf; __tmc_etr_enable_hw(drvdata); @@ -1134,7 +1152,7 @@ void tmc_etr_disable_hw(struct tmc_drvdata *drvdata) __tmc_etr_disable_hw(drvdata); /* Disable CATU device if this ETR is connected to one */ tmc_etr_disable_catu(drvdata); - coresight_disclaim_device(drvdata->base); + coresight_disclaim_device(drvdata->csdev); /* Reset the ETR buf used by hardware */ drvdata->etr_buf = NULL; } diff --git a/drivers/hwtracing/coresight/coresight-tpiu.c b/drivers/hwtracing/coresight/coresight-tpiu.c index 0ca39d905d0b3d70decf54ede4780f40b8fc2e1a..686ab7749f022ebb86c2900bc4351cb32c554a4f 100644 --- a/drivers/hwtracing/coresight/coresight-tpiu.c +++ b/drivers/hwtracing/coresight/coresight-tpiu.c @@ -81,6 +81,8 @@ static int tpiu_enable(struct coresight_device *csdev, u32 mode, void *__unused) static void tpiu_disable_hw(struct tpiu_drvdata *drvdata) { + struct csdev_access *csa = &drvdata->csdev->access; + CS_UNLOCK(drvdata->base); /* Clear formatter and stop on flush */ @@ -88,9 +90,9 @@ static void tpiu_disable_hw(struct tpiu_drvdata *drvdata) /* Generate manual flush */ writel_relaxed(FFCR_STOP_FI | FFCR_FON_MAN, drvdata->base + TPIU_FFCR); /* Wait for flush to complete */ - coresight_timeout(drvdata->base, TPIU_FFCR, FFCR_FON_MAN_BIT, 0); + coresight_timeout(csa, TPIU_FFCR, FFCR_FON_MAN_BIT, 0); /* Wait for formatter to stop */ - coresight_timeout(drvdata->base, TPIU_FFSR, FFSR_FT_STOPPED_BIT, 1); + coresight_timeout(csa, TPIU_FFSR, FFSR_FT_STOPPED_BIT, 1); CS_LOCK(drvdata->base); } @@ -149,6 +151,7 @@ static int tpiu_probe(struct amba_device *adev, const struct amba_id *id) return PTR_ERR(base); drvdata->base = base; + desc.access = CSDEV_ACCESS_IOMEM(base); /* Disable tpiu to support older devices */ tpiu_disable_hw(drvdata); diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c new file mode 100644 index 0000000000000000000000000000000000000000..277dfae76931bb91f109d5cc56a33b0a84ade1d3 --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -0,0 +1,1227 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This driver enables Trace Buffer Extension (TRBE) as a per-cpu coresight + * sink device could then pair with an appropriate per-cpu coresight source + * device (ETE) thus generating required trace data. Trace can be enabled + * via the perf framework. + * + * The AUX buffer handling is inspired from Arm SPE PMU driver. + * + * Copyright (C) 2020 ARM Ltd. + * + * Author: Anshuman Khandual + */ +#define DRVNAME "arm_trbe" + +#define pr_fmt(fmt) DRVNAME ": " fmt + +#include +#include "coresight-self-hosted-trace.h" +#include "coresight-trbe.h" + +#define PERF_IDX2OFF(idx, buf) ((idx) % ((buf)->nr_pages << PAGE_SHIFT)) + +/* + * A padding packet that will help the user space tools + * in skipping relevant sections in the captured trace + * data which could not be decoded. TRBE doesn't support + * formatting the trace data, unlike the legacy CoreSight + * sinks and thus we use ETE trace packets to pad the + * sections of the buffer. + */ +#define ETE_IGNORE_PACKET 0x70 + +/* + * Minimum amount of meaningful trace will contain: + * A-Sync, Trace Info, Trace On, Address, Atom. + * This is about 44bytes of ETE trace. To be on + * the safer side, we assume 64bytes is the minimum + * space required for a meaningful session, before + * we hit a "WRAP" event. + */ +#define TRBE_TRACE_MIN_BUF_SIZE 64 + +enum trbe_fault_action { + TRBE_FAULT_ACT_WRAP, + TRBE_FAULT_ACT_SPURIOUS, + TRBE_FAULT_ACT_FATAL, +}; + +struct trbe_buf { + /* + * Even though trbe_base represents vmap() + * mapped allocated buffer's start address, + * it's being as unsigned long for various + * arithmetic and comparision operations & + * also to be consistent with trbe_write & + * trbe_limit sibling pointers. + */ + unsigned long trbe_base; + /* The base programmed into the TRBE */ + unsigned long trbe_hw_base; + unsigned long trbe_limit; + unsigned long trbe_write; + int nr_pages; + void **pages; + bool snapshot; + struct trbe_cpudata *cpudata; +}; + +/* + * struct trbe_cpudata: TRBE instance specific data + * @trbe_flag - TRBE dirty/access flag support + * @trbe_hw_align - Actual TRBE alignment required for TRBPTR_EL1. + * @trbe_align - Software alignment used for the TRBPTR_EL1. + * @cpu - CPU this TRBE belongs to. + * @mode - Mode of current operation. (perf/disabled) + * @drvdata - TRBE specific drvdata + */ +struct trbe_cpudata { + bool trbe_flag; + u64 trbe_hw_align; + u64 trbe_align; + int cpu; + enum cs_mode mode; + struct trbe_buf *buf; + struct trbe_drvdata *drvdata; +}; + +struct trbe_drvdata { + struct trbe_cpudata __percpu *cpudata; + struct perf_output_handle * __percpu *handle; + struct hlist_node hotplug_node; + int irq; + cpumask_t supported_cpus; + enum cpuhp_state trbe_online; + struct platform_device *pdev; +}; + +static int trbe_alloc_node(struct perf_event *event) +{ + if (event->cpu == -1) + return NUMA_NO_NODE; + return cpu_to_node(event->cpu); +} + +static void trbe_drain_buffer(void) +{ + tsb_csync(); + dsb(nsh); +} + +static void trbe_drain_and_disable_local(void) +{ + u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); + + trbe_drain_buffer(); + + /* + * Disable the TRBE without clearing LIMITPTR which + * might be required for fetching the buffer limits. + */ + trblimitr &= ~TRBLIMITR_ENABLE; + write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1); + isb(); +} + +static void trbe_reset_local(void) +{ + trbe_drain_and_disable_local(); + write_sysreg_s(0, SYS_TRBLIMITR_EL1); + write_sysreg_s(0, SYS_TRBPTR_EL1); + write_sysreg_s(0, SYS_TRBBASER_EL1); + write_sysreg_s(0, SYS_TRBSR_EL1); +} + +static void trbe_report_wrap_event(struct perf_output_handle *handle) +{ + /* + * Mark the buffer to indicate that there was a WRAP event by + * setting the COLLISION flag. This indicates to the user that + * the TRBE trace collection was stopped without stopping the + * ETE and thus there might be some amount of trace that was + * lost between the time the WRAP was detected and the IRQ + * was consumed by the CPU. + * + * Setting the TRUNCATED flag would move the event to STOPPED + * state unnecessarily, even when there is space left in the + * ring buffer. Using the COLLISION flag doesn't have this side + * effect. We only set TRUNCATED flag when there is no space + * left in the ring buffer. + */ + perf_aux_output_flag(handle, PERF_AUX_FLAG_COLLISION); +} + +static void trbe_stop_and_truncate_event(struct perf_output_handle *handle) +{ + struct trbe_buf *buf = etm_perf_sink_config(handle); + + /* + * We cannot proceed with the buffer collection and we + * do not have any data for the current session. The + * etm_perf driver expects to close out the aux_buffer + * at event_stop(). So disable the TRBE here and leave + * the update_buffer() to return a 0 size. + */ + trbe_drain_and_disable_local(); + perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); + perf_aux_output_end(handle, 0); + *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL; +} + +/* + * TRBE Buffer Management + * + * The TRBE buffer spans from the base pointer till the limit pointer. When enabled, + * it starts writing trace data from the write pointer onward till the limit pointer. + * When the write pointer reaches the address just before the limit pointer, it gets + * wrapped around again to the base pointer. This is called a TRBE wrap event, which + * generates a maintenance interrupt when operated in WRAP or FILL mode. This driver + * uses FILL mode, where the TRBE stops the trace collection at wrap event. The IRQ + * handler updates the AUX buffer and re-enables the TRBE with updated WRITE and + * LIMIT pointers. + * + * Wrap around with an IRQ + * ------ < ------ < ------- < ----- < ----- + * | | + * ------ > ------ > ------- > ----- > ----- + * + * +---------------+-----------------------+ + * | | | + * +---------------+-----------------------+ + * Base Pointer Write Pointer Limit Pointer + * + * The base and limit pointers always needs to be PAGE_SIZE aligned. But the write + * pointer can be aligned to the implementation defined TRBE trace buffer alignment + * as captured in trbe_cpudata->trbe_align. + * + * + * head tail wakeup + * +---------------------------------------+----- ~ ~ ------ + * |$$$$$$$|################|$$$$$$$$$$$$$$| | + * +---------------------------------------+----- ~ ~ ------ + * Base Pointer Write Pointer Limit Pointer + * + * The perf_output_handle indices (head, tail, wakeup) are monotonically increasing + * values which tracks all the driver writes and user reads from the perf auxiliary + * buffer. Generally [head..tail] is the area where the driver can write into unless + * the wakeup is behind the tail. Enabled TRBE buffer span needs to be adjusted and + * configured depending on the perf_output_handle indices, so that the driver does + * not override into areas in the perf auxiliary buffer which is being or yet to be + * consumed from the user space. The enabled TRBE buffer area is a moving subset of + * the allocated perf auxiliary buffer. + */ + +static void __trbe_pad_buf(struct trbe_buf *buf, u64 offset, int len) +{ + memset((void *)buf->trbe_base + offset, ETE_IGNORE_PACKET, len); +} + +static void trbe_pad_buf(struct perf_output_handle *handle, int len) +{ + struct trbe_buf *buf = etm_perf_sink_config(handle); + u64 head = PERF_IDX2OFF(handle->head, buf); + + __trbe_pad_buf(buf, head, len); + if (!buf->snapshot) + perf_aux_output_skip(handle, len); +} + +static unsigned long trbe_snapshot_offset(struct perf_output_handle *handle) +{ + struct trbe_buf *buf = etm_perf_sink_config(handle); + + /* + * The ETE trace has alignment synchronization packets allowing + * the decoder to reset in case of an overflow or corruption. + * So we can use the entire buffer for the snapshot mode. + */ + return buf->nr_pages * PAGE_SIZE; +} + +/* + * TRBE Limit Calculation + * + * The following markers are used to illustrate various TRBE buffer situations. + * + * $$$$ - Data area, unconsumed captured trace data, not to be overridden + * #### - Free area, enabled, trace will be written + * %%%% - Free area, disabled, trace will not be written + * ==== - Free area, padded with ETE_IGNORE_PACKET, trace will be skipped + */ +static unsigned long __trbe_normal_offset(struct perf_output_handle *handle) +{ + struct trbe_buf *buf = etm_perf_sink_config(handle); + struct trbe_cpudata *cpudata = buf->cpudata; + const u64 bufsize = buf->nr_pages * PAGE_SIZE; + u64 limit = bufsize; + u64 head, tail, wakeup; + + head = PERF_IDX2OFF(handle->head, buf); + + /* + * head + * ------->| + * | + * head TRBE align tail + * +----|-------|---------------|-------+ + * |$$$$|=======|###############|$$$$$$$| + * +----|-------|---------------|-------+ + * trbe_base trbe_base + nr_pages + * + * Perf aux buffer output head position can be misaligned depending on + * various factors including user space reads. In case misaligned, head + * needs to be aligned before TRBE can be configured. Pad the alignment + * gap with ETE_IGNORE_PACKET bytes that will be ignored by user tools + * and skip this section thus advancing the head. + */ + if (!IS_ALIGNED(head, cpudata->trbe_align)) { + unsigned long delta = roundup(head, cpudata->trbe_align) - head; + + delta = min(delta, handle->size); + trbe_pad_buf(handle, delta); + head = PERF_IDX2OFF(handle->head, buf); + } + + /* + * head = tail (size = 0) + * +----|-------------------------------+ + * |$$$$|$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$ | + * +----|-------------------------------+ + * trbe_base trbe_base + nr_pages + * + * Perf aux buffer does not have any space for the driver to write into. + */ + if (!handle->size) + return 0; + + /* Compute the tail and wakeup indices now that we've aligned head */ + tail = PERF_IDX2OFF(handle->head + handle->size, buf); + wakeup = PERF_IDX2OFF(handle->wakeup, buf); + + /* + * Lets calculate the buffer area which TRBE could write into. There + * are three possible scenarios here. Limit needs to be aligned with + * PAGE_SIZE per the TRBE requirement. Always avoid clobbering the + * unconsumed data. + * + * 1) head < tail + * + * head tail + * +----|-----------------------|-------+ + * |$$$$|#######################|$$$$$$$| + * +----|-----------------------|-------+ + * trbe_base limit trbe_base + nr_pages + * + * TRBE could write into [head..tail] area. Unless the tail is right at + * the end of the buffer, neither an wrap around nor an IRQ is expected + * while being enabled. + * + * 2) head == tail + * + * head = tail (size > 0) + * +----|-------------------------------+ + * |%%%%|###############################| + * +----|-------------------------------+ + * trbe_base limit = trbe_base + nr_pages + * + * TRBE should just write into [head..base + nr_pages] area even though + * the entire buffer is empty. Reason being, when the trace reaches the + * end of the buffer, it will just wrap around with an IRQ giving an + * opportunity to reconfigure the buffer. + * + * 3) tail < head + * + * tail head + * +----|-----------------------|-------+ + * |%%%%|$$$$$$$$$$$$$$$$$$$$$$$|#######| + * +----|-----------------------|-------+ + * trbe_base limit = trbe_base + nr_pages + * + * TRBE should just write into [head..base + nr_pages] area even though + * the [trbe_base..tail] is also empty. Reason being, when the trace + * reaches the end of the buffer, it will just wrap around with an IRQ + * giving an opportunity to reconfigure the buffer. + */ + if (head < tail) + limit = round_down(tail, PAGE_SIZE); + + /* + * Wakeup may be arbitrarily far into the future. If it's not in the + * current generation, either we'll wrap before hitting it, or it's + * in the past and has been handled already. + * + * If there's a wakeup before we wrap, arrange to be woken up by the + * page boundary following it. Keep the tail boundary if that's lower. + * + * head wakeup tail + * +----|---------------|-------|-------+ + * |$$$$|###############|%%%%%%%|$$$$$$$| + * +----|---------------|-------|-------+ + * trbe_base limit trbe_base + nr_pages + */ + if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) + limit = min(limit, round_up(wakeup, PAGE_SIZE)); + + /* + * There are two situation when this can happen i.e limit is before + * the head and hence TRBE cannot be configured. + * + * 1) head < tail (aligned down with PAGE_SIZE) and also they are both + * within the same PAGE size range. + * + * PAGE_SIZE + * |----------------------| + * + * limit head tail + * +------------|------|--------|-------+ + * |$$$$$$$$$$$$$$$$$$$|========|$$$$$$$| + * +------------|------|--------|-------+ + * trbe_base trbe_base + nr_pages + * + * 2) head < wakeup (aligned up with PAGE_SIZE) < tail and also both + * head and wakeup are within same PAGE size range. + * + * PAGE_SIZE + * |----------------------| + * + * limit head wakeup tail + * +----|------|-------|--------|-------+ + * |$$$$$$$$$$$|=======|========|$$$$$$$| + * +----|------|-------|--------|-------+ + * trbe_base trbe_base + nr_pages + */ + if (limit > head) + return limit; + + trbe_pad_buf(handle, handle->size); + return 0; +} + +static unsigned long trbe_normal_offset(struct perf_output_handle *handle) +{ + struct trbe_buf *buf = etm_perf_sink_config(handle); + u64 limit = __trbe_normal_offset(handle); + u64 head = PERF_IDX2OFF(handle->head, buf); + + /* + * If the head is too close to the limit and we don't + * have space for a meaningful run, we rather pad it + * and start fresh. + */ + while (limit && (limit - head < TRBE_TRACE_MIN_BUF_SIZE)) { + trbe_pad_buf(handle, limit - head); + limit = __trbe_normal_offset(handle); + head = PERF_IDX2OFF(handle->head, buf); + } + return limit; +} + +static unsigned long compute_trbe_buffer_limit(struct perf_output_handle *handle) +{ + struct trbe_buf *buf = etm_perf_sink_config(handle); + unsigned long offset; + + if (buf->snapshot) + offset = trbe_snapshot_offset(handle); + else + offset = trbe_normal_offset(handle); + return buf->trbe_base + offset; +} + +static void clr_trbe_status(void) +{ + u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1); + + WARN_ON(is_trbe_enabled()); + trbsr &= ~TRBSR_IRQ; + trbsr &= ~TRBSR_TRG; + trbsr &= ~TRBSR_WRAP; + trbsr &= ~(TRBSR_EC_MASK << TRBSR_EC_SHIFT); + trbsr &= ~(TRBSR_BSC_MASK << TRBSR_BSC_SHIFT); + trbsr &= ~TRBSR_STOP; + write_sysreg_s(trbsr, SYS_TRBSR_EL1); +} + +static void set_trbe_limit_pointer_enabled(unsigned long addr) +{ + u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); + + WARN_ON(!IS_ALIGNED(addr, (1UL << TRBLIMITR_LIMIT_SHIFT))); + WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE)); + + trblimitr &= ~TRBLIMITR_NVM; + trblimitr &= ~(TRBLIMITR_FILL_MODE_MASK << TRBLIMITR_FILL_MODE_SHIFT); + trblimitr &= ~(TRBLIMITR_TRIG_MODE_MASK << TRBLIMITR_TRIG_MODE_SHIFT); + trblimitr &= ~(TRBLIMITR_LIMIT_MASK << TRBLIMITR_LIMIT_SHIFT); + + /* + * Fill trace buffer mode is used here while configuring the + * TRBE for trace capture. In this particular mode, the trace + * collection is stopped and a maintenance interrupt is raised + * when the current write pointer wraps. This pause in trace + * collection gives the software an opportunity to capture the + * trace data in the interrupt handler, before reconfiguring + * the TRBE. + */ + trblimitr |= (TRBE_FILL_MODE_FILL & TRBLIMITR_FILL_MODE_MASK) << TRBLIMITR_FILL_MODE_SHIFT; + + /* + * Trigger mode is not used here while configuring the TRBE for + * the trace capture. Hence just keep this in the ignore mode. + */ + trblimitr |= (TRBE_TRIG_MODE_IGNORE & TRBLIMITR_TRIG_MODE_MASK) << + TRBLIMITR_TRIG_MODE_SHIFT; + trblimitr |= (addr & PAGE_MASK); + + trblimitr |= TRBLIMITR_ENABLE; + write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1); + + /* Synchronize the TRBE enable event */ + isb(); +} + +static void trbe_enable_hw(struct trbe_buf *buf) +{ + WARN_ON(buf->trbe_hw_base < buf->trbe_base); + WARN_ON(buf->trbe_write < buf->trbe_hw_base); + WARN_ON(buf->trbe_write >= buf->trbe_limit); + set_trbe_disabled(); + isb(); + clr_trbe_status(); + set_trbe_base_pointer(buf->trbe_hw_base); + set_trbe_write_pointer(buf->trbe_write); + + /* + * Synchronize all the register updates + * till now before enabling the TRBE. + */ + isb(); + set_trbe_limit_pointer_enabled(buf->trbe_limit); +} + +static enum trbe_fault_action trbe_get_fault_act(u64 trbsr) +{ + int ec = get_trbe_ec(trbsr); + int bsc = get_trbe_bsc(trbsr); + + WARN_ON(is_trbe_running(trbsr)); + if (is_trbe_trg(trbsr) || is_trbe_abort(trbsr)) + return TRBE_FAULT_ACT_FATAL; + + if ((ec == TRBE_EC_STAGE1_ABORT) || (ec == TRBE_EC_STAGE2_ABORT)) + return TRBE_FAULT_ACT_FATAL; + + if (is_trbe_wrap(trbsr) && (ec == TRBE_EC_OTHERS) && (bsc == TRBE_BSC_FILLED)) { + if (get_trbe_write_pointer() == get_trbe_base_pointer()) + return TRBE_FAULT_ACT_WRAP; + } + return TRBE_FAULT_ACT_SPURIOUS; +} + +static unsigned long trbe_get_trace_size(struct perf_output_handle *handle, + struct trbe_buf *buf, bool wrap) +{ + u64 write; + u64 start_off, end_off; + + /* + * If the TRBE has wrapped around the write pointer has + * wrapped and should be treated as limit. + */ + if (wrap) + write = get_trbe_limit_pointer(); + else + write = get_trbe_write_pointer(); + + /* + * TRBE may use a different base address than the base + * of the ring buffer. Thus use the beginning of the ring + * buffer to compute the offsets. + */ + end_off = write - buf->trbe_base; + start_off = PERF_IDX2OFF(handle->head, buf); + + if (WARN_ON_ONCE(end_off < start_off)) + return 0; + return (end_off - start_off); +} + +static void *arm_trbe_alloc_buffer(struct coresight_device *csdev, + struct perf_event *event, void **pages, + int nr_pages, bool snapshot) +{ + struct trbe_buf *buf; + struct page **pglist; + int i; + + /* + * TRBE LIMIT and TRBE WRITE pointers must be page aligned. But with + * just a single page, there would not be any room left while writing + * into a partially filled TRBE buffer after the page size alignment. + * Hence restrict the minimum buffer size as two pages. + */ + if (nr_pages < 2) + return NULL; + + buf = kzalloc_node(sizeof(*buf), GFP_KERNEL, trbe_alloc_node(event)); + if (!buf) + return ERR_PTR(-ENOMEM); + + pglist = kcalloc(nr_pages, sizeof(*pglist), GFP_KERNEL); + if (!pglist) { + kfree(buf); + return ERR_PTR(-ENOMEM); + } + + for (i = 0; i < nr_pages; i++) + pglist[i] = virt_to_page(pages[i]); + + buf->trbe_base = (unsigned long)vmap(pglist, nr_pages, VM_MAP, PAGE_KERNEL); + if (!buf->trbe_base) { + kfree(pglist); + kfree(buf); + return ERR_PTR(-ENOMEM); + } + buf->trbe_limit = buf->trbe_base + nr_pages * PAGE_SIZE; + buf->trbe_write = buf->trbe_base; + buf->snapshot = snapshot; + buf->nr_pages = nr_pages; + buf->pages = pages; + kfree(pglist); + return buf; +} + +static void arm_trbe_free_buffer(void *config) +{ + struct trbe_buf *buf = config; + + vunmap((void *)buf->trbe_base); + kfree(buf); +} + +static unsigned long arm_trbe_update_buffer(struct coresight_device *csdev, + struct perf_output_handle *handle, + void *config) +{ + struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); + struct trbe_buf *buf = config; + enum trbe_fault_action act; + unsigned long size, status; + unsigned long flags; + bool wrap = false; + + WARN_ON(buf->cpudata != cpudata); + WARN_ON(cpudata->cpu != smp_processor_id()); + WARN_ON(cpudata->drvdata != drvdata); + if (cpudata->mode != CS_MODE_PERF) + return 0; + + /* + * We are about to disable the TRBE. And this could in turn + * fill up the buffer triggering, an IRQ. This could be consumed + * by the PE asynchronously, causing a race here against + * the IRQ handler in closing out the handle. So, let us + * make sure the IRQ can't trigger while we are collecting + * the buffer. We also make sure that a WRAP event is handled + * accordingly. + */ + local_irq_save(flags); + + /* + * If the TRBE was disabled due to lack of space in the AUX buffer or a + * spurious fault, the driver leaves it disabled, truncating the buffer. + * Since the etm_perf driver expects to close out the AUX buffer, the + * driver skips it. Thus, just pass in 0 size here to indicate that the + * buffer was truncated. + */ + if (!is_trbe_enabled()) { + size = 0; + goto done; + } + /* + * perf handle structure needs to be shared with the TRBE IRQ handler for + * capturing trace data and restarting the handle. There is a probability + * of an undefined reference based crash when etm event is being stopped + * while a TRBE IRQ also getting processed. This happens due the release + * of perf handle via perf_aux_output_end() in etm_event_stop(). Stopping + * the TRBE here will ensure that no IRQ could be generated when the perf + * handle gets freed in etm_event_stop(). + */ + trbe_drain_and_disable_local(); + + /* Check if there is a pending interrupt and handle it here */ + status = read_sysreg_s(SYS_TRBSR_EL1); + if (is_trbe_irq(status)) { + + /* + * Now that we are handling the IRQ here, clear the IRQ + * from the status, to let the irq handler know that it + * is taken care of. + */ + clr_trbe_irq(); + isb(); + + act = trbe_get_fault_act(status); + /* + * If this was not due to a WRAP event, we have some + * errors and as such buffer is empty. + */ + if (act != TRBE_FAULT_ACT_WRAP) { + size = 0; + goto done; + } + + trbe_report_wrap_event(handle); + wrap = true; + } + + size = trbe_get_trace_size(handle, buf, wrap); + +done: + local_irq_restore(flags); + + if (buf->snapshot) + handle->head += size; + return size; +} + +static int __arm_trbe_enable(struct trbe_buf *buf, + struct perf_output_handle *handle) +{ + perf_aux_output_flag(handle, PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW); + buf->trbe_limit = compute_trbe_buffer_limit(handle); + buf->trbe_write = buf->trbe_base + PERF_IDX2OFF(handle->head, buf); + if (buf->trbe_limit == buf->trbe_base) { + trbe_stop_and_truncate_event(handle); + return -ENOSPC; + } + /* Set the base of the TRBE to the buffer base */ + buf->trbe_hw_base = buf->trbe_base; + *this_cpu_ptr(buf->cpudata->drvdata->handle) = handle; + trbe_enable_hw(buf); + return 0; +} + +static int arm_trbe_enable(struct coresight_device *csdev, u32 mode, void *data) +{ + struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); + struct perf_output_handle *handle = data; + struct trbe_buf *buf = etm_perf_sink_config(handle); + + WARN_ON(cpudata->cpu != smp_processor_id()); + WARN_ON(cpudata->drvdata != drvdata); + if (mode != CS_MODE_PERF) + return -EINVAL; + + cpudata->buf = buf; + cpudata->mode = mode; + buf->cpudata = cpudata; + + return __arm_trbe_enable(buf, handle); +} + +static int arm_trbe_disable(struct coresight_device *csdev) +{ + struct trbe_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); + struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev); + struct trbe_buf *buf = cpudata->buf; + + WARN_ON(buf->cpudata != cpudata); + WARN_ON(cpudata->cpu != smp_processor_id()); + WARN_ON(cpudata->drvdata != drvdata); + if (cpudata->mode != CS_MODE_PERF) + return -EINVAL; + + trbe_drain_and_disable_local(); + buf->cpudata = NULL; + cpudata->buf = NULL; + cpudata->mode = CS_MODE_DISABLED; + return 0; +} + +static void trbe_handle_spurious(struct perf_output_handle *handle) +{ + u64 limitr = read_sysreg_s(SYS_TRBLIMITR_EL1); + + /* + * If the IRQ was spurious, simply re-enable the TRBE + * back without modifying the buffer parameters to + * retain the trace collected so far. + */ + limitr |= TRBLIMITR_ENABLE; + write_sysreg_s(limitr, SYS_TRBLIMITR_EL1); + isb(); +} + +static int trbe_handle_overflow(struct perf_output_handle *handle) +{ + struct perf_event *event = handle->event; + struct trbe_buf *buf = etm_perf_sink_config(handle); + unsigned long size; + struct etm_event_data *event_data; + + size = trbe_get_trace_size(handle, buf, true); + if (buf->snapshot) + handle->head += size; + + trbe_report_wrap_event(handle); + perf_aux_output_end(handle, size); + event_data = perf_aux_output_begin(handle, event); + if (!event_data) { + /* + * We are unable to restart the trace collection, + * thus leave the TRBE disabled. The etm-perf driver + * is able to detect this with a disconnected handle + * (handle->event = NULL). + */ + trbe_drain_and_disable_local(); + *this_cpu_ptr(buf->cpudata->drvdata->handle) = NULL; + return -EINVAL; + } + + return __arm_trbe_enable(buf, handle); +} + +static bool is_perf_trbe(struct perf_output_handle *handle) +{ + struct trbe_buf *buf = etm_perf_sink_config(handle); + struct trbe_cpudata *cpudata = buf->cpudata; + struct trbe_drvdata *drvdata = cpudata->drvdata; + int cpu = smp_processor_id(); + + WARN_ON(buf->trbe_hw_base != get_trbe_base_pointer()); + WARN_ON(buf->trbe_limit != get_trbe_limit_pointer()); + + if (cpudata->mode != CS_MODE_PERF) + return false; + + if (cpudata->cpu != cpu) + return false; + + if (!cpumask_test_cpu(cpu, &drvdata->supported_cpus)) + return false; + + return true; +} + +static irqreturn_t arm_trbe_irq_handler(int irq, void *dev) +{ + struct perf_output_handle **handle_ptr = dev; + struct perf_output_handle *handle = *handle_ptr; + enum trbe_fault_action act; + u64 status; + bool truncated = false; + u64 trfcr; + + /* Reads to TRBSR_EL1 is fine when TRBE is active */ + status = read_sysreg_s(SYS_TRBSR_EL1); + /* + * If the pending IRQ was handled by update_buffer callback + * we have nothing to do here. + */ + if (!is_trbe_irq(status)) + return IRQ_NONE; + + /* Prohibit the CPU from tracing before we disable the TRBE */ + trfcr = cpu_prohibit_trace(); + /* + * Ensure the trace is visible to the CPUs and + * any external aborts have been resolved. + */ + trbe_drain_and_disable_local(); + clr_trbe_irq(); + isb(); + + if (WARN_ON_ONCE(!handle) || !perf_get_aux(handle)) + return IRQ_NONE; + + if (!is_perf_trbe(handle)) + return IRQ_NONE; + + act = trbe_get_fault_act(status); + switch (act) { + case TRBE_FAULT_ACT_WRAP: + truncated = !!trbe_handle_overflow(handle); + break; + case TRBE_FAULT_ACT_SPURIOUS: + trbe_handle_spurious(handle); + break; + case TRBE_FAULT_ACT_FATAL: + trbe_stop_and_truncate_event(handle); + truncated = true; + break; + } + + /* + * If the buffer was truncated, ensure perf callbacks + * have completed, which will disable the event. + * + * Otherwise, restore the trace filter controls to + * allow the tracing. + */ + if (truncated) + irq_work_run(); + else + write_trfcr(trfcr); + + return IRQ_HANDLED; +} + +static const struct coresight_ops_sink arm_trbe_sink_ops = { + .enable = arm_trbe_enable, + .disable = arm_trbe_disable, + .alloc_buffer = arm_trbe_alloc_buffer, + .free_buffer = arm_trbe_free_buffer, + .update_buffer = arm_trbe_update_buffer, +}; + +static const struct coresight_ops arm_trbe_cs_ops = { + .sink_ops = &arm_trbe_sink_ops, +}; + +static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct trbe_cpudata *cpudata = dev_get_drvdata(dev); + + return sprintf(buf, "%llx\n", cpudata->trbe_hw_align); +} +static DEVICE_ATTR_RO(align); + +static ssize_t flag_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct trbe_cpudata *cpudata = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", cpudata->trbe_flag); +} +static DEVICE_ATTR_RO(flag); + +static struct attribute *arm_trbe_attrs[] = { + &dev_attr_align.attr, + &dev_attr_flag.attr, + NULL, +}; + +static const struct attribute_group arm_trbe_group = { + .attrs = arm_trbe_attrs, +}; + +static const struct attribute_group *arm_trbe_groups[] = { + &arm_trbe_group, + NULL, +}; + +static void arm_trbe_enable_cpu(void *info) +{ + struct trbe_drvdata *drvdata = info; + + trbe_reset_local(); + enable_percpu_irq(drvdata->irq, IRQ_TYPE_NONE); +} + +static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cpu) +{ + struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu); + struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu); + struct coresight_desc desc = { 0 }; + struct device *dev; + + if (WARN_ON(trbe_csdev)) + return; + + /* If the TRBE was not probed on the CPU, we shouldn't be here */ + if (WARN_ON(!cpudata->drvdata)) + return; + + dev = &cpudata->drvdata->pdev->dev; + desc.name = devm_kasprintf(dev, GFP_KERNEL, "trbe%d", cpu); + if (IS_ERR(desc.name)) + goto cpu_clear; + + desc.type = CORESIGHT_DEV_TYPE_SINK; + desc.subtype.sink_subtype = CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM; + desc.ops = &arm_trbe_cs_ops; + desc.pdata = dev_get_platdata(dev); + desc.groups = arm_trbe_groups; + desc.dev = dev; + trbe_csdev = coresight_register(&desc); + if (IS_ERR(trbe_csdev)) + goto cpu_clear; + + dev_set_drvdata(&trbe_csdev->dev, cpudata); + coresight_set_percpu_sink(cpu, trbe_csdev); + return; +cpu_clear: + cpumask_clear_cpu(cpu, &drvdata->supported_cpus); +} + +static void arm_trbe_probe_cpu(void *info) +{ + struct trbe_drvdata *drvdata = info; + int cpu = smp_processor_id(); + struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu); + u64 trbidr; + + if (WARN_ON(!cpudata)) + goto cpu_clear; + + if (!is_trbe_available()) { + pr_err("TRBE is not implemented on cpu %d\n", cpu); + goto cpu_clear; + } + + trbidr = read_sysreg_s(SYS_TRBIDR_EL1); + if (!is_trbe_programmable(trbidr)) { + pr_err("TRBE is owned in higher exception level on cpu %d\n", cpu); + goto cpu_clear; + } + + cpudata->trbe_hw_align = 1ULL << get_trbe_address_align(trbidr); + if (cpudata->trbe_hw_align > SZ_2K) { + pr_err("Unsupported alignment on cpu %d\n", cpu); + goto cpu_clear; + } + + cpudata->trbe_align = cpudata->trbe_hw_align; + cpudata->trbe_flag = get_trbe_flag_update(trbidr); + cpudata->cpu = cpu; + cpudata->drvdata = drvdata; + return; +cpu_clear: + cpumask_clear_cpu(cpu, &drvdata->supported_cpus); +} + +static void arm_trbe_remove_coresight_cpu(void *info) +{ + int cpu = smp_processor_id(); + struct trbe_drvdata *drvdata = info; + struct trbe_cpudata *cpudata = per_cpu_ptr(drvdata->cpudata, cpu); + struct coresight_device *trbe_csdev = coresight_get_percpu_sink(cpu); + + disable_percpu_irq(drvdata->irq); + trbe_reset_local(); + if (trbe_csdev) { + coresight_unregister(trbe_csdev); + cpudata->drvdata = NULL; + coresight_set_percpu_sink(cpu, NULL); + } +} + +static int arm_trbe_probe_coresight(struct trbe_drvdata *drvdata) +{ + int cpu; + + drvdata->cpudata = alloc_percpu(typeof(*drvdata->cpudata)); + if (!drvdata->cpudata) + return -ENOMEM; + + for_each_cpu(cpu, &drvdata->supported_cpus) { + /* If we fail to probe the CPU, let us defer it to hotplug callbacks */ + if (smp_call_function_single(cpu, arm_trbe_probe_cpu, drvdata, 1)) + continue; + if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) + arm_trbe_register_coresight_cpu(drvdata, cpu); + if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) + smp_call_function_single(cpu, arm_trbe_enable_cpu, drvdata, 1); + } + return 0; +} + +static int arm_trbe_remove_coresight(struct trbe_drvdata *drvdata) +{ + int cpu; + + for_each_cpu(cpu, &drvdata->supported_cpus) + smp_call_function_single(cpu, arm_trbe_remove_coresight_cpu, drvdata, 1); + free_percpu(drvdata->cpudata); + return 0; +} + +static int arm_trbe_cpu_startup(unsigned int cpu, struct hlist_node *node) +{ + struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node); + + if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) { + + /* + * If this CPU was not probed for TRBE, + * initialize it now. + */ + if (!coresight_get_percpu_sink(cpu)) { + arm_trbe_probe_cpu(drvdata); + if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) + arm_trbe_register_coresight_cpu(drvdata, cpu); + if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) + arm_trbe_enable_cpu(drvdata); + } else { + arm_trbe_enable_cpu(drvdata); + } + } + return 0; +} + +static int arm_trbe_cpu_teardown(unsigned int cpu, struct hlist_node *node) +{ + struct trbe_drvdata *drvdata = hlist_entry_safe(node, struct trbe_drvdata, hotplug_node); + + if (cpumask_test_cpu(cpu, &drvdata->supported_cpus)) { + disable_percpu_irq(drvdata->irq); + trbe_reset_local(); + } + return 0; +} + +static int arm_trbe_probe_cpuhp(struct trbe_drvdata *drvdata) +{ + enum cpuhp_state trbe_online; + int ret; + + trbe_online = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DRVNAME, + arm_trbe_cpu_startup, arm_trbe_cpu_teardown); + if (trbe_online < 0) + return trbe_online; + + ret = cpuhp_state_add_instance(trbe_online, &drvdata->hotplug_node); + if (ret) { + cpuhp_remove_multi_state(trbe_online); + return ret; + } + drvdata->trbe_online = trbe_online; + return 0; +} + +static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata) +{ + cpuhp_remove_multi_state(drvdata->trbe_online); +} + +static int arm_trbe_probe_irq(struct platform_device *pdev, + struct trbe_drvdata *drvdata) +{ + int ret; + + drvdata->irq = platform_get_irq(pdev, 0); + if (drvdata->irq < 0) { + pr_err("IRQ not found for the platform device\n"); + return drvdata->irq; + } + + if (!irq_is_percpu(drvdata->irq)) { + pr_err("IRQ is not a PPI\n"); + return -EINVAL; + } + + if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus)) + return -EINVAL; + + drvdata->handle = alloc_percpu(struct perf_output_handle *); + if (!drvdata->handle) + return -ENOMEM; + + ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle); + if (ret) { + free_percpu(drvdata->handle); + return ret; + } + return 0; +} + +static void arm_trbe_remove_irq(struct trbe_drvdata *drvdata) +{ + free_percpu_irq(drvdata->irq, drvdata->handle); + free_percpu(drvdata->handle); +} + +static int arm_trbe_device_probe(struct platform_device *pdev) +{ + struct coresight_platform_data *pdata; + struct trbe_drvdata *drvdata; + struct device *dev = &pdev->dev; + int ret; + + drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); + if (!drvdata) + return -ENOMEM; + + pdata = coresight_get_platform_data(dev); + if (IS_ERR(pdata)) + return PTR_ERR(pdata); + + dev_set_drvdata(dev, drvdata); + dev->platform_data = pdata; + drvdata->pdev = pdev; + ret = arm_trbe_probe_irq(pdev, drvdata); + if (ret) + return ret; + + ret = arm_trbe_probe_coresight(drvdata); + if (ret) + goto probe_failed; + + ret = arm_trbe_probe_cpuhp(drvdata); + if (ret) + goto cpuhp_failed; + + return 0; +cpuhp_failed: + arm_trbe_remove_coresight(drvdata); +probe_failed: + arm_trbe_remove_irq(drvdata); + return ret; +} + +static int arm_trbe_device_remove(struct platform_device *pdev) +{ + struct trbe_drvdata *drvdata = platform_get_drvdata(pdev); + + arm_trbe_remove_cpuhp(drvdata); + arm_trbe_remove_coresight(drvdata); + arm_trbe_remove_irq(drvdata); + return 0; +} + +static const struct of_device_id arm_trbe_of_match[] = { + { .compatible = "arm,trace-buffer-extension"}, + {}, +}; +MODULE_DEVICE_TABLE(of, arm_trbe_of_match); + +static struct platform_driver arm_trbe_driver = { + .driver = { + .name = DRVNAME, + .of_match_table = of_match_ptr(arm_trbe_of_match), + .suppress_bind_attrs = true, + }, + .probe = arm_trbe_device_probe, + .remove = arm_trbe_device_remove, +}; + +static int __init arm_trbe_init(void) +{ + int ret; + + if (arm64_kernel_unmapped_at_el0()) { + pr_err("TRBE wouldn't work if kernel gets unmapped at EL0\n"); + return -EOPNOTSUPP; + } + + ret = platform_driver_register(&arm_trbe_driver); + if (!ret) + return 0; + + pr_err("Error registering %s platform driver\n", DRVNAME); + return ret; +} + +static void __exit arm_trbe_exit(void) +{ + platform_driver_unregister(&arm_trbe_driver); +} +module_init(arm_trbe_init); +module_exit(arm_trbe_exit); + +MODULE_AUTHOR("Anshuman Khandual "); +MODULE_DESCRIPTION("Arm Trace Buffer Extension (TRBE) driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/hwtracing/coresight/coresight-trbe.h b/drivers/hwtracing/coresight/coresight-trbe.h new file mode 100644 index 0000000000000000000000000000000000000000..abf3e36082f0d2355311435d783986b638525977 --- /dev/null +++ b/drivers/hwtracing/coresight/coresight-trbe.h @@ -0,0 +1,152 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This contains all required hardware related helper functions for + * Trace Buffer Extension (TRBE) driver in the coresight framework. + * + * Copyright (C) 2020 ARM Ltd. + * + * Author: Anshuman Khandual + */ +#include +#include +#include +#include +#include +#include +#include + +#include "coresight-etm-perf.h" + +static inline bool is_trbe_available(void) +{ + u64 aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1); + unsigned int trbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_TRBE_SHIFT); + + return trbe >= 0b0001; +} + +static inline bool is_trbe_enabled(void) +{ + u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); + + return trblimitr & TRBLIMITR_ENABLE; +} + +#define TRBE_EC_OTHERS 0 +#define TRBE_EC_STAGE1_ABORT 36 +#define TRBE_EC_STAGE2_ABORT 37 + +static inline int get_trbe_ec(u64 trbsr) +{ + return (trbsr >> TRBSR_EC_SHIFT) & TRBSR_EC_MASK; +} + +#define TRBE_BSC_NOT_STOPPED 0 +#define TRBE_BSC_FILLED 1 +#define TRBE_BSC_TRIGGERED 2 + +static inline int get_trbe_bsc(u64 trbsr) +{ + return (trbsr >> TRBSR_BSC_SHIFT) & TRBSR_BSC_MASK; +} + +static inline void clr_trbe_irq(void) +{ + u64 trbsr = read_sysreg_s(SYS_TRBSR_EL1); + + trbsr &= ~TRBSR_IRQ; + write_sysreg_s(trbsr, SYS_TRBSR_EL1); +} + +static inline bool is_trbe_irq(u64 trbsr) +{ + return trbsr & TRBSR_IRQ; +} + +static inline bool is_trbe_trg(u64 trbsr) +{ + return trbsr & TRBSR_TRG; +} + +static inline bool is_trbe_wrap(u64 trbsr) +{ + return trbsr & TRBSR_WRAP; +} + +static inline bool is_trbe_abort(u64 trbsr) +{ + return trbsr & TRBSR_ABORT; +} + +static inline bool is_trbe_running(u64 trbsr) +{ + return !(trbsr & TRBSR_STOP); +} + +#define TRBE_TRIG_MODE_STOP 0 +#define TRBE_TRIG_MODE_IRQ 1 +#define TRBE_TRIG_MODE_IGNORE 3 + +#define TRBE_FILL_MODE_FILL 0 +#define TRBE_FILL_MODE_WRAP 1 +#define TRBE_FILL_MODE_CIRCULAR_BUFFER 3 + +static inline void set_trbe_disabled(void) +{ + u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); + + trblimitr &= ~TRBLIMITR_ENABLE; + write_sysreg_s(trblimitr, SYS_TRBLIMITR_EL1); +} + +static inline bool get_trbe_flag_update(u64 trbidr) +{ + return trbidr & TRBIDR_FLAG; +} + +static inline bool is_trbe_programmable(u64 trbidr) +{ + return !(trbidr & TRBIDR_PROG); +} + +static inline int get_trbe_address_align(u64 trbidr) +{ + return (trbidr >> TRBIDR_ALIGN_SHIFT) & TRBIDR_ALIGN_MASK; +} + +static inline unsigned long get_trbe_write_pointer(void) +{ + return read_sysreg_s(SYS_TRBPTR_EL1); +} + +static inline void set_trbe_write_pointer(unsigned long addr) +{ + WARN_ON(is_trbe_enabled()); + write_sysreg_s(addr, SYS_TRBPTR_EL1); +} + +static inline unsigned long get_trbe_limit_pointer(void) +{ + u64 trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1); + unsigned long addr = trblimitr & (TRBLIMITR_LIMIT_MASK << TRBLIMITR_LIMIT_SHIFT); + + WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE)); + return addr; +} + +static inline unsigned long get_trbe_base_pointer(void) +{ + u64 trbbaser = read_sysreg_s(SYS_TRBBASER_EL1); + unsigned long addr = trbbaser & (TRBBASER_BASE_MASK << TRBBASER_BASE_SHIFT); + + WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE)); + return addr; +} + +static inline void set_trbe_base_pointer(unsigned long addr) +{ + WARN_ON(is_trbe_enabled()); + WARN_ON(!IS_ALIGNED(addr, (1UL << TRBBASER_BASE_SHIFT))); + WARN_ON(!IS_ALIGNED(addr, PAGE_SIZE)); + write_sysreg_s(addr, SYS_TRBBASER_EL1); +} diff --git a/include/linux/coresight-pmu.h b/include/linux/coresight-pmu.h index b0e35eec6499bf4367da462c1f180c77821336e2..0c2cee05c4e3046badf95ad3bcdc2ac6b425822f 100644 --- a/include/linux/coresight-pmu.h +++ b/include/linux/coresight-pmu.h @@ -8,7 +8,8 @@ #define _LINUX_CORESIGHT_PMU_H #define CORESIGHT_ETM_PMU_NAME "cs_etm" -#define CORESIGHT_ETM_PMU_SEED 0x10 +#define CORESIGHT_ETM_PMU_SEED 0x1 +#define CORESIGHT_ETM_CSID_MAX 0x70 /* ETMv3.5/PTM's ETMCR config bit */ #define ETM_OPT_CYCACC 12 @@ -30,7 +31,7 @@ static inline int coresight_get_trace_id(int cpu) * the common convention is to have data trace IDs be I(N) + 1, * set instruction trace IDs as a function of the CPU number. */ - return (CORESIGHT_ETM_PMU_SEED + (cpu * 2)); + return (CORESIGHT_ETM_PMU_SEED + (cpu * 2)) % CORESIGHT_ETM_CSID_MAX; } #endif diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 7d3c87e5b97c3c940a31b6d9b722fbed8892d830..85008a65e21f6f5dc3afaeb1fc7d45f710bc517c 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -7,6 +7,7 @@ #define _LINUX_CORESIGHT_H #include +#include #include #include @@ -49,6 +50,7 @@ enum coresight_dev_subtype_sink { CORESIGHT_DEV_SUBTYPE_SINK_PORT, CORESIGHT_DEV_SUBTYPE_SINK_BUFFER, CORESIGHT_DEV_SUBTYPE_SINK_SYSMEM, + CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM, }; enum coresight_dev_subtype_link { @@ -114,6 +116,32 @@ struct coresight_platform_data { struct coresight_connection *conns; }; +/** + * struct csdev_access - Abstraction of a CoreSight device access. + * + * @io_mem : True if the device has memory mapped I/O + * @base : When io_mem == true, base address of the component + * @read : Read from the given "offset" of the given instance. + * @write : Write "val" to the given "offset". + */ +struct csdev_access { + bool io_mem; + union { + void __iomem *base; + struct { + u64 (*read)(u32 offset, bool relaxed, bool _64bit); + void (*write)(u64 val, u32 offset, bool relaxed, + bool _64bit); + }; + }; +}; + +#define CSDEV_ACCESS_IOMEM(_addr) \ + ((struct csdev_access) { \ + .io_mem = true, \ + .base = (_addr), \ + }) + /** * struct coresight_desc - description of a component required from drivers * @type: as defined by @coresight_dev_type. @@ -125,6 +153,7 @@ struct coresight_platform_data { * @groups: operations specific to this component. These will end up * in the component's sysfs sub-directory. * @name: name for the coresight device, also shown under sysfs. + * @access: Describe access to the device */ struct coresight_desc { enum coresight_dev_type type; @@ -134,6 +163,7 @@ struct coresight_desc { struct device *dev; const struct attribute_group **groups; const char *name; + struct csdev_access access; }; /** @@ -173,7 +203,8 @@ struct coresight_sysfs_link { * @type: as defined by @coresight_dev_type. * @subtype: as defined by @coresight_dev_subtype. * @ops: generic operations for this component, as defined - by @coresight_ops. + * by @coresight_ops. + * @access: Device i/o access abstraction for this device. * @dev: The device entity associated to this component. * @refcnt: keep track of what is in use. * @orphan: true if the component has connections that haven't been linked. @@ -195,6 +226,7 @@ struct coresight_device { enum coresight_dev_type type; union coresight_dev_subtype subtype; const struct coresight_ops *ops; + struct csdev_access access; struct device dev; atomic_t *refcnt; bool orphan; @@ -326,23 +358,145 @@ struct coresight_ops { }; #if IS_ENABLED(CONFIG_CORESIGHT) + +static inline u32 csdev_access_relaxed_read32(struct csdev_access *csa, + u32 offset) +{ + if (likely(csa->io_mem)) + return readl_relaxed(csa->base + offset); + + return csa->read(offset, true, false); +} + +static inline u32 csdev_access_read32(struct csdev_access *csa, u32 offset) +{ + if (likely(csa->io_mem)) + return readl(csa->base + offset); + + return csa->read(offset, false, false); +} + +static inline void csdev_access_relaxed_write32(struct csdev_access *csa, + u32 val, u32 offset) +{ + if (likely(csa->io_mem)) + writel_relaxed(val, csa->base + offset); + else + csa->write(val, offset, true, false); +} + +static inline void csdev_access_write32(struct csdev_access *csa, u32 val, u32 offset) +{ + if (likely(csa->io_mem)) + writel(val, csa->base + offset); + else + csa->write(val, offset, false, false); +} + +#ifdef CONFIG_64BIT + +static inline u64 csdev_access_relaxed_read64(struct csdev_access *csa, + u32 offset) +{ + if (likely(csa->io_mem)) + return readq_relaxed(csa->base + offset); + + return csa->read(offset, true, true); +} + +static inline u64 csdev_access_read64(struct csdev_access *csa, u32 offset) +{ + if (likely(csa->io_mem)) + return readq(csa->base + offset); + + return csa->read(offset, false, true); +} + +static inline void csdev_access_relaxed_write64(struct csdev_access *csa, + u64 val, u32 offset) +{ + if (likely(csa->io_mem)) + writeq_relaxed(val, csa->base + offset); + else + csa->write(val, offset, true, true); +} + +static inline void csdev_access_write64(struct csdev_access *csa, u64 val, u32 offset) +{ + if (likely(csa->io_mem)) + writeq(val, csa->base + offset); + else + csa->write(val, offset, false, true); +} + +#else /* !CONFIG_64BIT */ + +static inline u64 csdev_access_relaxed_read64(struct csdev_access *csa, + u32 offset) +{ + WARN_ON(1); + return 0; +} + +static inline u64 csdev_access_read64(struct csdev_access *csa, u32 offset) +{ + WARN_ON(1); + return 0; +} + +static inline void csdev_access_relaxed_write64(struct csdev_access *csa, + u64 val, u32 offset) +{ + WARN_ON(1); +} + +static inline void csdev_access_write64(struct csdev_access *csa, u64 val, u32 offset) +{ + WARN_ON(1); +} +#endif /* CONFIG_64BIT */ + +static inline bool coresight_is_percpu_source(struct coresight_device *csdev) +{ + return csdev && (csdev->type == CORESIGHT_DEV_TYPE_SOURCE) && + (csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_PROC); +} + +static inline bool coresight_is_percpu_sink(struct coresight_device *csdev) +{ + return csdev && (csdev->type == CORESIGHT_DEV_TYPE_SINK) && + (csdev->subtype.sink_subtype == CORESIGHT_DEV_SUBTYPE_SINK_PERCPU_SYSMEM); +} + extern struct coresight_device * coresight_register(struct coresight_desc *desc); extern void coresight_unregister(struct coresight_device *csdev); extern int coresight_enable(struct coresight_device *csdev); extern void coresight_disable(struct coresight_device *csdev); -extern int coresight_timeout(void __iomem *addr, u32 offset, +extern int coresight_timeout(struct csdev_access *csa, u32 offset, int position, int value); -extern int coresight_claim_device(void __iomem *base); -extern int coresight_claim_device_unlocked(void __iomem *base); +extern int coresight_claim_device(struct coresight_device *csdev); +extern int coresight_claim_device_unlocked(struct coresight_device *csdev); -extern void coresight_disclaim_device(void __iomem *base); -extern void coresight_disclaim_device_unlocked(void __iomem *base); +extern void coresight_disclaim_device(struct coresight_device *csdev); +extern void coresight_disclaim_device_unlocked(struct coresight_device *csdev); extern char *coresight_alloc_device_name(struct coresight_dev_list *devs, struct device *dev); extern bool coresight_loses_context_with_cpu(struct device *dev); + +u32 coresight_relaxed_read32(struct coresight_device *csdev, u32 offset); +u32 coresight_read32(struct coresight_device *csdev, u32 offset); +void coresight_write32(struct coresight_device *csdev, u32 val, u32 offset); +void coresight_relaxed_write32(struct coresight_device *csdev, + u32 val, u32 offset); +u64 coresight_relaxed_read64(struct coresight_device *csdev, u32 offset); +u64 coresight_read64(struct coresight_device *csdev, u32 offset); +void coresight_relaxed_write64(struct coresight_device *csdev, + u64 val, u32 offset); +void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset); + #else static inline struct coresight_device * coresight_register(struct coresight_desc *desc) { return NULL; } @@ -350,29 +504,78 @@ static inline void coresight_unregister(struct coresight_device *csdev) {} static inline int coresight_enable(struct coresight_device *csdev) { return -ENOSYS; } static inline void coresight_disable(struct coresight_device *csdev) {} -static inline int coresight_timeout(void __iomem *addr, u32 offset, - int position, int value) { return 1; } -static inline int coresight_claim_device_unlocked(void __iomem *base) + +static inline int coresight_timeout(struct csdev_access *csa, u32 offset, + int position, int value) +{ + return 1; +} + +static inline int coresight_claim_device_unlocked(struct coresight_device *csdev) { return -EINVAL; } -static inline int coresight_claim_device(void __iomem *base) +static inline int coresight_claim_device(struct coresight_device *csdev) { return -EINVAL; } -static inline void coresight_disclaim_device(void __iomem *base) {} -static inline void coresight_disclaim_device_unlocked(void __iomem *base) {} +static inline void coresight_disclaim_device(struct coresight_device *csdev) {} +static inline void coresight_disclaim_device_unlocked(struct coresight_device *csdev) {} static inline bool coresight_loses_context_with_cpu(struct device *dev) { return false; } -#endif + +static inline u32 coresight_relaxed_read32(struct coresight_device *csdev, u32 offset) +{ + WARN_ON_ONCE(1); + return 0; +} + +static inline u32 coresight_read32(struct coresight_device *csdev, u32 offset) +{ + WARN_ON_ONCE(1); + return 0; +} + +static inline void coresight_write32(struct coresight_device *csdev, u32 val, u32 offset) +{ +} + +static inline void coresight_relaxed_write32(struct coresight_device *csdev, + u32 val, u32 offset) +{ +} + +static inline u64 coresight_relaxed_read64(struct coresight_device *csdev, + u32 offset) +{ + WARN_ON_ONCE(1); + return 0; +} + +static inline u64 coresight_read64(struct coresight_device *csdev, u32 offset) +{ + WARN_ON_ONCE(1); + return 0; +} + +static inline void coresight_relaxed_write64(struct coresight_device *csdev, + u64 val, u32 offset) +{ +} + +static inline void coresight_write64(struct coresight_device *csdev, u64 val, u32 offset) +{ +} + +#endif /* IS_ENABLED(CONFIG_CORESIGHT) */ extern int coresight_get_cpu(struct device *dev); struct coresight_platform_data *coresight_get_platform_data(struct device *dev); -#endif +#endif /* _LINUX_COREISGHT_H */ diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h index b95d3c485d27e10f5a1b04354a1bb4ba7cc41146..33bdc13b81cb8bf2ae93298bb8f0d26fe39a064c 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h @@ -1101,10 +1101,15 @@ enum perf_callchain_context { /** * PERF_RECORD_AUX::flags bits */ -#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */ -#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */ -#define PERF_AUX_FLAG_PARTIAL 0x04 /* record contains gaps */ -#define PERF_AUX_FLAG_COLLISION 0x08 /* sample collided with another */ +#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */ +#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */ +#define PERF_AUX_FLAG_PARTIAL 0x04 /* record contains gaps */ +#define PERF_AUX_FLAG_COLLISION 0x08 /* sample collided with another */ +#define PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK 0xff00 /* PMU specific trace format type */ + +/* CoreSight PMU AUX buffer formats */ +#define PERF_AUX_FLAG_CORESIGHT_FORMAT_CORESIGHT 0x0000 /* Default for backward compatibility */ +#define PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW 0x0100 /* Raw format of the source */ #define PERF_FLAG_FD_NO_GROUP (1UL << 0) #define PERF_FLAG_FD_OUTPUT (1UL << 1) diff --git a/tools/build/feature/test-libopencsd.c b/tools/build/feature/test-libopencsd.c index 1547bc2c09505ff80d863e0e2909eb77da26b9f4..52c790b0317b1ee6506c2bfc5b3bb86745213539 100644 --- a/tools/build/feature/test-libopencsd.c +++ b/tools/build/feature/test-libopencsd.c @@ -4,9 +4,9 @@ /* * Check OpenCSD library version is sufficient to provide required features */ -#define OCSD_MIN_VER ((0 << 16) | (14 << 8) | (0)) +#define OCSD_MIN_VER ((1 << 16) | (0 << 8) | (0)) #if !defined(OCSD_VER_NUM) || (OCSD_VER_NUM < OCSD_MIN_VER) -#error "OpenCSD >= 0.14.0 is required" +#error "OpenCSD >= 1.0.0 is required" #endif int main(void) diff --git a/tools/include/linux/coresight-pmu.h b/tools/include/linux/coresight-pmu.h index b0e35eec6499bf4367da462c1f180c77821336e2..0c2cee05c4e3046badf95ad3bcdc2ac6b425822f 100644 --- a/tools/include/linux/coresight-pmu.h +++ b/tools/include/linux/coresight-pmu.h @@ -8,7 +8,8 @@ #define _LINUX_CORESIGHT_PMU_H #define CORESIGHT_ETM_PMU_NAME "cs_etm" -#define CORESIGHT_ETM_PMU_SEED 0x10 +#define CORESIGHT_ETM_PMU_SEED 0x1 +#define CORESIGHT_ETM_CSID_MAX 0x70 /* ETMv3.5/PTM's ETMCR config bit */ #define ETM_OPT_CYCACC 12 @@ -30,7 +31,7 @@ static inline int coresight_get_trace_id(int cpu) * the common convention is to have data trace IDs be I(N) + 1, * set instruction trace IDs as a function of the CPU number. */ - return (CORESIGHT_ETM_PMU_SEED + (cpu * 2)); + return (CORESIGHT_ETM_PMU_SEED + (cpu * 2)) % CORESIGHT_ETM_CSID_MAX; } #endif diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c index cd007cc9c283a638b06008a656951a9a6b84e76d..3f4bc40504772acc94ac6cfcc38a000f42854bbe 100644 --- a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c @@ -419,19 +419,10 @@ cs_etm_decoder__buffer_range(struct cs_etm_queue *etmq, packet->last_instr_subtype = elem->last_i_subtype; packet->last_instr_cond = elem->last_instr_cond; - switch (elem->last_i_type) { - case OCSD_INSTR_BR: - case OCSD_INSTR_BR_INDIRECT: + if (elem->last_i_type == OCSD_INSTR_BR || elem->last_i_type == OCSD_INSTR_BR_INDIRECT) packet->last_instr_taken_branch = elem->last_instr_exec; - break; - case OCSD_INSTR_ISB: - case OCSD_INSTR_DSB_DMB: - case OCSD_INSTR_WFI_WFE: - case OCSD_INSTR_OTHER: - default: + else packet->last_instr_taken_branch = false; - break; - } packet->last_instr_size = elem->last_instr_sz; @@ -572,6 +563,8 @@ static ocsd_datapath_resp_t cs_etm_decoder__gen_trace_elem_printer( case OCSD_GEN_TRC_ELEM_EVENT: case OCSD_GEN_TRC_ELEM_SWTRACE: case OCSD_GEN_TRC_ELEM_CUSTOM: + case OCSD_GEN_TRC_ELEM_SYNC_MARKER: + case OCSD_GEN_TRC_ELEM_MEMTRANS: default: break; }