提交 2ff2b289 编写于 作者: L Linus Torvalds

Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull perf changes from Ingo Molnar:
 "Lots of changes:

   - (much) improved assembly annotation support in perf report, with
     jump visualization, searching, navigation, visual output
     improvements and more.

    - kernel support for AMD IBS PMU hardware features.  Notably 'perf
      record -e cycles:p' and 'perf top -e cycles:p' should work without
      skid now, like PEBS does on the Intel side, because it takes
      advantage of IBS transparently.

    - the libtracevents library: it is the first step towards unifying
      tracing tooling and perf, and it also gives a tracing library for
      external tools like powertop to rely on.

    - infrastructure: various improvements and refactoring of the UI
      modules and related code

    - infrastructure: cleanup and simplification of the profiling
      targets code (--uid, --pid, --tid, --cpu, --all-cpus, etc.)

    - tons of robustness fixes all around

    - various ftrace updates: speedups, cleanups, robustness
      improvements.

    - typing 'make' in tools/ will now give you a menu of projects to
      build and a short help text to explain what each does.

    - ... and lots of other changes I forgot to list.

  The perf record make bzImage + perf report regression you reported
  should be fixed."

* 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (166 commits)
  tracing: Remove kernel_lock annotations
  tracing: Fix initial buffer_size_kb state
  ring-buffer: Merge separate resize loops
  perf evsel: Create events initially disabled -- again
  perf tools: Split term type into value type and term type
  perf hists: Fix callchain ip printf format
  perf target: Add uses_mmap field
  ftrace: Remove selecting FRAME_POINTER with FUNCTION_TRACER
  ftrace/x86: Have x86 ftrace use the ftrace_modify_all_code()
  ftrace: Make ftrace_modify_all_code() global for archs to use
  ftrace: Return record ip addr for ftrace_location()
  ftrace: Consolidate ftrace_location() and ftrace_text_reserved()
  ftrace: Speed up search by skipping pages by address
  ftrace: Remove extra helper functions
  ftrace: Sort all function addresses, not just per page
  tracing: change CPU ring buffer state from tracing_cpumask
  tracing: Check return value of tracing_dentry_percpu()
  ring-buffer: Reset head page before running self test
  ring-buffer: Add integrity check at end of iter read
  ring-buffer: Make addition of pages in ring buffer atomic
  ...
...@@ -1471,6 +1471,13 @@ kernelrelease: ...@@ -1471,6 +1471,13 @@ kernelrelease:
kernelversion: kernelversion:
@echo $(KERNELVERSION) @echo $(KERNELVERSION)
# Clear a bunch of variables before executing the submake
tools/: FORCE
$(Q)$(MAKE) LDFLAGS= MAKEFLAGS= -C $(src)/tools/
tools/%: FORCE
$(Q)$(MAKE) LDFLAGS= MAKEFLAGS= -C $(src)/tools/ $*
# Single targets # Single targets
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Single targets are compatible with: # Single targets are compatible with:
......
...@@ -824,7 +824,6 @@ static void alpha_perf_event_irq_handler(unsigned long la_ptr, ...@@ -824,7 +824,6 @@ static void alpha_perf_event_irq_handler(unsigned long la_ptr,
idx = la_ptr; idx = la_ptr;
perf_sample_data_init(&data, 0);
for (j = 0; j < cpuc->n_events; j++) { for (j = 0; j < cpuc->n_events; j++) {
if (cpuc->current_idx[j] == idx) if (cpuc->current_idx[j] == idx)
break; break;
...@@ -848,7 +847,7 @@ static void alpha_perf_event_irq_handler(unsigned long la_ptr, ...@@ -848,7 +847,7 @@ static void alpha_perf_event_irq_handler(unsigned long la_ptr,
hwc = &event->hw; hwc = &event->hw;
alpha_perf_event_update(event, hwc, idx, alpha_pmu->pmc_max_period[idx]+1); alpha_perf_event_update(event, hwc, idx, alpha_pmu->pmc_max_period[idx]+1);
data.period = event->hw.last_period; perf_sample_data_init(&data, 0, hwc->last_period);
if (alpha_perf_event_set_period(event, hwc, idx)) { if (alpha_perf_event_set_period(event, hwc, idx)) {
if (perf_event_overflow(event, &data, regs)) { if (perf_event_overflow(event, &data, regs)) {
......
...@@ -11,7 +11,7 @@ CONFIG_KALLSYMS_EXTRA_PASS=y ...@@ -11,7 +11,7 @@ CONFIG_KALLSYMS_EXTRA_PASS=y
# CONFIG_TIMERFD is not set # CONFIG_TIMERFD is not set
# CONFIG_EVENTFD is not set # CONFIG_EVENTFD is not set
# CONFIG_AIO is not set # CONFIG_AIO is not set
CONFIG_PERF_COUNTERS=y CONFIG_PERF_EVENTS=y
# CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_VM_EVENT_COUNTERS is not set
# CONFIG_SLUB_DEBUG is not set # CONFIG_SLUB_DEBUG is not set
# CONFIG_COMPAT_BRK is not set # CONFIG_COMPAT_BRK is not set
......
...@@ -489,8 +489,6 @@ armv6pmu_handle_irq(int irq_num, ...@@ -489,8 +489,6 @@ armv6pmu_handle_irq(int irq_num,
*/ */
armv6_pmcr_write(pmcr); armv6_pmcr_write(pmcr);
perf_sample_data_init(&data, 0);
cpuc = &__get_cpu_var(cpu_hw_events); cpuc = &__get_cpu_var(cpu_hw_events);
for (idx = 0; idx < cpu_pmu->num_events; ++idx) { for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
struct perf_event *event = cpuc->events[idx]; struct perf_event *event = cpuc->events[idx];
...@@ -509,7 +507,7 @@ armv6pmu_handle_irq(int irq_num, ...@@ -509,7 +507,7 @@ armv6pmu_handle_irq(int irq_num,
hwc = &event->hw; hwc = &event->hw;
armpmu_event_update(event, hwc, idx); armpmu_event_update(event, hwc, idx);
data.period = event->hw.last_period; perf_sample_data_init(&data, 0, hwc->last_period);
if (!armpmu_event_set_period(event, hwc, idx)) if (!armpmu_event_set_period(event, hwc, idx))
continue; continue;
......
...@@ -1077,8 +1077,6 @@ static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev) ...@@ -1077,8 +1077,6 @@ static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
*/ */
regs = get_irq_regs(); regs = get_irq_regs();
perf_sample_data_init(&data, 0);
cpuc = &__get_cpu_var(cpu_hw_events); cpuc = &__get_cpu_var(cpu_hw_events);
for (idx = 0; idx < cpu_pmu->num_events; ++idx) { for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
struct perf_event *event = cpuc->events[idx]; struct perf_event *event = cpuc->events[idx];
...@@ -1097,7 +1095,7 @@ static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev) ...@@ -1097,7 +1095,7 @@ static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
hwc = &event->hw; hwc = &event->hw;
armpmu_event_update(event, hwc, idx); armpmu_event_update(event, hwc, idx);
data.period = event->hw.last_period; perf_sample_data_init(&data, 0, hwc->last_period);
if (!armpmu_event_set_period(event, hwc, idx)) if (!armpmu_event_set_period(event, hwc, idx))
continue; continue;
......
...@@ -248,8 +248,6 @@ xscale1pmu_handle_irq(int irq_num, void *dev) ...@@ -248,8 +248,6 @@ xscale1pmu_handle_irq(int irq_num, void *dev)
regs = get_irq_regs(); regs = get_irq_regs();
perf_sample_data_init(&data, 0);
cpuc = &__get_cpu_var(cpu_hw_events); cpuc = &__get_cpu_var(cpu_hw_events);
for (idx = 0; idx < cpu_pmu->num_events; ++idx) { for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
struct perf_event *event = cpuc->events[idx]; struct perf_event *event = cpuc->events[idx];
...@@ -263,7 +261,7 @@ xscale1pmu_handle_irq(int irq_num, void *dev) ...@@ -263,7 +261,7 @@ xscale1pmu_handle_irq(int irq_num, void *dev)
hwc = &event->hw; hwc = &event->hw;
armpmu_event_update(event, hwc, idx); armpmu_event_update(event, hwc, idx);
data.period = event->hw.last_period; perf_sample_data_init(&data, 0, hwc->last_period);
if (!armpmu_event_set_period(event, hwc, idx)) if (!armpmu_event_set_period(event, hwc, idx))
continue; continue;
...@@ -588,8 +586,6 @@ xscale2pmu_handle_irq(int irq_num, void *dev) ...@@ -588,8 +586,6 @@ xscale2pmu_handle_irq(int irq_num, void *dev)
regs = get_irq_regs(); regs = get_irq_regs();
perf_sample_data_init(&data, 0);
cpuc = &__get_cpu_var(cpu_hw_events); cpuc = &__get_cpu_var(cpu_hw_events);
for (idx = 0; idx < cpu_pmu->num_events; ++idx) { for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
struct perf_event *event = cpuc->events[idx]; struct perf_event *event = cpuc->events[idx];
...@@ -603,7 +599,7 @@ xscale2pmu_handle_irq(int irq_num, void *dev) ...@@ -603,7 +599,7 @@ xscale2pmu_handle_irq(int irq_num, void *dev)
hwc = &event->hw; hwc = &event->hw;
armpmu_event_update(event, hwc, idx); armpmu_event_update(event, hwc, idx);
data.period = event->hw.last_period; perf_sample_data_init(&data, 0, hwc->last_period);
if (!armpmu_event_set_period(event, hwc, idx)) if (!armpmu_event_set_period(event, hwc, idx))
continue; continue;
......
...@@ -1325,7 +1325,7 @@ static int mipsxx_pmu_handle_shared_irq(void) ...@@ -1325,7 +1325,7 @@ static int mipsxx_pmu_handle_shared_irq(void)
regs = get_irq_regs(); regs = get_irq_regs();
perf_sample_data_init(&data, 0); perf_sample_data_init(&data, 0, 0);
switch (counters) { switch (counters) {
#define HANDLE_COUNTER(n) \ #define HANDLE_COUNTER(n) \
......
...@@ -32,7 +32,7 @@ CONFIG_RD_LZMA=y ...@@ -32,7 +32,7 @@ CONFIG_RD_LZMA=y
CONFIG_INITRAMFS_COMPRESSION_GZIP=y CONFIG_INITRAMFS_COMPRESSION_GZIP=y
CONFIG_KALLSYMS_ALL=y CONFIG_KALLSYMS_ALL=y
CONFIG_EMBEDDED=y CONFIG_EMBEDDED=y
CONFIG_PERF_COUNTERS=y CONFIG_PERF_EVENTS=y
CONFIG_PROFILING=y CONFIG_PROFILING=y
CONFIG_OPROFILE=y CONFIG_OPROFILE=y
CONFIG_KPROBES=y CONFIG_KPROBES=y
......
...@@ -8,7 +8,7 @@ CONFIG_BLK_DEV_INITRD=y ...@@ -8,7 +8,7 @@ CONFIG_BLK_DEV_INITRD=y
# CONFIG_CC_OPTIMIZE_FOR_SIZE is not set # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set
CONFIG_EXPERT=y CONFIG_EXPERT=y
# CONFIG_ELF_CORE is not set # CONFIG_ELF_CORE is not set
CONFIG_PERF_COUNTERS=y CONFIG_PERF_EVENTS=y
# CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_VM_EVENT_COUNTERS is not set
CONFIG_SLAB=y CONFIG_SLAB=y
CONFIG_MODULES=y CONFIG_MODULES=y
......
...@@ -9,7 +9,7 @@ CONFIG_BLK_DEV_INITRD=y ...@@ -9,7 +9,7 @@ CONFIG_BLK_DEV_INITRD=y
# CONFIG_CC_OPTIMIZE_FOR_SIZE is not set # CONFIG_CC_OPTIMIZE_FOR_SIZE is not set
CONFIG_EXPERT=y CONFIG_EXPERT=y
# CONFIG_ELF_CORE is not set # CONFIG_ELF_CORE is not set
CONFIG_PERF_COUNTERS=y CONFIG_PERF_EVENTS=y
# CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_VM_EVENT_COUNTERS is not set
CONFIG_SLAB=y CONFIG_SLAB=y
CONFIG_MODULES=y CONFIG_MODULES=y
......
...@@ -1299,8 +1299,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, ...@@ -1299,8 +1299,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val,
if (record) { if (record) {
struct perf_sample_data data; struct perf_sample_data data;
perf_sample_data_init(&data, ~0ULL); perf_sample_data_init(&data, ~0ULL, event->hw.last_period);
data.period = event->hw.last_period;
if (event->attr.sample_type & PERF_SAMPLE_ADDR) if (event->attr.sample_type & PERF_SAMPLE_ADDR)
perf_get_data_addr(regs, &data.addr); perf_get_data_addr(regs, &data.addr);
......
...@@ -613,8 +613,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, ...@@ -613,8 +613,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val,
if (record) { if (record) {
struct perf_sample_data data; struct perf_sample_data data;
perf_sample_data_init(&data, 0); perf_sample_data_init(&data, 0, event->hw.last_period);
data.period = event->hw.last_period;
if (perf_event_overflow(event, &data, regs)) if (perf_event_overflow(event, &data, regs))
fsl_emb_pmu_stop(event, 0); fsl_emb_pmu_stop(event, 0);
......
...@@ -5,7 +5,7 @@ CONFIG_BSD_PROCESS_ACCT=y ...@@ -5,7 +5,7 @@ CONFIG_BSD_PROCESS_ACCT=y
CONFIG_IKCONFIG=y CONFIG_IKCONFIG=y
CONFIG_IKCONFIG_PROC=y CONFIG_IKCONFIG_PROC=y
CONFIG_LOG_BUF_SHIFT=16 CONFIG_LOG_BUF_SHIFT=16
CONFIG_PERF_COUNTERS=y CONFIG_PERF_EVENTS=y
# CONFIG_COMPAT_BRK is not set # CONFIG_COMPAT_BRK is not set
CONFIG_SLAB=y CONFIG_SLAB=y
CONFIG_PROFILING=y CONFIG_PROFILING=y
......
...@@ -5,7 +5,7 @@ CONFIG_SYSVIPC=y ...@@ -5,7 +5,7 @@ CONFIG_SYSVIPC=y
CONFIG_POSIX_MQUEUE=y CONFIG_POSIX_MQUEUE=y
CONFIG_LOG_BUF_SHIFT=18 CONFIG_LOG_BUF_SHIFT=18
CONFIG_BLK_DEV_INITRD=y CONFIG_BLK_DEV_INITRD=y
CONFIG_PERF_COUNTERS=y CONFIG_PERF_EVENTS=y
# CONFIG_COMPAT_BRK is not set # CONFIG_COMPAT_BRK is not set
CONFIG_SLAB=y CONFIG_SLAB=y
CONFIG_PROFILING=y CONFIG_PROFILING=y
......
...@@ -1296,8 +1296,6 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, ...@@ -1296,8 +1296,6 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
regs = args->regs; regs = args->regs;
perf_sample_data_init(&data, 0);
cpuc = &__get_cpu_var(cpu_hw_events); cpuc = &__get_cpu_var(cpu_hw_events);
/* If the PMU has the TOE IRQ enable bits, we need to do a /* If the PMU has the TOE IRQ enable bits, we need to do a
...@@ -1321,7 +1319,7 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, ...@@ -1321,7 +1319,7 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
if (val & (1ULL << 31)) if (val & (1ULL << 31))
continue; continue;
data.period = event->hw.last_period; perf_sample_data_init(&data, 0, hwc->last_period);
if (!sparc_perf_event_set_period(event, hwc, idx)) if (!sparc_perf_event_set_period(event, hwc, idx))
continue; continue;
......
...@@ -40,7 +40,6 @@ config X86 ...@@ -40,7 +40,6 @@ config X86
select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_GRAPH_TRACER
select HAVE_FUNCTION_GRAPH_FP_TEST select HAVE_FUNCTION_GRAPH_FP_TEST
select HAVE_FUNCTION_TRACE_MCOUNT_TEST select HAVE_FUNCTION_TRACE_MCOUNT_TEST
select HAVE_FTRACE_NMI_ENTER if DYNAMIC_FTRACE
select HAVE_SYSCALL_TRACEPOINTS select HAVE_SYSCALL_TRACEPOINTS
select HAVE_KVM select HAVE_KVM
select HAVE_ARCH_KGDB select HAVE_ARCH_KGDB
......
...@@ -34,6 +34,7 @@ ...@@ -34,6 +34,7 @@
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
extern void mcount(void); extern void mcount(void);
extern int modifying_ftrace_code;
static inline unsigned long ftrace_call_adjust(unsigned long addr) static inline unsigned long ftrace_call_adjust(unsigned long addr)
{ {
...@@ -50,6 +51,8 @@ struct dyn_arch_ftrace { ...@@ -50,6 +51,8 @@ struct dyn_arch_ftrace {
/* No extra data needed for x86 */ /* No extra data needed for x86 */
}; };
int ftrace_int3_handler(struct pt_regs *regs);
#endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* CONFIG_DYNAMIC_FTRACE */
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
#endif /* CONFIG_FUNCTION_TRACER */ #endif /* CONFIG_FUNCTION_TRACER */
......
...@@ -134,6 +134,8 @@ ...@@ -134,6 +134,8 @@
#define MSR_AMD64_IBSFETCHCTL 0xc0011030 #define MSR_AMD64_IBSFETCHCTL 0xc0011030
#define MSR_AMD64_IBSFETCHLINAD 0xc0011031 #define MSR_AMD64_IBSFETCHLINAD 0xc0011031
#define MSR_AMD64_IBSFETCHPHYSAD 0xc0011032 #define MSR_AMD64_IBSFETCHPHYSAD 0xc0011032
#define MSR_AMD64_IBSFETCH_REG_COUNT 3
#define MSR_AMD64_IBSFETCH_REG_MASK ((1UL<<MSR_AMD64_IBSFETCH_REG_COUNT)-1)
#define MSR_AMD64_IBSOPCTL 0xc0011033 #define MSR_AMD64_IBSOPCTL 0xc0011033
#define MSR_AMD64_IBSOPRIP 0xc0011034 #define MSR_AMD64_IBSOPRIP 0xc0011034
#define MSR_AMD64_IBSOPDATA 0xc0011035 #define MSR_AMD64_IBSOPDATA 0xc0011035
...@@ -141,8 +143,11 @@ ...@@ -141,8 +143,11 @@
#define MSR_AMD64_IBSOPDATA3 0xc0011037 #define MSR_AMD64_IBSOPDATA3 0xc0011037
#define MSR_AMD64_IBSDCLINAD 0xc0011038 #define MSR_AMD64_IBSDCLINAD 0xc0011038
#define MSR_AMD64_IBSDCPHYSAD 0xc0011039 #define MSR_AMD64_IBSDCPHYSAD 0xc0011039
#define MSR_AMD64_IBSOP_REG_COUNT 7
#define MSR_AMD64_IBSOP_REG_MASK ((1UL<<MSR_AMD64_IBSOP_REG_COUNT)-1)
#define MSR_AMD64_IBSCTL 0xc001103a #define MSR_AMD64_IBSCTL 0xc001103a
#define MSR_AMD64_IBSBRTARGET 0xc001103b #define MSR_AMD64_IBSBRTARGET 0xc001103b
#define MSR_AMD64_IBS_REG_COUNT_MAX 8 /* includes MSR_AMD64_IBSBRTARGET */
/* Fam 15h MSRs */ /* Fam 15h MSRs */
#define MSR_F15H_PERF_CTL 0xc0010200 #define MSR_F15H_PERF_CTL 0xc0010200
......
...@@ -158,6 +158,7 @@ struct x86_pmu_capability { ...@@ -158,6 +158,7 @@ struct x86_pmu_capability {
#define IBS_CAPS_OPCNT (1U<<4) #define IBS_CAPS_OPCNT (1U<<4)
#define IBS_CAPS_BRNTRGT (1U<<5) #define IBS_CAPS_BRNTRGT (1U<<5)
#define IBS_CAPS_OPCNTEXT (1U<<6) #define IBS_CAPS_OPCNTEXT (1U<<6)
#define IBS_CAPS_RIPINVALIDCHK (1U<<7)
#define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \ #define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \
| IBS_CAPS_FETCHSAM \ | IBS_CAPS_FETCHSAM \
...@@ -170,21 +171,28 @@ struct x86_pmu_capability { ...@@ -170,21 +171,28 @@ struct x86_pmu_capability {
#define IBSCTL_LVT_OFFSET_VALID (1ULL<<8) #define IBSCTL_LVT_OFFSET_VALID (1ULL<<8)
#define IBSCTL_LVT_OFFSET_MASK 0x0F #define IBSCTL_LVT_OFFSET_MASK 0x0F
/* IbsFetchCtl bits/masks */ /* ibs fetch bits/masks */
#define IBS_FETCH_RAND_EN (1ULL<<57) #define IBS_FETCH_RAND_EN (1ULL<<57)
#define IBS_FETCH_VAL (1ULL<<49) #define IBS_FETCH_VAL (1ULL<<49)
#define IBS_FETCH_ENABLE (1ULL<<48) #define IBS_FETCH_ENABLE (1ULL<<48)
#define IBS_FETCH_CNT 0xFFFF0000ULL #define IBS_FETCH_CNT 0xFFFF0000ULL
#define IBS_FETCH_MAX_CNT 0x0000FFFFULL #define IBS_FETCH_MAX_CNT 0x0000FFFFULL
/* IbsOpCtl bits */ /* ibs op bits/masks */
/* lower 4 bits of the current count are ignored: */
#define IBS_OP_CUR_CNT (0xFFFF0ULL<<32)
#define IBS_OP_CNT_CTL (1ULL<<19) #define IBS_OP_CNT_CTL (1ULL<<19)
#define IBS_OP_VAL (1ULL<<18) #define IBS_OP_VAL (1ULL<<18)
#define IBS_OP_ENABLE (1ULL<<17) #define IBS_OP_ENABLE (1ULL<<17)
#define IBS_OP_MAX_CNT 0x0000FFFFULL #define IBS_OP_MAX_CNT 0x0000FFFFULL
#define IBS_OP_MAX_CNT_EXT 0x007FFFFFULL /* not a register bit mask */ #define IBS_OP_MAX_CNT_EXT 0x007FFFFFULL /* not a register bit mask */
#define IBS_RIP_INVALID (1ULL<<38)
#ifdef CONFIG_X86_LOCAL_APIC
extern u32 get_ibs_caps(void); extern u32 get_ibs_caps(void);
#else
static inline u32 get_ibs_caps(void) { return 0; }
#endif
#ifdef CONFIG_PERF_EVENTS #ifdef CONFIG_PERF_EVENTS
extern void perf_events_lapic_init(void); extern void perf_events_lapic_init(void);
......
...@@ -484,9 +484,6 @@ static int __x86_pmu_event_init(struct perf_event *event) ...@@ -484,9 +484,6 @@ static int __x86_pmu_event_init(struct perf_event *event)
/* mark unused */ /* mark unused */
event->hw.extra_reg.idx = EXTRA_REG_NONE; event->hw.extra_reg.idx = EXTRA_REG_NONE;
/* mark not used */
event->hw.extra_reg.idx = EXTRA_REG_NONE;
event->hw.branch_reg.idx = EXTRA_REG_NONE; event->hw.branch_reg.idx = EXTRA_REG_NONE;
return x86_pmu.hw_config(event); return x86_pmu.hw_config(event);
...@@ -1186,8 +1183,6 @@ int x86_pmu_handle_irq(struct pt_regs *regs) ...@@ -1186,8 +1183,6 @@ int x86_pmu_handle_irq(struct pt_regs *regs)
int idx, handled = 0; int idx, handled = 0;
u64 val; u64 val;
perf_sample_data_init(&data, 0);
cpuc = &__get_cpu_var(cpu_hw_events); cpuc = &__get_cpu_var(cpu_hw_events);
/* /*
...@@ -1222,7 +1217,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs) ...@@ -1222,7 +1217,7 @@ int x86_pmu_handle_irq(struct pt_regs *regs)
* event overflow * event overflow
*/ */
handled++; handled++;
data.period = event->hw.last_period; perf_sample_data_init(&data, 0, event->hw.last_period);
if (!x86_perf_event_set_period(event)) if (!x86_perf_event_set_period(event))
continue; continue;
......
...@@ -134,8 +134,13 @@ static u64 amd_pmu_event_map(int hw_event) ...@@ -134,8 +134,13 @@ static u64 amd_pmu_event_map(int hw_event)
static int amd_pmu_hw_config(struct perf_event *event) static int amd_pmu_hw_config(struct perf_event *event)
{ {
int ret = x86_pmu_hw_config(event); int ret;
/* pass precise event sampling to ibs: */
if (event->attr.precise_ip && get_ibs_caps())
return -ENOENT;
ret = x86_pmu_hw_config(event);
if (ret) if (ret)
return ret; return ret;
...@@ -205,10 +210,8 @@ static void amd_put_event_constraints(struct cpu_hw_events *cpuc, ...@@ -205,10 +210,8 @@ static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
* when we come here * when we come here
*/ */
for (i = 0; i < x86_pmu.num_counters; i++) { for (i = 0; i < x86_pmu.num_counters; i++) {
if (nb->owners[i] == event) { if (cmpxchg(nb->owners + i, event, NULL) == event)
cmpxchg(nb->owners+i, event, NULL);
break; break;
}
} }
} }
......
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
#include <linux/perf_event.h> #include <linux/perf_event.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/pci.h> #include <linux/pci.h>
#include <linux/ptrace.h>
#include <asm/apic.h> #include <asm/apic.h>
...@@ -16,36 +17,591 @@ static u32 ibs_caps; ...@@ -16,36 +17,591 @@ static u32 ibs_caps;
#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD)
static struct pmu perf_ibs; #include <linux/kprobes.h>
#include <linux/hardirq.h>
#include <asm/nmi.h>
#define IBS_FETCH_CONFIG_MASK (IBS_FETCH_RAND_EN | IBS_FETCH_MAX_CNT)
#define IBS_OP_CONFIG_MASK IBS_OP_MAX_CNT
enum ibs_states {
IBS_ENABLED = 0,
IBS_STARTED = 1,
IBS_STOPPING = 2,
IBS_MAX_STATES,
};
struct cpu_perf_ibs {
struct perf_event *event;
unsigned long state[BITS_TO_LONGS(IBS_MAX_STATES)];
};
struct perf_ibs {
struct pmu pmu;
unsigned int msr;
u64 config_mask;
u64 cnt_mask;
u64 enable_mask;
u64 valid_mask;
u64 max_period;
unsigned long offset_mask[1];
int offset_max;
struct cpu_perf_ibs __percpu *pcpu;
u64 (*get_count)(u64 config);
};
struct perf_ibs_data {
u32 size;
union {
u32 data[0]; /* data buffer starts here */
u32 caps;
};
u64 regs[MSR_AMD64_IBS_REG_COUNT_MAX];
};
static int
perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *hw_period)
{
s64 left = local64_read(&hwc->period_left);
s64 period = hwc->sample_period;
int overflow = 0;
/*
* If we are way outside a reasonable range then just skip forward:
*/
if (unlikely(left <= -period)) {
left = period;
local64_set(&hwc->period_left, left);
hwc->last_period = period;
overflow = 1;
}
if (unlikely(left < (s64)min)) {
left += period;
local64_set(&hwc->period_left, left);
hwc->last_period = period;
overflow = 1;
}
/*
* If the hw period that triggers the sw overflow is too short
* we might hit the irq handler. This biases the results.
* Thus we shorten the next-to-last period and set the last
* period to the max period.
*/
if (left > max) {
left -= max;
if (left > max)
left = max;
else if (left < min)
left = min;
}
*hw_period = (u64)left;
return overflow;
}
static int
perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width)
{
struct hw_perf_event *hwc = &event->hw;
int shift = 64 - width;
u64 prev_raw_count;
u64 delta;
/*
* Careful: an NMI might modify the previous event value.
*
* Our tactic to handle this is to first atomically read and
* exchange a new raw count - then add that new-prev delta
* count to the generic event atomically:
*/
prev_raw_count = local64_read(&hwc->prev_count);
if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
new_raw_count) != prev_raw_count)
return 0;
/*
* Now we have the new raw value and have updated the prev
* timestamp already. We can now calculate the elapsed delta
* (event-)time and add that to the generic event.
*
* Careful, not all hw sign-extends above the physical width
* of the count.
*/
delta = (new_raw_count << shift) - (prev_raw_count << shift);
delta >>= shift;
local64_add(delta, &event->count);
local64_sub(delta, &hwc->period_left);
return 1;
}
static struct perf_ibs perf_ibs_fetch;
static struct perf_ibs perf_ibs_op;
static struct perf_ibs *get_ibs_pmu(int type)
{
if (perf_ibs_fetch.pmu.type == type)
return &perf_ibs_fetch;
if (perf_ibs_op.pmu.type == type)
return &perf_ibs_op;
return NULL;
}
/*
* Use IBS for precise event sampling:
*
* perf record -a -e cpu-cycles:p ... # use ibs op counting cycle count
* perf record -a -e r076:p ... # same as -e cpu-cycles:p
* perf record -a -e r0C1:p ... # use ibs op counting micro-ops
*
* IbsOpCntCtl (bit 19) of IBS Execution Control Register (IbsOpCtl,
* MSRC001_1033) is used to select either cycle or micro-ops counting
* mode.
*
* The rip of IBS samples has skid 0. Thus, IBS supports precise
* levels 1 and 2 and the PERF_EFLAGS_EXACT is set. In rare cases the
* rip is invalid when IBS was not able to record the rip correctly.
* We clear PERF_EFLAGS_EXACT and take the rip from pt_regs then.
*
*/
static int perf_ibs_precise_event(struct perf_event *event, u64 *config)
{
switch (event->attr.precise_ip) {
case 0:
return -ENOENT;
case 1:
case 2:
break;
default:
return -EOPNOTSUPP;
}
switch (event->attr.type) {
case PERF_TYPE_HARDWARE:
switch (event->attr.config) {
case PERF_COUNT_HW_CPU_CYCLES:
*config = 0;
return 0;
}
break;
case PERF_TYPE_RAW:
switch (event->attr.config) {
case 0x0076:
*config = 0;
return 0;
case 0x00C1:
*config = IBS_OP_CNT_CTL;
return 0;
}
break;
default:
return -ENOENT;
}
return -EOPNOTSUPP;
}
static int perf_ibs_init(struct perf_event *event) static int perf_ibs_init(struct perf_event *event)
{ {
if (perf_ibs.type != event->attr.type) struct hw_perf_event *hwc = &event->hw;
struct perf_ibs *perf_ibs;
u64 max_cnt, config;
int ret;
perf_ibs = get_ibs_pmu(event->attr.type);
if (perf_ibs) {
config = event->attr.config;
} else {
perf_ibs = &perf_ibs_op;
ret = perf_ibs_precise_event(event, &config);
if (ret)
return ret;
}
if (event->pmu != &perf_ibs->pmu)
return -ENOENT; return -ENOENT;
if (config & ~perf_ibs->config_mask)
return -EINVAL;
if (hwc->sample_period) {
if (config & perf_ibs->cnt_mask)
/* raw max_cnt may not be set */
return -EINVAL;
if (!event->attr.sample_freq && hwc->sample_period & 0x0f)
/*
* lower 4 bits can not be set in ibs max cnt,
* but allowing it in case we adjust the
* sample period to set a frequency.
*/
return -EINVAL;
hwc->sample_period &= ~0x0FULL;
if (!hwc->sample_period)
hwc->sample_period = 0x10;
} else {
max_cnt = config & perf_ibs->cnt_mask;
config &= ~perf_ibs->cnt_mask;
event->attr.sample_period = max_cnt << 4;
hwc->sample_period = event->attr.sample_period;
}
if (!hwc->sample_period)
return -EINVAL;
/*
* If we modify hwc->sample_period, we also need to update
* hwc->last_period and hwc->period_left.
*/
hwc->last_period = hwc->sample_period;
local64_set(&hwc->period_left, hwc->sample_period);
hwc->config_base = perf_ibs->msr;
hwc->config = config;
return 0; return 0;
} }
static int perf_ibs_set_period(struct perf_ibs *perf_ibs,
struct hw_perf_event *hwc, u64 *period)
{
int overflow;
/* ignore lower 4 bits in min count: */
overflow = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period);
local64_set(&hwc->prev_count, 0);
return overflow;
}
static u64 get_ibs_fetch_count(u64 config)
{
return (config & IBS_FETCH_CNT) >> 12;
}
static u64 get_ibs_op_count(u64 config)
{
u64 count = 0;
if (config & IBS_OP_VAL)
count += (config & IBS_OP_MAX_CNT) << 4; /* cnt rolled over */
if (ibs_caps & IBS_CAPS_RDWROPCNT)
count += (config & IBS_OP_CUR_CNT) >> 32;
return count;
}
static void
perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event,
u64 *config)
{
u64 count = perf_ibs->get_count(*config);
/*
* Set width to 64 since we do not overflow on max width but
* instead on max count. In perf_ibs_set_period() we clear
* prev count manually on overflow.
*/
while (!perf_event_try_update(event, count, 64)) {
rdmsrl(event->hw.config_base, *config);
count = perf_ibs->get_count(*config);
}
}
static inline void perf_ibs_enable_event(struct perf_ibs *perf_ibs,
struct hw_perf_event *hwc, u64 config)
{
wrmsrl(hwc->config_base, hwc->config | config | perf_ibs->enable_mask);
}
/*
* Erratum #420 Instruction-Based Sampling Engine May Generate
* Interrupt that Cannot Be Cleared:
*
* Must clear counter mask first, then clear the enable bit. See
* Revision Guide for AMD Family 10h Processors, Publication #41322.
*/
static inline void perf_ibs_disable_event(struct perf_ibs *perf_ibs,
struct hw_perf_event *hwc, u64 config)
{
config &= ~perf_ibs->cnt_mask;
wrmsrl(hwc->config_base, config);
config &= ~perf_ibs->enable_mask;
wrmsrl(hwc->config_base, config);
}
/*
* We cannot restore the ibs pmu state, so we always needs to update
* the event while stopping it and then reset the state when starting
* again. Thus, ignoring PERF_EF_RELOAD and PERF_EF_UPDATE flags in
* perf_ibs_start()/perf_ibs_stop() and instead always do it.
*/
static void perf_ibs_start(struct perf_event *event, int flags)
{
struct hw_perf_event *hwc = &event->hw;
struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
u64 period;
if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
return;
WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
hwc->state = 0;
perf_ibs_set_period(perf_ibs, hwc, &period);
set_bit(IBS_STARTED, pcpu->state);
perf_ibs_enable_event(perf_ibs, hwc, period >> 4);
perf_event_update_userpage(event);
}
static void perf_ibs_stop(struct perf_event *event, int flags)
{
struct hw_perf_event *hwc = &event->hw;
struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
u64 config;
int stopping;
stopping = test_and_clear_bit(IBS_STARTED, pcpu->state);
if (!stopping && (hwc->state & PERF_HES_UPTODATE))
return;
rdmsrl(hwc->config_base, config);
if (stopping) {
set_bit(IBS_STOPPING, pcpu->state);
perf_ibs_disable_event(perf_ibs, hwc, config);
WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
hwc->state |= PERF_HES_STOPPED;
}
if (hwc->state & PERF_HES_UPTODATE)
return;
/*
* Clear valid bit to not count rollovers on update, rollovers
* are only updated in the irq handler.
*/
config &= ~perf_ibs->valid_mask;
perf_ibs_event_update(perf_ibs, event, &config);
hwc->state |= PERF_HES_UPTODATE;
}
static int perf_ibs_add(struct perf_event *event, int flags) static int perf_ibs_add(struct perf_event *event, int flags)
{ {
struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
if (test_and_set_bit(IBS_ENABLED, pcpu->state))
return -ENOSPC;
event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
pcpu->event = event;
if (flags & PERF_EF_START)
perf_ibs_start(event, PERF_EF_RELOAD);
return 0; return 0;
} }
static void perf_ibs_del(struct perf_event *event, int flags) static void perf_ibs_del(struct perf_event *event, int flags)
{ {
struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu);
struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
if (!test_and_clear_bit(IBS_ENABLED, pcpu->state))
return;
perf_ibs_stop(event, PERF_EF_UPDATE);
pcpu->event = NULL;
perf_event_update_userpage(event);
} }
static struct pmu perf_ibs = { static void perf_ibs_read(struct perf_event *event) { }
.event_init= perf_ibs_init,
.add= perf_ibs_add, static struct perf_ibs perf_ibs_fetch = {
.del= perf_ibs_del, .pmu = {
.task_ctx_nr = perf_invalid_context,
.event_init = perf_ibs_init,
.add = perf_ibs_add,
.del = perf_ibs_del,
.start = perf_ibs_start,
.stop = perf_ibs_stop,
.read = perf_ibs_read,
},
.msr = MSR_AMD64_IBSFETCHCTL,
.config_mask = IBS_FETCH_CONFIG_MASK,
.cnt_mask = IBS_FETCH_MAX_CNT,
.enable_mask = IBS_FETCH_ENABLE,
.valid_mask = IBS_FETCH_VAL,
.max_period = IBS_FETCH_MAX_CNT << 4,
.offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK },
.offset_max = MSR_AMD64_IBSFETCH_REG_COUNT,
.get_count = get_ibs_fetch_count,
}; };
static struct perf_ibs perf_ibs_op = {
.pmu = {
.task_ctx_nr = perf_invalid_context,
.event_init = perf_ibs_init,
.add = perf_ibs_add,
.del = perf_ibs_del,
.start = perf_ibs_start,
.stop = perf_ibs_stop,
.read = perf_ibs_read,
},
.msr = MSR_AMD64_IBSOPCTL,
.config_mask = IBS_OP_CONFIG_MASK,
.cnt_mask = IBS_OP_MAX_CNT,
.enable_mask = IBS_OP_ENABLE,
.valid_mask = IBS_OP_VAL,
.max_period = IBS_OP_MAX_CNT << 4,
.offset_mask = { MSR_AMD64_IBSOP_REG_MASK },
.offset_max = MSR_AMD64_IBSOP_REG_COUNT,
.get_count = get_ibs_op_count,
};
static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
{
struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
struct perf_event *event = pcpu->event;
struct hw_perf_event *hwc = &event->hw;
struct perf_sample_data data;
struct perf_raw_record raw;
struct pt_regs regs;
struct perf_ibs_data ibs_data;
int offset, size, check_rip, offset_max, throttle = 0;
unsigned int msr;
u64 *buf, *config, period;
if (!test_bit(IBS_STARTED, pcpu->state)) {
/*
* Catch spurious interrupts after stopping IBS: After
* disabling IBS there could be still incomming NMIs
* with samples that even have the valid bit cleared.
* Mark all this NMIs as handled.
*/
return test_and_clear_bit(IBS_STOPPING, pcpu->state) ? 1 : 0;
}
msr = hwc->config_base;
buf = ibs_data.regs;
rdmsrl(msr, *buf);
if (!(*buf++ & perf_ibs->valid_mask))
return 0;
config = &ibs_data.regs[0];
perf_ibs_event_update(perf_ibs, event, config);
perf_sample_data_init(&data, 0, hwc->last_period);
if (!perf_ibs_set_period(perf_ibs, hwc, &period))
goto out; /* no sw counter overflow */
ibs_data.caps = ibs_caps;
size = 1;
offset = 1;
check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK));
if (event->attr.sample_type & PERF_SAMPLE_RAW)
offset_max = perf_ibs->offset_max;
else if (check_rip)
offset_max = 2;
else
offset_max = 1;
do {
rdmsrl(msr + offset, *buf++);
size++;
offset = find_next_bit(perf_ibs->offset_mask,
perf_ibs->offset_max,
offset + 1);
} while (offset < offset_max);
ibs_data.size = sizeof(u64) * size;
regs = *iregs;
if (check_rip && (ibs_data.regs[2] & IBS_RIP_INVALID)) {
regs.flags &= ~PERF_EFLAGS_EXACT;
} else {
instruction_pointer_set(&regs, ibs_data.regs[1]);
regs.flags |= PERF_EFLAGS_EXACT;
}
if (event->attr.sample_type & PERF_SAMPLE_RAW) {
raw.size = sizeof(u32) + ibs_data.size;
raw.data = ibs_data.data;
data.raw = &raw;
}
throttle = perf_event_overflow(event, &data, &regs);
out:
if (throttle)
perf_ibs_disable_event(perf_ibs, hwc, *config);
else
perf_ibs_enable_event(perf_ibs, hwc, period >> 4);
perf_event_update_userpage(event);
return 1;
}
static int __kprobes
perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
{
int handled = 0;
handled += perf_ibs_handle_irq(&perf_ibs_fetch, regs);
handled += perf_ibs_handle_irq(&perf_ibs_op, regs);
if (handled)
inc_irq_stat(apic_perf_irqs);
return handled;
}
static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)
{
struct cpu_perf_ibs __percpu *pcpu;
int ret;
pcpu = alloc_percpu(struct cpu_perf_ibs);
if (!pcpu)
return -ENOMEM;
perf_ibs->pcpu = pcpu;
ret = perf_pmu_register(&perf_ibs->pmu, name, -1);
if (ret) {
perf_ibs->pcpu = NULL;
free_percpu(pcpu);
}
return ret;
}
static __init int perf_event_ibs_init(void) static __init int perf_event_ibs_init(void)
{ {
if (!ibs_caps) if (!ibs_caps)
return -ENODEV; /* ibs not supported by the cpu */ return -ENODEV; /* ibs not supported by the cpu */
perf_pmu_register(&perf_ibs, "ibs", -1); perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch");
if (ibs_caps & IBS_CAPS_OPCNT)
perf_ibs_op.config_mask |= IBS_OP_CNT_CTL;
perf_ibs_pmu_init(&perf_ibs_op, "ibs_op");
register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs");
printk(KERN_INFO "perf: AMD IBS detected (0x%08x)\n", ibs_caps); printk(KERN_INFO "perf: AMD IBS detected (0x%08x)\n", ibs_caps);
return 0; return 0;
......
...@@ -1027,8 +1027,6 @@ static int intel_pmu_handle_irq(struct pt_regs *regs) ...@@ -1027,8 +1027,6 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
u64 status; u64 status;
int handled; int handled;
perf_sample_data_init(&data, 0);
cpuc = &__get_cpu_var(cpu_hw_events); cpuc = &__get_cpu_var(cpu_hw_events);
/* /*
...@@ -1082,7 +1080,7 @@ static int intel_pmu_handle_irq(struct pt_regs *regs) ...@@ -1082,7 +1080,7 @@ static int intel_pmu_handle_irq(struct pt_regs *regs)
if (!intel_pmu_save_and_restart(event)) if (!intel_pmu_save_and_restart(event))
continue; continue;
data.period = event->hw.last_period; perf_sample_data_init(&data, 0, event->hw.last_period);
if (has_branch_stack(event)) if (has_branch_stack(event))
data.br_stack = &cpuc->lbr_stack; data.br_stack = &cpuc->lbr_stack;
......
...@@ -316,8 +316,7 @@ int intel_pmu_drain_bts_buffer(void) ...@@ -316,8 +316,7 @@ int intel_pmu_drain_bts_buffer(void)
ds->bts_index = ds->bts_buffer_base; ds->bts_index = ds->bts_buffer_base;
perf_sample_data_init(&data, 0); perf_sample_data_init(&data, 0, event->hw.last_period);
data.period = event->hw.last_period;
regs.ip = 0; regs.ip = 0;
/* /*
...@@ -564,8 +563,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event, ...@@ -564,8 +563,7 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
if (!intel_pmu_save_and_restart(event)) if (!intel_pmu_save_and_restart(event))
return; return;
perf_sample_data_init(&data, 0); perf_sample_data_init(&data, 0, event->hw.last_period);
data.period = event->hw.last_period;
/* /*
* We use the interrupt regs as a base because the PEBS record * We use the interrupt regs as a base because the PEBS record
......
...@@ -1005,8 +1005,6 @@ static int p4_pmu_handle_irq(struct pt_regs *regs) ...@@ -1005,8 +1005,6 @@ static int p4_pmu_handle_irq(struct pt_regs *regs)
int idx, handled = 0; int idx, handled = 0;
u64 val; u64 val;
perf_sample_data_init(&data, 0);
cpuc = &__get_cpu_var(cpu_hw_events); cpuc = &__get_cpu_var(cpu_hw_events);
for (idx = 0; idx < x86_pmu.num_counters; idx++) { for (idx = 0; idx < x86_pmu.num_counters; idx++) {
...@@ -1034,10 +1032,12 @@ static int p4_pmu_handle_irq(struct pt_regs *regs) ...@@ -1034,10 +1032,12 @@ static int p4_pmu_handle_irq(struct pt_regs *regs)
handled += overflow; handled += overflow;
/* event overflow for sure */ /* event overflow for sure */
data.period = event->hw.last_period; perf_sample_data_init(&data, 0, hwc->last_period);
if (!x86_perf_event_set_period(event)) if (!x86_perf_event_set_period(event))
continue; continue;
if (perf_event_overflow(event, &data, regs)) if (perf_event_overflow(event, &data, regs))
x86_pmu_stop(event, 0); x86_pmu_stop(event, 0);
} }
......
...@@ -24,40 +24,21 @@ ...@@ -24,40 +24,21 @@
#include <trace/syscall.h> #include <trace/syscall.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/kprobes.h>
#include <asm/ftrace.h> #include <asm/ftrace.h>
#include <asm/nops.h> #include <asm/nops.h>
#include <asm/nmi.h>
#ifdef CONFIG_DYNAMIC_FTRACE #ifdef CONFIG_DYNAMIC_FTRACE
/*
* modifying_code is set to notify NMIs that they need to use
* memory barriers when entering or exiting. But we don't want
* to burden NMIs with unnecessary memory barriers when code
* modification is not being done (which is most of the time).
*
* A mutex is already held when ftrace_arch_code_modify_prepare
* and post_process are called. No locks need to be taken here.
*
* Stop machine will make sure currently running NMIs are done
* and new NMIs will see the updated variable before we need
* to worry about NMIs doing memory barriers.
*/
static int modifying_code __read_mostly;
static DEFINE_PER_CPU(int, save_modifying_code);
int ftrace_arch_code_modify_prepare(void) int ftrace_arch_code_modify_prepare(void)
{ {
set_kernel_text_rw(); set_kernel_text_rw();
set_all_modules_text_rw(); set_all_modules_text_rw();
modifying_code = 1;
return 0; return 0;
} }
int ftrace_arch_code_modify_post_process(void) int ftrace_arch_code_modify_post_process(void)
{ {
modifying_code = 0;
set_all_modules_text_ro(); set_all_modules_text_ro();
set_kernel_text_ro(); set_kernel_text_ro();
return 0; return 0;
...@@ -90,134 +71,6 @@ static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr) ...@@ -90,134 +71,6 @@ static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
return calc.code; return calc.code;
} }
/*
* Modifying code must take extra care. On an SMP machine, if
* the code being modified is also being executed on another CPU
* that CPU will have undefined results and possibly take a GPF.
* We use kstop_machine to stop other CPUS from exectuing code.
* But this does not stop NMIs from happening. We still need
* to protect against that. We separate out the modification of
* the code to take care of this.
*
* Two buffers are added: An IP buffer and a "code" buffer.
*
* 1) Put the instruction pointer into the IP buffer
* and the new code into the "code" buffer.
* 2) Wait for any running NMIs to finish and set a flag that says
* we are modifying code, it is done in an atomic operation.
* 3) Write the code
* 4) clear the flag.
* 5) Wait for any running NMIs to finish.
*
* If an NMI is executed, the first thing it does is to call
* "ftrace_nmi_enter". This will check if the flag is set to write
* and if it is, it will write what is in the IP and "code" buffers.
*
* The trick is, it does not matter if everyone is writing the same
* content to the code location. Also, if a CPU is executing code
* it is OK to write to that code location if the contents being written
* are the same as what exists.
*/
#define MOD_CODE_WRITE_FLAG (1 << 31) /* set when NMI should do the write */
static atomic_t nmi_running = ATOMIC_INIT(0);
static int mod_code_status; /* holds return value of text write */
static void *mod_code_ip; /* holds the IP to write to */
static const void *mod_code_newcode; /* holds the text to write to the IP */
static unsigned nmi_wait_count;
static atomic_t nmi_update_count = ATOMIC_INIT(0);
int ftrace_arch_read_dyn_info(char *buf, int size)
{
int r;
r = snprintf(buf, size, "%u %u",
nmi_wait_count,
atomic_read(&nmi_update_count));
return r;
}
static void clear_mod_flag(void)
{
int old = atomic_read(&nmi_running);
for (;;) {
int new = old & ~MOD_CODE_WRITE_FLAG;
if (old == new)
break;
old = atomic_cmpxchg(&nmi_running, old, new);
}
}
static void ftrace_mod_code(void)
{
/*
* Yes, more than one CPU process can be writing to mod_code_status.
* (and the code itself)
* But if one were to fail, then they all should, and if one were
* to succeed, then they all should.
*/
mod_code_status = probe_kernel_write(mod_code_ip, mod_code_newcode,
MCOUNT_INSN_SIZE);
/* if we fail, then kill any new writers */
if (mod_code_status)
clear_mod_flag();
}
void ftrace_nmi_enter(void)
{
__this_cpu_write(save_modifying_code, modifying_code);
if (!__this_cpu_read(save_modifying_code))
return;
if (atomic_inc_return(&nmi_running) & MOD_CODE_WRITE_FLAG) {
smp_rmb();
ftrace_mod_code();
atomic_inc(&nmi_update_count);
}
/* Must have previous changes seen before executions */
smp_mb();
}
void ftrace_nmi_exit(void)
{
if (!__this_cpu_read(save_modifying_code))
return;
/* Finish all executions before clearing nmi_running */
smp_mb();
atomic_dec(&nmi_running);
}
static void wait_for_nmi_and_set_mod_flag(void)
{
if (!atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG))
return;
do {
cpu_relax();
} while (atomic_cmpxchg(&nmi_running, 0, MOD_CODE_WRITE_FLAG));
nmi_wait_count++;
}
static void wait_for_nmi(void)
{
if (!atomic_read(&nmi_running))
return;
do {
cpu_relax();
} while (atomic_read(&nmi_running));
nmi_wait_count++;
}
static inline int static inline int
within(unsigned long addr, unsigned long start, unsigned long end) within(unsigned long addr, unsigned long start, unsigned long end)
{ {
...@@ -238,26 +91,7 @@ do_ftrace_mod_code(unsigned long ip, const void *new_code) ...@@ -238,26 +91,7 @@ do_ftrace_mod_code(unsigned long ip, const void *new_code)
if (within(ip, (unsigned long)_text, (unsigned long)_etext)) if (within(ip, (unsigned long)_text, (unsigned long)_etext))
ip = (unsigned long)__va(__pa(ip)); ip = (unsigned long)__va(__pa(ip));
mod_code_ip = (void *)ip; return probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE);
mod_code_newcode = new_code;
/* The buffers need to be visible before we let NMIs write them */
smp_mb();
wait_for_nmi_and_set_mod_flag();
/* Make sure all running NMIs have finished before we write the code */
smp_mb();
ftrace_mod_code();
/* Make sure the write happens before clearing the bit */
smp_mb();
clear_mod_flag();
wait_for_nmi();
return mod_code_status;
} }
static const unsigned char *ftrace_nop_replace(void) static const unsigned char *ftrace_nop_replace(void)
...@@ -334,6 +168,336 @@ int ftrace_update_ftrace_func(ftrace_func_t func) ...@@ -334,6 +168,336 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
return ret; return ret;
} }
int modifying_ftrace_code __read_mostly;
/*
* A breakpoint was added to the code address we are about to
* modify, and this is the handle that will just skip over it.
* We are either changing a nop into a trace call, or a trace
* call to a nop. While the change is taking place, we treat
* it just like it was a nop.
*/
int ftrace_int3_handler(struct pt_regs *regs)
{
if (WARN_ON_ONCE(!regs))
return 0;
if (!ftrace_location(regs->ip - 1))
return 0;
regs->ip += MCOUNT_INSN_SIZE - 1;
return 1;
}
static int ftrace_write(unsigned long ip, const char *val, int size)
{
/*
* On x86_64, kernel text mappings are mapped read-only with
* CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
* of the kernel text mapping to modify the kernel text.
*
* For 32bit kernels, these mappings are same and we can use
* kernel identity mapping to modify code.
*/
if (within(ip, (unsigned long)_text, (unsigned long)_etext))
ip = (unsigned long)__va(__pa(ip));
return probe_kernel_write((void *)ip, val, size);
}
static int add_break(unsigned long ip, const char *old)
{
unsigned char replaced[MCOUNT_INSN_SIZE];
unsigned char brk = BREAKPOINT_INSTRUCTION;
if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
return -EFAULT;
/* Make sure it is what we expect it to be */
if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
return -EINVAL;
if (ftrace_write(ip, &brk, 1))
return -EPERM;
return 0;
}
static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned const char *old;
unsigned long ip = rec->ip;
old = ftrace_call_replace(ip, addr);
return add_break(rec->ip, old);
}
static int add_brk_on_nop(struct dyn_ftrace *rec)
{
unsigned const char *old;
old = ftrace_nop_replace();
return add_break(rec->ip, old);
}
static int add_breakpoints(struct dyn_ftrace *rec, int enable)
{
unsigned long ftrace_addr;
int ret;
ret = ftrace_test_record(rec, enable);
ftrace_addr = (unsigned long)FTRACE_ADDR;
switch (ret) {
case FTRACE_UPDATE_IGNORE:
return 0;
case FTRACE_UPDATE_MAKE_CALL:
/* converting nop to call */
return add_brk_on_nop(rec);
case FTRACE_UPDATE_MAKE_NOP:
/* converting a call to a nop */
return add_brk_on_call(rec, ftrace_addr);
}
return 0;
}
/*
* On error, we need to remove breakpoints. This needs to
* be done caefully. If the address does not currently have a
* breakpoint, we know we are done. Otherwise, we look at the
* remaining 4 bytes of the instruction. If it matches a nop
* we replace the breakpoint with the nop. Otherwise we replace
* it with the call instruction.
*/
static int remove_breakpoint(struct dyn_ftrace *rec)
{
unsigned char ins[MCOUNT_INSN_SIZE];
unsigned char brk = BREAKPOINT_INSTRUCTION;
const unsigned char *nop;
unsigned long ftrace_addr;
unsigned long ip = rec->ip;
/* If we fail the read, just give up */
if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
return -EFAULT;
/* If this does not have a breakpoint, we are done */
if (ins[0] != brk)
return -1;
nop = ftrace_nop_replace();
/*
* If the last 4 bytes of the instruction do not match
* a nop, then we assume that this is a call to ftrace_addr.
*/
if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
/*
* For extra paranoidism, we check if the breakpoint is on
* a call that would actually jump to the ftrace_addr.
* If not, don't touch the breakpoint, we make just create
* a disaster.
*/
ftrace_addr = (unsigned long)FTRACE_ADDR;
nop = ftrace_call_replace(ip, ftrace_addr);
if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
return -EINVAL;
}
return probe_kernel_write((void *)ip, &nop[0], 1);
}
static int add_update_code(unsigned long ip, unsigned const char *new)
{
/* skip breakpoint */
ip++;
new++;
if (ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1))
return -EPERM;
return 0;
}
static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned long ip = rec->ip;
unsigned const char *new;
new = ftrace_call_replace(ip, addr);
return add_update_code(ip, new);
}
static int add_update_nop(struct dyn_ftrace *rec)
{
unsigned long ip = rec->ip;
unsigned const char *new;
new = ftrace_nop_replace();
return add_update_code(ip, new);
}
static int add_update(struct dyn_ftrace *rec, int enable)
{
unsigned long ftrace_addr;
int ret;
ret = ftrace_test_record(rec, enable);
ftrace_addr = (unsigned long)FTRACE_ADDR;
switch (ret) {
case FTRACE_UPDATE_IGNORE:
return 0;
case FTRACE_UPDATE_MAKE_CALL:
/* converting nop to call */
return add_update_call(rec, ftrace_addr);
case FTRACE_UPDATE_MAKE_NOP:
/* converting a call to a nop */
return add_update_nop(rec);
}
return 0;
}
static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned long ip = rec->ip;
unsigned const char *new;
new = ftrace_call_replace(ip, addr);
if (ftrace_write(ip, new, 1))
return -EPERM;
return 0;
}
static int finish_update_nop(struct dyn_ftrace *rec)
{
unsigned long ip = rec->ip;
unsigned const char *new;
new = ftrace_nop_replace();
if (ftrace_write(ip, new, 1))
return -EPERM;
return 0;
}
static int finish_update(struct dyn_ftrace *rec, int enable)
{
unsigned long ftrace_addr;
int ret;
ret = ftrace_update_record(rec, enable);
ftrace_addr = (unsigned long)FTRACE_ADDR;
switch (ret) {
case FTRACE_UPDATE_IGNORE:
return 0;
case FTRACE_UPDATE_MAKE_CALL:
/* converting nop to call */
return finish_update_call(rec, ftrace_addr);
case FTRACE_UPDATE_MAKE_NOP:
/* converting a call to a nop */
return finish_update_nop(rec);
}
return 0;
}
static void do_sync_core(void *data)
{
sync_core();
}
static void run_sync(void)
{
int enable_irqs = irqs_disabled();
/* We may be called with interrupts disbled (on bootup). */
if (enable_irqs)
local_irq_enable();
on_each_cpu(do_sync_core, NULL, 1);
if (enable_irqs)
local_irq_disable();
}
void ftrace_replace_code(int enable)
{
struct ftrace_rec_iter *iter;
struct dyn_ftrace *rec;
const char *report = "adding breakpoints";
int count = 0;
int ret;
for_ftrace_rec_iter(iter) {
rec = ftrace_rec_iter_record(iter);
ret = add_breakpoints(rec, enable);
if (ret)
goto remove_breakpoints;
count++;
}
run_sync();
report = "updating code";
for_ftrace_rec_iter(iter) {
rec = ftrace_rec_iter_record(iter);
ret = add_update(rec, enable);
if (ret)
goto remove_breakpoints;
}
run_sync();
report = "removing breakpoints";
for_ftrace_rec_iter(iter) {
rec = ftrace_rec_iter_record(iter);
ret = finish_update(rec, enable);
if (ret)
goto remove_breakpoints;
}
run_sync();
return;
remove_breakpoints:
ftrace_bug(ret, rec ? rec->ip : 0);
printk(KERN_WARNING "Failed on %s (%d):\n", report, count);
for_ftrace_rec_iter(iter) {
rec = ftrace_rec_iter_record(iter);
remove_breakpoint(rec);
}
}
void arch_ftrace_update_code(int command)
{
modifying_ftrace_code++;
ftrace_modify_all_code(command);
modifying_ftrace_code--;
}
int __init ftrace_dyn_arch_init(void *data) int __init ftrace_dyn_arch_init(void *data)
{ {
/* The return code is retured via data */ /* The return code is retured via data */
......
...@@ -84,7 +84,7 @@ __setup("unknown_nmi_panic", setup_unknown_nmi_panic); ...@@ -84,7 +84,7 @@ __setup("unknown_nmi_panic", setup_unknown_nmi_panic);
#define nmi_to_desc(type) (&nmi_desc[type]) #define nmi_to_desc(type) (&nmi_desc[type])
static int notrace __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b) static int __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
{ {
struct nmi_desc *desc = nmi_to_desc(type); struct nmi_desc *desc = nmi_to_desc(type);
struct nmiaction *a; struct nmiaction *a;
...@@ -166,7 +166,7 @@ void unregister_nmi_handler(unsigned int type, const char *name) ...@@ -166,7 +166,7 @@ void unregister_nmi_handler(unsigned int type, const char *name)
} }
EXPORT_SYMBOL_GPL(unregister_nmi_handler); EXPORT_SYMBOL_GPL(unregister_nmi_handler);
static notrace __kprobes void static __kprobes void
pci_serr_error(unsigned char reason, struct pt_regs *regs) pci_serr_error(unsigned char reason, struct pt_regs *regs)
{ {
/* check to see if anyone registered against these types of errors */ /* check to see if anyone registered against these types of errors */
...@@ -197,7 +197,7 @@ pci_serr_error(unsigned char reason, struct pt_regs *regs) ...@@ -197,7 +197,7 @@ pci_serr_error(unsigned char reason, struct pt_regs *regs)
outb(reason, NMI_REASON_PORT); outb(reason, NMI_REASON_PORT);
} }
static notrace __kprobes void static __kprobes void
io_check_error(unsigned char reason, struct pt_regs *regs) io_check_error(unsigned char reason, struct pt_regs *regs)
{ {
unsigned long i; unsigned long i;
...@@ -228,7 +228,7 @@ io_check_error(unsigned char reason, struct pt_regs *regs) ...@@ -228,7 +228,7 @@ io_check_error(unsigned char reason, struct pt_regs *regs)
outb(reason, NMI_REASON_PORT); outb(reason, NMI_REASON_PORT);
} }
static notrace __kprobes void static __kprobes void
unknown_nmi_error(unsigned char reason, struct pt_regs *regs) unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
{ {
int handled; int handled;
...@@ -270,7 +270,7 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs) ...@@ -270,7 +270,7 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
static DEFINE_PER_CPU(bool, swallow_nmi); static DEFINE_PER_CPU(bool, swallow_nmi);
static DEFINE_PER_CPU(unsigned long, last_nmi_rip); static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
static notrace __kprobes void default_do_nmi(struct pt_regs *regs) static __kprobes void default_do_nmi(struct pt_regs *regs)
{ {
unsigned char reason = 0; unsigned char reason = 0;
int handled; int handled;
......
...@@ -50,6 +50,7 @@ ...@@ -50,6 +50,7 @@
#include <asm/processor.h> #include <asm/processor.h>
#include <asm/debugreg.h> #include <asm/debugreg.h>
#include <linux/atomic.h> #include <linux/atomic.h>
#include <asm/ftrace.h>
#include <asm/traps.h> #include <asm/traps.h>
#include <asm/desc.h> #include <asm/desc.h>
#include <asm/i387.h> #include <asm/i387.h>
...@@ -303,8 +304,13 @@ do_general_protection(struct pt_regs *regs, long error_code) ...@@ -303,8 +304,13 @@ do_general_protection(struct pt_regs *regs, long error_code)
} }
/* May run on IST stack. */ /* May run on IST stack. */
dotraplinkage void __kprobes do_int3(struct pt_regs *regs, long error_code) dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_code)
{ {
#ifdef CONFIG_DYNAMIC_FTRACE
/* ftrace must be first, everything else may cause a recursive crash */
if (unlikely(modifying_ftrace_code) && ftrace_int3_handler(regs))
return;
#endif
#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP, if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
SIGTRAP) == NOTIFY_STOP) SIGTRAP) == NOTIFY_STOP)
......
...@@ -486,8 +486,8 @@ ...@@ -486,8 +486,8 @@
CPU_DISCARD(init.data) \ CPU_DISCARD(init.data) \
MEM_DISCARD(init.data) \ MEM_DISCARD(init.data) \
KERNEL_CTORS() \ KERNEL_CTORS() \
*(.init.rodata) \
MCOUNT_REC() \ MCOUNT_REC() \
*(.init.rodata) \
FTRACE_EVENTS() \ FTRACE_EVENTS() \
TRACE_SYSCALLS() \ TRACE_SYSCALLS() \
DEV_DISCARD(init.rodata) \ DEV_DISCARD(init.rodata) \
......
...@@ -286,10 +286,16 @@ struct ftrace_rec_iter *ftrace_rec_iter_start(void); ...@@ -286,10 +286,16 @@ struct ftrace_rec_iter *ftrace_rec_iter_start(void);
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter); struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter);
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter); struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter);
#define for_ftrace_rec_iter(iter) \
for (iter = ftrace_rec_iter_start(); \
iter; \
iter = ftrace_rec_iter_next(iter))
int ftrace_update_record(struct dyn_ftrace *rec, int enable); int ftrace_update_record(struct dyn_ftrace *rec, int enable);
int ftrace_test_record(struct dyn_ftrace *rec, int enable); int ftrace_test_record(struct dyn_ftrace *rec, int enable);
void ftrace_run_stop_machine(int command); void ftrace_run_stop_machine(int command);
int ftrace_location(unsigned long ip); unsigned long ftrace_location(unsigned long ip);
extern ftrace_func_t ftrace_trace_function; extern ftrace_func_t ftrace_trace_function;
...@@ -308,11 +314,14 @@ ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable); ...@@ -308,11 +314,14 @@ ftrace_set_early_filter(struct ftrace_ops *ops, char *buf, int enable);
/* defined in arch */ /* defined in arch */
extern int ftrace_ip_converted(unsigned long ip); extern int ftrace_ip_converted(unsigned long ip);
extern int ftrace_dyn_arch_init(void *data); extern int ftrace_dyn_arch_init(void *data);
extern void ftrace_replace_code(int enable);
extern int ftrace_update_ftrace_func(ftrace_func_t func); extern int ftrace_update_ftrace_func(ftrace_func_t func);
extern void ftrace_caller(void); extern void ftrace_caller(void);
extern void ftrace_call(void); extern void ftrace_call(void);
extern void mcount_call(void); extern void mcount_call(void);
void ftrace_modify_all_code(int command);
#ifndef FTRACE_ADDR #ifndef FTRACE_ADDR
#define FTRACE_ADDR ((unsigned long)ftrace_caller) #define FTRACE_ADDR ((unsigned long)ftrace_caller)
#endif #endif
...@@ -485,8 +494,12 @@ static inline void __ftrace_enabled_restore(int enabled) ...@@ -485,8 +494,12 @@ static inline void __ftrace_enabled_restore(int enabled)
extern void trace_preempt_on(unsigned long a0, unsigned long a1); extern void trace_preempt_on(unsigned long a0, unsigned long a1);
extern void trace_preempt_off(unsigned long a0, unsigned long a1); extern void trace_preempt_off(unsigned long a0, unsigned long a1);
#else #else
static inline void trace_preempt_on(unsigned long a0, unsigned long a1) { } /*
static inline void trace_preempt_off(unsigned long a0, unsigned long a1) { } * Use defines instead of static inlines because some arches will make code out
* of the CALLER_ADDR, when we really want these to be a real nop.
*/
# define trace_preempt_on(a0, a1) do { } while (0)
# define trace_preempt_off(a0, a1) do { } while (0)
#endif #endif
#ifdef CONFIG_FTRACE_MCOUNT_RECORD #ifdef CONFIG_FTRACE_MCOUNT_RECORD
......
...@@ -480,15 +480,16 @@ do { \ ...@@ -480,15 +480,16 @@ do { \
#define trace_printk(fmt, args...) \ #define trace_printk(fmt, args...) \
do { \ do { \
static const char *trace_printk_fmt \
__attribute__((section("__trace_printk_fmt"))) = \
__builtin_constant_p(fmt) ? fmt : NULL; \
\
__trace_printk_check_format(fmt, ##args); \ __trace_printk_check_format(fmt, ##args); \
if (__builtin_constant_p(fmt)) { \
static const char *trace_printk_fmt \
__attribute__((section("__trace_printk_fmt"))) = \
__builtin_constant_p(fmt) ? fmt : NULL; \
\ \
if (__builtin_constant_p(fmt)) \
__trace_bprintk(_THIS_IP_, trace_printk_fmt, ##args); \ __trace_bprintk(_THIS_IP_, trace_printk_fmt, ##args); \
} else \ else \
__trace_printk(_THIS_IP_, fmt, ##args); \ __trace_printk(_THIS_IP_, fmt, ##args); \
} while (0) } while (0)
extern __printf(2, 3) extern __printf(2, 3)
......
...@@ -1084,10 +1084,8 @@ extern void perf_pmu_unregister(struct pmu *pmu); ...@@ -1084,10 +1084,8 @@ extern void perf_pmu_unregister(struct pmu *pmu);
extern int perf_num_counters(void); extern int perf_num_counters(void);
extern const char *perf_pmu_name(void); extern const char *perf_pmu_name(void);
extern void __perf_event_task_sched_in(struct task_struct *prev, extern void __perf_event_task_sched(struct task_struct *prev,
struct task_struct *task); struct task_struct *next);
extern void __perf_event_task_sched_out(struct task_struct *prev,
struct task_struct *next);
extern int perf_event_init_task(struct task_struct *child); extern int perf_event_init_task(struct task_struct *child);
extern void perf_event_exit_task(struct task_struct *child); extern void perf_event_exit_task(struct task_struct *child);
extern void perf_event_free_task(struct task_struct *task); extern void perf_event_free_task(struct task_struct *task);
...@@ -1132,11 +1130,14 @@ struct perf_sample_data { ...@@ -1132,11 +1130,14 @@ struct perf_sample_data {
struct perf_branch_stack *br_stack; struct perf_branch_stack *br_stack;
}; };
static inline void perf_sample_data_init(struct perf_sample_data *data, u64 addr) static inline void perf_sample_data_init(struct perf_sample_data *data,
u64 addr, u64 period)
{ {
/* remaining struct members initialized in perf_prepare_sample() */
data->addr = addr; data->addr = addr;
data->raw = NULL; data->raw = NULL;
data->br_stack = NULL; data->br_stack = NULL;
data->period = period;
} }
extern void perf_output_sample(struct perf_output_handle *handle, extern void perf_output_sample(struct perf_output_handle *handle,
...@@ -1204,20 +1205,13 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) ...@@ -1204,20 +1205,13 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
extern struct static_key_deferred perf_sched_events; extern struct static_key_deferred perf_sched_events;
static inline void perf_event_task_sched_in(struct task_struct *prev, static inline void perf_event_task_sched(struct task_struct *prev,
struct task_struct *task) struct task_struct *task)
{
if (static_key_false(&perf_sched_events.key))
__perf_event_task_sched_in(prev, task);
}
static inline void perf_event_task_sched_out(struct task_struct *prev,
struct task_struct *next)
{ {
perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0); perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, NULL, 0);
if (static_key_false(&perf_sched_events.key)) if (static_key_false(&perf_sched_events.key))
__perf_event_task_sched_out(prev, next); __perf_event_task_sched(prev, task);
} }
extern void perf_event_mmap(struct vm_area_struct *vma); extern void perf_event_mmap(struct vm_area_struct *vma);
...@@ -1292,11 +1286,8 @@ extern void perf_event_disable(struct perf_event *event); ...@@ -1292,11 +1286,8 @@ extern void perf_event_disable(struct perf_event *event);
extern void perf_event_task_tick(void); extern void perf_event_task_tick(void);
#else #else
static inline void static inline void
perf_event_task_sched_in(struct task_struct *prev, perf_event_task_sched(struct task_struct *prev,
struct task_struct *task) { } struct task_struct *task) { }
static inline void
perf_event_task_sched_out(struct task_struct *prev,
struct task_struct *next) { }
static inline int perf_event_init_task(struct task_struct *child) { return 0; } static inline int perf_event_init_task(struct task_struct *child) { return 0; }
static inline void perf_event_exit_task(struct task_struct *child) { } static inline void perf_event_exit_task(struct task_struct *child) { }
static inline void perf_event_free_task(struct task_struct *task) { } static inline void perf_event_free_task(struct task_struct *task) { }
......
...@@ -96,9 +96,11 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k ...@@ -96,9 +96,11 @@ __ring_buffer_alloc(unsigned long size, unsigned flags, struct lock_class_key *k
__ring_buffer_alloc((size), (flags), &__key); \ __ring_buffer_alloc((size), (flags), &__key); \
}) })
#define RING_BUFFER_ALL_CPUS -1
void ring_buffer_free(struct ring_buffer *buffer); void ring_buffer_free(struct ring_buffer *buffer);
int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size); int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size, int cpu);
void ring_buffer_change_overwrite(struct ring_buffer *buffer, int val); void ring_buffer_change_overwrite(struct ring_buffer *buffer, int val);
...@@ -129,7 +131,7 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts); ...@@ -129,7 +131,7 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts);
void ring_buffer_iter_reset(struct ring_buffer_iter *iter); void ring_buffer_iter_reset(struct ring_buffer_iter *iter);
int ring_buffer_iter_empty(struct ring_buffer_iter *iter); int ring_buffer_iter_empty(struct ring_buffer_iter *iter);
unsigned long ring_buffer_size(struct ring_buffer *buffer); unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu);
void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu); void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu);
void ring_buffer_reset(struct ring_buffer *buffer); void ring_buffer_reset(struct ring_buffer *buffer);
......
...@@ -1198,7 +1198,7 @@ menu "Kernel Performance Events And Counters" ...@@ -1198,7 +1198,7 @@ menu "Kernel Performance Events And Counters"
config PERF_EVENTS config PERF_EVENTS
bool "Kernel performance events and counters" bool "Kernel performance events and counters"
default y if (PROFILING || PERF_COUNTERS) default y if PROFILING
depends on HAVE_PERF_EVENTS depends on HAVE_PERF_EVENTS
select ANON_INODES select ANON_INODES
select IRQ_WORK select IRQ_WORK
...@@ -1225,18 +1225,6 @@ config PERF_EVENTS ...@@ -1225,18 +1225,6 @@ config PERF_EVENTS
Say Y if unsure. Say Y if unsure.
config PERF_COUNTERS
bool "Kernel performance counters (old config option)"
depends on HAVE_PERF_EVENTS
help
This config has been obsoleted by the PERF_EVENTS
config option - please see that one for details.
It has no effect on the kernel whether you enable
it or not, it is a compatibility placeholder.
Say N if unsure.
config DEBUG_PERF_USE_VMALLOC config DEBUG_PERF_USE_VMALLOC
default n default n
bool "Debug: use vmalloc to back perf mmap() buffers" bool "Debug: use vmalloc to back perf mmap() buffers"
......
...@@ -2039,8 +2039,8 @@ static void perf_event_context_sched_out(struct task_struct *task, int ctxn, ...@@ -2039,8 +2039,8 @@ static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
* accessing the event control register. If a NMI hits, then it will * accessing the event control register. If a NMI hits, then it will
* not restart the event. * not restart the event.
*/ */
void __perf_event_task_sched_out(struct task_struct *task, static void __perf_event_task_sched_out(struct task_struct *task,
struct task_struct *next) struct task_struct *next)
{ {
int ctxn; int ctxn;
...@@ -2279,8 +2279,8 @@ static void perf_branch_stack_sched_in(struct task_struct *prev, ...@@ -2279,8 +2279,8 @@ static void perf_branch_stack_sched_in(struct task_struct *prev,
* accessing the event control register. If a NMI hits, then it will * accessing the event control register. If a NMI hits, then it will
* keep the event running. * keep the event running.
*/ */
void __perf_event_task_sched_in(struct task_struct *prev, static void __perf_event_task_sched_in(struct task_struct *prev,
struct task_struct *task) struct task_struct *task)
{ {
struct perf_event_context *ctx; struct perf_event_context *ctx;
int ctxn; int ctxn;
...@@ -2305,6 +2305,12 @@ void __perf_event_task_sched_in(struct task_struct *prev, ...@@ -2305,6 +2305,12 @@ void __perf_event_task_sched_in(struct task_struct *prev,
perf_branch_stack_sched_in(prev, task); perf_branch_stack_sched_in(prev, task);
} }
void __perf_event_task_sched(struct task_struct *prev, struct task_struct *next)
{
__perf_event_task_sched_out(prev, next);
__perf_event_task_sched_in(prev, next);
}
static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count) static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
{ {
u64 frequency = event->attr.sample_freq; u64 frequency = event->attr.sample_freq;
...@@ -4957,7 +4963,7 @@ void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) ...@@ -4957,7 +4963,7 @@ void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
if (rctx < 0) if (rctx < 0)
return; return;
perf_sample_data_init(&data, addr); perf_sample_data_init(&data, addr, 0);
do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
...@@ -5215,7 +5221,7 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, ...@@ -5215,7 +5221,7 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
.data = record, .data = record,
}; };
perf_sample_data_init(&data, addr); perf_sample_data_init(&data, addr, 0);
data.raw = &raw; data.raw = &raw;
hlist_for_each_entry_rcu(event, node, head, hlist_entry) { hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
...@@ -5318,7 +5324,7 @@ void perf_bp_event(struct perf_event *bp, void *data) ...@@ -5318,7 +5324,7 @@ void perf_bp_event(struct perf_event *bp, void *data)
struct perf_sample_data sample; struct perf_sample_data sample;
struct pt_regs *regs = data; struct pt_regs *regs = data;
perf_sample_data_init(&sample, bp->attr.bp_addr); perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
if (!bp->hw.state && !perf_exclude_event(bp, regs)) if (!bp->hw.state && !perf_exclude_event(bp, regs))
perf_swevent_event(bp, 1, &sample, regs); perf_swevent_event(bp, 1, &sample, regs);
...@@ -5344,13 +5350,12 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) ...@@ -5344,13 +5350,12 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
event->pmu->read(event); event->pmu->read(event);
perf_sample_data_init(&data, 0); perf_sample_data_init(&data, 0, event->hw.last_period);
data.period = event->hw.last_period;
regs = get_irq_regs(); regs = get_irq_regs();
if (regs && !perf_exclude_event(event, regs)) { if (regs && !perf_exclude_event(event, regs)) {
if (!(event->attr.exclude_idle && is_idle_task(current))) if (!(event->attr.exclude_idle && is_idle_task(current)))
if (perf_event_overflow(event, &data, regs)) if (__perf_event_overflow(event, 1, &data, regs))
ret = HRTIMER_NORESTART; ret = HRTIMER_NORESTART;
} }
......
...@@ -1914,7 +1914,7 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev, ...@@ -1914,7 +1914,7 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev,
struct task_struct *next) struct task_struct *next)
{ {
sched_info_switch(prev, next); sched_info_switch(prev, next);
perf_event_task_sched_out(prev, next); perf_event_task_sched(prev, next);
fire_sched_out_preempt_notifiers(prev, next); fire_sched_out_preempt_notifiers(prev, next);
prepare_lock_switch(rq, next); prepare_lock_switch(rq, next);
prepare_arch_switch(next); prepare_arch_switch(next);
...@@ -1957,13 +1957,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev) ...@@ -1957,13 +1957,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
*/ */
prev_state = prev->state; prev_state = prev->state;
finish_arch_switch(prev); finish_arch_switch(prev);
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
local_irq_disable();
#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
perf_event_task_sched_in(prev, current);
#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
local_irq_enable();
#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
finish_lock_switch(rq, prev); finish_lock_switch(rq, prev);
finish_arch_post_lock_switch(); finish_arch_post_lock_switch();
......
...@@ -141,7 +141,6 @@ if FTRACE ...@@ -141,7 +141,6 @@ if FTRACE
config FUNCTION_TRACER config FUNCTION_TRACER
bool "Kernel Function Tracer" bool "Kernel Function Tracer"
depends on HAVE_FUNCTION_TRACER depends on HAVE_FUNCTION_TRACER
select FRAME_POINTER if !ARM_UNWIND && !PPC && !S390 && !MICROBLAZE
select KALLSYMS select KALLSYMS
select GENERIC_TRACER select GENERIC_TRACER
select CONTEXT_SWITCH_TRACER select CONTEXT_SWITCH_TRACER
......
...@@ -1383,44 +1383,73 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip) ...@@ -1383,44 +1383,73 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
static int ftrace_cmp_recs(const void *a, const void *b) static int ftrace_cmp_recs(const void *a, const void *b)
{ {
const struct dyn_ftrace *reca = a; const struct dyn_ftrace *key = a;
const struct dyn_ftrace *recb = b; const struct dyn_ftrace *rec = b;
if (reca->ip > recb->ip) if (key->flags < rec->ip)
return 1;
if (reca->ip < recb->ip)
return -1; return -1;
if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
return 1;
return 0; return 0;
} }
/** static unsigned long ftrace_location_range(unsigned long start, unsigned long end)
* ftrace_location - return true if the ip giving is a traced location
* @ip: the instruction pointer to check
*
* Returns 1 if @ip given is a pointer to a ftrace location.
* That is, the instruction that is either a NOP or call to
* the function tracer. It checks the ftrace internal tables to
* determine if the address belongs or not.
*/
int ftrace_location(unsigned long ip)
{ {
struct ftrace_page *pg; struct ftrace_page *pg;
struct dyn_ftrace *rec; struct dyn_ftrace *rec;
struct dyn_ftrace key; struct dyn_ftrace key;
key.ip = ip; key.ip = start;
key.flags = end; /* overload flags, as it is unsigned long */
for (pg = ftrace_pages_start; pg; pg = pg->next) { for (pg = ftrace_pages_start; pg; pg = pg->next) {
if (end < pg->records[0].ip ||
start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
continue;
rec = bsearch(&key, pg->records, pg->index, rec = bsearch(&key, pg->records, pg->index,
sizeof(struct dyn_ftrace), sizeof(struct dyn_ftrace),
ftrace_cmp_recs); ftrace_cmp_recs);
if (rec) if (rec)
return 1; return rec->ip;
} }
return 0; return 0;
} }
/**
* ftrace_location - return true if the ip giving is a traced location
* @ip: the instruction pointer to check
*
* Returns rec->ip if @ip given is a pointer to a ftrace location.
* That is, the instruction that is either a NOP or call to
* the function tracer. It checks the ftrace internal tables to
* determine if the address belongs or not.
*/
unsigned long ftrace_location(unsigned long ip)
{
return ftrace_location_range(ip, ip);
}
/**
* ftrace_text_reserved - return true if range contains an ftrace location
* @start: start of range to search
* @end: end of range to search (inclusive). @end points to the last byte to check.
*
* Returns 1 if @start and @end contains a ftrace location.
* That is, the instruction that is either a NOP or call to
* the function tracer. It checks the ftrace internal tables to
* determine if the address belongs or not.
*/
int ftrace_text_reserved(void *start, void *end)
{
unsigned long ret;
ret = ftrace_location_range((unsigned long)start,
(unsigned long)end);
return (int)!!ret;
}
static void __ftrace_hash_rec_update(struct ftrace_ops *ops, static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
int filter_hash, int filter_hash,
bool inc) bool inc)
...@@ -1520,35 +1549,6 @@ static void ftrace_hash_rec_enable(struct ftrace_ops *ops, ...@@ -1520,35 +1549,6 @@ static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
__ftrace_hash_rec_update(ops, filter_hash, 1); __ftrace_hash_rec_update(ops, filter_hash, 1);
} }
static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
{
if (ftrace_pages->index == ftrace_pages->size) {
/* We should have allocated enough */
if (WARN_ON(!ftrace_pages->next))
return NULL;
ftrace_pages = ftrace_pages->next;
}
return &ftrace_pages->records[ftrace_pages->index++];
}
static struct dyn_ftrace *
ftrace_record_ip(unsigned long ip)
{
struct dyn_ftrace *rec;
if (ftrace_disabled)
return NULL;
rec = ftrace_alloc_dyn_node(ip);
if (!rec)
return NULL;
rec->ip = ip;
return rec;
}
static void print_ip_ins(const char *fmt, unsigned char *p) static void print_ip_ins(const char *fmt, unsigned char *p)
{ {
int i; int i;
...@@ -1598,21 +1598,6 @@ void ftrace_bug(int failed, unsigned long ip) ...@@ -1598,21 +1598,6 @@ void ftrace_bug(int failed, unsigned long ip)
} }
} }
/* Return 1 if the address range is reserved for ftrace */
int ftrace_text_reserved(void *start, void *end)
{
struct dyn_ftrace *rec;
struct ftrace_page *pg;
do_for_each_ftrace_rec(pg, rec) {
if (rec->ip <= (unsigned long)end &&
rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
return 1;
} while_for_each_ftrace_rec();
return 0;
}
static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update) static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
{ {
unsigned long flag = 0UL; unsigned long flag = 0UL;
...@@ -1698,7 +1683,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable) ...@@ -1698,7 +1683,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
return -1; /* unknow ftrace bug */ return -1; /* unknow ftrace bug */
} }
static void ftrace_replace_code(int update) void __weak ftrace_replace_code(int enable)
{ {
struct dyn_ftrace *rec; struct dyn_ftrace *rec;
struct ftrace_page *pg; struct ftrace_page *pg;
...@@ -1708,7 +1693,7 @@ static void ftrace_replace_code(int update) ...@@ -1708,7 +1693,7 @@ static void ftrace_replace_code(int update)
return; return;
do_for_each_ftrace_rec(pg, rec) { do_for_each_ftrace_rec(pg, rec) {
failed = __ftrace_replace_code(rec, update); failed = __ftrace_replace_code(rec, enable);
if (failed) { if (failed) {
ftrace_bug(failed, rec->ip); ftrace_bug(failed, rec->ip);
/* Stop processing */ /* Stop processing */
...@@ -1826,22 +1811,27 @@ int __weak ftrace_arch_code_modify_post_process(void) ...@@ -1826,22 +1811,27 @@ int __weak ftrace_arch_code_modify_post_process(void)
return 0; return 0;
} }
static int __ftrace_modify_code(void *data) void ftrace_modify_all_code(int command)
{ {
int *command = data; if (command & FTRACE_UPDATE_CALLS)
if (*command & FTRACE_UPDATE_CALLS)
ftrace_replace_code(1); ftrace_replace_code(1);
else if (*command & FTRACE_DISABLE_CALLS) else if (command & FTRACE_DISABLE_CALLS)
ftrace_replace_code(0); ftrace_replace_code(0);
if (*command & FTRACE_UPDATE_TRACE_FUNC) if (command & FTRACE_UPDATE_TRACE_FUNC)
ftrace_update_ftrace_func(ftrace_trace_function); ftrace_update_ftrace_func(ftrace_trace_function);
if (*command & FTRACE_START_FUNC_RET) if (command & FTRACE_START_FUNC_RET)
ftrace_enable_ftrace_graph_caller(); ftrace_enable_ftrace_graph_caller();
else if (*command & FTRACE_STOP_FUNC_RET) else if (command & FTRACE_STOP_FUNC_RET)
ftrace_disable_ftrace_graph_caller(); ftrace_disable_ftrace_graph_caller();
}
static int __ftrace_modify_code(void *data)
{
int *command = data;
ftrace_modify_all_code(*command);
return 0; return 0;
} }
...@@ -2469,57 +2459,35 @@ static int ...@@ -2469,57 +2459,35 @@ static int
ftrace_avail_open(struct inode *inode, struct file *file) ftrace_avail_open(struct inode *inode, struct file *file)
{ {
struct ftrace_iterator *iter; struct ftrace_iterator *iter;
int ret;
if (unlikely(ftrace_disabled)) if (unlikely(ftrace_disabled))
return -ENODEV; return -ENODEV;
iter = kzalloc(sizeof(*iter), GFP_KERNEL); iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
if (!iter) if (iter) {
return -ENOMEM; iter->pg = ftrace_pages_start;
iter->ops = &global_ops;
iter->pg = ftrace_pages_start;
iter->ops = &global_ops;
ret = seq_open(file, &show_ftrace_seq_ops);
if (!ret) {
struct seq_file *m = file->private_data;
m->private = iter;
} else {
kfree(iter);
} }
return ret; return iter ? 0 : -ENOMEM;
} }
static int static int
ftrace_enabled_open(struct inode *inode, struct file *file) ftrace_enabled_open(struct inode *inode, struct file *file)
{ {
struct ftrace_iterator *iter; struct ftrace_iterator *iter;
int ret;
if (unlikely(ftrace_disabled)) if (unlikely(ftrace_disabled))
return -ENODEV; return -ENODEV;
iter = kzalloc(sizeof(*iter), GFP_KERNEL); iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
if (!iter) if (iter) {
return -ENOMEM; iter->pg = ftrace_pages_start;
iter->flags = FTRACE_ITER_ENABLED;
iter->pg = ftrace_pages_start; iter->ops = &global_ops;
iter->flags = FTRACE_ITER_ENABLED;
iter->ops = &global_ops;
ret = seq_open(file, &show_ftrace_seq_ops);
if (!ret) {
struct seq_file *m = file->private_data;
m->private = iter;
} else {
kfree(iter);
} }
return ret; return iter ? 0 : -ENOMEM;
} }
static void ftrace_filter_reset(struct ftrace_hash *hash) static void ftrace_filter_reset(struct ftrace_hash *hash)
...@@ -3688,22 +3656,36 @@ static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer) ...@@ -3688,22 +3656,36 @@ static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
return 0; return 0;
} }
static void ftrace_swap_recs(void *a, void *b, int size) static int ftrace_cmp_ips(const void *a, const void *b)
{
const unsigned long *ipa = a;
const unsigned long *ipb = b;
if (*ipa > *ipb)
return 1;
if (*ipa < *ipb)
return -1;
return 0;
}
static void ftrace_swap_ips(void *a, void *b, int size)
{ {
struct dyn_ftrace *reca = a; unsigned long *ipa = a;
struct dyn_ftrace *recb = b; unsigned long *ipb = b;
struct dyn_ftrace t; unsigned long t;
t = *reca; t = *ipa;
*reca = *recb; *ipa = *ipb;
*recb = t; *ipb = t;
} }
static int ftrace_process_locs(struct module *mod, static int ftrace_process_locs(struct module *mod,
unsigned long *start, unsigned long *start,
unsigned long *end) unsigned long *end)
{ {
struct ftrace_page *start_pg;
struct ftrace_page *pg; struct ftrace_page *pg;
struct dyn_ftrace *rec;
unsigned long count; unsigned long count;
unsigned long *p; unsigned long *p;
unsigned long addr; unsigned long addr;
...@@ -3715,8 +3697,11 @@ static int ftrace_process_locs(struct module *mod, ...@@ -3715,8 +3697,11 @@ static int ftrace_process_locs(struct module *mod,
if (!count) if (!count)
return 0; return 0;
pg = ftrace_allocate_pages(count); sort(start, count, sizeof(*start),
if (!pg) ftrace_cmp_ips, ftrace_swap_ips);
start_pg = ftrace_allocate_pages(count);
if (!start_pg)
return -ENOMEM; return -ENOMEM;
mutex_lock(&ftrace_lock); mutex_lock(&ftrace_lock);
...@@ -3729,7 +3714,7 @@ static int ftrace_process_locs(struct module *mod, ...@@ -3729,7 +3714,7 @@ static int ftrace_process_locs(struct module *mod,
if (!mod) { if (!mod) {
WARN_ON(ftrace_pages || ftrace_pages_start); WARN_ON(ftrace_pages || ftrace_pages_start);
/* First initialization */ /* First initialization */
ftrace_pages = ftrace_pages_start = pg; ftrace_pages = ftrace_pages_start = start_pg;
} else { } else {
if (!ftrace_pages) if (!ftrace_pages)
goto out; goto out;
...@@ -3740,11 +3725,11 @@ static int ftrace_process_locs(struct module *mod, ...@@ -3740,11 +3725,11 @@ static int ftrace_process_locs(struct module *mod,
ftrace_pages = ftrace_pages->next; ftrace_pages = ftrace_pages->next;
} }
ftrace_pages->next = pg; ftrace_pages->next = start_pg;
ftrace_pages = pg;
} }
p = start; p = start;
pg = start_pg;
while (p < end) { while (p < end) {
addr = ftrace_call_adjust(*p++); addr = ftrace_call_adjust(*p++);
/* /*
...@@ -3755,17 +3740,26 @@ static int ftrace_process_locs(struct module *mod, ...@@ -3755,17 +3740,26 @@ static int ftrace_process_locs(struct module *mod,
*/ */
if (!addr) if (!addr)
continue; continue;
if (!ftrace_record_ip(addr))
break; if (pg->index == pg->size) {
/* We should have allocated enough */
if (WARN_ON(!pg->next))
break;
pg = pg->next;
}
rec = &pg->records[pg->index++];
rec->ip = addr;
} }
/* These new locations need to be initialized */ /* We should have used all pages */
ftrace_new_pgs = pg; WARN_ON(pg->next);
/* Assign the last page to ftrace_pages */
ftrace_pages = pg;
/* Make each individual set of pages sorted by ips */ /* These new locations need to be initialized */
for (; pg; pg = pg->next) ftrace_new_pgs = start_pg;
sort(pg->records, pg->index, sizeof(struct dyn_ftrace),
ftrace_cmp_recs, ftrace_swap_recs);
/* /*
* We only need to disable interrupts on start up * We only need to disable interrupts on start up
......
此差异已折叠。
此差异已折叠。
...@@ -131,6 +131,7 @@ struct trace_array_cpu { ...@@ -131,6 +131,7 @@ struct trace_array_cpu {
atomic_t disabled; atomic_t disabled;
void *buffer_page; /* ring buffer spare */ void *buffer_page; /* ring buffer spare */
unsigned long entries;
unsigned long saved_latency; unsigned long saved_latency;
unsigned long critical_start; unsigned long critical_start;
unsigned long critical_end; unsigned long critical_end;
...@@ -152,7 +153,6 @@ struct trace_array_cpu { ...@@ -152,7 +153,6 @@ struct trace_array_cpu {
*/ */
struct trace_array { struct trace_array {
struct ring_buffer *buffer; struct ring_buffer *buffer;
unsigned long entries;
int cpu; int cpu;
int buffer_disabled; int buffer_disabled;
cycle_t time_start; cycle_t time_start;
...@@ -826,6 +826,8 @@ extern struct list_head ftrace_events; ...@@ -826,6 +826,8 @@ extern struct list_head ftrace_events;
extern const char *__start___trace_bprintk_fmt[]; extern const char *__start___trace_bprintk_fmt[];
extern const char *__stop___trace_bprintk_fmt[]; extern const char *__stop___trace_bprintk_fmt[];
void trace_printk_init_buffers(void);
#undef FTRACE_ENTRY #undef FTRACE_ENTRY
#define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter) \ #define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter) \
extern struct ftrace_event_call \ extern struct ftrace_event_call \
......
...@@ -51,6 +51,10 @@ void hold_module_trace_bprintk_format(const char **start, const char **end) ...@@ -51,6 +51,10 @@ void hold_module_trace_bprintk_format(const char **start, const char **end)
const char **iter; const char **iter;
char *fmt; char *fmt;
/* allocate the trace_printk per cpu buffers */
if (start != end)
trace_printk_init_buffers();
mutex_lock(&btrace_mutex); mutex_lock(&btrace_mutex);
for (iter = start; iter < end; iter++) { for (iter = start; iter < end; iter++) {
struct trace_bprintk_fmt *tb_fmt = lookup_format(*iter); struct trace_bprintk_fmt *tb_fmt = lookup_format(*iter);
......
include scripts/Makefile.include
help:
@echo 'Possible targets:'
@echo ''
@echo ' cpupower - a tool for all things x86 CPU power'
@echo ' firewire - the userspace part of nosy, an IEEE-1394 traffic sniffer'
@echo ' lguest - a minimal 32-bit x86 hypervisor'
@echo ' perf - Linux performance measurement and analysis tool'
@echo ' selftests - various kernel selftests'
@echo ' turbostat - Intel CPU idle stats and freq reporting tool'
@echo ' usb - USB testing tools'
@echo ' virtio - vhost test module'
@echo ' vm - misc vm tools'
@echo ' x86_energy_perf_policy - Intel energy policy tool'
@echo ''
@echo 'You can do:'
@echo ' $$ make -C tools/<tool>_install'
@echo ''
@echo ' from the kernel command line to build and install one of'
@echo ' the tools above'
@echo ''
@echo ' $$ make tools/install'
@echo ''
@echo ' installs all tools.'
@echo ''
@echo 'Cleaning targets:'
@echo ''
@echo ' all of the above with the "_clean" string appended cleans'
@echo ' the respective build directory.'
@echo ' clean: a summary clean target to clean _all_ folders'
cpupower: FORCE
$(QUIET_SUBDIR0)power/$@/ $(QUIET_SUBDIR1)
firewire lguest perf usb virtio vm: FORCE
$(QUIET_SUBDIR0)$@/ $(QUIET_SUBDIR1)
selftests: FORCE
$(QUIET_SUBDIR0)testing/$@/ $(QUIET_SUBDIR1)
turbostat x86_energy_perf_policy: FORCE
$(QUIET_SUBDIR0)power/x86/$@/ $(QUIET_SUBDIR1)
cpupower_install:
$(QUIET_SUBDIR0)power/$(@:_install=)/ $(QUIET_SUBDIR1) install
firewire_install lguest_install perf_install usb_install virtio_install vm_install:
$(QUIET_SUBDIR0)$(@:_install=)/ $(QUIET_SUBDIR1) install
selftests_install:
$(QUIET_SUBDIR0)testing/$(@:_clean=)/ $(QUIET_SUBDIR1) install
turbostat_install x86_energy_perf_policy_install:
$(QUIET_SUBDIR0)power/x86/$(@:_install=)/ $(QUIET_SUBDIR1) install
install: cpupower_install firewire_install lguest_install perf_install \
selftests_install turbostat_install usb_install virtio_install \
vm_install x86_energy_perf_policy_install
cpupower_clean:
$(QUIET_SUBDIR0)power/cpupower/ $(QUIET_SUBDIR1) clean
firewire_clean lguest_clean perf_clean usb_clean virtio_clean vm_clean:
$(QUIET_SUBDIR0)$(@:_clean=)/ $(QUIET_SUBDIR1) clean
selftests_clean:
$(QUIET_SUBDIR0)testing/$(@:_clean=)/ $(QUIET_SUBDIR1) clean
turbostat_clean x86_energy_perf_policy_clean:
$(QUIET_SUBDIR0)power/x86/$(@:_clean=)/ $(QUIET_SUBDIR1) clean
clean: cpupower_clean firewire_clean lguest_clean perf_clean selftests_clean \
turbostat_clean usb_clean virtio_clean vm_clean \
x86_energy_perf_policy_clean
.PHONY: FORCE
# trace-cmd version
EP_VERSION = 1
EP_PATCHLEVEL = 1
EP_EXTRAVERSION = 0
# file format version
FILE_VERSION = 6
MAKEFLAGS += --no-print-directory
# Makefiles suck: This macro sets a default value of $(2) for the
# variable named by $(1), unless the variable has been set by
# environment or command line. This is necessary for CC and AR
# because make sets default values, so the simpler ?= approach
# won't work as expected.
define allow-override
$(if $(or $(findstring environment,$(origin $(1))),\
$(findstring command line,$(origin $(1)))),,\
$(eval $(1) = $(2)))
endef
# Allow setting CC and AR, or setting CROSS_COMPILE as a prefix.
$(call allow-override,CC,$(CROSS_COMPILE)gcc)
$(call allow-override,AR,$(CROSS_COMPILE)ar)
EXT = -std=gnu99
INSTALL = install
# Use DESTDIR for installing into a different root directory.
# This is useful for building a package. The program will be
# installed in this directory as if it was the root directory.
# Then the build tool can move it later.
DESTDIR ?=
DESTDIR_SQ = '$(subst ','\'',$(DESTDIR))'
prefix ?= /usr/local
bindir_relative = bin
bindir = $(prefix)/$(bindir_relative)
man_dir = $(prefix)/share/man
man_dir_SQ = '$(subst ','\'',$(man_dir))'
html_install = $(prefix)/share/kernelshark/html
html_install_SQ = '$(subst ','\'',$(html_install))'
img_install = $(prefix)/share/kernelshark/html/images
img_install_SQ = '$(subst ','\'',$(img_install))'
export man_dir man_dir_SQ html_install html_install_SQ INSTALL
export img_install img_install_SQ
export DESTDIR DESTDIR_SQ
# copy a bit from Linux kbuild
ifeq ("$(origin V)", "command line")
VERBOSE = $(V)
endif
ifndef VERBOSE
VERBOSE = 0
endif
ifeq ("$(origin O)", "command line")
BUILD_OUTPUT := $(O)
endif
ifeq ($(BUILD_SRC),)
ifneq ($(BUILD_OUTPUT),)
define build_output
$(if $(VERBOSE:1=),@)$(MAKE) -C $(BUILD_OUTPUT) \
BUILD_SRC=$(CURDIR) -f $(CURDIR)/Makefile $1
endef
saved-output := $(BUILD_OUTPUT)
BUILD_OUTPUT := $(shell cd $(BUILD_OUTPUT) && /bin/pwd)
$(if $(BUILD_OUTPUT),, \
$(error output directory "$(saved-output)" does not exist))
all: sub-make
gui: force
$(call build_output, all_cmd)
$(filter-out gui,$(MAKECMDGOALS)): sub-make
sub-make: force
$(call build_output, $(MAKECMDGOALS))
# Leave processing to above invocation of make
skip-makefile := 1
endif # BUILD_OUTPUT
endif # BUILD_SRC
# We process the rest of the Makefile if this is the final invocation of make
ifeq ($(skip-makefile),)
srctree := $(if $(BUILD_SRC),$(BUILD_SRC),$(CURDIR))
objtree := $(CURDIR)
src := $(srctree)
obj := $(objtree)
export prefix bindir src obj
# Shell quotes
bindir_SQ = $(subst ','\'',$(bindir))
bindir_relative_SQ = $(subst ','\'',$(bindir_relative))
LIB_FILE = libtraceevent.a libtraceevent.so
CONFIG_INCLUDES =
CONFIG_LIBS =
CONFIG_FLAGS =
VERSION = $(EP_VERSION)
PATCHLEVEL = $(EP_PATCHLEVEL)
EXTRAVERSION = $(EP_EXTRAVERSION)
OBJ = $@
N =
export Q VERBOSE
EVENT_PARSE_VERSION = $(EP_VERSION).$(EP_PATCHLEVEL).$(EP_EXTRAVERSION)
INCLUDES = -I. -I/usr/local/include $(CONFIG_INCLUDES)
# Set compile option CFLAGS if not set elsewhere
CFLAGS ?= -g -Wall
# Append required CFLAGS
override CFLAGS += $(CONFIG_FLAGS) $(INCLUDES) $(PLUGIN_DIR_SQ)
override CFLAGS += $(udis86-flags)
ifeq ($(VERBOSE),1)
Q =
print_compile =
print_app_build =
print_fpic_compile =
print_shared_lib_compile =
print_plugin_obj_compile =
print_plugin_build =
print_install =
else
Q = @
print_compile = echo ' CC '$(OBJ);
print_app_build = echo ' BUILD '$(OBJ);
print_fpic_compile = echo ' CC FPIC '$(OBJ);
print_shared_lib_compile = echo ' BUILD SHARED LIB '$(OBJ);
print_plugin_obj_compile = echo ' CC PLUGIN OBJ '$(OBJ);
print_plugin_build = echo ' CC PLUGI '$(OBJ);
print_static_lib_build = echo ' BUILD STATIC LIB '$(OBJ);
print_install = echo ' INSTALL '$1' to $(DESTDIR_SQ)$2';
endif
do_fpic_compile = \
($(print_fpic_compile) \
$(CC) -c $(CFLAGS) $(EXT) -fPIC $< -o $@)
do_app_build = \
($(print_app_build) \
$(CC) $^ -rdynamic -o $@ $(CONFIG_LIBS) $(LIBS))
do_compile_shared_library = \
($(print_shared_lib_compile) \
$(CC) --shared $^ -o $@)
do_compile_plugin_obj = \
($(print_plugin_obj_compile) \
$(CC) -c $(CFLAGS) -fPIC -o $@ $<)
do_plugin_build = \
($(print_plugin_build) \
$(CC) $(CFLAGS) -shared -nostartfiles -o $@ $<)
do_build_static_lib = \
($(print_static_lib_build) \
$(RM) $@; $(AR) rcs $@ $^)
define do_compile
$(print_compile) \
$(CC) -c $(CFLAGS) $(EXT) $< -o $(obj)/$@;
endef
$(obj)/%.o: $(src)/%.c
$(Q)$(call do_compile)
%.o: $(src)/%.c
$(Q)$(call do_compile)
PEVENT_LIB_OBJS = event-parse.o trace-seq.o parse-filter.o parse-utils.o
ALL_OBJS = $(PEVENT_LIB_OBJS)
CMD_TARGETS = $(LIB_FILE)
TARGETS = $(CMD_TARGETS)
all: all_cmd
all_cmd: $(CMD_TARGETS)
libtraceevent.so: $(PEVENT_LIB_OBJS)
$(Q)$(do_compile_shared_library)
libtraceevent.a: $(PEVENT_LIB_OBJS)
$(Q)$(do_build_static_lib)
$(PEVENT_LIB_OBJS): %.o: $(src)/%.c
$(Q)$(do_fpic_compile)
define make_version.h
(echo '/* This file is automatically generated. Do not modify. */'; \
echo \#define VERSION_CODE $(shell \
expr $(VERSION) \* 256 + $(PATCHLEVEL)); \
echo '#define EXTRAVERSION ' $(EXTRAVERSION); \
echo '#define VERSION_STRING "'$(VERSION).$(PATCHLEVEL).$(EXTRAVERSION)'"'; \
echo '#define FILE_VERSION '$(FILE_VERSION); \
) > $1
endef
define update_version.h
($(call make_version.h, $@.tmp); \
if [ -r $@ ] && cmp -s $@ $@.tmp; then \
rm -f $@.tmp; \
else \
echo ' UPDATE $@'; \
mv -f $@.tmp $@; \
fi);
endef
ep_version.h: force
$(Q)$(N)$(call update_version.h)
VERSION_FILES = ep_version.h
define update_dir
(echo $1 > $@.tmp; \
if [ -r $@ ] && cmp -s $@ $@.tmp; then \
rm -f $@.tmp; \
else \
echo ' UPDATE $@'; \
mv -f $@.tmp $@; \
fi);
endef
## make deps
all_objs := $(sort $(ALL_OBJS))
all_deps := $(all_objs:%.o=.%.d)
define check_deps
$(CC) -M $(CFLAGS) $< > $@;
endef
$(gui_deps): ks_version.h
$(non_gui_deps): tc_version.h
$(all_deps): .%.d: $(src)/%.c
$(Q)$(call check_deps)
$(all_objs) : %.o : .%.d
dep_includes := $(wildcard $(all_deps))
ifneq ($(dep_includes),)
include $(dep_includes)
endif
tags: force
$(RM) tags
find . -name '*.[ch]' | xargs ctags --extra=+f --c-kinds=+px
TAGS: force
$(RM) TAGS
find . -name '*.[ch]' | xargs etags
define do_install
$(print_install) \
if [ ! -d '$(DESTDIR_SQ)$2' ]; then \
$(INSTALL) -d -m 755 '$(DESTDIR_SQ)$2'; \
fi; \
$(INSTALL) $1 '$(DESTDIR_SQ)$2'
endef
install_lib: all_cmd install_plugins install_python
$(Q)$(call do_install,$(LIB_FILE),$(bindir_SQ))
install: install_lib
clean:
$(RM) *.o *~ $(TARGETS) *.a *.so $(VERSION_FILES).*.d
$(RM) tags TAGS
endif # skip-makefile
PHONY += force
force:
# Declare the contents of the .PHONY variable as phony. We keep that
# information in a variable so we can use it in if_changed and friends.
.PHONY: $(PHONY)
此差异已折叠。
此差异已折叠。
/*
* Copyright (C) 2010 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation;
* version 2.1 of the License (not later!)
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*/
#ifndef __UTIL_H
#define __UTIL_H
#include <ctype.h>
/* Can be overridden */
void die(const char *fmt, ...);
void *malloc_or_die(unsigned int size);
void warning(const char *fmt, ...);
void pr_stat(const char *fmt, ...);
void vpr_stat(const char *fmt, va_list ap);
/* Always available */
void __die(const char *fmt, ...);
void __warning(const char *fmt, ...);
void __pr_stat(const char *fmt, ...);
void __vdie(const char *fmt, ...);
void __vwarning(const char *fmt, ...);
void __vpr_stat(const char *fmt, ...);
static inline char *strim(char *string)
{
char *ret;
if (!string)
return NULL;
while (*string) {
if (!isspace(*string))
break;
string++;
}
ret = string;
string = ret + strlen(ret) - 1;
while (string > ret) {
if (!isspace(*string))
break;
string--;
}
string[1] = 0;
return ret;
}
static inline int has_text(const char *text)
{
if (!text)
return 0;
while (*text) {
if (!isspace(*text))
return 1;
text++;
}
return 0;
}
#endif
此差异已折叠。
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include <errno.h>
#define __weak __attribute__((weak))
void __vdie(const char *fmt, va_list ap)
{
int ret = errno;
if (errno)
perror("trace-cmd");
else
ret = -1;
fprintf(stderr, " ");
vfprintf(stderr, fmt, ap);
fprintf(stderr, "\n");
exit(ret);
}
void __die(const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
__vdie(fmt, ap);
va_end(ap);
}
void __weak die(const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
__vdie(fmt, ap);
va_end(ap);
}
void __vwarning(const char *fmt, va_list ap)
{
if (errno)
perror("trace-cmd");
errno = 0;
fprintf(stderr, " ");
vfprintf(stderr, fmt, ap);
fprintf(stderr, "\n");
}
void __warning(const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
__vwarning(fmt, ap);
va_end(ap);
}
void __weak warning(const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
__vwarning(fmt, ap);
va_end(ap);
}
void __vpr_stat(const char *fmt, va_list ap)
{
vprintf(fmt, ap);
printf("\n");
}
void __pr_stat(const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
__vpr_stat(fmt, ap);
va_end(ap);
}
void __weak vpr_stat(const char *fmt, va_list ap)
{
__vpr_stat(fmt, ap);
}
void __weak pr_stat(const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
__vpr_stat(fmt, ap);
va_end(ap);
}
void __weak *malloc_or_die(unsigned int size)
{
void *data;
data = malloc(size);
if (!data)
die("malloc");
return data;
}
/*
* Copyright (C) 2009 Red Hat Inc, Steven Rostedt <srostedt@redhat.com>
*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation;
* version 2.1 of the License (not later!)
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*
* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdarg.h>
#include "event-parse.h"
#include "event-utils.h"
/*
* The TRACE_SEQ_POISON is to catch the use of using
* a trace_seq structure after it was destroyed.
*/
#define TRACE_SEQ_POISON ((void *)0xdeadbeef)
#define TRACE_SEQ_CHECK(s) \
do { \
if ((s)->buffer == TRACE_SEQ_POISON) \
die("Usage of trace_seq after it was destroyed"); \
} while (0)
/**
* trace_seq_init - initialize the trace_seq structure
* @s: a pointer to the trace_seq structure to initialize
*/
void trace_seq_init(struct trace_seq *s)
{
s->len = 0;
s->readpos = 0;
s->buffer_size = TRACE_SEQ_BUF_SIZE;
s->buffer = malloc_or_die(s->buffer_size);
}
/**
* trace_seq_destroy - free up memory of a trace_seq
* @s: a pointer to the trace_seq to free the buffer
*
* Only frees the buffer, not the trace_seq struct itself.
*/
void trace_seq_destroy(struct trace_seq *s)
{
if (!s)
return;
TRACE_SEQ_CHECK(s);
free(s->buffer);
s->buffer = TRACE_SEQ_POISON;
}
static void expand_buffer(struct trace_seq *s)
{
s->buffer_size += TRACE_SEQ_BUF_SIZE;
s->buffer = realloc(s->buffer, s->buffer_size);
if (!s->buffer)
die("Can't allocate trace_seq buffer memory");
}
/**
* trace_seq_printf - sequence printing of trace information
* @s: trace sequence descriptor
* @fmt: printf format string
*
* It returns 0 if the trace oversizes the buffer's free
* space, 1 otherwise.
*
* The tracer may use either sequence operations or its own
* copy to user routines. To simplify formating of a trace
* trace_seq_printf is used to store strings into a special
* buffer (@s). Then the output may be either used by
* the sequencer or pulled into another buffer.
*/
int
trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
{
va_list ap;
int len;
int ret;
TRACE_SEQ_CHECK(s);
try_again:
len = (s->buffer_size - 1) - s->len;
va_start(ap, fmt);
ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
va_end(ap);
if (ret >= len) {
expand_buffer(s);
goto try_again;
}
s->len += ret;
return 1;
}
/**
* trace_seq_vprintf - sequence printing of trace information
* @s: trace sequence descriptor
* @fmt: printf format string
*
* The tracer may use either sequence operations or its own
* copy to user routines. To simplify formating of a trace
* trace_seq_printf is used to store strings into a special
* buffer (@s). Then the output may be either used by
* the sequencer or pulled into another buffer.
*/
int
trace_seq_vprintf(struct trace_seq *s, const char *fmt, va_list args)
{
int len;
int ret;
TRACE_SEQ_CHECK(s);
try_again:
len = (s->buffer_size - 1) - s->len;
ret = vsnprintf(s->buffer + s->len, len, fmt, args);
if (ret >= len) {
expand_buffer(s);
goto try_again;
}
s->len += ret;
return len;
}
/**
* trace_seq_puts - trace sequence printing of simple string
* @s: trace sequence descriptor
* @str: simple string to record
*
* The tracer may use either the sequence operations or its own
* copy to user routines. This function records a simple string
* into a special buffer (@s) for later retrieval by a sequencer
* or other mechanism.
*/
int trace_seq_puts(struct trace_seq *s, const char *str)
{
int len;
TRACE_SEQ_CHECK(s);
len = strlen(str);
while (len > ((s->buffer_size - 1) - s->len))
expand_buffer(s);
memcpy(s->buffer + s->len, str, len);
s->len += len;
return len;
}
int trace_seq_putc(struct trace_seq *s, unsigned char c)
{
TRACE_SEQ_CHECK(s);
while (s->len >= (s->buffer_size - 1))
expand_buffer(s);
s->buffer[s->len++] = c;
return 1;
}
void trace_seq_terminate(struct trace_seq *s)
{
TRACE_SEQ_CHECK(s);
/* There's always one character left on the buffer */
s->buffer[s->len] = 0;
}
int trace_seq_do_printf(struct trace_seq *s)
{
TRACE_SEQ_CHECK(s);
return printf("%.*s", s->len, s->buffer);
}
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
normal = black, lightgray normal = black, lightgray
selected = lightgray, magenta selected = lightgray, magenta
code = blue, lightgray code = blue, lightgray
addr = magenta, lightgray
[tui] [tui]
......
ifeq ("$(origin O)", "command line") include ../scripts/Makefile.include
OUTPUT := $(O)/
endif
# The default target of this Makefile is... # The default target of this Makefile is...
all: all:
include config/utilities.mak include config/utilities.mak
ifneq ($(OUTPUT),)
# check that the output directory actually exists
OUTDIR := $(shell cd $(OUTPUT) && /bin/pwd)
$(if $(OUTDIR),, $(error output directory "$(OUTPUT)" does not exist))
endif
# Define V to have a more verbose compile. # Define V to have a more verbose compile.
# #
# Define O to save output files in a separate directory. # Define O to save output files in a separate directory.
...@@ -84,31 +76,6 @@ ifneq ($(WERROR),0) ...@@ -84,31 +76,6 @@ ifneq ($(WERROR),0)
CFLAGS_WERROR := -Werror CFLAGS_WERROR := -Werror
endif endif
#
# Include saner warnings here, which can catch bugs:
#
EXTRA_WARNINGS := -Wformat
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wformat-security
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wformat-y2k
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wshadow
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Winit-self
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wpacked
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wredundant-decls
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wstrict-aliasing=3
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wswitch-default
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wswitch-enum
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wno-system-headers
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wundef
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wwrite-strings
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wbad-function-cast
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wmissing-declarations
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wmissing-prototypes
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wnested-externs
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wold-style-definition
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wstrict-prototypes
EXTRA_WARNINGS := $(EXTRA_WARNINGS) -Wdeclaration-after-statement
ifeq ("$(origin DEBUG)", "command line") ifeq ("$(origin DEBUG)", "command line")
PERF_DEBUG = $(DEBUG) PERF_DEBUG = $(DEBUG)
endif endif
...@@ -182,7 +149,7 @@ endif ...@@ -182,7 +149,7 @@ endif
### --- END CONFIGURATION SECTION --- ### --- END CONFIGURATION SECTION ---
BASIC_CFLAGS = -Iutil/include -Iarch/$(ARCH)/include -I$(OUTPUT)/util -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE BASIC_CFLAGS = -Iutil/include -Iarch/$(ARCH)/include -I$(OUTPUT)/util -I$(EVENT_PARSE_DIR) -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE
BASIC_LDFLAGS = BASIC_LDFLAGS =
# Guard against environment variables # Guard against environment variables
...@@ -211,6 +178,17 @@ $(OUTPUT)python/perf.so: $(PYRF_OBJS) $(PYTHON_EXT_SRCS) $(PYTHON_EXT_DEPS) ...@@ -211,6 +178,17 @@ $(OUTPUT)python/perf.so: $(PYRF_OBJS) $(PYTHON_EXT_SRCS) $(PYTHON_EXT_DEPS)
SCRIPTS = $(patsubst %.sh,%,$(SCRIPT_SH)) SCRIPTS = $(patsubst %.sh,%,$(SCRIPT_SH))
EVENT_PARSE_DIR = ../lib/traceevent/
ifeq ("$(origin O)", "command line")
EP_PATH=$(OUTPUT)/
else
EP_PATH=$(EVENT_PARSE_DIR)/
endif
LIBPARSEVENT = $(EP_PATH)libtraceevent.a
EP_LIB := -L$(EP_PATH) -ltraceevent
# #
# Single 'perf' binary right now: # Single 'perf' binary right now:
# #
...@@ -333,6 +311,8 @@ LIB_H += util/cpumap.h ...@@ -333,6 +311,8 @@ LIB_H += util/cpumap.h
LIB_H += util/top.h LIB_H += util/top.h
LIB_H += $(ARCH_INCLUDE) LIB_H += $(ARCH_INCLUDE)
LIB_H += util/cgroup.h LIB_H += util/cgroup.h
LIB_H += $(EVENT_PARSE_DIR)event-parse.h
LIB_H += util/target.h
LIB_OBJS += $(OUTPUT)util/abspath.o LIB_OBJS += $(OUTPUT)util/abspath.o
LIB_OBJS += $(OUTPUT)util/alias.o LIB_OBJS += $(OUTPUT)util/alias.o
...@@ -394,6 +374,7 @@ LIB_OBJS += $(OUTPUT)util/util.o ...@@ -394,6 +374,7 @@ LIB_OBJS += $(OUTPUT)util/util.o
LIB_OBJS += $(OUTPUT)util/xyarray.o LIB_OBJS += $(OUTPUT)util/xyarray.o
LIB_OBJS += $(OUTPUT)util/cpumap.o LIB_OBJS += $(OUTPUT)util/cpumap.o
LIB_OBJS += $(OUTPUT)util/cgroup.o LIB_OBJS += $(OUTPUT)util/cgroup.o
LIB_OBJS += $(OUTPUT)util/target.o
BUILTIN_OBJS += $(OUTPUT)builtin-annotate.o BUILTIN_OBJS += $(OUTPUT)builtin-annotate.o
...@@ -429,7 +410,7 @@ BUILTIN_OBJS += $(OUTPUT)builtin-kvm.o ...@@ -429,7 +410,7 @@ BUILTIN_OBJS += $(OUTPUT)builtin-kvm.o
BUILTIN_OBJS += $(OUTPUT)builtin-test.o BUILTIN_OBJS += $(OUTPUT)builtin-test.o
BUILTIN_OBJS += $(OUTPUT)builtin-inject.o BUILTIN_OBJS += $(OUTPUT)builtin-inject.o
PERFLIBS = $(LIB_FILE) PERFLIBS = $(LIB_FILE) $(LIBPARSEVENT)
# Files needed for the python binding, perf.so # Files needed for the python binding, perf.so
# pyrf is just an internal name needed for all those wrappers. # pyrf is just an internal name needed for all those wrappers.
...@@ -506,22 +487,23 @@ else ...@@ -506,22 +487,23 @@ else
# Fedora has /usr/include/slang/slang.h, but ubuntu /usr/include/slang.h # Fedora has /usr/include/slang/slang.h, but ubuntu /usr/include/slang.h
BASIC_CFLAGS += -I/usr/include/slang BASIC_CFLAGS += -I/usr/include/slang
EXTLIBS += -lnewt -lslang EXTLIBS += -lnewt -lslang
LIB_OBJS += $(OUTPUT)util/ui/setup.o LIB_OBJS += $(OUTPUT)ui/setup.o
LIB_OBJS += $(OUTPUT)util/ui/browser.o LIB_OBJS += $(OUTPUT)ui/browser.o
LIB_OBJS += $(OUTPUT)util/ui/browsers/annotate.o LIB_OBJS += $(OUTPUT)ui/browsers/annotate.o
LIB_OBJS += $(OUTPUT)util/ui/browsers/hists.o LIB_OBJS += $(OUTPUT)ui/browsers/hists.o
LIB_OBJS += $(OUTPUT)util/ui/browsers/map.o LIB_OBJS += $(OUTPUT)ui/browsers/map.o
LIB_OBJS += $(OUTPUT)util/ui/helpline.o LIB_OBJS += $(OUTPUT)ui/helpline.o
LIB_OBJS += $(OUTPUT)util/ui/progress.o LIB_OBJS += $(OUTPUT)ui/progress.o
LIB_OBJS += $(OUTPUT)util/ui/util.o LIB_OBJS += $(OUTPUT)ui/util.o
LIB_H += util/ui/browser.h LIB_OBJS += $(OUTPUT)ui/tui/setup.o
LIB_H += util/ui/browsers/map.h LIB_H += ui/browser.h
LIB_H += util/ui/helpline.h LIB_H += ui/browsers/map.h
LIB_H += util/ui/keysyms.h LIB_H += ui/helpline.h
LIB_H += util/ui/libslang.h LIB_H += ui/keysyms.h
LIB_H += util/ui/progress.h LIB_H += ui/libslang.h
LIB_H += util/ui/util.h LIB_H += ui/progress.h
LIB_H += util/ui/ui.h LIB_H += ui/util.h
LIB_H += ui/ui.h
endif endif
endif endif
...@@ -535,7 +517,12 @@ else ...@@ -535,7 +517,12 @@ else
else else
BASIC_CFLAGS += $(shell pkg-config --cflags gtk+-2.0) BASIC_CFLAGS += $(shell pkg-config --cflags gtk+-2.0)
EXTLIBS += $(shell pkg-config --libs gtk+-2.0) EXTLIBS += $(shell pkg-config --libs gtk+-2.0)
LIB_OBJS += $(OUTPUT)util/gtk/browser.o LIB_OBJS += $(OUTPUT)ui/gtk/browser.o
LIB_OBJS += $(OUTPUT)ui/gtk/setup.o
# Make sure that it'd be included only once.
ifneq ($(findstring -DNO_NEWT_SUPPORT,$(BASIC_CFLAGS)),)
LIB_OBJS += $(OUTPUT)ui/setup.o
endif
endif endif
endif endif
...@@ -678,18 +665,6 @@ else ...@@ -678,18 +665,6 @@ else
endif endif
endif endif
ifneq ($(findstring $(MAKEFLAGS),s),s)
ifndef V
QUIET_CC = @echo ' ' CC $@;
QUIET_AR = @echo ' ' AR $@;
QUIET_LINK = @echo ' ' LINK $@;
QUIET_MKDIR = @echo ' ' MKDIR $@;
QUIET_GEN = @echo ' ' GEN $@;
QUIET_FLEX = @echo ' ' FLEX $@;
QUIET_BISON = @echo ' ' BISON $@;
endif
endif
ifdef ASCIIDOC8 ifdef ASCIIDOC8
export ASCIIDOC8 export ASCIIDOC8
endif endif
...@@ -800,16 +775,16 @@ $(OUTPUT)util/exec_cmd.o: util/exec_cmd.c $(OUTPUT)PERF-CFLAGS ...@@ -800,16 +775,16 @@ $(OUTPUT)util/exec_cmd.o: util/exec_cmd.c $(OUTPUT)PERF-CFLAGS
$(OUTPUT)util/config.o: util/config.c $(OUTPUT)PERF-CFLAGS $(OUTPUT)util/config.o: util/config.c $(OUTPUT)PERF-CFLAGS
$(QUIET_CC)$(CC) -o $@ -c $(ALL_CFLAGS) -DETC_PERFCONFIG='"$(ETC_PERFCONFIG_SQ)"' $< $(QUIET_CC)$(CC) -o $@ -c $(ALL_CFLAGS) -DETC_PERFCONFIG='"$(ETC_PERFCONFIG_SQ)"' $<
$(OUTPUT)util/ui/browser.o: util/ui/browser.c $(OUTPUT)PERF-CFLAGS $(OUTPUT)ui/browser.o: ui/browser.c $(OUTPUT)PERF-CFLAGS
$(QUIET_CC)$(CC) -o $@ -c $(ALL_CFLAGS) -DENABLE_SLFUTURE_CONST $< $(QUIET_CC)$(CC) -o $@ -c $(ALL_CFLAGS) -DENABLE_SLFUTURE_CONST $<
$(OUTPUT)util/ui/browsers/annotate.o: util/ui/browsers/annotate.c $(OUTPUT)PERF-CFLAGS $(OUTPUT)ui/browsers/annotate.o: ui/browsers/annotate.c $(OUTPUT)PERF-CFLAGS
$(QUIET_CC)$(CC) -o $@ -c $(ALL_CFLAGS) -DENABLE_SLFUTURE_CONST $< $(QUIET_CC)$(CC) -o $@ -c $(ALL_CFLAGS) -DENABLE_SLFUTURE_CONST $<
$(OUTPUT)util/ui/browsers/hists.o: util/ui/browsers/hists.c $(OUTPUT)PERF-CFLAGS $(OUTPUT)ui/browsers/hists.o: ui/browsers/hists.c $(OUTPUT)PERF-CFLAGS
$(QUIET_CC)$(CC) -o $@ -c $(ALL_CFLAGS) -DENABLE_SLFUTURE_CONST $< $(QUIET_CC)$(CC) -o $@ -c $(ALL_CFLAGS) -DENABLE_SLFUTURE_CONST $<
$(OUTPUT)util/ui/browsers/map.o: util/ui/browsers/map.c $(OUTPUT)PERF-CFLAGS $(OUTPUT)ui/browsers/map.o: ui/browsers/map.c $(OUTPUT)PERF-CFLAGS
$(QUIET_CC)$(CC) -o $@ -c $(ALL_CFLAGS) -DENABLE_SLFUTURE_CONST $< $(QUIET_CC)$(CC) -o $@ -c $(ALL_CFLAGS) -DENABLE_SLFUTURE_CONST $<
$(OUTPUT)util/rbtree.o: ../../lib/rbtree.c $(OUTPUT)PERF-CFLAGS $(OUTPUT)util/rbtree.o: ../../lib/rbtree.c $(OUTPUT)PERF-CFLAGS
...@@ -844,6 +819,10 @@ $(sort $(dir $(DIRECTORY_DEPS))): ...@@ -844,6 +819,10 @@ $(sort $(dir $(DIRECTORY_DEPS))):
$(LIB_FILE): $(LIB_OBJS) $(LIB_FILE): $(LIB_OBJS)
$(QUIET_AR)$(RM) $@ && $(AR) rcs $@ $(LIB_OBJS) $(QUIET_AR)$(RM) $@ && $(AR) rcs $@ $(LIB_OBJS)
# libparsevent.a
$(LIBPARSEVENT):
make -C $(EVENT_PARSE_DIR) $(COMMAND_O) libtraceevent.a
help: help:
@echo 'Perf make targets:' @echo 'Perf make targets:'
@echo ' doc - make *all* documentation (see below)' @echo ' doc - make *all* documentation (see below)'
......
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
...@@ -261,7 +261,7 @@ static void print_sample_start(struct perf_sample *sample, ...@@ -261,7 +261,7 @@ static void print_sample_start(struct perf_sample *sample,
struct perf_event_attr *attr) struct perf_event_attr *attr)
{ {
int type; int type;
struct event *event; struct event_format *event;
const char *evname = NULL; const char *evname = NULL;
unsigned long secs; unsigned long secs;
unsigned long usecs; unsigned long usecs;
......
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
此差异已折叠。
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册