/* * Intel Cache Quality-of-Service Monitoring (CQM) support. * * Based very, very heavily on work by Peter Zijlstra. */ #include #include #include #include "perf_event.h" #define MSR_IA32_PQR_ASSOC 0x0c8f #define MSR_IA32_QM_CTR 0x0c8e #define MSR_IA32_QM_EVTSEL 0x0c8d static unsigned int cqm_max_rmid = -1; static unsigned int cqm_l3_scale; /* supposedly cacheline size */ struct intel_cqm_state { raw_spinlock_t lock; int rmid; int cnt; }; static DEFINE_PER_CPU(struct intel_cqm_state, cqm_state); /* * Protects cache_cgroups and cqm_rmid_lru. */ static DEFINE_MUTEX(cache_mutex); /* * Groups of events that have the same target(s), one RMID per group. */ static LIST_HEAD(cache_groups); /* * Mask of CPUs for reading CQM values. We only need one per-socket. */ static cpumask_t cqm_cpumask; #define RMID_VAL_ERROR (1ULL << 63) #define RMID_VAL_UNAVAIL (1ULL << 62) #define QOS_L3_OCCUP_EVENT_ID (1 << 0) #define QOS_EVENT_MASK QOS_L3_OCCUP_EVENT_ID static u64 __rmid_read(unsigned long rmid) { u64 val; /* * Ignore the SDM, this thing is _NOTHING_ like a regular perfcnt, * it just says that to increase confusion. */ wrmsr(MSR_IA32_QM_EVTSEL, QOS_L3_OCCUP_EVENT_ID, rmid); rdmsrl(MSR_IA32_QM_CTR, val); /* * Aside from the ERROR and UNAVAIL bits, assume this thing returns * the number of cachelines tagged with @rmid. */ return val; } struct cqm_rmid_entry { u64 rmid; struct list_head list; }; /* * A least recently used list of RMIDs. * * Oldest entry at the head, newest (most recently used) entry at the * tail. This list is never traversed, it's only used to keep track of * the lru order. That is, we only pick entries of the head or insert * them on the tail. * * All entries on the list are 'free', and their RMIDs are not currently * in use. To mark an RMID as in use, remove its entry from the lru * list. * * This list is protected by cache_mutex. */ static LIST_HEAD(cqm_rmid_lru); /* * We use a simple array of pointers so that we can lookup a struct * cqm_rmid_entry in O(1). This alleviates the callers of __get_rmid() * and __put_rmid() from having to worry about dealing with struct * cqm_rmid_entry - they just deal with rmids, i.e. integers. * * Once this array is initialized it is read-only. No locks are required * to access it. * * All entries for all RMIDs can be looked up in the this array at all * times. */ static struct cqm_rmid_entry **cqm_rmid_ptrs; static inline struct cqm_rmid_entry *__rmid_entry(int rmid) { struct cqm_rmid_entry *entry; entry = cqm_rmid_ptrs[rmid]; WARN_ON(entry->rmid != rmid); return entry; } /* * Returns < 0 on fail. * * We expect to be called with cache_mutex held. */ static int __get_rmid(void) { struct cqm_rmid_entry *entry; lockdep_assert_held(&cache_mutex); if (list_empty(&cqm_rmid_lru)) return -EAGAIN; entry = list_first_entry(&cqm_rmid_lru, struct cqm_rmid_entry, list); list_del(&entry->list); return entry->rmid; } static void __put_rmid(int rmid) { struct cqm_rmid_entry *entry; lockdep_assert_held(&cache_mutex); entry = __rmid_entry(rmid); list_add_tail(&entry->list, &cqm_rmid_lru); } static int intel_cqm_setup_rmid_cache(void) { struct cqm_rmid_entry *entry; int r; cqm_rmid_ptrs = kmalloc(sizeof(struct cqm_rmid_entry *) * (cqm_max_rmid + 1), GFP_KERNEL); if (!cqm_rmid_ptrs) return -ENOMEM; for (r = 0; r <= cqm_max_rmid; r++) { struct cqm_rmid_entry *entry; entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto fail; INIT_LIST_HEAD(&entry->list); entry->rmid = r; cqm_rmid_ptrs[r] = entry; list_add_tail(&entry->list, &cqm_rmid_lru); } /* * RMID 0 is special and is always allocated. It's used for all * tasks that are not monitored. */ entry = __rmid_entry(0); list_del(&entry->list); return 0; fail: while (r--) kfree(cqm_rmid_ptrs[r]); kfree(cqm_rmid_ptrs); return -ENOMEM; } /* * Determine if @a and @b measure the same set of tasks. */ static bool __match_event(struct perf_event *a, struct perf_event *b) { if ((a->attach_state & PERF_ATTACH_TASK) != (b->attach_state & PERF_ATTACH_TASK)) return false; /* not task */ return true; /* if not task, we're machine wide */ } /* * Determine if @a's tasks intersect with @b's tasks */ static bool __conflict_event(struct perf_event *a, struct perf_event *b) { /* * If one of them is not a task, same story as above with cgroups. */ if (!(a->attach_state & PERF_ATTACH_TASK) || !(b->attach_state & PERF_ATTACH_TASK)) return true; /* * Must be non-overlapping. */ return false; } /* * Find a group and setup RMID. * * If we're part of a group, we use the group's RMID. */ static int intel_cqm_setup_event(struct perf_event *event, struct perf_event **group) { struct perf_event *iter; int rmid; list_for_each_entry(iter, &cache_groups, hw.cqm_groups_entry) { if (__match_event(iter, event)) { /* All tasks in a group share an RMID */ event->hw.cqm_rmid = iter->hw.cqm_rmid; *group = iter; return 0; } if (__conflict_event(iter, event)) return -EBUSY; } rmid = __get_rmid(); if (rmid < 0) return rmid; event->hw.cqm_rmid = rmid; return 0; } static void intel_cqm_event_read(struct perf_event *event) { unsigned long rmid = event->hw.cqm_rmid; u64 val; val = __rmid_read(rmid); /* * Ignore this reading on error states and do not update the value. */ if (val & (RMID_VAL_ERROR | RMID_VAL_UNAVAIL)) return; local64_set(&event->count, val); } static void intel_cqm_event_start(struct perf_event *event, int mode) { struct intel_cqm_state *state = this_cpu_ptr(&cqm_state); unsigned long rmid = event->hw.cqm_rmid; unsigned long flags; if (!(event->hw.cqm_state & PERF_HES_STOPPED)) return; event->hw.cqm_state &= ~PERF_HES_STOPPED; raw_spin_lock_irqsave(&state->lock, flags); if (state->cnt++) WARN_ON_ONCE(state->rmid != rmid); else WARN_ON_ONCE(state->rmid); state->rmid = rmid; wrmsrl(MSR_IA32_PQR_ASSOC, state->rmid); raw_spin_unlock_irqrestore(&state->lock, flags); } static void intel_cqm_event_stop(struct perf_event *event, int mode) { struct intel_cqm_state *state = this_cpu_ptr(&cqm_state); unsigned long flags; if (event->hw.cqm_state & PERF_HES_STOPPED) return; event->hw.cqm_state |= PERF_HES_STOPPED; raw_spin_lock_irqsave(&state->lock, flags); intel_cqm_event_read(event); if (!--state->cnt) { state->rmid = 0; wrmsrl(MSR_IA32_PQR_ASSOC, 0); } else { WARN_ON_ONCE(!state->rmid); } raw_spin_unlock_irqrestore(&state->lock, flags); } static int intel_cqm_event_add(struct perf_event *event, int mode) { int rmid; event->hw.cqm_state = PERF_HES_STOPPED; rmid = event->hw.cqm_rmid; WARN_ON_ONCE(!rmid); if (mode & PERF_EF_START) intel_cqm_event_start(event, mode); return 0; } static void intel_cqm_event_del(struct perf_event *event, int mode) { intel_cqm_event_stop(event, mode); } static void intel_cqm_event_destroy(struct perf_event *event) { struct perf_event *group_other = NULL; mutex_lock(&cache_mutex); /* * If there's another event in this group... */ if (!list_empty(&event->hw.cqm_group_entry)) { group_other = list_first_entry(&event->hw.cqm_group_entry, struct perf_event, hw.cqm_group_entry); list_del(&event->hw.cqm_group_entry); } /* * And we're the group leader.. */ if (!list_empty(&event->hw.cqm_groups_entry)) { /* * If there was a group_other, make that leader, otherwise * destroy the group and return the RMID. */ if (group_other) { list_replace(&event->hw.cqm_groups_entry, &group_other->hw.cqm_groups_entry); } else { int rmid = event->hw.cqm_rmid; __put_rmid(rmid); list_del(&event->hw.cqm_groups_entry); } } mutex_unlock(&cache_mutex); } static struct pmu intel_cqm_pmu; /* * XXX there's a bit of a problem in that we cannot simply do the one * event per node as one would want, since that one event would one get * scheduled on the one cpu. But we want to 'schedule' the RMID on all * CPUs. * * This means we want events for each CPU, however, that generates a lot * of duplicate values out to userspace -- this is not to be helped * unless we want to change the core code in some way. Fore more info, * see intel_cqm_event_read(). */ static int intel_cqm_event_init(struct perf_event *event) { struct perf_event *group = NULL; int err; if (event->attr.type != intel_cqm_pmu.type) return -ENOENT; if (event->attr.config & ~QOS_EVENT_MASK) return -EINVAL; if (event->cpu == -1) return -EINVAL; /* unsupported modes and filters */ if (event->attr.exclude_user || event->attr.exclude_kernel || event->attr.exclude_hv || event->attr.exclude_idle || event->attr.exclude_host || event->attr.exclude_guest || event->attr.sample_period) /* no sampling */ return -EINVAL; INIT_LIST_HEAD(&event->hw.cqm_group_entry); INIT_LIST_HEAD(&event->hw.cqm_groups_entry); event->destroy = intel_cqm_event_destroy; mutex_lock(&cache_mutex); err = intel_cqm_setup_event(event, &group); /* will also set rmid */ if (err) goto out; if (group) { list_add_tail(&event->hw.cqm_group_entry, &group->hw.cqm_group_entry); } else { list_add_tail(&event->hw.cqm_groups_entry, &cache_groups); } out: mutex_unlock(&cache_mutex); return err; } EVENT_ATTR_STR(llc_occupancy, intel_cqm_llc, "event=0x01"); EVENT_ATTR_STR(llc_occupancy.per-pkg, intel_cqm_llc_pkg, "1"); EVENT_ATTR_STR(llc_occupancy.unit, intel_cqm_llc_unit, "Bytes"); EVENT_ATTR_STR(llc_occupancy.scale, intel_cqm_llc_scale, NULL); EVENT_ATTR_STR(llc_occupancy.snapshot, intel_cqm_llc_snapshot, "1"); static struct attribute *intel_cqm_events_attr[] = { EVENT_PTR(intel_cqm_llc), EVENT_PTR(intel_cqm_llc_pkg), EVENT_PTR(intel_cqm_llc_unit), EVENT_PTR(intel_cqm_llc_scale), EVENT_PTR(intel_cqm_llc_snapshot), NULL, }; static struct attribute_group intel_cqm_events_group = { .name = "events", .attrs = intel_cqm_events_attr, }; PMU_FORMAT_ATTR(event, "config:0-7"); static struct attribute *intel_cqm_formats_attr[] = { &format_attr_event.attr, NULL, }; static struct attribute_group intel_cqm_format_group = { .name = "format", .attrs = intel_cqm_formats_attr, }; static const struct attribute_group *intel_cqm_attr_groups[] = { &intel_cqm_events_group, &intel_cqm_format_group, NULL, }; static struct pmu intel_cqm_pmu = { .attr_groups = intel_cqm_attr_groups, .task_ctx_nr = perf_sw_context, .event_init = intel_cqm_event_init, .add = intel_cqm_event_add, .del = intel_cqm_event_del, .start = intel_cqm_event_start, .stop = intel_cqm_event_stop, .read = intel_cqm_event_read, }; static inline void cqm_pick_event_reader(int cpu) { int phys_id = topology_physical_package_id(cpu); int i; for_each_cpu(i, &cqm_cpumask) { if (phys_id == topology_physical_package_id(i)) return; /* already got reader for this socket */ } cpumask_set_cpu(cpu, &cqm_cpumask); } static void intel_cqm_cpu_prepare(unsigned int cpu) { struct intel_cqm_state *state = &per_cpu(cqm_state, cpu); struct cpuinfo_x86 *c = &cpu_data(cpu); raw_spin_lock_init(&state->lock); state->rmid = 0; state->cnt = 0; WARN_ON(c->x86_cache_max_rmid != cqm_max_rmid); WARN_ON(c->x86_cache_occ_scale != cqm_l3_scale); } static void intel_cqm_cpu_exit(unsigned int cpu) { int phys_id = topology_physical_package_id(cpu); int i; /* * Is @cpu a designated cqm reader? */ if (!cpumask_test_and_clear_cpu(cpu, &cqm_cpumask)) return; for_each_online_cpu(i) { if (i == cpu) continue; if (phys_id == topology_physical_package_id(i)) { cpumask_set_cpu(i, &cqm_cpumask); break; } } } static int intel_cqm_cpu_notifier(struct notifier_block *nb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; switch (action & ~CPU_TASKS_FROZEN) { case CPU_UP_PREPARE: intel_cqm_cpu_prepare(cpu); break; case CPU_DOWN_PREPARE: intel_cqm_cpu_exit(cpu); break; case CPU_STARTING: cqm_pick_event_reader(cpu); break; } return NOTIFY_OK; } static const struct x86_cpu_id intel_cqm_match[] = { { .vendor = X86_VENDOR_INTEL, .feature = X86_FEATURE_CQM_OCCUP_LLC }, {} }; static int __init intel_cqm_init(void) { char *str, scale[20]; int i, cpu, ret; if (!x86_match_cpu(intel_cqm_match)) return -ENODEV; cqm_l3_scale = boot_cpu_data.x86_cache_occ_scale; /* * It's possible that not all resources support the same number * of RMIDs. Instead of making scheduling much more complicated * (where we have to match a task's RMID to a cpu that supports * that many RMIDs) just find the minimum RMIDs supported across * all cpus. * * Also, check that the scales match on all cpus. */ cpu_notifier_register_begin(); for_each_online_cpu(cpu) { struct cpuinfo_x86 *c = &cpu_data(cpu); if (c->x86_cache_max_rmid < cqm_max_rmid) cqm_max_rmid = c->x86_cache_max_rmid; if (c->x86_cache_occ_scale != cqm_l3_scale) { pr_err("Multiple LLC scale values, disabling\n"); ret = -EINVAL; goto out; } } snprintf(scale, sizeof(scale), "%u", cqm_l3_scale); str = kstrdup(scale, GFP_KERNEL); if (!str) { ret = -ENOMEM; goto out; } event_attr_intel_cqm_llc_scale.event_str = str; ret = intel_cqm_setup_rmid_cache(); if (ret) goto out; for_each_online_cpu(i) { intel_cqm_cpu_prepare(i); cqm_pick_event_reader(i); } __perf_cpu_notifier(intel_cqm_cpu_notifier); ret = perf_pmu_register(&intel_cqm_pmu, "intel_cqm", -1); if (ret) pr_err("Intel CQM perf registration failed: %d\n", ret); else pr_info("Intel CQM monitoring enabled\n"); out: cpu_notifier_register_done(); return ret; } device_initcall(intel_cqm_init);