diff --git a/arch/arm64/include/asm/mpam.h b/arch/arm64/include/asm/mpam.h index de00c141065f7bedb187ca0d807ec55176cc8eb2..b83f940e04328efd011c3d3ad49f3d06defa219b 100644 --- a/arch/arm64/include/asm/mpam.h +++ b/arch/arm64/include/asm/mpam.h @@ -8,6 +8,7 @@ #include #include +#include /* MPAM register */ #define SYS_MPAM0_EL1 sys_reg(3, 0, 10, 5, 1) @@ -97,9 +98,11 @@ */ #define VPMR_MAX_BITS (3) #define PARTID_MAX_SHIFT (0) +#define PARTID_MAX_MASK (MPAM_MASK(PARTID_BITS) << PARTID_MAX_SHIFT) #define HAS_HCR_SHIFT (PARTID_MAX_SHIFT + PARTID_BITS + 1) #define VPMR_MAX_SHIFT (HAS_HCR_SHIFT + 1) #define PMG_MAX_SHIFT (VPMR_MAX_SHIFT + VPMR_MAX_BITS + 11) +#define PMG_MAX_MASK (MPAM_MASK(PMG_BITS) << PMG_MAX_SHIFT) #define VPMR_MASK MPAM_MASK(VPMR_MAX_BITS) /* diff --git a/arch/arm64/kernel/mpam/mpam_device.c b/arch/arm64/kernel/mpam/mpam_device.c index 269a99695e6caa82ff35250dca5065217a699ec9..36ee7bf9e86225f3938e2d867e9a5ea3d56550ef 100644 --- a/arch/arm64/kernel/mpam/mpam_device.c +++ b/arch/arm64/kernel/mpam/mpam_device.c @@ -29,6 +29,9 @@ #include #include #include +#include +#include +#include #include "mpam_device.h" @@ -45,6 +48,75 @@ static LIST_HEAD(mpam_all_devices); /* Classes are the set of MSCs that make up components of the same type. */ LIST_HEAD(mpam_classes); +static DEFINE_MUTEX(mpam_cpuhp_lock); +static int mpam_cpuhp_state; + + +static inline int mpam_cpu_online(unsigned int cpu); +static inline int mpam_cpu_offline(unsigned int cpu); + +static struct mpam_sysprops_prop mpam_sysprops; + +/* + * mpam is enabled once all devices have been probed from CPU online callbacks, + * scheduled via this work_struct. + */ +static struct work_struct mpam_enable_work; + +/* + * This gets set if something terrible happens, it prevents future attempts + * to configure devices. + */ +static int mpam_broken; +static struct work_struct mpam_failed_work; + +static int mpam_device_probe(struct mpam_device *dev) +{ + return 0; +} + +/* + * Enable mpam once all devices have been probed. + * Scheduled by mpam_discovery_complete() once all devices have been created. + * Also scheduled when new devices are probed when new CPUs come online. + */ +static void __init mpam_enable(struct work_struct *work) +{ + unsigned long flags; + struct mpam_device *dev; + bool all_devices_probed = true; + + /* Have we probed all the devices? */ + mutex_lock(&mpam_devices_lock); + list_for_each_entry(dev, &mpam_all_devices, glbl_list) { + spin_lock_irqsave(&dev->lock, flags); + if (!dev->probed) + all_devices_probed = false; + spin_unlock_irqrestore(&dev->lock, flags); + + if (!all_devices_probed) + break; + } + mutex_unlock(&mpam_devices_lock); + + if (!all_devices_probed) + return; +} + +static void mpam_failed(struct work_struct *work) +{ + /* + * Make it look like all CPUs are offline. This also resets the + * cpu default values and disables interrupts. + */ + mutex_lock(&mpam_cpuhp_lock); + if (mpam_cpuhp_state) { + cpuhp_remove_state(mpam_cpuhp_state); + mpam_cpuhp_state = 0; + } + mutex_unlock(&mpam_cpuhp_lock); +} + static struct mpam_device * __init mpam_device_alloc(struct mpam_component *comp) { @@ -242,6 +314,28 @@ static int mpam_cpus_have_feature(void) return 1; } +/* + * get max partid from reading SYS_MPAMIDR_EL1. + */ +static inline u16 mpam_cpu_max_partid(void) +{ + u64 reg; + + reg = mpam_read_sysreg_s(SYS_MPAMIDR_EL1, "SYS_MPAMIDR_EL1"); + return reg & PARTID_MAX_MASK; +} + +/* + * get max pmg from reading SYS_MPAMIDR_EL1. + */ +static inline u16 mpam_cpu_max_pmg(void) +{ + u64 reg; + + reg = mpam_read_sysreg_s(SYS_MPAMIDR_EL1, "SYS_MPAMIDR_EL1"); + return (reg & PMG_MAX_MASK) >> PMG_MAX_SHIFT; +} + /* * prepare for initializing devices. */ @@ -250,14 +344,139 @@ int __init mpam_discovery_start(void) if (!mpam_cpus_have_feature()) return -EOPNOTSUPP; + mpam_sysprops.max_partid = mpam_cpu_max_partid(); + mpam_sysprops.max_pmg = mpam_cpu_max_pmg(); + + INIT_WORK(&mpam_enable_work, mpam_enable); + INIT_WORK(&mpam_failed_work, mpam_failed); + return 0; } -int __init mpam_discovery_complete(void) +static int __online_devices(struct mpam_component *comp, int cpu) +{ + int err = 0; + unsigned long flags; + struct mpam_device *dev; + bool new_device_probed = false; + + list_for_each_entry(dev, &comp->devices, comp_list) { + if (!cpumask_test_cpu(cpu, &dev->fw_affinity)) + continue; + + spin_lock_irqsave(&dev->lock, flags); + if (!dev->probed) { + err = mpam_device_probe(dev); + if (!err) + new_device_probed = true; + } + + cpumask_set_cpu(cpu, &dev->online_affinity); + spin_unlock_irqrestore(&dev->lock, flags); + + if (err) + return err; + } + + if (new_device_probed) + return 1; + + return 0; +} + +/* + * Firmware didn't give us an affinity, but a cache-id, if this cpu has that + * cache-id, update the fw_affinity for this component. + */ +static void +mpam_sync_cpu_cache_component_fw_affinity(struct mpam_class *class, int cpu) +{ + int cpu_cache_id; + struct mpam_component *comp; + + lockdep_assert_held(&mpam_devices_lock); /* we modify mpam_sysprops */ + + if (class->type != MPAM_CLASS_CACHE) + return; + + cpu_cache_id = cpu_to_node(cpu); + comp = mpam_component_get(class, cpu_cache_id, false); + + /* This cpu does not have a component of this class */ + if (IS_ERR(comp)) + return; + + cpumask_set_cpu(cpu, &comp->fw_affinity); + cpumask_set_cpu(cpu, &class->fw_affinity); +} + +static int mpam_cpu_online(unsigned int cpu) { + int err = 0; + struct mpam_class *class; + struct mpam_component *comp; + bool new_device_probed = false; + + mutex_lock(&mpam_devices_lock); + + list_for_each_entry(class, &mpam_classes, classes_list) { + mpam_sync_cpu_cache_component_fw_affinity(class, cpu); + + list_for_each_entry(comp, &class->components, class_list) { + if (!cpumask_test_cpu(cpu, &comp->fw_affinity)) + continue; + + err = __online_devices(comp, cpu); + if (err > 0) + new_device_probed = true; + if (err < 0) + break; // mpam_broken + } + } + + if (new_device_probed && err >= 0) + schedule_work(&mpam_enable_work); + + mutex_unlock(&mpam_devices_lock); + if (err < 0) { + if (!cmpxchg(&mpam_broken, err, 0)) + schedule_work(&mpam_failed_work); + return err; + } + return 0; } +static int mpam_cpu_offline(unsigned int cpu) +{ + struct mpam_device *dev; + + mutex_lock(&mpam_devices_lock); + list_for_each_entry(dev, &mpam_all_devices, glbl_list) + cpumask_clear_cpu(cpu, &dev->online_affinity); + + mutex_unlock(&mpam_devices_lock); + + return 0; +} + +int __init mpam_discovery_complete(void) +{ + int ret = 0; + + mutex_lock(&mpam_cpuhp_lock); + mpam_cpuhp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, + "mpam:online", mpam_cpu_online, + mpam_cpu_offline); + if (mpam_cpuhp_state <= 0) { + pr_err("Failed to register 'dyn' cpuhp callbacks"); + ret = -EINVAL; + } + mutex_unlock(&mpam_cpuhp_lock); + + return ret; +} + void __init mpam_discovery_failed(void) { struct mpam_class *class, *tmp; diff --git a/arch/arm64/kernel/mpam/mpam_device.h b/arch/arm64/kernel/mpam/mpam_device.h index ab986fb1911f382285393faa6226b0fdbcfe2d7e..7b8d9ae5a5484f6a8205a82d90b76d90174e0bce 100644 --- a/arch/arm64/kernel/mpam/mpam_device.h +++ b/arch/arm64/kernel/mpam/mpam_device.h @@ -89,4 +89,11 @@ struct mpam_class { struct list_head classes_list; }; +/* System wide properties */ +struct mpam_sysprops_prop { + u32 mpam_llc_size; + u16 max_partid; + u16 max_pmg; +}; + #endif /* _ASM_ARM64_MPAM_DEVICE_H */