diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 536c210836edd44727b955678254c23de5bed3c8..07c49be426c793d4cccad3ebfae3f1bf2548740d 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -141,6 +141,7 @@ CONFIG_CGROUP_SCHED=y CONFIG_QOS_SCHED=y # CONFIG_QOS_SCHED_SMT_EXPELLER is not set CONFIG_FAIR_GROUP_SCHED=y +CONFIG_QOS_SCHED_PRIO_LB=y CONFIG_CFS_BANDWIDTH=y CONFIG_RT_GROUP_SCHED=y CONFIG_CGROUP_PIDS=y diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig index bea07d020e1c0edcd3dd1c92aedff9c42314b5c3..4c0ff69b14bc1d94801cc6ee57b9b61a1da5ac96 100644 --- a/arch/x86/configs/openeuler_defconfig +++ b/arch/x86/configs/openeuler_defconfig @@ -160,6 +160,7 @@ CONFIG_CGROUP_SCHED=y CONFIG_QOS_SCHED=y # CONFIG_QOS_SCHED_SMT_EXPELLER is not set CONFIG_FAIR_GROUP_SCHED=y +CONFIG_QOS_SCHED_PRIO_LB=y CONFIG_CFS_BANDWIDTH=y CONFIG_RT_GROUP_SCHED=y CONFIG_CGROUP_PIDS=y diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index e3aec81fd92de8002e1722963a2351ff3d649155..0a25112d83b5274f62dc2796269ca12116a663e6 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h @@ -79,6 +79,10 @@ extern unsigned int sysctl_overload_detect_period; extern unsigned int sysctl_offline_wait_interval; #endif +#ifdef CONFIG_QOS_SCHED_PRIO_LB +extern unsigned int sysctl_sched_prio_load_balance_enabled; +#endif + #ifdef CONFIG_SCHED_AUTOGROUP extern unsigned int sysctl_sched_autogroup_enabled; #endif diff --git a/init/Kconfig b/init/Kconfig index b137a6c043ac6d0d9ac36e4cfef3a2aa54dca752..85225fd2dfe4181c33b7d7f61aa28060790073e3 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -986,6 +986,15 @@ config QOS_SCHED_SMT_EXPELLER This feature enable online tasks to expel offline tasks on the smt sibling cpus, and exclusively occupy CPU resources. +config QOS_SCHED_PRIO_LB + bool "Priority load balance for Qos scheduler" + depends on QOS_SCHED + default n + help + This feature enable priority load balance + for Qos scheduler, which prefer migrating online tasks + and migrating offline tasks secondly between CPUs. + config FAIR_GROUP_SCHED bool "Group scheduling for SCHED_OTHER" depends on CGROUP_SCHED diff --git a/kernel/sched/core.c b/kernel/sched/core.c index a2ea3b0ab334f7d110b5407fab145a83c095ec0e..b67d65bfddce02b59acb1c822283593c699f620f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -8273,6 +8273,9 @@ void __init sched_init(void) rq->max_idle_balance_cost = sysctl_sched_migration_cost; INIT_LIST_HEAD(&rq->cfs_tasks); +#ifdef CONFIG_QOS_SCHED_PRIO_LB + INIT_LIST_HEAD(&rq->cfs_offline_tasks); +#endif rq_attach_root(rq, &def_root_domain); #ifdef CONFIG_NO_HZ_COMMON diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index d3c4b945c0196204c80ff36f51189171d5592768..fe08832d38d83ab42b73de817e2d6857334ab678 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -139,6 +139,10 @@ unsigned int sysctl_offline_wait_interval = 100; /* in ms */ static int unthrottle_qos_cfs_rqs(int cpu); #endif +#ifdef CONFIG_QOS_SCHED_PRIO_LB +unsigned int sysctl_sched_prio_load_balance_enabled; +#endif + #ifdef CONFIG_QOS_SCHED_SMT_EXPELLER static DEFINE_PER_CPU(int, qos_smt_status); #endif @@ -2971,6 +2975,21 @@ static inline void update_scan_period(struct task_struct *p, int new_cpu) #endif /* CONFIG_NUMA_BALANCING */ +#ifdef CONFIG_QOS_SCHED_PRIO_LB +static void +adjust_rq_cfs_tasks(void (*list_op)(struct list_head *, struct list_head *), + struct rq *rq, + struct sched_entity *se) +{ + struct task_group *tg = task_group(task_of(se)); + + if (sysctl_sched_prio_load_balance_enabled && tg->qos_level == -1) + (*list_op)(&se->group_node, &rq->cfs_offline_tasks); + else + (*list_op)(&se->group_node, &rq->cfs_tasks); +} +#endif + static void account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { @@ -2980,7 +2999,11 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) struct rq *rq = rq_of(cfs_rq); account_numa_enqueue(rq, task_of(se)); +#ifdef CONFIG_QOS_SCHED_PRIO_LB + adjust_rq_cfs_tasks(list_add, rq, se); +#else list_add(&se->group_node, &rq->cfs_tasks); +#endif } #endif cfs_rq->nr_running++; @@ -7879,7 +7902,11 @@ done: __maybe_unused; * the list, so our cfs_tasks list becomes MRU * one. */ +#ifdef CONFIG_QOS_SCHED_PRIO_LB + adjust_rq_cfs_tasks(list_move, rq, &p->se); +#else list_move(&p->se.group_node, &rq->cfs_tasks); +#endif #endif if (hrtick_enabled(rq)) @@ -8249,6 +8276,14 @@ static int task_hot(struct task_struct *p, struct lb_env *env) &p->se == cfs_rq_of(&p->se)->last)) return 1; +#ifdef CONFIG_QOS_SCHED_PRIO_LB + /* Preempt sched idle cpu do not consider migration cost */ + if (sysctl_sched_prio_load_balance_enabled && + cpus_share_cache(env->src_cpu, env->dst_cpu) && + sched_idle_cpu(env->dst_cpu)) + return 0; +#endif + if (sysctl_sched_migration_cost == -1) return 1; @@ -8462,11 +8497,18 @@ static void detach_task(struct task_struct *p, struct rq *src_rq, int dst_cpu) static struct task_struct *detach_one_task(struct lb_env *env) { struct task_struct *p; + struct list_head *tasks = &env->src_rq->cfs_tasks; +#ifdef CONFIG_QOS_SCHED_PRIO_LB + int loop = 0; +#endif lockdep_assert_rq_held(env->src_rq); +#ifdef CONFIG_QOS_SCHED_PRIO_LB +again: +#endif list_for_each_entry_reverse(p, - &env->src_rq->cfs_tasks, se.group_node) { + tasks, se.group_node) { if (!can_migrate_task(p, env)) continue; @@ -8481,6 +8523,15 @@ static struct task_struct *detach_one_task(struct lb_env *env) schedstat_inc(env->sd->lb_gained[env->idle]); return p; } +#ifdef CONFIG_QOS_SCHED_PRIO_LB + if (sysctl_sched_prio_load_balance_enabled) { + loop++; + if (loop == 1) { + tasks = &env->src_rq->cfs_offline_tasks; + goto again; + } + } +#endif return NULL; } @@ -8498,12 +8549,18 @@ static int detach_tasks(struct lb_env *env) unsigned long util, load; struct task_struct *p; int detached = 0; +#ifdef CONFIG_QOS_SCHED_PRIO_LB + int loop = 0; +#endif lockdep_assert_rq_held(env->src_rq); if (env->imbalance <= 0) return 0; +#ifdef CONFIG_QOS_SCHED_PRIO_LB +again: +#endif while (!list_empty(tasks)) { /* * We don't want to steal all, otherwise we may be treated likewise, @@ -8605,6 +8662,22 @@ static int detach_tasks(struct lb_env *env) list_move(&p->se.group_node, tasks); } +#ifdef CONFIG_QOS_SCHED_PRIO_LB + if (sysctl_sched_prio_load_balance_enabled && env->imbalance > 0) { + /* + * Avoid offline tasks starve to death if env->loop exceed + * env->loop_max, so we should set env->loop to 0 and detach + * offline tasks again. + */ + if (env->loop > env->loop_max) + env->loop = 0; + loop++; + if (loop == 1) { + tasks = &env->src_rq->cfs_offline_tasks; + goto again; + } + } +#endif /* * Right now, this is one of only two places we collect this stat * so we can safely collect detach_one_task() stats here rather @@ -12124,7 +12197,11 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) * Move the next running task to the front of the list, so our * cfs_tasks list becomes MRU one. */ +#ifdef CONFIG_QOS_SCHED_PRIO_LB + adjust_rq_cfs_tasks(list_move, rq, se); +#else list_move(&se->group_node, &rq->cfs_tasks); +#endif } #endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 0ab8e2532f2d663e2974521a1c555b9f14a15446..32d4775e537d62ea1601e8fe58a8336c12400834 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1136,8 +1136,12 @@ struct rq { unsigned int core_forceidle_seq; #endif +#if defined(CONFIG_QOS_SCHED_PRIO_LB) && !defined(__GENKSYMS__) + struct list_head cfs_offline_tasks; +#else KABI_RESERVE(1) KABI_RESERVE(2) +#endif KABI_RESERVE(3) KABI_RESERVE(4) KABI_RESERVE(5) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 0a25c33e80a72c744d8250984bf2cc127cf7fb99..7919119e89b2416e78529551f2cd9b8ccc2ea427 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2717,6 +2717,17 @@ static struct ctl_table kern_table[] = { .extra1 = &one_hundred, .extra2 = &one_thousand, }, +#endif +#ifdef CONFIG_QOS_SCHED_PRIO_LB + { + .procname = "sched_prio_load_balance_enabled", + .data = &sysctl_sched_prio_load_balance_enabled, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, #endif { } };