From 6c7ab876dd0953cb448163bd4ec387f9f07f816a Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 23 Feb 2022 16:53:34 +0800 Subject: [PATCH] entry: Explicitly flush pending rcuog wakeup before last rescheduling point mainline inclusion from mainline-v5.12-rc1 commit 47b8ff194c1fd73d58dc339b597d466fe48c8958 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4U05V CVE: NA ------------------------------------------------------------------------- Following the idle loop model, cleanly check for pending rcuog wakeup before the last rescheduling point on resuming to user mode. This way we can avoid to do it from rcu_user_enter() with the last resort self-IPI hack that enforces rescheduling. Signed-off-by: Frederic Weisbecker Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: stable@vger.kernel.org Link: https://lkml.kernel.org/r/20210131230548.32970-5-frederic@kernel.org Conflicts: kernel/entry/common.c Signed-off-by: Zhen Lei Reviewed-by: Cheng Jian Signed-off-by: Zheng Zengkai --- kernel/entry/common.c | 7 +++++++ kernel/rcu/tree.c | 12 +++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/kernel/entry/common.c b/kernel/entry/common.c index df3c534dc138..18a29ca01bfe 100644 --- a/kernel/entry/common.c +++ b/kernel/entry/common.c @@ -178,6 +178,10 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, * enabled above. */ local_irq_disable_exit_to_user(); + + /* Check if any of the above work has queued a deferred wakeup */ + rcu_nocb_flush_deferred_wakeup(); + ti_work = READ_ONCE(current_thread_info()->flags); } @@ -191,6 +195,9 @@ static void exit_to_user_mode_prepare(struct pt_regs *regs) lockdep_assert_irqs_disabled(); + /* Flush pending rcuog wakeup before the last need_resched() check */ + rcu_nocb_flush_deferred_wakeup(); + if (unlikely((ti_work & EXIT_TO_USER_MODE_WORK) || sched_qos_cpu_overload())) ti_work = exit_to_user_mode_loop(regs, ti_work); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 25721617cdea..1f6c78aa7bfd 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -699,13 +699,15 @@ noinstr void rcu_user_enter(void) lockdep_assert_irqs_disabled(); /* - * We may be past the last rescheduling opportunity in the entry code. - * Trigger a self IPI that will fire and reschedule once we resume to - * user/guest mode. + * Other than generic entry implementation, we may be past the last + * rescheduling opportunity in the entry code. Trigger a self IPI + * that will fire and reschedule once we resume in user/guest mode. */ instrumentation_begin(); - if (do_nocb_deferred_wakeup(rdp) && need_resched()) - irq_work_queue(this_cpu_ptr(&late_wakeup_work)); + if (!IS_ENABLED(CONFIG_GENERIC_ENTRY) || (current->flags & PF_VCPU)) { + if (do_nocb_deferred_wakeup(rdp) && need_resched()) + irq_work_queue(this_cpu_ptr(&late_wakeup_work)); + } instrumentation_end(); rcu_eqs_enter(true); -- GitLab