提交 34404ca8 编写于 作者: P Paul E. McKenney

rcu: Move early-boot callbacks to no-CBs lists for no-CBs CPUs

When a CPU is first determined to be a no-CBs CPUs, this commit causes
any early boot callbacks to be moved to the no-CBs callback list,
allowing them to be invoked.
Signed-off-by: NPaul E. McKenney <paulmck@linux.vnet.ibm.com>
上级 39c8d313
...@@ -2851,6 +2851,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), ...@@ -2851,6 +2851,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
* and then drop through to queue the callback. * and then drop through to queue the callback.
*/ */
BUG_ON(cpu != -1); BUG_ON(cpu != -1);
WARN_ON_ONCE(!rcu_is_watching());
if (!likely(rdp->nxtlist)) if (!likely(rdp->nxtlist))
init_default_callback_list(rdp); init_default_callback_list(rdp);
} }
......
...@@ -2393,18 +2393,8 @@ void __init rcu_init_nohz(void) ...@@ -2393,18 +2393,8 @@ void __init rcu_init_nohz(void)
pr_info("\tPoll for callbacks from no-CBs CPUs.\n"); pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
for_each_rcu_flavor(rsp) { for_each_rcu_flavor(rsp) {
for_each_cpu(cpu, rcu_nocb_mask) { for_each_cpu(cpu, rcu_nocb_mask)
struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
/*
* If there are early callbacks, they will need
* to be moved to the nocb lists.
*/
WARN_ON_ONCE(rdp->nxttail[RCU_NEXT_TAIL] !=
&rdp->nxtlist &&
rdp->nxttail[RCU_NEXT_TAIL] != NULL);
init_nocb_callback_list(rdp);
}
rcu_organize_nocb_kthreads(rsp); rcu_organize_nocb_kthreads(rsp);
} }
} }
...@@ -2541,6 +2531,16 @@ static bool init_nocb_callback_list(struct rcu_data *rdp) ...@@ -2541,6 +2531,16 @@ static bool init_nocb_callback_list(struct rcu_data *rdp)
if (!rcu_is_nocb_cpu(rdp->cpu)) if (!rcu_is_nocb_cpu(rdp->cpu))
return false; return false;
/* If there are early-boot callbacks, move them to nocb lists. */
if (rdp->nxtlist) {
rdp->nocb_head = rdp->nxtlist;
rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
rdp->nxtlist = NULL;
rdp->qlen = 0;
rdp->qlen_lazy = 0;
}
rdp->nxttail[RCU_NEXT_TAIL] = NULL; rdp->nxttail[RCU_NEXT_TAIL] = NULL;
return true; return true;
} }
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册