To be sure we have no rcog wake ups that were lost, add a warning to cover the case where the rdp is overloaded with callbacks but no wake up was attempted.
Signed-off-by: Joel Fernandes <[email protected]> --- kernel/rcu/tree.c | 2 ++ kernel/rcu/tree.h | 1 + kernel/rcu/tree_nocb.h | 6 +++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 0c7710caf041..f3f790238805 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3783,6 +3783,8 @@ static void rcu_barrier_entrain(struct rcu_data *rdp) debug_rcu_head_unqueue(&rdp->barrier_head); rcu_barrier_trace(TPS("IRQNQ"), -1, rcu_state.barrier_sequence); } + if (wake_nocb) + rdp->nocb_gp_wake_attempt = true; rcu_nocb_unlock(rdp); if (wake_nocb) wake_nocb_gp(rdp, false); diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 83773560332e..91793b6b288e 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -257,6 +257,7 @@ struct rcu_data { unsigned long nocb_gp_loops; /* # passes through wait code. */ struct swait_queue_head nocb_gp_wq; /* For nocb kthreads to sleep on. */ bool nocb_cb_sleep; /* Is the nocb CB thread asleep? */ + bool nocb_gp_wake_attempt; /* Was a rcuog wakeup attempted? */ struct task_struct *nocb_cb_kthread; struct list_head nocb_head_rdp; /* * Head of rcu_data list in wakeup chain, diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index daff2756cd90..7e9d465c8ab1 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -546,6 +546,7 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, lazy_len = READ_ONCE(rdp->lazy_len); if (was_alldone) { rdp->qlen_last_fqs_check = len; + rdp->nocb_gp_wake_attempt = true; rcu_nocb_unlock(rdp); // Only lazy CBs in bypass list if (lazy_len && bypass_len == lazy_len) { @@ -563,7 +564,8 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, return; } else if (len > rdp->qlen_last_fqs_check + qhimark) { - /* ... or if many callbacks queued. */ + /* Callback overload condition. */ + WARN_ON_ONCE(!rdp->nocb_gp_wake_attempt); rdp->qlen_last_fqs_check = len; j = jiffies; if (j != rdp->nocb_gp_adv_time && @@ -688,6 +690,7 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) bypass_ncbs > 2 * qhimark)) { flush_bypass = true; } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { + rdp->nocb_gp_wake_attempt = false; rcu_nocb_unlock_irqrestore(rdp, flags); continue; /* No callbacks here, try next. */ } @@ -1254,6 +1257,7 @@ lazy_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) continue; } rcu_nocb_try_flush_bypass(rdp, jiffies); + rdp->nocb_gp_wake_attempt = true; rcu_nocb_unlock_irqrestore(rdp, flags); wake_nocb_gp(rdp, false); sc->nr_to_scan -= _count; -- 2.34.1
