From: "Paul E. McKenney" <[email protected]> This commit adds a check for failure to have fully deboosted a multi-segmented RCU reader at the end of the full read-side critical section. This check only happens for fully task-level readers, because a a handler might have interrupted an already-boosted task-level RCU reader, and a reader in that handler could then cause false positives. The first failed check (due to an RCU reader that was not immediately deboosted) causes a splat and a list of the segments making up that RCU reader. Subsequent failures fail silently, all in the name of keeping console output down to a dull roar.
Although most uses of RCU priority boosting serve as debugging aids, this might change, and in fact might already have changed. And allowing (for example) RCU priority boosting to persist until the next scheduler tick could cause an aggressively real-time system to miss sub-millisecond deadlines. So we do need to find this sort of problem during testing, and preferably not in the field. The name and type of the newly added rcu_torture_ops function pointer (named "->is_task_rcu_boosted()") may need to change should other end-of-reader checks be needed. But let's start simple. Signed-off-by: Paul E. McKenney <[email protected]> --- kernel/rcu/rcu.h | 7 +++++++ kernel/rcu/rcutorture.c | 22 ++++++++++++++++++++++ kernel/rcu/tree_plugin.h | 32 +++++++++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index fa6d30ce73d1..14faa11ef23c 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -695,4 +695,11 @@ static inline int rcu_stall_notifier_call_chain(unsigned long val, void *v) { re void synchronize_rcu_trivial_preempt(void); #endif // #ifdef CONFIG_TRIVIAL_PREEMPT_RCU +#if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +bool rcu_is_task_rcu_boosted(void); +#else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +static inline bool rcu_is_task_rcu_boosted(void) { return false; } +#endif // #else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) + + #endif /* __LINUX_RCU_H */ diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 42f94c62c18a..04c36fcde5cd 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -426,6 +426,7 @@ struct rcu_torture_ops { void (*format_gp_seqs)(unsigned long long seqs, char *cp, size_t len); void (*set_gpwrap_lag)(unsigned long lag); int (*get_gpwrap_count)(int cpu); + bool (*is_task_rcu_boosted)(void); long cbflood_max; int irq_capable; int can_boost; @@ -635,6 +636,7 @@ static struct rcu_torture_ops rcu_ops = { .format_gp_seqs = rcutorture_format_gp_seqs, .set_gpwrap_lag = rcu_set_gpwrap_lag, .get_gpwrap_count = rcu_get_gpwrap_count, + .is_task_rcu_boosted = rcu_is_task_rcu_boosted, .irq_capable = 1, .can_boost = IS_ENABLED(CONFIG_RCU_BOOST), .extendables = RCUTORTURE_MAX_EXTEND, @@ -2565,7 +2567,9 @@ static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp, */ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) { + static int firsttime = 1; int newstate; + unsigned int nsegs; struct rcu_torture_one_read_state rtors; WARN_ON_ONCE(!rcu_is_watching()); @@ -2577,6 +2581,24 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) return false; rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp); rcu_torture_one_read_end(&rtors, trsp); + // This splat will happen on systems built with CONFIG_IRQ_WORK=n + // and on systems where arch_irq_work_has_interrupt() returns false. + // It might also happen on systems using a short-duration clock + // interrupt instead of a self-IPI (powerpc, s390) or that use + // neither a self-IPI nor a short-duration clock interrupts + // (all architectures using the generic implementation + // of arch_irq_work_raise()). On such systems, RCU cannot + // guarantee to immediately deboost RCU readers when the outermost + // rcu_read_unlock() does not end the full segmented RCU read-side + // critical section. + if (WARN_ON_ONCE(cur_ops->is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() && + !in_serving_softirq() && !in_hardirq() && !in_nmi()) && + READ_ONCE(firsttime) && xchg(&firsttime, 0)) { + nsegs = rtors.rtrsp - rtors.rtseg; + nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS); + pr_alert("Slow-deboost rcutorture reader segments:\n"); + rcu_torture_dump_read_segs(rtors.rtseg, nsegs); + } return true; } diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index fbbabcf2c3a7..d495747e143f 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1093,7 +1093,7 @@ void rcu_read_unlock_strict(void) * * The in_atomic_preempt_off() check ensures that we come here holding * the last preempt_count (which will get dropped once we return to - * __rcu_read_unlock(). + * __rcu_read_unlock()). */ rdp = this_cpu_ptr(&rcu_data); rdp->cpu_no_qs.b.norm = false; @@ -1490,6 +1490,36 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ } +#ifdef CONFIG_RCU_TORTURE_TEST + +/* + * Is the current task RCU priority boosted? This is used by + * rcutorture to check that tasks are always deboosted once then exit + * an RCU read-side critical section, no matter how many overlapping + * segments of rcu_read_lock(), preempt_disable(), local_bh_disable(), + * or local_irq_disable() made up that reader. + * + * The lockless accesses in rt_mutex_owner(&rnp->boost_mtx.rtmutex) + * are safe because tasks release ->boost_mtx when they own it, they + * cannot be boosted unless current->rcu_blocked_node is non-NULL, + * current->rcu_blocked_node is modified only by the current task, + * rt_mutex_owner() uses READ_ONCE() on the ->owner field, and the owner + * switching among other tasks cannot force an equality comparison. + */ +bool rcu_is_task_rcu_boosted(void) +{ + struct rcu_node *rnp; + struct task_struct *t = current; + + rnp = t->rcu_blocked_node; + if (!rnp) + return false; + return rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t; +} +EXPORT_SYMBOL_GPL(rcu_is_task_rcu_boosted); + +#endif // #ifdef CONFIG_RCU_TORTURE_TEST + #else /* #ifdef CONFIG_RCU_BOOST */ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) -- 2.34.1
