From: Kunwu Chan <[email protected]> Add atomic SRCU operation checks and the associated state to Tiny SRCU.
An atomic SRCU domain does not use the normal SRCU callback and grace-period machinery. In particular, a callback queued with call_srcu() would never be processed. Use WARN_ON_ONCE() to reject call_srcu() and srcu_barrier() on atomic SRCU domains. For synchronize_srcu(), redirect atomic SRCU domains to synchronize_srcu_atomic(). Add srcu_reader_flavor to the Tiny SRCU state for these checks. Tiny SRCU does not currently set the flavor for atomic SRCU domains, but keeping the flavor in the common state allows the operation checks to enforce the restriction once atomic flavor tracking is enabled. Also initialize srcu_atomic_gp_flag, which was previously left uninitialized. Signed-off-by: Kunwu Chan <[email protected]> --- include/linux/srcutiny.h | 1 + kernel/rcu/srcutiny.c | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 47a368f945e3..2b293336525a 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -20,6 +20,7 @@ struct srcu_struct { u8 srcu_gp_running; /* GP workqueue running? */ u8 srcu_gp_waiting; /* GP waiting for readers? */ u8 srcu_atomic_gp_flag; /* Serialize atomic GP work.*/ + u8 srcu_reader_flavor; /* Values: SRCU_READ_FLAVOR_.* */ unsigned long srcu_idx; /* Current reader array element in bit 0x2. */ unsigned long srcu_idx_max; /* Furthest future srcu_idx request. */ struct swait_queue_head srcu_wq; diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 26ea4bfbeaf2..22f7716cbb0e 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -41,6 +41,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) ssp->srcu_cb_tail = &ssp->srcu_cb_head; ssp->srcu_gp_running = false; ssp->srcu_gp_waiting = false; + ssp->srcu_atomic_gp_flag = 0; ssp->srcu_idx = 0; ssp->srcu_idx_max = 0; INIT_WORK(&ssp->srcu_work, srcu_drive_gp); @@ -289,6 +290,9 @@ EXPORT_SYMBOL_GPL(srcu_defer_drain); void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, rcu_callback_t func) { + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) + return; + if (should_rcu_defer()) { /* A re-entrant call_srcu() during the drain would livelock it. */ if (READ_ONCE(srcu_defer_draining) && !in_nmi()) { @@ -319,6 +323,11 @@ void synchronize_srcu(struct srcu_struct *ssp) { struct rcu_synchronize rs; + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) { + synchronize_srcu_atomic(ssp); + return; + } + srcu_lock_sync(&ssp->dep_map); RCU_LOCKDEP_WARN(lockdep_is_held(ssp) || @@ -415,6 +424,9 @@ EXPORT_SYMBOL_GPL(synchronize_srcu_atomic); /* Register any deferred callbacks, then wait for all in-flight ones. */ void srcu_barrier(struct srcu_struct *ssp) { + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) + return; + __srcu_defer_drain(ssp); synchronize_srcu(ssp); } -- 2.43.0
