Initially (and perhaps forever), call_srcu() will not be available for atomic srcu_struct structures. There is therefore no reason to transition such a structure to big, because the main purpose of such a transition is to reduce lock contention for concurrent SRCU callback queueing.
This commit therefore adds an is_atomic parameter to both the check_init_srcu_struct() and init_srcu_struct_fields() functions, which suppresses the initialization-time transition to big that is enabled by default on large systems. It will still be possible to force a transition using rcutorture as a destructive test. This might (or might not) be adjusted later. [ paulmck: Apply feedback from Kunwu Chan. ] Signed-off-by: Paul E. McKenney <[email protected]> --- kernel/rcu/srcutree.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index dc063eb49b0d..c611a7168c70 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -238,8 +238,10 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) * Initialize non-compile-time initialized fields, including the * associated srcu_node and srcu_data structures. The is_static parameter * tells us that ->sda has already been wired up to srcu_data. + * The is_atomic parameter tells us that there is no reason to + * ever transition to big. */ -static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) +static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static, bool is_atomic) { if (!is_static) ssp->srcu_sup = kzalloc_obj(*ssp->srcu_sup); @@ -267,7 +269,8 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) init_srcu_struct_data(ssp); ssp->srcu_sup->srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL; ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns(); - if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { + if (!is_atomic && + READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { if (!preemptible()) WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC); else if (init_srcu_struct_nodes(ssp, GFP_KERNEL)) @@ -301,7 +304,7 @@ __init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock /* Don't re-initialize a lock while it is held. */ debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); lockdep_init_map(&ssp->dep_map, name, key, 0); - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, false); } int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, @@ -343,7 +346,7 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); int init_srcu_struct_generic(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = 0; - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, false); } EXPORT_SYMBOL_GPL(init_srcu_struct_generic); @@ -360,7 +363,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_generic); int init_srcu_struct_fast(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST; - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, false); } EXPORT_SYMBOL_GPL(init_srcu_struct_fast); @@ -378,7 +381,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_fast); int init_srcu_struct_fast_updown(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST_UPDOWN; - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, false); } EXPORT_SYMBOL_GPL(init_srcu_struct_fast_updown); @@ -470,9 +473,11 @@ static void raw_spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsign * done with compile-time initialization, so this check is added * to each update-side SRCU primitive. Use ssp->lock, which -is- * compile-time initialized, to resolve races involving multiple - * CPUs trying to garner first-use privileges. + * CPUs trying to garner first-use privileges. The is_atomic + * parameter tells us that there will never be a reason to + * transition to big. */ -static void check_init_srcu_struct(struct srcu_struct *ssp) +static void check_init_srcu_struct(struct srcu_struct *ssp, bool is_atomic) { unsigned long flags; @@ -484,7 +489,7 @@ static void check_init_srcu_struct(struct srcu_struct *ssp) raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); return; } - init_srcu_struct_fields(ssp, true); + init_srcu_struct_fields(ssp, true, is_atomic); raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); } @@ -1279,7 +1284,7 @@ static bool srcu_should_expedite(struct srcu_struct *ssp) unsigned long t; unsigned long tlast; - check_init_srcu_struct(ssp); + check_init_srcu_struct(ssp, false); /* If _lite() readers, don't do unsolicited expediting. */ if (this_cpu_read(ssp->sda->srcu_reader_flavor) & SRCU_READ_FLAVOR_SLOWGP) return false; @@ -1338,7 +1343,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, struct srcu_node *sdp_mynode; int ss_state; - check_init_srcu_struct(ssp); + check_init_srcu_struct(ssp, false); /* * While starting a new grace period, make sure we are in an * SRCU read-side critical section so that the grace-period @@ -1617,7 +1622,7 @@ static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm) if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) return; might_sleep(); - check_init_srcu_struct(ssp); + check_init_srcu_struct(ssp, false); init_completion(&rcu.completion); init_rcu_head_on_stack(&rcu.head); __call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm); @@ -1827,7 +1832,7 @@ void srcu_barrier(struct srcu_struct *ssp) int idx; unsigned long s; - check_init_srcu_struct(ssp); + check_init_srcu_struct(ssp, false); /* * Register any deferred callbacks before snapshotting the sequence. The -- 2.40.1
