This commit extracts the tail-pointer cleanup and segment compaction logic from rcu_segcblist_advance() into a new static helper function, rcu_segcblist_advance_compact(). This shared logic will be reused by the upcoming srcu_segcblist_advance() standalone implementation, which cannot call the core rcu_segcblist_advance() because that function will use RCU-specific globals.
No functional change. Reviewed-by: Paul E. McKenney <[email protected]> Signed-off-by: Puranjay Mohan <[email protected]> --- kernel/rcu/rcu_segcblist.c | 50 ++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index da39d818b01b..421f1dadb5e5 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -462,13 +462,43 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail); } +/* + * Clean up and compact the segmented callback list after callbacks have been + * advanced to the RCU_DONE_TAIL segment. The @i parameter is the index of the + * first segment that was NOT advanced (i.e., the segment after the last one + * moved to RCU_DONE_TAIL). This function fixes up tail pointers and compacts + * any gaps left by the moved segments. + */ +static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i) +{ + int j; + + /* Clean up tail pointers that might have been misordered above. */ + for (j = RCU_WAIT_TAIL; j < i; j++) + WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]); + + /* + * Callbacks moved, so there might be an empty RCU_WAIT_TAIL + * and a non-empty RCU_NEXT_READY_TAIL. If so, copy the + * RCU_NEXT_READY_TAIL segment to fill the RCU_WAIT_TAIL gap + * created by the now-ready-to-invoke segments. + */ + for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) { + if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL]) + break; /* No more callbacks. */ + WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]); + rcu_segcblist_move_seglen(rsclp, i, j); + rsclp->gp_seq[j] = rsclp->gp_seq[i]; + } +} + /* * Advance the callbacks in the specified rcu_segcblist structure based * on the current value passed in for the grace-period counter. */ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) { - int i, j; + int i; WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) @@ -489,23 +519,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) if (i == RCU_WAIT_TAIL) return; - /* Clean up tail pointers that might have been misordered above. */ - for (j = RCU_WAIT_TAIL; j < i; j++) - WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]); - - /* - * Callbacks moved, so there might be an empty RCU_WAIT_TAIL - * and a non-empty RCU_NEXT_READY_TAIL. If so, copy the - * RCU_NEXT_READY_TAIL segment to fill the RCU_WAIT_TAIL gap - * created by the now-ready-to-invoke segments. - */ - for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) { - if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL]) - break; /* No more callbacks. */ - WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]); - rcu_segcblist_move_seglen(rsclp, i, j); - rsclp->gp_seq[j] = rsclp->gp_seq[i]; - } + rcu_segcblist_advance_compact(rsclp, i); } /* -- 2.52.0
