Preparation for the following patch that adds tail caching. For simplicity, hlist_nulls_del_init_rcu() is now always used, which is also valid during rotation, because the entry is immediately reinserted under `memcg_lru.lock`.
Signed-off-by: Max Kellermann <[email protected]> --- mm/vmscan.c | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index f11491ee9ed5..3d4bc12cf18e 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -4379,6 +4379,29 @@ enum { MEMCG_LRU_YOUNG, }; +static void lru_gen_add_memcg(struct lru_gen_memcg *memcg_lru, + struct lru_gen_folio *lrugen, + int gen, int bin, bool at_head) +{ + struct hlist_nulls_head *head = &memcg_lru->fifo[gen][bin]; + + lockdep_assert_held(&memcg_lru->lock); + + if (at_head) + hlist_nulls_add_head_rcu(&lrugen->list, head); + else + hlist_nulls_add_tail_rcu(&lrugen->list, head); +} + +static void lru_gen_del_memcg(struct lru_gen_memcg *memcg_lru, + struct lru_gen_folio *lrugen, + int gen) +{ + lockdep_assert_held(&memcg_lru->lock); + + hlist_nulls_del_init_rcu(&lrugen->list); +} + static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op) { int seg; @@ -4409,12 +4432,9 @@ static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op) WRITE_ONCE(lruvec->lrugen.seg, seg); WRITE_ONCE(lruvec->lrugen.gen, new); - hlist_nulls_del_rcu(&lruvec->lrugen.list); - - if (op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD) - hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); - else - hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]); + lru_gen_del_memcg(&pgdat->memcg_lru, &lruvec->lrugen, old); + lru_gen_add_memcg(&pgdat->memcg_lru, &lruvec->lrugen, new, bin, + op == MEMCG_LRU_HEAD || op == MEMCG_LRU_OLD); pgdat->memcg_lru.nr_memcgs[old]--; pgdat->memcg_lru.nr_memcgs[new]++; @@ -4445,7 +4465,7 @@ void lru_gen_online_memcg(struct mem_cgroup *memcg) lruvec->lrugen.gen = gen; - hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]); + lru_gen_add_memcg(&pgdat->memcg_lru, &lruvec->lrugen, gen, bin, false); pgdat->memcg_lru.nr_memcgs[gen]++; spin_unlock_irq(&pgdat->memcg_lru.lock); @@ -4479,7 +4499,7 @@ void lru_gen_release_memcg(struct mem_cgroup *memcg) gen = lruvec->lrugen.gen; - hlist_nulls_del_init_rcu(&lruvec->lrugen.list); + lru_gen_del_memcg(&pgdat->memcg_lru, &lruvec->lrugen, gen); pgdat->memcg_lru.nr_memcgs[gen]--; if (!pgdat->memcg_lru.nr_memcgs[gen] && gen == get_memcg_gen(pgdat->memcg_lru.seq)) -- 2.47.3
