[PATCH v2 2/6] mm: prepare to premature release of per-node lruvec_stat_cpu

[Roman Gushchin]

Similar to the memcg's vmstats_percpu, per-memcg per-node stats
consists of percpu- and atomic counterparts, and we do expect
that both coexist during the whole life-cycle of the memcg.

To prepare for a premature release of percpu per-node data,
let's pretend that lruvec_stat_cpu is a rcu-protected pointer,
which can be NULL. This patch adds corresponding checks whenever
required.

Signed-off-by: Roman Gushchin <g...@fb.com>
Acked-by: Johannes Weiner <han...@cmpxchg.org>
---
 include/linux/memcontrol.h | 21 +++++++++++++++------
 mm/memcontrol.c            | 14 +++++++++++---
 2 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 05ca77767c6a..8ac04632002a 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -126,7 +126,7 @@ struct memcg_shrinker_map {
 struct mem_cgroup_per_node {
        struct lruvec           lruvec;
 
-       struct lruvec_stat __percpu *lruvec_stat_cpu;
+       struct lruvec_stat __rcu /* __percpu */ *lruvec_stat_cpu;
        atomic_long_t           lruvec_stat[NR_VM_NODE_STAT_ITEMS];
 
        unsigned long           lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
@@ -682,6 +682,7 @@ static inline unsigned long lruvec_page_state(struct lruvec 
*lruvec,
 static inline void __mod_lruvec_state(struct lruvec *lruvec,
                                      enum node_stat_item idx, int val)
 {
+       struct lruvec_stat __percpu *lruvec_stat_cpu;
        struct mem_cgroup_per_node *pn;
        long x;
 
@@ -697,12 +698,20 @@ static inline void __mod_lruvec_state(struct lruvec 
*lruvec,
        __mod_memcg_state(pn->memcg, idx, val);
 
        /* Update lruvec */
-       x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
-       if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
-               atomic_long_add(x, &pn->lruvec_stat[idx]);
-               x = 0;
+       rcu_read_lock();
+       lruvec_stat_cpu = (struct lruvec_stat __percpu *)
+               rcu_dereference(pn->lruvec_stat_cpu);
+       if (likely(lruvec_stat_cpu)) {
+               x = val + __this_cpu_read(lruvec_stat_cpu->count[idx]);
+               if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
+                       atomic_long_add(x, &pn->lruvec_stat[idx]);
+                       x = 0;
+               }
+               __this_cpu_write(lruvec_stat_cpu->count[idx], x);
+       } else {
+               atomic_long_add(val, &pn->lruvec_stat[idx]);
        }
-       __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
+       rcu_read_unlock();
 }
 
 static inline void mod_lruvec_state(struct lruvec *lruvec,
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 803c772f354b..5ef4098f3f8d 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2122,6 +2122,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
 static int memcg_hotplug_cpu_dead(unsigned int cpu)
 {
        struct memcg_vmstats_percpu __percpu *vmstats_percpu;
+       struct lruvec_stat __percpu *lruvec_stat_cpu;
        struct memcg_stock_pcp *stock;
        struct mem_cgroup *memcg;
 
@@ -2152,7 +2153,12 @@ static int memcg_hotplug_cpu_dead(unsigned int cpu)
                                struct mem_cgroup_per_node *pn;
 
                                pn = mem_cgroup_nodeinfo(memcg, nid);
-                               x = 
this_cpu_xchg(pn->lruvec_stat_cpu->count[i], 0);
+
+                               lruvec_stat_cpu = (struct lruvec_stat __percpu*)
+                                       rcu_dereference(pn->lruvec_stat_cpu);
+                               if (!lruvec_stat_cpu)
+                                       continue;
+                               x = this_cpu_xchg(lruvec_stat_cpu->count[i], 0);
                                if (x)
                                        atomic_long_add(x, &pn->lruvec_stat[i]);
                        }
@@ -4414,6 +4420,7 @@ struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
 
 static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
 {
+       struct lruvec_stat __percpu *lruvec_stat_cpu;
        struct mem_cgroup_per_node *pn;
        int tmp = node;
        /*
@@ -4430,11 +4437,12 @@ static int alloc_mem_cgroup_per_node_info(struct 
mem_cgroup *memcg, int node)
        if (!pn)
                return 1;
 
-       pn->lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
-       if (!pn->lruvec_stat_cpu) {
+       lruvec_stat_cpu = alloc_percpu(struct lruvec_stat);
+       if (!lruvec_stat_cpu) {
                kfree(pn);
                return 1;
        }
+       rcu_assign_pointer(pn->lruvec_stat_cpu, lruvec_stat_cpu);
 
        lruvec_init(&pn->lruvec);
        pn->usage_in_excess = 0;
-- 
2.20.1