Add mem_cgroup_dmem_charge() and mem_cgroup_dmem_uncharge() to allow dmem pool allocations to optionally be double-charged against the memory controller. Take the struct cgroup from the dmem pool's css as there is no convenient object exported to represent these allocations. These will resolve the effective memory css from that cgroup and perform the charge.
Introduce a MEMCG_DMEM stat counter to memory.stat to make the cgroup's dmem charge visible. Signed-off-by: Eric Chanudet <[email protected]> --- include/linux/memcontrol.h | 16 ++++++++++++ mm/memcontrol.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index dc3fa687759b45748b2acee6d7f43da325eb50c1..8e1d49b87fb64e6114f3eb920293e14920290fe7 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -39,6 +39,7 @@ enum memcg_stat_item { MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP, + MEMCG_DMEM, MEMCG_NR_STAT, }; @@ -1872,6 +1873,21 @@ static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg) } #endif +#if defined(CONFIG_MEMCG) && defined(CONFIG_CGROUP_DMEM) +bool mem_cgroup_dmem_charge(struct cgroup *cgrp, unsigned int nr_pages, + gfp_t gfp_mask); +void mem_cgroup_dmem_uncharge(struct cgroup *cgrp, unsigned int nr_pages); +#else +static inline bool mem_cgroup_dmem_charge(struct cgroup *cgrp, + unsigned int nr_pages, gfp_t gfp_mask) +{ + return true; +} +static inline void mem_cgroup_dmem_uncharge(struct cgroup *cgrp, + unsigned int nr_pages) +{ +} +#endif /* Cgroup v1-related declarations */ diff --git a/mm/memcontrol.c b/mm/memcontrol.c index c03d4787d466803db49cdaa90e6d6ba426b7afe2..91a7ac16b6eac2d6c3700b6885a068bf8b640706 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -433,6 +433,7 @@ static const unsigned int memcg_stat_items[] = { MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, MEMCG_ZSWAP_INCOMP, + MEMCG_DMEM, }; #define NR_MEMCG_NODE_STAT_ITEMS ARRAY_SIZE(memcg_node_stat_items) @@ -1606,6 +1607,9 @@ static const struct memory_stat memory_stats[] = { #ifdef CONFIG_NUMA_BALANCING { "pgpromote_success", PGPROMOTE_SUCCESS }, #endif +#ifdef CONFIG_CGROUP_DMEM + { "dmem", MEMCG_DMEM }, +#endif }; /* The actual unit of the state item, not the same as the output unit */ @@ -5909,6 +5913,67 @@ static struct cftype zswap_files[] = { }; #endif /* CONFIG_ZSWAP */ +#ifdef CONFIG_CGROUP_DMEM +/** + * mem_cgroup_dmem_charge - charge memcg for a dmem pool allocation + * @cgrp: cgroup of the dmem pool + * @nr_pages: number of pages to charge + * @gfp_mask: reclaim mode + * + * Charges @nr_pages to @memcg. Returns %true if the charge fit within + * @memcg's configured limit, %false if it doesn't. + */ +bool mem_cgroup_dmem_charge(struct cgroup *cgrp, unsigned int nr_pages, + gfp_t gfp_mask) +{ + struct cgroup_subsys_state *mem_css; + struct mem_cgroup *memcg; + + /* CGROUP_DMEM and MEMCG guarantees this cannot be NULL. */ + mem_css = cgroup_get_e_css(cgrp, &memory_cgrp_subsys); + + /* Use the memcg, if any, of the dmem cgroup. */ + memcg = mem_cgroup_from_css(mem_css); + if (!memcg || mem_cgroup_is_root(memcg)) { + css_put(mem_css); + return false; + } + + if (try_charge_memcg(memcg, gfp_mask, nr_pages)) { + css_put(mem_css); + return false; + } + + mod_memcg_state(memcg, MEMCG_DMEM, nr_pages); + css_put(mem_css); + return true; +} + +/** + * mem_cgroup_dmem_uncharge - uncharge memcg from a dmem pool allocation + * @cgrp: cgroup of the dmem pool + * @nr_pages: number of pages to uncharge + */ +void mem_cgroup_dmem_uncharge(struct cgroup *cgrp, unsigned int nr_pages) +{ + struct cgroup_subsys_state *mem_css; + struct mem_cgroup *memcg; + + /* CGROUP_DMEM and MEMCG guarantees this cannot be NULL. */ + mem_css = cgroup_get_e_css(cgrp, &memory_cgrp_subsys); + + memcg = mem_cgroup_from_css(mem_css); + if (!memcg || mem_cgroup_is_root(memcg)) { + css_put(mem_css); + return; + } + + mod_memcg_state(memcg, MEMCG_DMEM, -nr_pages); + refill_stock(memcg, nr_pages); + css_put(mem_css); +} +#endif /* CONFIG_CGROUP_DMEM */ + static int __init mem_cgroup_swap_init(void) { if (mem_cgroup_disabled()) -- 2.52.0
