This patch enhances sysfs by adding two new pseudo-files aimed to help monitor memory utilization over HTTP api.
One of the new files located under /sys/osv/memory/free_page_ranges gives more detailed insight about free_page_ranges that holds all free registered physical memory at given point in time like in this example: huge 0001 002114469888 03 0002 000000040960 Each row shows information about free page ranges for given size range - huge (>= 256MB), 16, 15 .. 1 where the number (order) is log2() of the minimum of the corresponding range size in 4K pages. For example 3 represents page ranges of size between 16K - 32K. The second column displays number of page ranges for given size order (always 1 for huge) and last column displays total number of bytes for given range order. For more information please read https://github.com/cloudius-systems/osv/wiki/Managing-Memory-Pages. Second new file located under /sys/osv/memory/pools gives detailed information about L1 local memory pools and L2 global memory pool like in this example: global l2 (in batches) 64 16 48 24 cpu 0 l1 (in pages) 512 128 384 158 cpu 1 l1 (in pages) 512 128 384 255 cpu 2 l1 (in pages) 512 128 384 251 cpu 3 l1 (in pages) 512 128 384 000 The last 4 columns show respectively max, low watermark, high watermark and current number of pages or batches of pages for given L1 or L2 pool. For more information please read https://github.com/cloudius-systems/osv/wiki/Memory-Management#high-level-layer. Signed-off-by: Waldemar Kozaczuk <[email protected]> --- core/mempool.cc | 58 ++++++++++++++++++++++++++++++++++++++++- fs/sysfs/sysfs_vnops.cc | 51 ++++++++++++++++++++++++++++++++++++ include/osv/mempool.hh | 21 +++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/core/mempool.cc b/core/mempool.cc index 50f5e877..e557d04f 100644 --- a/core/mempool.cc +++ b/core/mempool.cc @@ -541,7 +541,7 @@ void reclaimer::wait_for_memory(size_t mem) class page_range_allocator { public: - static constexpr unsigned max_order = 16; + static constexpr unsigned max_order = page_ranges_max_order; page_range_allocator() : _deferred_free(nullptr) { } @@ -571,6 +571,22 @@ public: return size; } + void stats(stats::page_ranges_stats& stats) const { + stats.order[max_order].ranges_num = _free_huge.size(); + stats.order[max_order].bytes = 0; + for (auto& pr : _free_huge) { + stats.order[max_order].bytes += pr.size; + } + + for (auto order = max_order; order--;) { + stats.order[order].ranges_num = _free[order].size(); + stats.order[order].bytes = 0; + for (auto& pr : _free[order]) { + stats.order[order].bytes += pr.size; + } + } + } + private: template<bool UseBitmap = true> void insert(page_range& pr) { @@ -822,6 +838,15 @@ void page_range_allocator::for_each(unsigned min_order, Func f) } } +namespace stats { + void get_page_ranges_stats(page_ranges_stats &stats) + { + WITH_LOCK(free_page_ranges_lock) { + free_page_ranges.stats(stats); + } + } +} + static void* mapped_malloc_large(size_t size, size_t offset) { //TODO: For now pre-populate the memory, in future consider doing lazy population @@ -1123,6 +1148,8 @@ static size_t large_object_size(void *obj) namespace page_pool { +static std::vector<stats::pool_stats> l1_pool_stats; + // L1-pool (Percpu page buffer pool) // // if nr < max * 1 / 4 @@ -1137,6 +1164,7 @@ struct l1 { : _fill_thread(sched::thread::make([] { fill_thread(); }, sched::thread::attr().pin(cpu).name(osv::sprintf("page_pool_l1_%d", cpu->id)))) { + cpu_id = cpu->id; _fill_thread->start(); } @@ -1160,12 +1188,15 @@ struct l1 { void* pop() { assert(nr); + l1_pool_stats[cpu_id]._nr = nr - 1; return _pages[--nr]; } void push(void* page) { assert(nr < 512); _pages[nr++] = page; + l1_pool_stats[cpu_id]._nr = nr; + } void* top() { return _pages[nr - 1]; } void wake_thread() { _fill_thread->wake(); } @@ -1177,6 +1208,7 @@ struct l1 { static constexpr size_t watermark_lo = max * 1 / 4; static constexpr size_t watermark_hi = max * 3 / 4; size_t nr = 0; + unsigned int cpu_id; private: std::unique_ptr<sched::thread> _fill_thread; @@ -1266,6 +1298,14 @@ public: return true; } + void stats(stats::pool_stats &stats) + { + stats._nr = get_nr(); + stats._max = _max; + stats._watermark_lo = _watermark_lo; + stats._watermark_hi = _watermark_hi; + } + void fill_thread(); void refill(); void unfill(); @@ -1291,6 +1331,7 @@ static sched::cpu::notifier _notifier([] () { if (smp_allocator_cnt++ == sched::cpus.size()) { smp_allocator = true; } + l1_pool_stats.resize(sched::cpus.size()); }); static inline l1& get_l1() { @@ -1469,6 +1510,21 @@ void l2::free_batch(page_batch& batch) } +namespace stats { + void get_global_l2_stats(pool_stats &stats) + { + page_pool::global_l2.stats(stats); + } + + void get_l1_stats(unsigned int cpu_id, pool_stats &stats) + { + stats._nr = page_pool::l1_pool_stats[cpu_id]._nr; + stats._max = page_pool::l1::max; + stats._watermark_lo = page_pool::l1::watermark_lo; + stats._watermark_hi = page_pool::l1::watermark_hi; + } +} + static void* early_alloc_page() { WITH_LOCK(free_page_ranges_lock) { diff --git a/fs/sysfs/sysfs_vnops.cc b/fs/sysfs/sysfs_vnops.cc index df24ecb0..15636f92 100644 --- a/fs/sysfs/sysfs_vnops.cc +++ b/fs/sysfs/sysfs_vnops.cc @@ -8,6 +8,8 @@ #include <unistd.h> #include <osv/mount.h> #include <mntent.h> +#include <osv/printf.hh> +#include <osv/mempool.hh> #include "fs/pseudofs/pseudofs.hh" @@ -30,6 +32,47 @@ static string sysfs_distance() return std::string("10"); } +using namespace memory; +static string sysfs_free_page_ranges() +{ + stats::page_ranges_stats stats; + stats::get_page_ranges_stats(stats); + + std::ostringstream os; + if (stats.order[page_ranges_max_order].ranges_num) { + osv::fprintf(os, "huge %04d %012ld\n", //TODO: Show in GB/MB/KB + stats.order[page_ranges_max_order].ranges_num, stats.order[page_ranges_max_order].bytes); + } + + for (int order = page_ranges_max_order; order--; ) { + if (stats.order[order].ranges_num) { + osv::fprintf(os, " %02d %04d %012ld\n", + order + 1, stats.order[order].ranges_num, stats.order[order].bytes); + } + } + + return os.str(); +} + +static string sysfs_memory_pools() +{ + stats::pool_stats stats; + stats::get_global_l2_stats(stats); + + std::ostringstream os; + osv::fprintf(os, "global l2 (in batches) %02d %02d %02d %02d\n", + stats._max, stats._watermark_lo, stats._watermark_hi, stats._nr); + + for (auto cpu : sched::cpus) { + stats::pool_stats stats; + stats::get_l1_stats(cpu->id, stats); + osv::fprintf(os, "cpu %d l1 (in pages) %03d %03d %03d %03d\n", + cpu->id, stats._max, stats._watermark_lo, stats._watermark_hi, stats._nr); + } + + return os.str(); +} + static int sysfs_mount(mount* mp, const char *dev, int flags, const void* data) { @@ -49,8 +92,16 @@ sysfs_mount(mount* mp, const char *dev, int flags, const void* data) auto devices = make_shared<pseudo_dir_node>(inode_count++); devices->add("system", system); + auto memory = make_shared<pseudo_dir_node>(inode_count++); + memory->add("free_page_ranges", inode_count++, sysfs_free_page_ranges); + memory->add("pools", inode_count++, sysfs_memory_pools); + + auto osv_extension = make_shared<pseudo_dir_node>(inode_count++); + osv_extension->add("memory", memory); + auto* root = new pseudo_dir_node(vp->v_ino); root->add("devices", devices); + root->add("osv", osv_extension); vp->v_data = static_cast<void*>(root); diff --git a/include/osv/mempool.hh b/include/osv/mempool.hh index 10fe5602..33a9af68 100644 --- a/include/osv/mempool.hh +++ b/include/osv/mempool.hh @@ -188,6 +188,8 @@ private: ssize_t bytes_until_normal() { return bytes_until_normal(pressure_level()); } }; +const unsigned page_ranges_max_order = 16; + namespace stats { size_t free(); size_t total(); @@ -195,6 +197,25 @@ namespace stats { size_t jvm_heap(); void on_jvm_heap_alloc(size_t mem); void on_jvm_heap_free(size_t mem); + + struct page_ranges_stats { + struct { + size_t bytes; + size_t ranges_num; + } order[page_ranges_max_order + 1]; + }; + + void get_page_ranges_stats(page_ranges_stats &stats); + + struct pool_stats { + size_t _max; + size_t _nr; + size_t _watermark_lo; + size_t _watermark_hi; + }; + + void get_global_l2_stats(pool_stats &stats); + void get_l1_stats(unsigned int cpu_id, stats::pool_stats &stats); } class phys_contiguous_memory final { -- 2.20.1 -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To view this discussion on the web visit https://groups.google.com/d/msgid/osv-dev/20200408035745.29894-1-jwkozaczuk%40gmail.com.
