This patch enhances sysfs by adding two new pseudo-files aimed to help monitor memory utilization over HTTP api.
One of the new files located under /sys/osv/memory/free_page_ranges gives more detailed insight about free_page_ranges that holds all free registered physical memory at given point in time like in this example: huge 0001 002114469888 03 0002 000000040960 Each row shows information about free page ranges for given size range - huge (>= 256MB), 16, 15 .. 1 where the number (order) is log2() of the minimum of the corresponding range size in 4K pages. For example 3 represents page ranges of size between 16K - 32K. The second column displays number of page ranges for given size order (always 1 for huge) and last column displays total number of bytes for given range order. For more information please read https://github.com/cloudius-systems/osv/wiki/Managing-Memory-Pages. Second new file located under /sys/osv/memory/pools gives detailed information about L1 local memory pools and L2 global memory pool like in this example: global l2 (in batches) 64 16 48 24 cpu 0 l1 (in pages) 512 128 384 158 cpu 1 l1 (in pages) 512 128 384 255 cpu 2 l1 (in pages) 512 128 384 251 cpu 3 l1 (in pages) 512 128 384 000 The last 4 columns show respectively max, low watermark, high watermark and current number of pages or batches of pages for given L1 or L2 pool. For more information please read https://github.com/cloudius-systems/osv/wiki/Memory-Management#high-level-layer. Signed-off-by: Waldemar Kozaczuk <jwkozac...@gmail.com> --- core/mempool.cc | 58 ++++++++++++++++++++++++++++++++++++++++- fs/sysfs/sysfs_vnops.cc | 51 ++++++++++++++++++++++++++++++++++++ include/osv/mempool.hh | 21 +++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/core/mempool.cc b/core/mempool.cc index 50f5e877..e557d04f 100644 --- a/core/mempool.cc +++ b/core/mempool.cc @@ -541,7 +541,7 @@ void reclaimer::wait_for_memory(size_t mem) class page_range_allocator { public: - static constexpr unsigned max_order = 16; + static constexpr unsigned max_order = page_ranges_max_order; page_range_allocator() : _deferred_free(nullptr) { } @@ -571,6 +571,22 @@ public: return size; } + void stats(stats::page_ranges_stats& stats) const { + stats.order[max_order].ranges_num = _free_huge.size(); + stats.order[max_order].bytes = 0; + for (auto& pr : _free_huge) { + stats.order[max_order].bytes += pr.size; + } + + for (auto order = max_order; order--;) { + stats.order[order].ranges_num = _free[order].size(); + stats.order[order].bytes = 0; + for (auto& pr : _free[order]) { + stats.order[order].bytes += pr.size; + } + } + } + private: template<bool UseBitmap = true> void insert(page_range& pr) { @@ -822,6 +838,15 @@ void page_range_allocator::for_each(unsigned min_order, Func f) } } +namespace stats { + void get_page_ranges_stats(page_ranges_stats &stats) + { + WITH_LOCK(free_page_ranges_lock) { + free_page_ranges.stats(stats); + } + } +} + static void* mapped_malloc_large(size_t size, size_t offset) { //TODO: For now pre-populate the memory, in future consider doing lazy population @@ -1123,6 +1148,8 @@ static size_t large_object_size(void *obj) namespace page_pool { +static std::vector<stats::pool_stats> l1_pool_stats; + // L1-pool (Percpu page buffer pool) // // if nr < max * 1 / 4 @@ -1137,6 +1164,7 @@ struct l1 { : _fill_thread(sched::thread::make([] { fill_thread(); }, sched::thread::attr().pin(cpu).name(osv::sprintf("page_pool_l1_%d", cpu->id)))) { + cpu_id = cpu->id; _fill_thread->start(); } @@ -1160,12 +1188,15 @@ struct l1 { void* pop() { assert(nr); + l1_pool_stats[cpu_id]._nr = nr - 1; return _pages[--nr]; } void push(void* page) { assert(nr < 512); _pages[nr++] = page; + l1_pool_stats[cpu_id]._nr = nr; + } void* top() { return _pages[nr - 1]; } void wake_thread() { _fill_thread->wake(); } @@ -1177,6 +1208,7 @@ struct l1 { static constexpr size_t watermark_lo = max * 1 / 4; static constexpr size_t watermark_hi = max * 3 / 4; size_t nr = 0; + unsigned int cpu_id; private: std::unique_ptr<sched::thread> _fill_thread; @@ -1266,6 +1298,14 @@ public: return true; } + void stats(stats::pool_stats &stats) + { + stats._nr = get_nr(); + stats._max = _max; + stats._watermark_lo = _watermark_lo; + stats._watermark_hi = _watermark_hi; + } + void fill_thread(); void refill(); void unfill(); @@ -1291,6 +1331,7 @@ static sched::cpu::notifier _notifier([] () { if (smp_allocator_cnt++ == sched::cpus.size()) { smp_allocator = true; } + l1_pool_stats.resize(sched::cpus.size()); }); static inline l1& get_l1() { @@ -1469,6 +1510,21 @@ void l2::free_batch(page_batch& batch) } +namespace stats { + void get_global_l2_stats(pool_stats &stats) + { + page_pool::global_l2.stats(stats); + } + + void get_l1_stats(unsigned int cpu_id, pool_stats &stats) + { + stats._nr = page_pool::l1_pool_stats[cpu_id]._nr; + stats._max = page_pool::l1::max; + stats._watermark_lo = page_pool::l1::watermark_lo; + stats._watermark_hi = page_pool::l1::watermark_hi; + } +} + static void* early_alloc_page() { WITH_LOCK(free_page_ranges_lock) { diff --git a/fs/sysfs/sysfs_vnops.cc b/fs/sysfs/sysfs_vnops.cc index df24ecb0..15636f92 100644 --- a/fs/sysfs/sysfs_vnops.cc +++ b/fs/sysfs/sysfs_vnops.cc @@ -8,6 +8,8 @@ #include <unistd.h> #include <osv/mount.h> #include <mntent.h> +#include <osv/printf.hh> +#include <osv/mempool.hh> #include "fs/pseudofs/pseudofs.hh" @@ -30,6 +32,47 @@ static string sysfs_distance() return std::string("10"); } +using namespace memory; +static string sysfs_free_page_ranges() +{ + stats::page_ranges_stats stats; + stats::get_page_ranges_stats(stats); + + std::ostringstream os; + if (stats.order[page_ranges_max_order].ranges_num) { + osv::fprintf(os, "huge %04d %012ld\n", //TODO: Show in GB/MB/KB + stats.order[page_ranges_max_order].ranges_num, stats.order[page_ranges_max_order].bytes); + } + + for (int order = page_ranges_max_order; order--; ) { + if (stats.order[order].ranges_num) { + osv::fprintf(os, " %02d %04d %012ld\n", + order + 1, stats.order[order].ranges_num, stats.order[order].bytes); + } + } + + return os.str(); +} + +static string sysfs_memory_pools() +{ + stats::pool_stats stats; + stats::get_global_l2_stats(stats); + + std::ostringstream os; + osv::fprintf(os, "global l2 (in batches) %02d %02d %02d %02d\n", + stats._max, stats._watermark_lo, stats._watermark_hi, stats._nr); + + for (auto cpu : sched::cpus) { + stats::pool_stats stats; + stats::get_l1_stats(cpu->id, stats); + osv::fprintf(os, "cpu %d l1 (in pages) %03d %03d %03d %03d\n", + cpu->id, stats._max, stats._watermark_lo, stats._watermark_hi, stats._nr); + } + + return os.str(); +} + static int sysfs_mount(mount* mp, const char *dev, int flags, const void* data) { @@ -49,8 +92,16 @@ sysfs_mount(mount* mp, const char *dev, int flags, const void* data) auto devices = make_shared<pseudo_dir_node>(inode_count++); devices->add("system", system); + auto memory = make_shared<pseudo_dir_node>(inode_count++); + memory->add("free_page_ranges", inode_count++, sysfs_free_page_ranges); + memory->add("pools", inode_count++, sysfs_memory_pools); + + auto osv_extension = make_shared<pseudo_dir_node>(inode_count++); + osv_extension->add("memory", memory); + auto* root = new pseudo_dir_node(vp->v_ino); root->add("devices", devices); + root->add("osv", osv_extension); vp->v_data = static_cast<void*>(root); diff --git a/include/osv/mempool.hh b/include/osv/mempool.hh index 10fe5602..33a9af68 100644 --- a/include/osv/mempool.hh +++ b/include/osv/mempool.hh @@ -188,6 +188,8 @@ private: ssize_t bytes_until_normal() { return bytes_until_normal(pressure_level()); } }; +const unsigned page_ranges_max_order = 16; + namespace stats { size_t free(); size_t total(); @@ -195,6 +197,25 @@ namespace stats { size_t jvm_heap(); void on_jvm_heap_alloc(size_t mem); void on_jvm_heap_free(size_t mem); + + struct page_ranges_stats { + struct { + size_t bytes; + size_t ranges_num; + } order[page_ranges_max_order + 1]; + }; + + void get_page_ranges_stats(page_ranges_stats &stats); + + struct pool_stats { + size_t _max; + size_t _nr; + size_t _watermark_lo; + size_t _watermark_hi; + }; + + void get_global_l2_stats(pool_stats &stats); + void get_l1_stats(unsigned int cpu_id, stats::pool_stats &stats); } class phys_contiguous_memory final { -- 2.20.1 -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to osv-dev+unsubscr...@googlegroups.com. To view this discussion on the web visit https://groups.google.com/d/msgid/osv-dev/20200408035745.29894-1-jwkozaczuk%40gmail.com.
