From: Waldemar Kozaczuk <[email protected]> Committer: Nadav Har'El <[email protected]> Branch: master
external x64: move java balloon code out of kernel into the module java-base The main motivation of this patch is to eliminate compile time dependencies in kernel on JDK headers. It achieves it by moving java balloon shrinker implementation that requires java headers to compile, from kernel (java/jvm) to modules/java-base. Given that kernel code in mmu, mempool and mman depended on jvm_balloon_shrinker directly, it was necessary to create new abstraction/interface - jvm_balloon_api (class with pure virtual methods) - which kernel code would use to interact with jvm balloon code if enabled. The code moved from java/jvm to modules/java-base/balloon would provide implementation of jvm_balloon_api hiding all direct interaction with JNI (Java Native Interface). In essence some functions in jvm_balloon.cc have been adopted into methods of jvm_balloon_api_impl class - singleton implementing interface defined by jvm_balloon_api. References #743 Signed-off-by: Waldemar Kozaczuk <[email protected]> Message-Id: <[email protected]> --- diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -319,7 +319,6 @@ COMMON = $(autodepend) -g -Wall -Wno-pointer-arith $(CFLAGS_WERROR) -Wformat=0 - $(kernel-defines) \ -fno-omit-frame-pointer $(compiler-specific) \ -include compiler/include/intrinsics.hh \ - $(do-sys-includes) \ $(arch-cflags) $(conf-opt) $(acpi-defines) $(tracing-flags) $(gcc-sysroot) \ $(configuration) -D__OSV__ -D__XEN_INTERFACE_VERSION__="0x00030207" -DARCH_STRING=$(ARCH_STR) $(EXTRA_FLAGS) ifeq ($(gcc_include_env), external) @@ -383,11 +382,8 @@ $(out)/%.o: %.s $(makedir) $(q-build-so) -sys-includes = $(jdkbase)/include $(jdkbase)/include/linux autodepend = -MD -MT $@ -MP -do-sys-includes = $(foreach inc, $(sys-includes), -isystem $(inc)) - tools := tools/mkfs/mkfs.so tools/cpiod/cpiod.so $(out)/tools/%.o: COMMON += -fPIC @@ -816,7 +812,6 @@ drivers += drivers/clock.o drivers += drivers/clock-common.o drivers += drivers/clockevent.o drivers += core/elf.o -drivers += java/jvm/jvm_balloon.o drivers += drivers/random.o drivers += drivers/zfs.o drivers += drivers/null.o diff --git a/core/mempool.cc b/core/mempool.cc --- a/core/mempool.cc +++ b/core/mempool.cc @@ -31,7 +31,6 @@ #include <osv/shrinker.h> #include <osv/defer.hh> #include <osv/dbg-alloc.hh> -#include "java/jvm/jvm_balloon.hh" #include <boost/dynamic_bitset.hpp> #include <boost/lockfree/stack.hpp> #include <boost/lockfree/policies.hpp> @@ -442,7 +441,9 @@ static void on_free(size_t mem) static void on_alloc(size_t mem) { free_memory.fetch_sub(mem); - jvm_balloon_adjust_memory(min_emergency_pool_size); + if (balloon_api) { + balloon_api->adjust_memory(min_emergency_pool_size); + } if ((stats::free() + stats::jvm_heap()) < watermark_lo) { reclaimer_thread.wake(); } @@ -1030,7 +1031,9 @@ void reclaimer::_do_reclaim() } } - jvm_balloon_voluntary_return(); + if (balloon_api) { + balloon_api->voluntary_return(); + } } } } @@ -1976,4 +1979,14 @@ void free_phys_contiguous_aligned(void* p) free_large(p); } +bool throttling_needed() +{ + if (!balloon_api) { + return false; + } + + return balloon_api->ballooning(); +} + +jvm_balloon_api *balloon_api = nullptr; } diff --git a/core/mmu.cc b/core/mmu.cc --- a/core/mmu.cc +++ b/core/mmu.cc @@ -22,7 +22,6 @@ #include <osv/error.h> #include <osv/trace.hh> #include <stack> -#include "java/jvm/jvm_balloon.hh" #include <fs/fs.hh> #include <osv/file.h> #include "dump.hh" @@ -1551,7 +1550,7 @@ error jvm_balloon_vma::sync(uintptr_t start, uintptr_t end) void jvm_balloon_vma::fault(uintptr_t fault_addr, exception_frame *ef) { - if (jvm_balloon_fault(_balloon, ef, this)) { + if (memory::balloon_api && memory::balloon_api->fault(_balloon, ef, this)) { return; } // Can only reach this case if we are doing partial copies @@ -1578,8 +1577,8 @@ jvm_balloon_vma::~jvm_balloon_vma() if (_effective_jvm_addr) { // Can't just use size(), because although rare, the source and destination can // have different alignments - auto end = align_down(_effective_jvm_addr + balloon_size, balloon_alignment); - auto s = end - align_up(_effective_jvm_addr, balloon_alignment); + auto end = align_down(_effective_jvm_addr + memory::balloon_size, memory::balloon_alignment); + auto s = end - align_up(_effective_jvm_addr, memory::balloon_alignment); mmu::map_jvm(_effective_jvm_addr, s, mmu::huge_page_size, _balloon); } } diff --git a/include/osv/mempool.hh b/include/osv/mempool.hh --- a/include/osv/mempool.hh +++ b/include/osv/mempool.hh @@ -283,6 +283,31 @@ public: /// Hold to mark self as a memory reclaimer extern reclaimer_lock_type reclaimer_lock; +// We will divide the balloon in units of 128Mb. That should increase the likelyhood +// of having hugepages mapped in and out of it. +// +// Using constant sized balloons should help with the process of giving memory +// back to the JVM, since we don't need to search the list of balloons until +// we find a balloon of the desired size: any will do. +constexpr size_t balloon_size = (128ULL << 20); +// FIXME: Can probably do better than this. We are counting 4Mb before 1Gb to +// account for ROMs and the such. 4Mb is probably too much (in kvm with no vga +// we lose around 400k), but it doesn't hurt. +constexpr size_t balloon_min_memory = (1ULL << 30) - (4 << 20); +constexpr size_t balloon_alignment = mmu::huge_page_size; + +class jvm_balloon_api { +public: + jvm_balloon_api() {}; + virtual ~jvm_balloon_api() {}; + virtual void return_heap(size_t mem) = 0; + virtual void adjust_memory(size_t threshold) = 0; + virtual void voluntary_return() = 0; + virtual bool fault(balloon_ptr b, exception_frame *ef, mmu::jvm_balloon_vma *vma) = 0; + virtual bool ballooning() = 0; +}; + +extern jvm_balloon_api *balloon_api; } #endif diff --git a/libc/mman.cc b/libc/mman.cc --- a/libc/mman.cc +++ b/libc/mman.cc @@ -15,7 +15,6 @@ #include "osv/mount.h" #include "libc/libc.hh" #include <safe-ptr.hh> -#include <java/jvm/jvm_balloon.hh> TRACEPOINT(trace_memory_mmap, "addr=%p, length=%d, prot=%d, flags=%d, fd=%d, offset=%d", void *, size_t, int, int, int, off_t); TRACEPOINT(trace_memory_mmap_err, "%d", int); @@ -150,7 +149,9 @@ void *mmap(void *addr, size_t length, int prot, int flags, // it this way now because it is simpler and I don't expect that to // ever be harmful. mmap_flags |= mmu::mmap_jvm_heap; - memory::return_jvm_heap(length); + if (memory::balloon_api) { + memory::balloon_api->return_heap(length); + } } try { ret = mmu::map_anon(addr, length, mmap_flags, mmap_perm); diff --git a/modules/httpserver-api/api/os.cc b/modules/httpserver-api/api/os.cc --- a/modules/httpserver-api/api/os.cc +++ b/modules/httpserver-api/api/os.cc @@ -19,7 +19,7 @@ #include <api/unistd.h> #include <osv/commands.hh> #include <algorithm> -#include "java/jvm/balloon_api.hh" +#include "../java-base/balloon/balloon_api.hh" extern char debug_buffer[DEBUG_BUFFER_SIZE]; diff --git a/modules/java-base/Makefile b/modules/java-base/Makefile --- a/modules/java-base/Makefile +++ b/modules/java-base/Makefile @@ -3,7 +3,7 @@ include common.gmk ifeq ($(arch),aarch64) java-targets := else -java-targets := obj/jni/monitor.so obj/jvm/jni_helpers.o obj/jvm/java_api.o +java-targets := obj/jni/monitor.so obj/jvm/jni_helpers.o obj/jvm/java_api.o obj/balloon/jvm_balloon.o endif module: all @@ -14,6 +14,7 @@ init: @echo " MKDIRS" $(call very-quiet, mkdir -p obj/jni) $(call very-quiet, mkdir -p obj/jvm) + $(call very-quiet, mkdir -p obj/balloon) .PHONY: init clean: diff --git a/modules/java-base/balloon/balloon_api.hh b/modules/java-base/balloon/balloon_api.hh --- a/modules/java-base/balloon/balloon_api.hh +++ b/modules/java-base/balloon/balloon_api.hh null diff --git a/modules/java-base/balloon/jvm_balloon.cc b/modules/java-base/balloon/jvm_balloon.cc --- a/modules/java-base/balloon/jvm_balloon.cc +++ b/modules/java-base/balloon/jvm_balloon.cc @@ -29,8 +29,6 @@ TRACEPOINT(trace_jvm_balloon_close, "from=%p, to=%p, condition=%s", uintptr_t, uintptr_t, const char *); -jvm_balloon_shrinker *balloon_shrinker = nullptr; - namespace memory { // If we are under pressure, we will end up setting the voluntary return flag @@ -45,65 +43,48 @@ void reserve_jvm_heap(size_t mem) jvm_heap_allowance.fetch_sub(mem, std::memory_order_relaxed); } -void return_jvm_heap(size_t mem) +void jvm_balloon_api_impl::return_heap(size_t mem) { jvm_heap_allowance.fetch_add(mem, std::memory_order_relaxed); balloon_voluntary_return = true; } ssize_t jvm_heap_reserved() { - if (!balloon_shrinker) { + if (!memory::balloon_api) { return 0; } return (stats::free() + stats::jvm_heap()) - jvm_heap_allowance.load(std::memory_order_relaxed); } -void jvm_balloon_adjust_memory(size_t threshold) +void jvm_balloon_api_impl::adjust_memory(size_t threshold) { - if (!balloon_shrinker) { - return; - } - // Core of the reservation system: // The heap allowance starts as the initial memory that is reserved to // the JVM. It means how much it can eventually use, and it is completely // dissociated with the amount of memory it is using now. When we balloon, // that number goes down, and when we return the balloon back, it goes // up again. if (jvm_heap_reserved() <= static_cast<ssize_t>(threshold)) { - balloon_shrinker->request_memory(1); - } -} - -bool throttling_needed() -{ - if (!balloon_shrinker) { - return false; + _balloon_shrinker->request_memory(1); } - - return balloon_shrinker->ballooning(); } -}; -void jvm_balloon_voluntary_return() +void jvm_balloon_api_impl::voluntary_return() { - if (!balloon_shrinker) { - return; - } - // If we freed memory and now we have more than a balloon + 20 % worth of // reserved memory, give it back to the Java Heap. This is because it is a // lot harder to react to JVM memory shortages than it is to react to OSv // memory shortages - which are effectively under our control. Don't doing // this can result in Heap exhaustions in situations where JVM allocation // rates are very high and memory is tight - if ((memory::jvm_heap_reserved() > 6 * static_cast<ssize_t>(balloon_size/5)) && + if ((memory::jvm_heap_reserved() > 6 * static_cast<ssize_t>(memory::balloon_size/5)) && memory::balloon_voluntary_return.exchange(false)) { - balloon_shrinker->release_memory(1); + _balloon_shrinker->release_memory(1); } } +}; class balloon { public: @@ -127,7 +108,7 @@ class balloon { jobject _jref; unsigned int _alignment; - size_t _balloon_size = balloon_size; + size_t _balloon_size = memory::balloon_size; }; mutex balloons_lock; @@ -136,7 +117,7 @@ std::list<balloon_ptr> balloons; namespace memory { ssize_t get_balloon_size() { WITH_LOCK(balloons_lock) { - return balloons.size() * balloon_size; + return balloons.size() * memory::balloon_size; } } } @@ -154,7 +135,7 @@ ulong balloon::empty_area(balloon_ptr b) return ret; } -balloon::balloon(unsigned char *jvm_addr, jobject jref, int alignment = mmu::huge_page_size, size_t size = balloon_size) +balloon::balloon(unsigned char *jvm_addr, jobject jref, int alignment = mmu::huge_page_size, size_t size = memory::balloon_size) : _jvm_addr(jvm_addr), _jref(jref), _alignment(alignment), _balloon_size(size) { assert(mutex_owned(&balloons_lock)); @@ -172,7 +153,7 @@ void balloon::release(JNIEnv *env) // No need to remap. Will happen automatically when JVM touches it again env->DeleteGlobalRef(_jref); - memory::return_jvm_heap(minimum_size()); + memory::balloon_api->return_heap(minimum_size()); trace_jvm_balloon_free(); } @@ -229,7 +210,7 @@ size_t jvm_balloon_shrinker::_request_memory(JNIEnv *env, size_t size) size_t ret = 0; do { - jbyteArray array = env->NewByteArray(balloon_size); + jbyteArray array = env->NewByteArray(memory::balloon_size); jthrowable exc = env->ExceptionOccurred(); if (exc) { env->ExceptionClear(); @@ -313,12 +294,12 @@ void jvm_balloon_shrinker::_thread_loop() break; } size_t freed = 1; - while ((memory::jvm_heap_reserved() < static_cast<ssize_t>(balloon_size)) && (freed != 0)) { - uint64_t to_free = balloon_size - memory::jvm_heap_reserved(); + while ((memory::jvm_heap_reserved() < static_cast<ssize_t>(memory::balloon_size)) && (freed != 0)) { + uint64_t to_free = memory::balloon_size - memory::jvm_heap_reserved(); freed = arc_sized_adjust(to_free); } - if (memory::jvm_heap_reserved() >= static_cast<ssize_t>(balloon_size)) { + if (memory::jvm_heap_reserved() >= static_cast<ssize_t>(memory::balloon_size)) { _pending_release.fetch_sub(1); _release_memory(env, memory::jvm_heap_reserved()); } else { @@ -349,7 +330,8 @@ void jvm_balloon_shrinker::_thread_loop() // part. That means copying the part that comes before the balloon, playing // with the maps for the balloon itself, and then finish copying the part that // comes after the balloon. -bool jvm_balloon_fault(balloon_ptr b, exception_frame *ef, mmu::jvm_balloon_vma *vma) +namespace memory { +bool jvm_balloon_api_impl::fault(balloon_ptr b, exception_frame *ef, mmu::jvm_balloon_vma *vma) { if (!ef || mmu::is_page_fault_write_exclusive(ef->get_error())) { if (vma->effective_jvm_addr()) { @@ -424,6 +406,7 @@ bool jvm_balloon_fault(balloon_ptr b, exception_frame *ef, mmu::jvm_balloon_vma #endif /* !AARCH64_PORT_STUB */ return true; } +}; jvm_balloon_shrinker::jvm_balloon_shrinker(JavaVM_ *vm) : _vm(vm) @@ -446,7 +429,7 @@ jvm_balloon_shrinker::jvm_balloon_shrinker(JavaVM_ *vm) // down towards number. This is because if the JVM is very short on // memory, it can quickly fill up the new balloon and may not have time // for a new GC cycle. - _soft_max_balloons = (_total_heap / ( 2 * balloon_size)) - 1; + _soft_max_balloons = (_total_heap / ( 2 * memory::balloon_size)) - 1; auto monmethod = env->GetStaticMethodID(monitor, "MonitorGC", "(J)V"); env->CallStaticVoidMethod(monitor, monmethod, this); @@ -461,8 +444,6 @@ jvm_balloon_shrinker::jvm_balloon_shrinker(JavaVM_ *vm) _detach(status); - balloon_shrinker = this; - // This cannot be a sched::thread because it may call into JNI functions, // if the JVM balloon is registered as a shrinker. It expects the full pthread // API to be functional, and for sched::threads it is not. @@ -471,7 +452,21 @@ jvm_balloon_shrinker::jvm_balloon_shrinker(JavaVM_ *vm) tmp.detach(); } -jvm_balloon_shrinker::~jvm_balloon_shrinker() +namespace memory { +jvm_balloon_api_impl::jvm_balloon_api_impl(JavaVM *jvm) { - balloon_shrinker = nullptr; + _balloon_shrinker = new jvm_balloon_shrinker(jvm); + balloon_api = this; } + +jvm_balloon_api_impl::~jvm_balloon_api_impl() +{ + delete _balloon_shrinker; + balloon_api = nullptr; +} + +bool jvm_balloon_api_impl::ballooning() +{ + return _balloon_shrinker->ballooning(); +} +}; diff --git a/modules/java-base/balloon/jvm_balloon.hh b/modules/java-base/balloon/jvm_balloon.hh --- a/modules/java-base/balloon/jvm_balloon.hh +++ b/modules/java-base/balloon/jvm_balloon.hh @@ -15,26 +15,13 @@ #include <osv/condvar.h> #include <atomic> -// We will divide the balloon in units of 128Mb. That should increase the likelyhood -// of having hugepages mapped in and out of it. -// -// Using constant sized balloons should help with the process of giving memory -// back to the JVM, since we don't need to search the list of balloons until -// we find a balloon of the desired size: any will do. -constexpr size_t balloon_size = (128ULL << 20); -// FIXME: Can probably do better than this. We are counting 4Mb before 1Gb to -// account for ROMs and the such. 4Mb is probably too much (in kvm with no vga -// we lose around 400k), but it doesn't hurt. -constexpr size_t balloon_min_memory = (1ULL << 30) - (4 << 20); -constexpr size_t balloon_alignment = mmu::huge_page_size; - class jvm_balloon_shrinker { public: explicit jvm_balloon_shrinker(JavaVM *vm); void request_memory(size_t s) { _pending.fetch_add(s); _blocked.wake_one(); } void release_memory(size_t s) { _pending_release.fetch_add(s); _blocked.wake_one(); } bool ballooning() { return _pending.load() > 0; } - virtual ~jvm_balloon_shrinker(); + virtual ~jvm_balloon_shrinker() {}; private: void _release_memory(JNIEnv *env, size_t s); size_t _request_memory(JNIEnv *env, size_t s); @@ -50,13 +37,18 @@ private: std::atomic<size_t> _pending_release = {0}; }; -bool jvm_balloon_fault(balloon_ptr b, exception_frame *ef, mmu::jvm_balloon_vma *vma); - namespace memory { - void return_jvm_heap(size_t size); - void reserve_jvm_heap(size_t size); - ssize_t jvm_heap_reserved(); - void jvm_balloon_adjust_memory(size_t threshold); +class jvm_balloon_api_impl : public jvm_balloon_api { +public: + explicit jvm_balloon_api_impl(JavaVM *jvm); + virtual ~jvm_balloon_api_impl(); + virtual void return_heap(size_t mem); + virtual void adjust_memory(size_t threshold); + virtual void voluntary_return(); + virtual bool fault(balloon_ptr b, exception_frame *ef, mmu::jvm_balloon_vma *vma); + virtual bool ballooning(); +private: + jvm_balloon_shrinker *_balloon_shrinker; }; -void jvm_balloon_voluntary_return(); +} #endif diff --git a/modules/java-base/java.cc b/modules/java-base/java.cc --- a/modules/java-base/java.cc +++ b/modules/java-base/java.cc @@ -12,7 +12,7 @@ #include <unistd.h> #include <regex> #include <osv/debug.hh> -#include <java/jvm/jvm_balloon.hh> +#include "balloon/jvm_balloon.hh" #include <osv/mempool.hh> #include "jvm/java_api.hh" #include "osv/version.hh" @@ -174,9 +174,9 @@ static int java_main(int argc, char **argv) size_t auto_heap = 0; #if 0 // Do not use total(), since that won't reflect the whole memory for the - // machine. It then becomes counterintuitive to tell the user what is the + // machine. It then becomes counter intuitive to tell the user what is the // minimum he has to set to balloon - if (!has_xmx && (memory::phys_mem_size >= balloon_min_memory)) { + if (!has_xmx && (memory::phys_mem_size >= memory::balloon_min_memory)) { auto_heap = std::min(memory::stats::free(), memory::stats::max_no_reclaim()) >> 20; options.push_back(mkoption("-Xmx%dM", auto_heap)); if (!has_xms) { @@ -252,8 +252,8 @@ static int java_main(int argc, char **argv) // Manually setting the heap size is viewed as a declaration of intent. In // that case, we'll leave the user alone. This may be revisited in the // future, but it is certainly the safest option. - std::unique_ptr<jvm_balloon_shrinker> - balloon(auto_heap == 0 ? nullptr : new jvm_balloon_shrinker(jvm)); + std::unique_ptr<memory::jvm_balloon_api_impl> + balloon(auto_heap == 0 ? nullptr : new memory::jvm_balloon_api_impl(jvm)); env->CallStaticVoidMethod(mainclass, mainmethod, args); diff --git a/modules/java-base/jni/monitor.cc b/modules/java-base/jni/monitor.cc --- a/modules/java-base/jni/monitor.cc +++ b/modules/java-base/jni/monitor.cc @@ -1,6 +1,6 @@ #include <jni.h> #include "monitor.hh" -#include "java/jvm/jvm_balloon.hh" +#include "../balloon/jvm_balloon.hh" /* * Class: io_osv_OSvGCMonitor @@ -10,5 +10,5 @@ JNIEXPORT void JNICALL Java_io_osv_OSvGCMonitor_NotifyOSv(JNIEnv *env, jclass mon, jlong handle, jlong qty) { jvm_balloon_shrinker *shrinker = (jvm_balloon_shrinker *)handle; - shrinker->release_memory((qty / balloon_size) + !!(qty % balloon_size)); + shrinker->release_memory((qty / memory::balloon_size) + !!(qty % memory::balloon_size)); } diff --git a/modules/java-isolated/Makefile b/modules/java-isolated/Makefile --- a/modules/java-isolated/Makefile +++ b/modules/java-isolated/Makefile @@ -11,7 +11,7 @@ endif obj/java.o: $(java-base-path)/java.cc | init $(call quiet, $(CXX) $(CXXFLAGS) -o $@ -c $(java-base-path)/java.cc -MMD, CXX $@) -obj/java.so: obj/java.o $(java-base-path)/obj/jvm/java_api.o $(java-base-path)/obj/jvm/jni_helpers.o +obj/java.so: obj/java.o $(java-base-path)/obj/jvm/java_api.o $(java-base-path)/obj/jvm/jni_helpers.o $(java-base-path)/obj/balloon/jvm_balloon.o $(call quiet, $(CXX) $(CXXFLAGS) -shared -o $@ $^, LINK $@) init: diff --git a/modules/java-non-isolated/Makefile b/modules/java-non-isolated/Makefile --- a/modules/java-non-isolated/Makefile +++ b/modules/java-non-isolated/Makefile @@ -11,7 +11,8 @@ endif obj/java_non_isolated.o: $(java-base-path)/java.cc | init $(call quiet, $(CXX) $(CXXFLAGS) -DRUN_JAVA_NON_ISOLATED -o $@ -c $(java-base-path)/java.cc -MMD, CXX $@) -obj/java_non_isolated.so: obj/java_non_isolated.o $(java-base-path)/obj/jvm/java_api.o $(java-base-path)/obj/jvm/jni_helpers.o +obj/java_non_isolated.so: obj/java_non_isolated.o $(java-base-path)/obj/jvm/java_api.o \ + $(java-base-path)/obj/jvm/jni_helpers.o $(java-base-path)/obj/balloon/jvm_balloon.o $(call quiet, $(CXX) $(CXXFLAGS) -shared -o $@ $^, LINK $@) init: diff --git a/modules/java-tests/Makefile b/modules/java-tests/Makefile --- a/modules/java-tests/Makefile +++ b/modules/java-tests/Makefile @@ -10,7 +10,7 @@ endif obj/java_isolated.o: $(SRC)/modules/java-base/java.cc | init $(call quiet, $(CXX) $(CXXFLAGS) -o $@ -c $(SRC)/modules/java-base/java.cc -MMD, CXX $@) -obj/java_isolated.so: obj/java_isolated.o $(java-base-path)/obj/jvm/java_api.o $(java-base-path)/obj/jvm/jni_helpers.o +obj/java_isolated.so: obj/java_isolated.o $(java-base-path)/obj/jvm/java_api.o $(java-base-path)/obj/jvm/jni_helpers.o $(java-base-path)/obj/balloon/jvm_balloon.o $(call quiet, $(CXX) $(CXXFLAGS) -shared -o $@ $^, LINK $@) init: -- You received this message because you are subscribed to the Google Groups "OSv Development" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To view this discussion on the web visit https://groups.google.com/d/msgid/osv-dev/000000000000c1121c059cb790e6%40google.com.
