Supported basic auto image gc in the agent. Review: https://reviews.apache.org/r/64267
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c4741b9b Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c4741b9b Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c4741b9b Branch: refs/heads/master Commit: c4741b9bd4c86e63ffd98f28975fb7aa54f25861 Parents: e9125dd Author: Gilbert Song <[email protected]> Authored: Sun Nov 26 15:12:34 2017 -0500 Committer: Gilbert Song <[email protected]> Committed: Wed Dec 6 12:04:19 2017 -0800 ---------------------------------------------------------------------- src/slave/slave.cpp | 61 ++++++++++++++++++++++++++++++++++++++++++++++++ src/slave/slave.hpp | 8 +++++++ 2 files changed, 69 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/c4741b9b/src/slave/slave.cpp ---------------------------------------------------------------------- diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp index 4927001..dd830d9 100644 --- a/src/slave/slave.cpp +++ b/src/slave/slave.cpp @@ -620,6 +620,21 @@ void Slave::initialize() // a very large disk_watch_interval). delay(flags.disk_watch_interval, self(), &Slave::checkDiskUsage); + // Start image store disk monitoring. Please note that image layers + // garbage collection is only enabled if the agent flag `--image_gc_config` + // is set. + // TODO(gilbert): Consider move the image auto GC logic to containerizers + // respectively. For now, it is only enabled for the Mesos Containerizer. + if (flags.image_gc_config.isSome() && + flags.image_providers.isSome() && + strings::contains(flags.containerizers, "mesos")) { + delay( + Nanoseconds( + flags.image_gc_config->image_disk_watch_interval().nanoseconds()), + self(), + &Slave::checkImageDiskUsage); + } + startTime = Clock::now(); // Install protobuf handlers. @@ -6194,6 +6209,52 @@ Try<Nothing> Slave::compatible( } +// TODO(gilbert): Consider to read the Image GC config dynamically. +// For now, the Image GC config can only be updated after the agent +// restarts. +void Slave::checkImageDiskUsage() +{ + // TODO(gilbert): Container image gc is supported for docker image + // in Mesos Containerizer for now. Add more image store gc supports + // if necessary. + Future<double>(::fs::usage(flags.docker_store_dir)) + .onAny(defer(self(), &Slave::_checkImageDiskUsage, lambda::_1)); +} + + +void Slave::_checkImageDiskUsage(const Future<double>& usage) +{ + CHECK(flags.image_gc_config.isSome()); + + if (!usage.isReady()) { + LOG(ERROR) << "Failed to get image store disk usage: " + << (usage.isFailed() ? usage.failure() : "future discarded"); + } else { + LOG(INFO) << "Current docker image store disk usage: " + << std::setiosflags(std::ios::fixed) << std::setprecision(2) + << 100 * usage.get() << "%."; + + if ((flags.image_gc_config->image_disk_headroom() + usage.get()) > 1.0) { + LOG(INFO) << "Image store disk usage exceeds the threshold '" + << 100 * (1.0 - flags.image_gc_config->image_disk_headroom()) + << "%'. Container Image GC is triggered."; + + vector<Image> excludedImages( + flags.image_gc_config->excluded_images().begin(), + flags.image_gc_config->excluded_images().end()); + + containerizer->pruneImages(excludedImages); + } + } + + delay( + Nanoseconds( + flags.image_gc_config->image_disk_watch_interval().nanoseconds()), + self(), + &Slave::checkImageDiskUsage); +} + + Future<Nothing> Slave::recover(const Try<state::State>& state) { if (state.isError()) { http://git-wip-us.apache.org/repos/asf/mesos/blob/c4741b9b/src/slave/slave.hpp ---------------------------------------------------------------------- diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp index 06afd52..fc762fb 100644 --- a/src/slave/slave.hpp +++ b/src/slave/slave.hpp @@ -322,6 +322,10 @@ public: // and os calls. void _checkDiskUsage(const process::Future<double>& usage); + // Garbage collect image layers based on the disk usage of image + // store. + void _checkImageDiskUsage(const process::Future<double>& usage); + // Invoked whenever the detector detects a change in masters. // Made public for testing purposes. void detected(const process::Future<Option<MasterInfo>>& _master); @@ -430,6 +434,10 @@ public: // Checks the current disk usage and schedules for gc as necessary. void checkDiskUsage(); + // Checks the current container image disk usage and trigger image + // gc if necessary. + void checkImageDiskUsage(); + // Recovers the slave, task status update manager and isolator. process::Future<Nothing> recover(const Try<state::State>& state);
