Allow docker containers to be scheduled for removal
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ac385583 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ac385583 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ac385583 Branch: refs/heads/docker_symlink Commit: ac3855832ea18354ce118d91759b4d6c0ee85b72 Parents: 6e3e12d Author: Timothy Chen <[email protected]> Authored: Thu Oct 16 17:07:21 2014 -0700 Committer: Timothy Chen <[email protected]> Committed: Tue Oct 28 00:58:22 2014 -0700 ---------------------------------------------------------------------- src/slave/constants.cpp | 1 + src/slave/constants.hpp | 3 +++ src/slave/containerizer/docker.cpp | 28 ++++++++++++++++++++++++++-- src/slave/flags.hpp | 8 ++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/ac385583/src/slave/constants.cpp ---------------------------------------------------------------------- diff --git a/src/slave/constants.cpp b/src/slave/constants.cpp index e1da5c0..30e3f6b 100644 --- a/src/slave/constants.cpp +++ b/src/slave/constants.cpp @@ -49,6 +49,7 @@ const std::string DEFAULT_PORTS = "[31000-32000]"; #ifdef WITH_NETWORK_ISOLATOR const uint16_t DEFAULT_EPHEMERAL_PORTS_PER_CONTAINER = 1024; #endif +const Duration DOCKER_CLEANUP_DELAY = Hours(6); Duration MASTER_PING_TIMEOUT() { http://git-wip-us.apache.org/repos/asf/mesos/blob/ac385583/src/slave/constants.hpp ---------------------------------------------------------------------- diff --git a/src/slave/constants.hpp b/src/slave/constants.hpp index 9030871..7062db8 100644 --- a/src/slave/constants.hpp +++ b/src/slave/constants.hpp @@ -94,6 +94,9 @@ const Bytes DEFAULT_EXECUTOR_MEM = Megabytes(32); extern const uint16_t DEFAULT_EPHEMERAL_PORTS_PER_CONTAINER; #endif +// Default duration that docker containers will be cleaned after exit. +extern const Duration DOCKER_CLEANUP_DELAY; + // If no pings received within this timeout, then the slave will // trigger a re-detection of the master to cause a re-registration. Duration MASTER_PING_TIMEOUT(); http://git-wip-us.apache.org/repos/asf/mesos/blob/ac385583/src/slave/containerizer/docker.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp index e322cd5..eb55ba9 100644 --- a/src/slave/containerizer/docker.cpp +++ b/src/slave/containerizer/docker.cpp @@ -23,6 +23,7 @@ #include <process/check.hpp> #include <process/defer.hpp> +#include <process/delay.hpp> #include <process/io.hpp> #include <process/reap.hpp> #include <process/subprocess.hpp> @@ -204,6 +205,9 @@ private: // container destroy. void reaped(const ContainerID& containerId); + // Removes the docker container. + void cleanup(const std::string& container); + const Flags flags; Docker docker; @@ -1522,8 +1526,16 @@ void DockerContainerizerProcess::_destroy( LOG(INFO) << "Running docker kill on container '" << containerId << "'"; - docker.kill(container->name(), true) - .onAny(defer(self(), &Self::__destroy, containerId, killed, lambda::_1)); + if (killed) { + docker.kill(container->name(), false) + .onAny(defer(self(), &Self::__destroy, containerId, killed, lambda::_1)); + } else { + // If the container exited normally, skip docker kill so logs can + // still finish forwarding from the container. This is due to + // a docker bug that is sometimes not writing out stdout output + //if kill/stop is called on an already exited container. + __destroy(containerId, killed, Nothing()); + } } @@ -1547,6 +1559,9 @@ void DockerContainerizerProcess::__destroy( (kill.isFailed() ? kill.failure() : "discarded future")); containers_.erase(containerId); + + delay(flags.docker_cleanup_delay, self(), &Self::cleanup, container->name()); + delete container; return; @@ -1582,6 +1597,9 @@ void DockerContainerizerProcess::___destroy( container->termination.set(termination); containers_.erase(containerId); + + delay(flags.docker_cleanup_delay, self(), &Self::cleanup, container->name()); + delete container; } @@ -1605,6 +1623,12 @@ void DockerContainerizerProcess::reaped(const ContainerID& containerId) } +void DockerContainerizerProcess::cleanup(const string& container) +{ + docker.rm(container, true); +} + + } // namespace slave { } // namespace internal { } // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/ac385583/src/slave/flags.hpp ---------------------------------------------------------------------- diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp index f7a8cde..3bc623c 100644 --- a/src/slave/flags.hpp +++ b/src/slave/flags.hpp @@ -291,6 +291,7 @@ public: "The default container image to use if not specified by a task,\n" "when using external containerizer.\n"); + // Docker containerizer flags. add(&Flags::docker, "docker", "The absolute path to the docker executable for docker\n" @@ -303,6 +304,12 @@ public: "sandbox is mapped to.\n", "/mnt/mesos/sandbox"); + add(&Flags::docker_cleanup_delay, + "docker_cleanup_delay", + "The amount of time to wait before cleaning up\n" + "docker containers (e.g., 3days, 2weeks, etc).\n", + DOCKER_CLEANUP_DELAY); + add(&Flags::default_container_info, "default_container_info", "JSON formatted ContainerInfo that will be included into\n" @@ -437,6 +444,7 @@ public: Option<std::string> default_container_image; std::string docker; std::string docker_sandbox_directory; + Duration docker_cleanup_delay; Option<ContainerInfo> default_container_info; #ifdef WITH_NETWORK_ISOLATOR uint16_t ephemeral_ports_per_container;
