Schedule docker containers for removal. Instead of removing docker containers right after reap, schedule it to be removed later.
Review: https://reviews.apache.org/r/26861 Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/20b0225f Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/20b0225f Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/20b0225f Branch: refs/heads/master Commit: 20b0225fcf1ed84c0f518ae51b856f43c044782f Parents: e8554e5 Author: Timothy Chen <[email protected]> Authored: Fri Oct 31 16:41:07 2014 -0700 Committer: Timothy Chen <[email protected]> Committed: Fri Oct 31 16:48:41 2014 -0700 ---------------------------------------------------------------------- src/slave/constants.cpp | 1 + src/slave/constants.hpp | 3 +++ src/slave/containerizer/docker.cpp | 28 ++++++++++++++++++++++++++-- src/slave/flags.hpp | 8 ++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/20b0225f/src/slave/constants.cpp ---------------------------------------------------------------------- diff --git a/src/slave/constants.cpp b/src/slave/constants.cpp index e1da5c0..d6ad78c 100644 --- a/src/slave/constants.cpp +++ b/src/slave/constants.cpp @@ -49,6 +49,7 @@ const std::string DEFAULT_PORTS = "[31000-32000]"; #ifdef WITH_NETWORK_ISOLATOR const uint16_t DEFAULT_EPHEMERAL_PORTS_PER_CONTAINER = 1024; #endif +const Duration DOCKER_REMOVE_DELAY = Hours(6); Duration MASTER_PING_TIMEOUT() { http://git-wip-us.apache.org/repos/asf/mesos/blob/20b0225f/src/slave/constants.hpp ---------------------------------------------------------------------- diff --git a/src/slave/constants.hpp b/src/slave/constants.hpp index 9030871..701dd89 100644 --- a/src/slave/constants.hpp +++ b/src/slave/constants.hpp @@ -94,6 +94,9 @@ const Bytes DEFAULT_EXECUTOR_MEM = Megabytes(32); extern const uint16_t DEFAULT_EPHEMERAL_PORTS_PER_CONTAINER; #endif +// Default duration that docker containers will be removed after exit. +extern const Duration DOCKER_REMOVE_DELAY; + // If no pings received within this timeout, then the slave will // trigger a re-detection of the master to cause a re-registration. Duration MASTER_PING_TIMEOUT(); http://git-wip-us.apache.org/repos/asf/mesos/blob/20b0225f/src/slave/containerizer/docker.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp index d6617a9..7e5ff37 100644 --- a/src/slave/containerizer/docker.cpp +++ b/src/slave/containerizer/docker.cpp @@ -23,6 +23,7 @@ #include <process/check.hpp> #include <process/defer.hpp> +#include <process/delay.hpp> #include <process/io.hpp> #include <process/reap.hpp> #include <process/subprocess.hpp> @@ -204,6 +205,9 @@ private: // container destroy. void reaped(const ContainerID& containerId); + // Removes the docker container. + void remove(const std::string& container); + const Flags flags; Docker docker; @@ -1522,8 +1526,16 @@ void DockerContainerizerProcess::_destroy( LOG(INFO) << "Running docker kill on container '" << containerId << "'"; - docker.kill(container->name(), true) - .onAny(defer(self(), &Self::__destroy, containerId, killed, lambda::_1)); + if (killed) { + docker.kill(container->name(), false) + .onAny(defer(self(), &Self::__destroy, containerId, killed, lambda::_1)); + } else { + // If the container exited normally, skip docker kill so logs can + // still finish forwarding from the container. This is due to + // a docker bug that is sometimes not writing out stdout output + //if kill/stop is called on an already exited container. + __destroy(containerId, killed, Nothing()); + } } @@ -1547,6 +1559,9 @@ void DockerContainerizerProcess::__destroy( (kill.isFailed() ? kill.failure() : "discarded future")); containers_.erase(containerId); + + delay(flags.docker_remove_delay, self(), &Self::remove, container->name()); + delete container; return; @@ -1582,6 +1597,9 @@ void DockerContainerizerProcess::___destroy( container->termination.set(termination); containers_.erase(containerId); + + delay(flags.docker_remove_delay, self(), &Self::remove, container->name()); + delete container; } @@ -1605,6 +1623,12 @@ void DockerContainerizerProcess::reaped(const ContainerID& containerId) } +void DockerContainerizerProcess::remove(const string& container) +{ + docker.rm(container, true); +} + + } // namespace slave { } // namespace internal { } // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/20b0225f/src/slave/flags.hpp ---------------------------------------------------------------------- diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp index f7a8cde..319c002 100644 --- a/src/slave/flags.hpp +++ b/src/slave/flags.hpp @@ -291,6 +291,7 @@ public: "The default container image to use if not specified by a task,\n" "when using external containerizer.\n"); + // Docker containerizer flags. add(&Flags::docker, "docker", "The absolute path to the docker executable for docker\n" @@ -303,6 +304,12 @@ public: "sandbox is mapped to.\n", "/mnt/mesos/sandbox"); + add(&Flags::docker_remove_delay, + "docker_remove_delay", + "The amount of time to wait before removing docker containers\n" + "(e.g., 3days, 2weeks, etc).\n", + DOCKER_REMOVE_DELAY); + add(&Flags::default_container_info, "default_container_info", "JSON formatted ContainerInfo that will be included into\n" @@ -437,6 +444,7 @@ public: Option<std::string> default_container_image; std::string docker; std::string docker_sandbox_directory; + Duration docker_remove_delay; Option<ContainerInfo> default_container_info; #ifdef WITH_NETWORK_ISOLATOR uint16_t ephemeral_ports_per_container;
