Added cgroups based statistics to Docker containerizer. Review: https://reviews.apache.org/r/36326
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/867ff24b Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/867ff24b Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/867ff24b Branch: refs/heads/master Commit: 867ff24ba91de54a50fe6b0bcf006152c017791a Parents: 9a5eaca Author: Jojy Varghese <[email protected]> Authored: Tue Jul 14 17:10:25 2015 -0700 Committer: Timothy Chen <[email protected]> Committed: Tue Jul 14 17:57:55 2015 -0700 ---------------------------------------------------------------------- src/slave/containerizer/docker.cpp | 142 +++++++++++++++++++------- src/slave/containerizer/docker.hpp | 8 +- src/tests/docker_containerizer_tests.cpp | 1 + 3 files changed, 105 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/867ff24b/src/slave/containerizer/docker.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp index cfb6017..e710876 100644 --- a/src/slave/containerizer/docker.cpp +++ b/src/slave/containerizer/docker.cpp @@ -1202,71 +1202,135 @@ Future<ResourceStatistics> DockerContainerizerProcess::usage( return Failure("Container is being removed: " + stringify(containerId)); } + auto collectUsage = [this, containerId]( + pid_t pid) -> Future<ResourceStatistics> { + // First make sure container is still there. + if (!containers_.contains(containerId)) { + return Failure("Container has been destroyed: " + stringify(containerId)); + } + + Container* container = containers_[containerId]; + + if (container->state == Container::DESTROYING) { + return Failure("Container is being removed: " + stringify(containerId)); + } + + const Try<ResourceStatistics> cgroupStats = cgroupsStatistics(pid); + if (cgroupStats.isError()) { + return Failure("Failed to collect cgroup stats: " + cgroupStats.error()); + } + + ResourceStatistics result = cgroupStats.get(); + + // Set the resource allocations. + const Resources& resource = container->resources; + const Option<Bytes> mem = resource.mem(); + if (mem.isSome()) { + result.set_mem_limit_bytes(mem.get().bytes()); + } + + const Option<double> cpus = resource.cpus(); + if (cpus.isSome()) { + result.set_cpus_limit(cpus.get()); + } + + return result; + }; + // Skip inspecting the docker container if we already have the pid. if (container->pid.isSome()) { - return __usage(containerId, container->pid.get()); + return collectUsage(container->pid.get()); } return docker->inspect(container->name()) - .then(defer(self(), &Self::_usage, containerId, lambda::_1)); + .then(defer( + self(), + [this, containerId, collectUsage] + (const Docker::Container& _container) -> Future<ResourceStatistics> { + const Option<pid_t> pid = _container.pid; + if (pid.isNone()) { + return Failure("Container is not running"); + } + + if (!containers_.contains(containerId)) { + return Failure( + "Container has been destroyed:" + stringify(containerId)); + } + + Container* container = containers_[containerId]; + + // Update the container's pid now. We ran inspect because we didn't have + // a pid for the container. + container->pid = pid; + + return collectUsage(pid.get()); + })); #endif // __linux__ } -Future<ResourceStatistics> DockerContainerizerProcess::_usage( - const ContainerID& containerId, - const Docker::Container& _container) +Try<ResourceStatistics> DockerContainerizerProcess::cgroupsStatistics( + pid_t pid) const { - if (!containers_.contains(containerId)) { - return Failure("Container has been destroyed:" + stringify(containerId)); - } - - Container* container = containers_[containerId]; +#ifndef __linux__ + return Error("Does not support cgroups on non-linux platform"); +#else + const Result<string> cpuHierarchy = cgroups::hierarchy("cpuacct"); + const Result<string> memHierarchy = cgroups::hierarchy("memory"); - if (container->state == Container::DESTROYING) { - return Failure("Container is being removed: " + stringify(containerId)); + if (cpuHierarchy.isError()) { + return Error( + "Failed to determine the cgroup 'cpu' subsystem hierarchy: " + + cpuHierarchy.error()); } - Option<pid_t> pid = _container.pid; - if (pid.isNone()) { - return Failure("Container is not running"); + if (memHierarchy.isError()) { + return Error( + "Failed to determine the cgroup 'memory' subsystem hierarchy: " + + memHierarchy.error()); } - container->pid = pid; - - return __usage(containerId, pid.get()); -} + const Result<string> cpuCgroup = cgroups::cpuacct::cgroup(pid); + if (cpuCgroup.isError()) { + return Error( + "Failed to determine cgroup for the 'cpu' subsystem: " + + cpuCgroup.error()); + } + const Result<string> memCgroup = cgroups::memory::cgroup(pid); + if (memCgroup.isError()) { + return Error( + "Failed to determine cgroup for the 'memory' subsystem: " + + memCgroup.error()); + } -Future<ResourceStatistics> DockerContainerizerProcess::__usage( - const ContainerID& containerId, - pid_t pid) -{ - Container* container = containers_[containerId]; + const Try<cgroups::cpuacct::Stats> cpuAcctStat = + cgroups::cpuacct::stat(cpuHierarchy.get(), cpuCgroup.get()); - // Note that here getting the root pid is enough because - // the root process acts as an 'init' process in the docker - // container, so no other child processes will escape it. - Try<ResourceStatistics> statistics = mesos::internal::usage(pid, true, true); - if (statistics.isError()) { - return Failure(statistics.error()); + if (cpuAcctStat.isError()) { + return Error("Failed to get cpu.stat: " + cpuAcctStat.error()); } - ResourceStatistics result = statistics.get(); + const Try<hashmap<string, uint64_t>> memStats = + cgroups::stat(memHierarchy.get(), memCgroup.get(), "memory.stat"); - // Set the resource allocations. - const Resources& resource = container->resources; - Option<Bytes> mem = resource.mem(); - if (mem.isSome()) { - result.set_mem_limit_bytes(mem.get().bytes()); + if (memStats.isError()) { + return Error( + "Error getting memory statistics from cgroups memory subsystem: " + + memStats.error()); } - Option<double> cpus = resource.cpus(); - if (cpus.isSome()) { - result.set_cpus_limit(cpus.get()); + if (!memStats.get().contains("rss")) { + return Error("cgroups memory stats does not contain 'rss' data"); } + ResourceStatistics result; + result.set_cpus_system_time_secs(cpuAcctStat.get().system.secs()); + result.set_cpus_user_time_secs(cpuAcctStat.get().user.secs()); + result.set_mem_rss_bytes(memStats.get().at("rss")); + return result; +#endif // __linux__ } http://git-wip-us.apache.org/repos/asf/mesos/blob/867ff24b/src/slave/containerizer/docker.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp index 9a7a951..e43d300 100644 --- a/src/slave/containerizer/docker.hpp +++ b/src/slave/containerizer/docker.hpp @@ -219,13 +219,7 @@ private: const Resources& resources, pid_t pid); - Future<ResourceStatistics> _usage( - const ContainerID& containerId, - const Docker::Container& container); - - Future<ResourceStatistics> __usage( - const ContainerID& containerId, - pid_t pid); + Try<ResourceStatistics> cgroupsStatistics(pid_t pid) const; // Call back for when the executor exits. This will trigger // container destroy. http://git-wip-us.apache.org/repos/asf/mesos/blob/867ff24b/src/tests/docker_containerizer_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/docker_containerizer_tests.cpp b/src/tests/docker_containerizer_tests.cpp index a3da786..77dbeaa 100644 --- a/src/tests/docker_containerizer_tests.cpp +++ b/src/tests/docker_containerizer_tests.cpp @@ -961,6 +961,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Usage) statistics.mem_limit_bytes()); EXPECT_LT(0, statistics.cpus_user_time_secs()); EXPECT_LT(0, statistics.cpus_system_time_secs()); + EXPECT_GT(statistics.mem_rss_bytes(), 0); Future<containerizer::Termination> termination = dockerContainerizer.wait(containerId.get());
