This is an automated email from the ASF dual-hosted git repository. qianzhang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 9ba0866b6c8033f627a4a15a6727934bb8178de4 Author: Qian Zhang <[email protected]> AuthorDate: Fri Jan 17 15:20:19 2020 +0800 Set resource limits and OOM score adjustment in Docker executor. Review: https://reviews.apache.org/r/72022 --- src/docker/docker.cpp | 103 ++++++++++++++++++++++++++++++++++++++---------- src/docker/docker.hpp | 13 +++++- src/docker/executor.cpp | 4 +- 3 files changed, 96 insertions(+), 24 deletions(-) diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp index 04fb8d0..a5f15dd 100644 --- a/src/docker/docker.cpp +++ b/src/docker/docker.cpp @@ -14,6 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include <cmath> #include <map> #include <mutex> #include <utility> @@ -54,6 +55,8 @@ #include "linux/cgroups.hpp" #endif // __linux__ +#include "slave/containerizer/mesos/utils.hpp" + #include "slave/containerizer/mesos/isolators/cgroups/constants.hpp" #include "slave/constants.hpp" @@ -625,11 +628,12 @@ Try<Docker::RunOptions> Docker::RunOptions::create( const string& name, const string& sandboxDirectory, const string& mappedDirectory, - const Option<Resources>& resources, + const Option<Resources>& resourceRequests, bool enableCfsQuota, const Option<map<string, string>>& env, const Option<vector<Device>>& devices, - const Option<ContainerDNSInfo>& defaultContainerDNS) + const Option<ContainerDNSInfo>& defaultContainerDNS, + const Option<google::protobuf::Map<string, Value::Scalar>>& resourceLimits) { if (!containerInfo.has_docker()) { return Error("No docker info found in container info"); @@ -640,26 +644,75 @@ Try<Docker::RunOptions> Docker::RunOptions::create( RunOptions options; options.privileged = dockerInfo.privileged(); - if (resources.isSome()) { - // TODO(yifan): Support other resources (e.g. disk). - Option<double> cpus = resources->cpus(); - if (cpus.isSome()) { - options.cpuShares = std::max( - static_cast<uint64_t>(CPU_SHARES_PER_CPU * cpus.get()), - MIN_CPU_SHARES); + Option<double> cpuRequest, cpuLimit, memLimit; + Option<Bytes> memRequest; - if (enableCfsQuota) { - const Duration quota = - std::max(CPU_CFS_PERIOD * cpus.get(), MIN_CPU_CFS_QUOTA); + if (resourceRequests.isSome()) { + // TODO(yifan): Support other resources (e.g. disk). + cpuRequest = resourceRequests->cpus(); + memRequest = resourceRequests->mem(); + } - options.cpuQuota = static_cast<uint64_t>(quota.us()); + if (resourceLimits.isSome()) { + foreach (auto&& limit, resourceLimits.get()) { + if (limit.first == "cpus") { + cpuLimit = limit.second.value(); + } else if (limit.first == "mem") { + memLimit = limit.second.value(); } } + } + + if (cpuRequest.isSome()) { + options.cpuShares = std::max( + static_cast<uint64_t>(CPU_SHARES_PER_CPU * cpuRequest.get()), + MIN_CPU_SHARES); + } + + // Set the `--cpu-quota` option to CPU limit (if it is not an infinite + // value) or to CPU request if the flag `--cgroups_enable_cfs` is true. + // If CPU limit is infinite, `--cpu-quota` will not be set at all which + // means the Docker container will run with infinite CPU quota. + if (cpuLimit.isSome()) { + if (!std::isinf(cpuLimit.get())) { + const Duration quota = + std::max(CPU_CFS_PERIOD * cpuLimit.get(), MIN_CPU_CFS_QUOTA); + + options.cpuQuota = static_cast<uint64_t>(quota.us()); + } + } else if (enableCfsQuota && cpuRequest.isSome()) { + const Duration quota = + std::max(CPU_CFS_PERIOD * cpuRequest.get(), MIN_CPU_CFS_QUOTA); + + options.cpuQuota = static_cast<uint64_t>(quota.us()); + } + + // Set the `--memory` option to memory limit (if it is not an infinite + // value) or to memory request. If memory limits is infinite, `--memory` + // will not be set at all which means the Docker container will run with + // infinite memory limit. + if (memLimit.isSome()) { + if (!std::isinf(memLimit.get())) { + options.memory = + std::max(Megabytes(static_cast<uint64_t>(memLimit.get())), MIN_MEMORY); + } + + if (memRequest.isSome()) { + options.memoryReservation = std::max(memRequest.get(), MIN_MEMORY); + + if (memRequest.get() < Megabytes(static_cast<uint64_t>(memLimit.get()))) { + Try<int> oomScoreAdj = calculateOOMScoreAdj(memRequest.get()); + if (oomScoreAdj.isError()) { + return Error( + "Failed to calculate OOM score adjustment: " + + oomScoreAdj.error()); + } - Option<Bytes> mem = resources->mem(); - if (mem.isSome()) { - options.memory = std::max(mem.get(), MIN_MEMORY); + options.oomScoreAdj = oomScoreAdj.get(); + } } + } else if (memRequest.isSome()) { + options.memory = std::max(memRequest.get(), MIN_MEMORY); } if (env.isSome()) { @@ -681,11 +734,11 @@ Try<Docker::RunOptions> Docker::RunOptions::create( options.env["MESOS_SANDBOX"] = mappedDirectory; options.env["MESOS_CONTAINER_NAME"] = name; - if (resources.isSome()) { + if (resourceRequests.isSome()) { // Set the `MESOS_ALLOCATION_ROLE` environment variable. Please note // that tasks and executors are not allowed to mix resources allocated // to different roles, see MESOS-6636. - const Resource resource = *resources->begin(); + const Resource resource = *resourceRequests->begin(); options.env["MESOS_ALLOCATION_ROLE"] = resource.allocation_info().role(); } @@ -848,11 +901,11 @@ Try<Docker::RunOptions> Docker::RunOptions::create( "user-defined networks"); } - if (!resources.isSome()) { + if (!resourceRequests.isSome()) { return Error("Port mappings require resources"); } - Option<Value::Ranges> portRanges = resources->ports(); + Option<Value::Ranges> portRanges = resourceRequests->ports(); if (!portRanges.isSome()) { return Error("Port mappings require port resources"); @@ -1027,11 +1080,21 @@ Future<Option<int>> Docker::run( argv.push_back(stringify(options.cpuQuota.get())); } + if (options.memoryReservation.isSome()) { + argv.push_back("--memory-reservation"); + argv.push_back(stringify(options.memoryReservation->bytes())); + } + if (options.memory.isSome()) { argv.push_back("--memory"); argv.push_back(stringify(options.memory->bytes())); } + if (options.oomScoreAdj.isSome()) { + argv.push_back("--oom-score-adj"); + argv.push_back(stringify(options.oomScoreAdj.get())); + } + foreachpair(const string& key, const string& value, options.env) { argv.push_back("-e"); argv.push_back(key + "=" + value); diff --git a/src/docker/docker.hpp b/src/docker/docker.hpp index b48d894..d8d56f7 100644 --- a/src/docker/docker.hpp +++ b/src/docker/docker.hpp @@ -183,11 +183,14 @@ public: const std::string& containerName, const std::string& sandboxDirectory, const std::string& mappedDirectory, - const Option<mesos::Resources>& resources = None(), + const Option<mesos::Resources>& resourceRequests = None(), bool enableCfsQuota = false, const Option<std::map<std::string, std::string>>& env = None(), const Option<std::vector<Device>>& devices = None(), - const Option<mesos::internal::ContainerDNSInfo>& defaultContainerDNS = None()); // NOLINT(whitespace/line_length) + const Option<mesos::internal::ContainerDNSInfo>& + defaultContainerDNS = None(), + const Option<google::protobuf::Map<std::string, mesos::Value::Scalar>>& + resourceLimits = None()); // "--privileged" option. bool privileged; @@ -198,9 +201,15 @@ public: // "--cpu-quota" option. Option<uint64_t> cpuQuota; + // "--memory-reservation" options. + Option<Bytes> memoryReservation; + // "--memory" option. Option<Bytes> memory; + // "--oom-score-adj" option. + Option<int> oomScoreAdj; + // Environment variable overrides. These overrides will be passed // to docker container through "--env-file" option. std::map<std::string, std::string> env; diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp index ebbbc0d..8ab165b 100644 --- a/src/docker/executor.cpp +++ b/src/docker/executor.cpp @@ -208,8 +208,8 @@ public: cgroupsEnableCfs, taskEnvironment, None(), // No extra devices. - defaultContainerDNS - ); + defaultContainerDNS, + task.limits()); if (runOptions.isError()) { // TODO(alexr): Use `protobuf::createTaskStatus()`
