This is an automated email from the ASF dual-hosted git repository. mzhu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 386b1fe99bb9d10af2abaca4832bf584b6181799 Author: Meng Zhu <[email protected]> AuthorDate: Sun Dec 30 20:55:06 2018 -0800 Extended `min_allocatable_resources` flag to cover non-scalar resources. The master flag min_allocatable_resources currently only supports scalar resources. Sometimes it is desirable to control the quantities of non-scalar resources in the offer as well, for example, to ensure offers always contain port resources. This patch extended the flag to cover non-scalar resources. For RANGES and SET type resources, their quantities are the number of different instances in the range or set. For example, range:[1-5] has a quantity of 5 and set:{a,b} has a quantity of 2. This patch also updated affected tests and modified the existing test for min_allocatable_resources to cover the updated use cases. Review: https://reviews.apache.org/r/69603 --- docs/configuration/master.md | 22 ++++++---- include/mesos/allocator/allocator.hpp | 5 ++- src/master/allocator/mesos/hierarchical.cpp | 6 +-- src/master/flags.cpp | 22 ++++++---- src/master/master.cpp | 58 +++++++++++-------------- src/tests/hierarchical_allocator_benchmarks.cpp | 13 ++++-- src/tests/hierarchical_allocator_tests.cpp | 13 ++++-- src/tests/master_allocator_tests.cpp | 19 ++++---- 8 files changed, 88 insertions(+), 70 deletions(-) diff --git a/docs/configuration/master.md b/docs/configuration/master.md index 83b83b0..7859d80 100644 --- a/docs/configuration/master.md +++ b/docs/configuration/master.md @@ -207,15 +207,19 @@ load an alternate allocator module using <code>--modules</code>. --min_allocatable_resources=VALUE </td> <td> -One or more sets of resources that define the minimum allocatable -resources for the allocator. The allocator will only offer resources -that contain at least one of the specified sets. The resources in each -set should be delimited by semicolons, and the sets should be delimited -by the pipe character. -(Example: <code>disk:1|cpu:1;mem:32</code> -configures the allocator to only offer resources if they contain a disk -resource of at least 1 megabyte, or if they contain both 1 cpu and -32 megabytes of memory.) (default: cpus:0.01|mem:32). +One or more sets of resource quantities that define the minimum allocatable +resource for the allocator. The allocator will only offer resources that +meets the quantity requirement of at least one of the specified sets. For +`SCALAR` type resources, its quantity is its scalar value. For `RANGES` and +`SET` type, their quantities are the number of different instances in the +range or set. For example, `range:[1-5]` has a quantity of 5 and `set:{a,b}` +has a quantity of 2. The resources in each set should be delimited by +semicolons (acting as logical AND), and each set should be delimited by the +pipe character (acting as logical OR). +(Example: `disk:1|cpu:1;mem:32;port:1` configures the allocator to only offer +resources if they contain a disk resource of at least 1 megabyte, or if they +at least contain 1 cpu, 32 megabytes of memory and 1 port.) +(default: cpus:0.01|mem:32). </td> </tr> diff --git a/include/mesos/allocator/allocator.hpp b/include/mesos/allocator/allocator.hpp index 2a6849b..b58371b 100644 --- a/include/mesos/allocator/allocator.hpp +++ b/include/mesos/allocator/allocator.hpp @@ -38,6 +38,8 @@ #include <stout/option.hpp> #include <stout/try.hpp> +#include "common/resource_quantities.hpp" + namespace mesos { namespace allocator { @@ -50,7 +52,8 @@ struct Options Option<std::set<std::string>> fairnessExcludeResourceNames = None(); bool filterGpuResources = true; Option<DomainInfo> domain = None(); - Option<std::vector<Resources>> minAllocatableResources = None(); + Option<std::vector<mesos::internal::ResourceQuantities>> + minAllocatableResources = None(); size_t maxCompletedFrameworks = 0; bool publishPerFrameworkMetrics = true; }; diff --git a/src/master/allocator/mesos/hierarchical.cpp b/src/master/allocator/mesos/hierarchical.cpp index cc8ab91..40621ec 100644 --- a/src/master/allocator/mesos/hierarchical.cpp +++ b/src/master/allocator/mesos/hierarchical.cpp @@ -41,6 +41,7 @@ #include <stout/stringify.hpp> #include "common/protobuf_utils.hpp" +#include "common/resource_quantities.hpp" using std::set; using std::string; @@ -2430,11 +2431,10 @@ bool HierarchicalAllocatorProcess::allocatable(const Resources& resources) return true; } - Resources quantity = resources.createStrippedScalarQuantity(); foreach ( - const Resources& minResources, + const ResourceQuantities& resourceQuantities, CHECK_NOTNONE(options.minAllocatableResources)) { - if (quantity.contains(minResources)) { + if (resources.contains(resourceQuantities)) { return true; } } diff --git a/src/master/flags.cpp b/src/master/flags.cpp index f9b68bc..9288714 100644 --- a/src/master/flags.cpp +++ b/src/master/flags.cpp @@ -488,14 +488,20 @@ mesos::internal::master::Flags::Flags() add(&Flags::min_allocatable_resources, "min_allocatable_resources", - "One or more sets of resources that define the minimum allocatable\n" - "resources for the allocator. The allocator will only offer resources\n" - "that contain at least one of the specified sets. The resources in\n" - "each set should be delimited by semicolons, and the sets should be\n" - "delimited by the pipe character.\n" - "(Example: `disk:1|cpu:1;mem:32` configures the allocator to only offer\n" - "resources if they contain a disk resource of at least 1 megabyte, or\n" - "if they contain both 1 cpu and 32 megabytes of memory.)\n", + "One or more sets of resource quantities that define the minimum\n" + "allocatable resources for the allocator. The allocator will only offer\n" + "resources that meets the quantity requirement of at least one of the\n" + "specified sets. For `SCALAR` type resources, its quantity is its\n" + "scalar value. For `RANGES` and `SET` type, their quantities are the\n" + "number of different instances in the range or set. For example,\n" + "`range:[1-5]` has a quantity of 5 and `set:{a,b}` has a quantity of 2.\n" + "The resources in each set should be delimited by semicolons (acting as\n" + "logical AND), and each set should be delimited by the pipe character\n" + "(acting as logical OR).\n" + "(Example: `disk:1|cpu:1;mem:32;port:1` configures the allocator to\n" + "only offer resources if they contain a disk resource of at least\n" + "1 megabyte, or if they at least contain 1 cpu, 32 megabytes of memory\n" + "and 1 port.)\n", "cpus:" + stringify(MIN_CPUS) + "|mem:" + stringify((double)MIN_MEM.bytes() / Bytes::MEGABYTES)); diff --git a/src/master/master.cpp b/src/master/master.cpp index b4faf2b..49b6e5c 100644 --- a/src/master/master.cpp +++ b/src/master/master.cpp @@ -79,6 +79,7 @@ #include "common/build.hpp" #include "common/http.hpp" #include "common/protobuf_utils.hpp" +#include "common/resource_quantities.hpp" #include "common/status_utils.hpp" #include "credentials/credentials.hpp" @@ -149,6 +150,7 @@ using mesos::master::contender::MasterContender; using mesos::master::detector::MasterDetector; +using mesos::internal::ResourceQuantities; static bool isValidFailoverTimeout(const FrameworkInfo& frameworkInfo); @@ -720,39 +722,31 @@ void Master::initialize() } // Parse min_allocatable_resources. - Try<vector<Resources>> minAllocatableResources = - [](const string& resourceString) -> Try<vector<Resources>> { - vector<Resources> result; - - foreach (const string& token, strings::tokenize(resourceString, "|")) { - Try<vector<Resource>> resourceVector = - Resources::fromSimpleString(token); - - if (resourceVector.isError()) { - return Error(resourceVector.error()); - } - - result.push_back(Resources(CHECK_NOTERROR(resourceVector))); - } - - return result; - }(flags.min_allocatable_resources); - - if (minAllocatableResources.isError()) { - EXIT(EXIT_FAILURE) << "Error parsing min_allocatable_resources: '" - << flags.min_allocatable_resources - << "': " << minAllocatableResources.error(); - } - - // Validate that configured minimum resources are "pure" scalar quantities. + vector<ResourceQuantities> minAllocatableResources; foreach ( - const Resources& resources, CHECK_NOTERROR(minAllocatableResources)) { - if (!Resources::isScalarQuantity(resources)) { - EXIT(EXIT_FAILURE) << "Invalid min_allocatable_resources: '" - << flags.min_allocatable_resources << "': " - << "minimum allocatable resources should only" - << "have name, type (scalar) and value set"; + const string& token, + strings::tokenize(flags.min_allocatable_resources, "|")) { + Try<ResourceQuantities> resourceQuantities = + ResourceQuantities::fromString(token); + + if (resourceQuantities.isError()) { + EXIT(EXIT_FAILURE) << "Error parsing min_allocatable_resources '" + << flags.min_allocatable_resources + << "': " << resourceQuantities.error(); + } + + // We check the configuration against first-class resources and warn + // against possible mis-configuration (e.g. typo). + set<string> firstClassResources = {"cpus", "mem", "disk", "ports", "gpus"}; + for (auto it = resourceQuantities->begin(); it != resourceQuantities->end(); + ++it) { + if (firstClassResources.count(it->first) == 0) { + LOG(WARNING) << "Non-first-class resource '" << it->first + << "' is configured as part of min_allocatable_resources"; + } } + + minAllocatableResources.push_back(resourceQuantities.get()); } // Initialize the allocator options. @@ -763,7 +757,7 @@ void Master::initialize() flags.fair_sharing_excluded_resource_names; options.filterGpuResources = flags.filter_gpu_resources; options.domain = flags.domain; - options.minAllocatableResources = CHECK_NOTERROR(minAllocatableResources); + options.minAllocatableResources = minAllocatableResources; options.maxCompletedFrameworks = flags.max_completed_frameworks; options.publishPerFrameworkMetrics = flags.publish_per_framework_metrics; diff --git a/src/tests/hierarchical_allocator_benchmarks.cpp b/src/tests/hierarchical_allocator_benchmarks.cpp index 527a25d..a240a2b 100644 --- a/src/tests/hierarchical_allocator_benchmarks.cpp +++ b/src/tests/hierarchical_allocator_benchmarks.cpp @@ -32,6 +32,8 @@ #include <stout/hashmap.hpp> #include <stout/stopwatch.hpp> +#include "common/resource_quantities.hpp" + #include "master/constants.hpp" #include "master/allocator/mesos/hierarchical.hpp" @@ -44,6 +46,8 @@ using mesos::internal::master::MIN_MEM; using mesos::internal::master::allocator::HierarchicalDRFAllocator; +using mesos::internal::ResourceQuantities; + using mesos::internal::slave::AGENT_CAPABILITIES; using mesos::allocator::Allocator; @@ -137,10 +141,11 @@ struct BenchmarkConfig frameworkSorter(frameworkSorter_), allocationInterval(allocationInterval_) { + minAllocatableResources.push_back(CHECK_NOTERROR( + ResourceQuantities::fromString("cpus:" + stringify(MIN_CPUS)))); minAllocatableResources.push_back( - CHECK_NOTERROR(Resources::parse("cpus:" + stringify(MIN_CPUS)))); - minAllocatableResources.push_back(CHECK_NOTERROR(Resources::parse( - "mem:" + stringify((double)MIN_MEM.bytes() / Bytes::MEGABYTES)))); + CHECK_NOTERROR(ResourceQuantities::fromString( + "mem:" + stringify((double)MIN_MEM.bytes() / Bytes::MEGABYTES)))); } string allocator; @@ -149,7 +154,7 @@ struct BenchmarkConfig Duration allocationInterval; - vector<Resources> minAllocatableResources; + vector<ResourceQuantities> minAllocatableResources; vector<FrameworkProfile> frameworkProfiles; vector<AgentProfile> agentProfiles; diff --git a/src/tests/hierarchical_allocator_tests.cpp b/src/tests/hierarchical_allocator_tests.cpp index a3a6d7b..cc88afb 100644 --- a/src/tests/hierarchical_allocator_tests.cpp +++ b/src/tests/hierarchical_allocator_tests.cpp @@ -40,6 +40,8 @@ #include <stout/stopwatch.hpp> #include <stout/utils.hpp> +#include "common/resource_quantities.hpp" + #include "master/constants.hpp" #include "master/flags.hpp" @@ -61,6 +63,8 @@ using mesos::internal::protobuf::createLabel; using mesos::internal::slave::AGENT_CAPABILITIES; +using mesos::internal::ResourceQuantities; + using mesos::allocator::Allocator; using process::Clock; @@ -176,11 +180,12 @@ protected: }; } - vector<Resources> minAllocatableResources; + vector<ResourceQuantities> minAllocatableResources; + minAllocatableResources.push_back(CHECK_NOTERROR( + ResourceQuantities::fromString("cpus:" + stringify(MIN_CPUS)))); minAllocatableResources.push_back( - CHECK_NOTERROR(Resources::parse("cpus:" + stringify(MIN_CPUS)))); - minAllocatableResources.push_back(CHECK_NOTERROR(Resources::parse( - "mem:" + stringify((double)MIN_MEM.bytes() / Bytes::MEGABYTES)))); + CHECK_NOTERROR(ResourceQuantities::fromString( + "mem:" + stringify((double)MIN_MEM.bytes() / Bytes::MEGABYTES)))); Options options; options.allocationInterval = flags.allocation_interval; diff --git a/src/tests/master_allocator_tests.cpp b/src/tests/master_allocator_tests.cpp index 9dbab18..e960757 100644 --- a/src/tests/master_allocator_tests.cpp +++ b/src/tests/master_allocator_tests.cpp @@ -1243,7 +1243,8 @@ TYPED_TEST(MasterAllocatorTest, MinAllocatableResources) Clock::pause(); master::Flags masterFlags = this->CreateMasterFlags(); - masterFlags.min_allocatable_resources = "cpus:1;mem:100|disk:100"; + masterFlags.min_allocatable_resources = + "cpus:1;mem:100|disk:100|cpus:1;ports:10"; TestAllocator<TypeParam> allocator; @@ -1258,10 +1259,10 @@ TYPED_TEST(MasterAllocatorTest, MinAllocatableResources) // Add agents with non-allocatable resources. vector<string> nonAllocatableAgentResources = { - "cpus:1;mem:0;disk:0", - "cpus:0;mem:100;disk:0", - "cpus:0.5;mem:200;disk:0", - "cpus:0;mem:0;disk:50"}; + "cpus:1;mem:10;disk:10;ports:[1-5]", + "cpus:0.5;mem:100;disk:10;ports:[6-10]", + "cpus:0.5;mem:200;disk:10;ports:[11-15]", + "cpus:0.5;mem:10;disk:50;ports:[16-20]"}; foreach (const string& resourcesString, nonAllocatableAgentResources) { Future<SlaveRegisteredMessage> slaveRegisteredMessage = @@ -1284,10 +1285,10 @@ TYPED_TEST(MasterAllocatorTest, MinAllocatableResources) // Add agents with allocatable resources. vector<string> allocatableAgentResources = { - "cpus:1;mem:100;disk:0", - "cpus:0;mem:0;disk:100", - "cpus:0.5;mem:0;disk:100", - "cpus:1;mem:100;disk:50", + "cpus:1;mem:100;disk:0;ports:[1-5]", // contains `cpus:1;mem:100` + "cpus:0.5;mem:10;disk:100;ports:[1-5]", // contains `disk:100` + "cpus:1;mem:100;disk:50", // contains `cpus:1;mem:100` + "cpus:1;mem:10;disk:10;ports:[1-10]", // contains `cpus:1;ports:10` }; hashset<SlaveID> allocatableAgents;
