This is an automated email from the ASF dual-hosted git repository. mzhu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 0433acf23822f691e18b45557fc76521b43053ad Author: Meng Zhu <[email protected]> AuthorDate: Wed Mar 6 16:21:40 2019 -0800 Added `class ResourceLimits`. Similar to `ResourceQuantities`. `ResourceLimits` is an efficient collection of resource quantities. The main difference lies in the absence semantic. Absent entry in `ResourceQuantities` implies zero quantity while in `ResourceLimits`, it implies no limit (i.e. infinite amount). Also, due to the absence semantic, zero values in `ResourceLimits` are not dropped. Review: https://reviews.apache.org/r/70151 --- src/common/resource_quantities.cpp | 156 +++++++++++++++++++++++++++++++++++++ src/common/resource_quantities.hpp | 89 ++++++++++++++++++++- 2 files changed, 242 insertions(+), 3 deletions(-) diff --git a/src/common/resource_quantities.cpp b/src/common/resource_quantities.cpp index 9e0b291..95fcbca 100644 --- a/src/common/resource_quantities.cpp +++ b/src/common/resource_quantities.cpp @@ -279,5 +279,161 @@ void ResourceQuantities::add(const string& name, const Value::Scalar& scalar) } +// This function tries to be consistent with `Resources::fromSimpleString()`. +// We trim the whitespace around the pair and in the number but whitespace in +// "c p us:10" are preserved and will be parsed to {"c p us", 10}. +Try<ResourceLimits> ResourceLimits::fromString(const string& text) +{ + ResourceLimits result; + + foreach (const string& token, strings::tokenize(text, ";")) { + vector<string> pair = strings::tokenize(token, ":"); + if (pair.size() != 2) { + return Error("Failed to parse '" + token + "': missing or extra ':'"); + } + + Try<Value> value = values::parse(pair[1]); + if (value.isError()) { + return Error( + "Failed to parse '" + pair[1] + "' to limit: " + value.error()); + } + + if (value->type() != Value::SCALAR) { + return Error( + "Failed to parse '" + pair[1] + "' to limit:" + " only scalar values are allowed"); + } + + if (value->scalar().value() < 0) { + return Error( + "Failed to parse '" + pair[1] + "' to limit:" + " negative values are not allowed"); + } else if (value->scalar().value() >= 0) { + // Zero value is preserved. + // And duplicate names are not allowed. + const string name = strings::trim(pair[0]); + if (result.get(name) != None()) { + return Error("Failed to parse '" + pair[1] + "' to limit:" + " duplicate names are not allowed"); + } + result.set(name, value->scalar()); + } + } + + return result; +} + + +ResourceLimits::ResourceLimits() +{ + // Pre-reserve space for first-class resources. + // [cpus, disk, gpus, mem, ports] + limits.reserve(5u); +} + + +ResourceLimits::ResourceLimits( + const google::protobuf::Map<std::string, Value::Scalar>& map) +{ + // Use `auto` in place of `protobuf::MapPair<string, Value::Scalar>` + // below since `foreach` is a macro and cannot contain angle brackets. + foreach (auto&& limit, map) { + set(limit.first, limit.second); + } +} + + +ResourceLimits::const_iterator ResourceLimits::begin() +{ + return static_cast<const std::vector<std::pair<std::string, Value::Scalar>>&>( + limits) + .begin(); +} + + +ResourceLimits::const_iterator ResourceLimits::end() +{ + return static_cast<const std::vector<std::pair<std::string, Value::Scalar>>&>( + limits) + .end(); +} + + +Option<Value::Scalar> ResourceLimits::get(const string& name) const +{ + // Don't bother binary searching since + // we don't expect a large number of elements. + foreach (auto&& limit, limits) { + if (limit.first == name) { + return limit.second; + } else if (limit.first > name) { + // We can return early since we keep names in alphabetical order. + break; + } + } + + return None(); +} + + +bool ResourceLimits::contains(const ResourceLimits& right) const +{ + size_t leftIndex = 0u; + size_t rightIndex = 0u; + + // Since limits are sorted in alphabetical order, we can walk them + // at the same time. + while (leftIndex < size() && rightIndex < right.size()) { + const pair<string, Value::Scalar>& left_ = limits.at(leftIndex); + const pair<string, Value::Scalar>& right_ = right.limits.at(rightIndex); + + if (left_.first < right_.first) { + // Left has a finite limit but right has no limit. + return false; + } else if (left_.first > right_.first) { + // Left has a no limit but right has finite limit. + ++rightIndex; + } else { + // Left and right both have finite limits. + if (left_.second < right_.second) { + return false; + } + leftIndex++; + rightIndex++; + } + } + + if (leftIndex < size()) { + // Left has finite limits for resources that right has no limits for. + return false; + } + + return true; +} + + +void ResourceLimits::set( + const std::string& name, const Value::Scalar& scalar) +{ + // Find the location to insert while maintaining + // alphabetical ordering. Don't bother binary searching + // since we don't expect a large number of limits. + auto it = limits.begin(); + for (; it != limits.end(); ++it) { + if (it->first == name) { + // Overwrite if it exists. + it->second = scalar; + return; + } + + if (it->first > name) { + break; + } + } + + it = limits.insert(it, std::make_pair(name, scalar)); +} + + } // namespace internal { } // namespace mesos { diff --git a/src/common/resource_quantities.hpp b/src/common/resource_quantities.hpp index 9605205..f774f70 100644 --- a/src/common/resource_quantities.hpp +++ b/src/common/resource_quantities.hpp @@ -37,9 +37,6 @@ namespace internal { // // Absent resource entries imply there is no (zero) such resources. // -// TODO(mzhu): Add `class ResourceLimits` where absence means infinite amount -// of such resources. -// // Notes on handling negativity and arithmetic operations: values are guaranteed // to be positive. This is achieved by construction validation and no public // mutation interfaces. During construction, `Value::Scalar` arguments must be @@ -127,6 +124,92 @@ private: }; +// An efficient collection of resource limits. All values are guaranteed +// to be non-negative and finite. +// +// E.g. [("cpus", 4.5), ("gpus", 0.1), ("ports", 1000)] +// +// Difference with `class ResourceQuantities`: +// The main difference resides in the semantics of absent entries and, in turn, +// whether zero values are preserved. +// +// (1) Absent entry in `ResourceQuantities` implies zero quantity. While in +// `ResourceLimits`, absence implies no limit (i.e. infinite amount). This +// affects the semantics of the contains operation. +// +// (2) Zero value preservation: `ResourceLimits` keeps zero value entries +// while `ResourceQuantities` silently drops them. This is in accordance with +// the absence semantic. Zero value and absence are equivalent in +// `ResourceQuantities`. While in `ResourceLimits`, they are not. +class ResourceLimits +{ +public: + // Parse an input string of semicolon separated "name:number" pairs. + // Entries with zero values will be preserved (unlike `ResourceQuantities`). + // Duplicate names are NOT allowed in the input, otherwise an `Error` will + // be returned. + // + // Example: "cpus:10;mem:1024;ports:0" + // "cpus:10; mem:1024; ports:0" + // NOTE: we will trim the whitespace around the pair and in the number. + // However, whitespace in "c p us:10" are preserved and will be parsed to + // {"c p us", 10}. This is consistent with `Resources::fromSimpleString()`. + // + // Numbers must be non-negative and finite, otherwise an `Error` + // will be returned. + static Try<ResourceLimits> fromString(const std::string& text); + + ResourceLimits(); + + explicit ResourceLimits( + const google::protobuf::Map<std::string, Value::Scalar>& map); + + ResourceLimits(const ResourceLimits& that) = default; + ResourceLimits(ResourceLimits&& that) = default; + + ResourceLimits& operator=(const ResourceLimits& that) = default; + ResourceLimits& operator=(ResourceLimits&& that) = default; + + typedef std::vector<std::pair<std::string, Value::Scalar>>::const_iterator + iterator; + typedef std::vector<std::pair<std::string, Value::Scalar>>::const_iterator + const_iterator; + + // NOTE: Non-`const` `iterator`, `begin()` and `end()` are __intentionally__ + // defined with `const` semantics in order to prevent mutation during + // iteration. Mutation is allowed with the `[]` operator. + const_iterator begin(); + const_iterator end(); + + const_iterator begin() const { return limits.begin(); } + const_iterator end() const { return limits.end(); } + + size_t size() const { return limits.size(); }; + + // Returns the limit of the resource with the given name. + // If there is no explicit limit for the resource, return `None()`. + // Note, `None()` implies that the limit of the resource is infinite. + Option<Value::Scalar> get(const std::string& name) const; + + // Due to the absence-means-infinite semantic of limits, absent entries, + // intuitively, are considered to contain entries with finite scalar values. + // For example: + // `[ ]`` will always contain any other `ResourceLimits`. + // `[("cpu":1)]` contains `[("mem":1)]` and vice versa. + // `[("cpu":1)]` will not contain `[("cpu":2)]`. + bool contains(const ResourceLimits& right) const; + +private: + // Set the limit of the resource with `name` to `scalar`. + // Note, the existing limit of the resource will be overwritten. + void set(const std::string& name, const Value::Scalar& scalar); + + // List of name limit pairs sorted by name. + // Arithmetic and comparison operations benefit from this sorting. + std::vector<std::pair<std::string, Value::Scalar>> limits; +}; + + } // namespace internal { } // namespace mesos {
