This is an automated email from the ASF dual-hosted git repository. bmahler pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git
commit b6b615ef275ad0c2a0739fd686a51bbaa5f0c5dd Author: Ian Downes <[email protected]> AuthorDate: Tue Jul 25 11:37:49 2017 +0100 Introduce HTB class. Introduce support for HTB class and expose configuration. --- src/linux/routing/queueing/class.hpp | 61 ++++++ src/linux/routing/queueing/htb.cpp | 163 ++++++++++++++-- src/linux/routing/queueing/htb.hpp | 74 +++++++ src/linux/routing/queueing/internal.hpp | 328 +++++++++++++++++++++++++++++++- 4 files changed, 611 insertions(+), 15 deletions(-) diff --git a/src/linux/routing/queueing/class.hpp b/src/linux/routing/queueing/class.hpp new file mode 100644 index 000000000..396c185e5 --- /dev/null +++ b/src/linux/routing/queueing/class.hpp @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __LINUX_ROUTING_QUEUEING_CLASS_HPP__ +#define __LINUX_ROUTING_QUEUEING_CLASS_HPP__ + +#include <string> + +#include <netlink/route/class.h> + +#include <stout/option.hpp> + +#include "linux/routing/handle.hpp" +#include "linux/routing/queueing/internal.hpp" + +namespace routing { + +template <> +inline void cleanup(struct rtnl_class* cls) +{ + rtnl_class_put(cls); +} + +namespace queueing { + +template <typename Config> +struct Class +{ + Class( + const std::string& _kind, + const Handle& _parent, + const Option<Handle>& _handle, + const Config& _config) + : kind(_kind), + parent(_parent), + handle(_handle), + config(_config) {} + + std::string kind; + Handle parent; + Option<Handle> handle; + Config config; +}; + +} // namespace queueing { +} // namespace routing { + +#endif // __LINUX_ROUTING_QUEUEING_CLASS_HPP__ diff --git a/src/linux/routing/queueing/htb.cpp b/src/linux/routing/queueing/htb.cpp index 464120214..5e5f0710b 100644 --- a/src/linux/routing/queueing/htb.cpp +++ b/src/linux/routing/queueing/htb.cpp @@ -19,6 +19,10 @@ #include <netlink/route/qdisc.h> #include <netlink/route/tc.h> +#include <netlink/route/link/veth.h> + +#include <netlink/route/qdisc/htb.h> + #include <stout/error.hpp> #include <stout/none.hpp> #include <stout/nothing.hpp> @@ -36,15 +40,14 @@ namespace queueing { namespace htb { -// TODO(cwang): The htb queueing discipline configuration is not -// exposed to the user because we use all the default parameters -// currently. -struct Config {}; +// NOTE: The htb queueing discipline configuration is not exposed to +// the user but the queueing class configuration is. +struct DisciplineConfig {}; } // namespace htb { ///////////////////////////////////////////////// -// Type specific {en}decoding functions. +// Type specific {en}decoding functions for disciplines and classes. ///////////////////////////////////////////////// namespace internal { @@ -53,9 +56,9 @@ namespace internal { // libnl queueing discipline 'qdisc'. Each type of queueing discipline // needs to implement this function. template <> -Try<Nothing> encode<htb::Config>( +Try<Nothing> encode<htb::DisciplineConfig>( const Netlink<struct rtnl_qdisc>& qdisc, - const htb::Config& config) + const htb::DisciplineConfig& config) { return Nothing(); } @@ -66,16 +69,84 @@ Try<Nothing> encode<htb::Config>( // to implement this function. Returns None if the libnl queueing // discipline is not an htb queueing discipline. template <> -Result<htb::Config> decode<htb::Config>( +Result<htb::DisciplineConfig> decode<htb::DisciplineConfig>( const Netlink<struct rtnl_qdisc>& qdisc) { - if (rtnl_tc_get_kind(TC_CAST(qdisc.get())) != htb::KIND) { + if (rtnl_tc_get_kind(TC_CAST(qdisc.get())) != string(htb::KIND)) { return None(); } - return htb::Config(); + return htb::DisciplineConfig(); +} + +namespace cls { + +// Encodes an htb queueing class configuration into the libnl queueing +// class 'cls'. Each type of queueing class needs to implement this +// function. +template<> +Try<Nothing> encode<htb::cls::Config>( + const Netlink<struct rtnl_class>& cls, + const htb::cls::Config& config) +{ + int error = rtnl_htb_set_rate(cls.get(), config.rate); + if (error != 0) { + return Error(string(nl_geterror(error))); + } + + if (config.ceil.isSome()) { + error = rtnl_htb_set_ceil(cls.get(), config.ceil.get()); + if (error != 0) { + return Error(string(nl_geterror(error))); + } + } + + if (config.burst.isSome()) { + // NOTE: The libnl documentation is incorrect/confusing. The + // correct buffer for sending at the ceil rate is rbuffer, *not* + // the cbuffer. + // https://www.infradead.org/~tgr/libnl/doc/api/group__qdisc__htb.html + // https://linux.die.net/man/8/tc-htb + error = rtnl_htb_set_rbuffer(cls.get(), config.burst.get()); + if (error != 0) { + return Error(string(nl_geterror(error))); + } + } + + return Nothing(); } + +// Decodes the htb queueing class configuration from the libnl +// queueing class 'cls'. Each type of queueing class needs to +// implement this function. Returns None if the libnl queueing class +// is not an htb queueing class. +template<> +Result<htb::cls::Config> decode<htb::cls::Config>( + const Netlink<struct rtnl_class>& cls) +{ + if (rtnl_tc_get_kind(TC_CAST(cls.get())) != string(htb::KIND)) { + return None(); + } + + htb::cls::Config config; + + uint32_t rate = rtnl_htb_get_rate(cls.get()); + uint32_t ceil = rtnl_htb_get_ceil(cls.get()); + // NOTE: The libnl documentation is incorrect/confusing. The + // correct buffer for sending at the ceil rate is rbuffer, *not* + // the cbuffer. + // https://www.infradead.org/~tgr/libnl/doc/api/group__qdisc__htb.html + // https://linux.die.net/man/8/tc-htb + uint32_t burst = rtnl_htb_get_rbuffer(cls.get()); + + return htb::cls::Config( + rate, + (ceil > 0) ? Option<uint32_t>(ceil) : None(), + (burst > 0) ? Option<uint32_t>(burst) : None()); +} + +} // namespace cls { } // namespace internal { ///////////////////////////////////////////////// @@ -97,11 +168,11 @@ Try<bool> create( { return internal::create( link, - Discipline<Config>( + Discipline<DisciplineConfig>( KIND, parent, handle, - Config())); + DisciplineConfig())); } @@ -119,6 +190,74 @@ Result<hashmap<string, uint64_t>> statistics( } +namespace cls { + +void json(JSON::ObjectWriter* writer, const Config& config) +{ + writer->field("rate", config.rate); + if (config.ceil.isSome()) { + writer->field("ceil", config.ceil.get()); + } + if (config.burst.isSome()) { + writer->field("burst", config.burst.get()); + } +} + +Try<bool> exists(const string& link, const Handle& classid) +{ + return internal::cls::exists(link, classid, KIND); +} + + +Try<bool> create( + const string& link, + const Handle& parent, + const Option<Handle>& handle, + const Config& config) +{ + return internal::cls::create( + link, + Class<Config>( + KIND, + parent, + handle, + config)); +} + + +Try<bool> update( + const std::string& link, + const Handle& classid, + const Config& config) +{ + return internal::cls::update( + link, + Class<Config>( + KIND, + Handle(classid.primary(), 0), + classid, + config)); +} + + +Result<Config> getConfig( + const std::string& link, + const Handle& classid) +{ + return internal::cls::getConfig<Config>( + link, + classid, + KIND); +} + + +Try<bool> remove(const string& link, const Handle& classid) +{ + return internal::cls::remove(link, classid, KIND); +} + + +} // namespace cls { } // namespace htb { } // namespace queueing { } // namespace routing { diff --git a/src/linux/routing/queueing/htb.hpp b/src/linux/routing/queueing/htb.hpp index 857646190..e5b4f5722 100644 --- a/src/linux/routing/queueing/htb.hpp +++ b/src/linux/routing/queueing/htb.hpp @@ -21,7 +21,9 @@ #include <string> +#include <stout/bytes.hpp> #include <stout/hashmap.hpp> +#include <stout/json.hpp> #include <stout/try.hpp> #include "linux/routing/handle.hpp" @@ -64,6 +66,78 @@ Result<hashmap<std::string, uint64_t>> statistics( const Handle& parent); +namespace cls { + +struct Config +{ + Config( + uint32_t _rate, + Option<uint32_t> _ceil = None(), + Option<uint32_t> _burst = None()) + : rate(_rate), + ceil(_ceil), + burst(_burst) {} + + Config() { rate = 0; } + + bool operator==(const Config& that) const + { + return rate == that.rate && + ceil == that.ceil && + burst == that.burst; + } + + // Normal rate limit. The size of the normal rate bucket is not + // exposed and will be computed by the kernel. + uint32_t rate; + // Burst limit. + Option<uint32_t> ceil; + // Size of the burst bucket. + Option<uint32_t> burst; +}; + + +void json(JSON::ObjectWriter* writer, const Config& config); + + +// Returns true if there exists an htb queueing class on the egress +// side of the link. +Try<bool> exists( + const std::string& link, + const Handle& classid); + + +// Creates a new htb queueing class on the link. Returns false if an +// htb queueing class already exists which prevents the creation. +Try<bool> create( + const std::string& link, + const Handle& parent, + const Option<Handle> & handle, + const Config& config); + + +// Update an existing htb queueing class on the link. Returns false if +// the htb queueing class is not found. +Try<bool> update( + const std::string& link, + const Handle& classid, + const Config& config); + + +// Returns the configuration of the HTB class, None() if the link or +// class do not exist or an error if the result cannot be determined. +Result<Config> getConfig( + const std::string& link, + const Handle& classid); + + +// Removes the htb queueing class from the link. Returns false if the +// htb queueing class is not found. +Try<bool> remove( + const std::string& link, + const Handle& classid); + +} // namespace cls { } // namespace htb { } // namespace queueing { } // namespace routing { diff --git a/src/linux/routing/queueing/internal.hpp b/src/linux/routing/queueing/internal.hpp index a7b460b3b..0875d48f1 100644 --- a/src/linux/routing/queueing/internal.hpp +++ b/src/linux/routing/queueing/internal.hpp @@ -22,6 +22,7 @@ #include <netlink/object.h> #include <netlink/socket.h> +#include <netlink/route/class.h> #include <netlink/route/link.h> #include <netlink/route/qdisc.h> #include <netlink/route/tc.h> @@ -41,6 +42,7 @@ #include "linux/routing/link/internal.hpp" +#include "linux/routing/queueing/class.hpp" #include "linux/routing/queueing/discipline.hpp" #include "linux/routing/queueing/statistics.hpp" @@ -56,7 +58,7 @@ namespace queueing { namespace internal { ///////////////////////////////////////////////// -// Helpers for {en}decoding. +// Helpers for {en}decoding queueing disciplines. ///////////////////////////////////////////////// // Forward declaration. Each type of queueing discipline needs to @@ -118,7 +120,69 @@ Try<Netlink<struct rtnl_qdisc>> encodeDiscipline( } ///////////////////////////////////////////////// -// Helpers for internal APIs. +// Helpers for {en}decoding queueing classes. +///////////////////////////////////////////////// + +namespace cls { +// Forward declaration. Each type of queueing class needs to implement +// this function to encode its type specific configurations into the +// libnl queueing class (rtnl_class). +template <typename Config> +Try<Nothing> encode( + const Netlink<struct rtnl_class>& cls, + const Config& config); + + +// Forward declaration. Each type of queueing class needs to implement +// this function to decode its type specific configurations from the +// libnl queueing class (rtnl_class). Returns None if the libnl +// queueing class does not match the specific queueing class type. +template <typename Config> +Result<Config> decode(const Netlink<struct rtnl_class>& cls); + + +// Encodes a queueing class (in our representation) to a libnl +// queueing class (rtnl_class). We use templating here so that it works +// for any type of queueing class. +template <typename Config> +Try<Netlink<struct rtnl_class>> encode( + const Netlink<struct rtnl_link>& link, + const Class<Config>& config) +{ + struct rtnl_class* c = rtnl_class_alloc(); + if (c == nullptr) { + return Error("Failed to allocate a libnl class"); + } + + Netlink<struct rtnl_class> cls(c); + + rtnl_tc_set_link(TC_CAST(cls.get()), link.get()); + rtnl_tc_set_parent(TC_CAST(cls.get()), config.parent.get()); + + if (config.handle.isSome()) { + rtnl_tc_set_handle(TC_CAST(cls.get()), config.handle.get().get()); + } + + int error = rtnl_tc_set_kind(TC_CAST(cls.get()), config.kind.c_str()); + if (error != 0) { + return Error( + "Failed to set the kind of the queueing class: " + + std::string(nl_geterror(error))); + } + + // Perform queueuing class specific encoding. + Try<Nothing> encoding = encode(cls, config.config); + if (encoding.isError()) { + return Error("Failed to encode the queueing class: " + encoding.error()); + } + + return cls; +} + +} // namespace cls { + +///////////////////////////////////////////////// +// Helpers for internal APIs for queueing disciplines. ///////////////////////////////////////////////// // Returns all the libnl queue discipline (rtnl_qdisc) on the link. @@ -182,8 +246,84 @@ inline Result<Netlink<struct rtnl_qdisc>> getQdisc( return None(); } + +///////////////////////////////////////////////// +// Helpers for internal APIs for queueing classes. +///////////////////////////////////////////////// + +namespace cls { + +// Returns all the libnl queueing classes (rtnl_class) on the link. +inline Try<std::vector<Netlink<struct rtnl_class>>> getClasses( + const Netlink<struct rtnl_link>& link) +{ + Try<Netlink<struct nl_sock>> socket = routing::socket(); + if (socket.isError()) { + return Error(socket.error()); + } + + // Dump all the queuing classes from the kernel. + struct nl_cache* c = nullptr; + int error = rtnl_class_alloc_cache( + socket.get().get(), + rtnl_link_get_ifindex(link.get()), + &c); + + if (error != 0) { + return Error( + "Failed to get queueing class info from kernel: " + + std::string(nl_geterror(error))); + } + + Netlink<struct nl_cache> cache(c); + + std::vector<Netlink<struct rtnl_class>> results; + + for (struct nl_object* o = nl_cache_get_first(cache.get()); + o != nullptr; + o = nl_cache_get_next(o)) { + if (rtnl_tc_get_ifindex(TC_CAST(o)) == rtnl_link_get_ifindex(link.get())) { + // NOTE: We increment the reference counter here because + // 'cache' will be freed when this function finishes and we + // want this object's lifetime to be longer than this + // function. + nl_object_get(o); + + results.push_back(Netlink<struct rtnl_class>((struct rtnl_class*) o)); + } + } + + return results; +} + + +// Returns the libnl queueing class (rtnl_class) with the given class +// ID that matches the specified queueing class kind on the link. +// Returns None if no match is found. +inline Result<Netlink<struct rtnl_class>> getClass( + const Netlink<struct rtnl_link>& link, + const Handle& classid, + const std::string& kind) +{ + Try<std::vector<Netlink<struct rtnl_class>>> classes = getClasses(link); + if (classes.isError()) { + return Error(classes.error()); + } + + foreach (const Netlink<struct rtnl_class>& cls, classes.get()) { + if (rtnl_tc_get_handle(TC_CAST(cls.get())) == classid.get() && + rtnl_tc_get_kind(TC_CAST(cls.get())) == kind) { + return cls; + } + } + + return None(); +} + +} // namespace cls { + ///////////////////////////////////////////////// -// Internal queueing APIs. +// Internal queueing APIs for queueing disciplines. ///////////////////////////////////////////////// // Returns true if there exists a queueing discipline attached to the @@ -295,6 +435,188 @@ inline Try<bool> remove( return true; } +///////////////////////////////////////////////// +// Internal queueing APIs for queueing classes. +///////////////////////////////////////////////// + +namespace cls { + +// Returns true if there exists a queueing class attached to the given +// parent that matches the specified queueing class kind on the link. +inline Try<bool> exists( + const std::string& _link, + const Handle& classid, + const std::string& kind) +{ + Result<Netlink<struct rtnl_link>> link = link::internal::get(_link); + if (link.isError()) { + return Error(link.error()); + } else if (link.isNone()) { + return false; + } + + Result<Netlink<struct rtnl_class>> cls = getClass(link.get(), classid, kind); + if (cls.isError()) { + return Error(cls.error()); + } + + return cls.isSome(); +} + + +// Creates a new queueing class on the link. Returns false if +// a queueing class attached to the same parent with the same +// configuration already exists on the link. We use template here so +// that it works for any type of queueing class. +template <typename Config> +Try<bool> create( + const std::string& _link, + const Class<Config>& config) +{ + Result<Netlink<struct rtnl_link>> link = link::internal::get(_link); + if (link.isError()) { + return Error(link.error()); + } else if (link.isNone()) { + return Error("Link '" + _link + "' was not found"); + } + + Try<Netlink<struct rtnl_class>> cls = encode(link.get(), config); + if (cls.isError()) { + return Error("Failed to encode the queueing class: " + cls.error()); + } + + Try<Netlink<struct nl_sock>> socket = routing::socket(); + if (socket.isError()) { + return Error(socket.error()); + } + + // The flag NLM_F_EXCL tells libnl that if the class already exists + // this function should return error. + int error = rtnl_class_add( + socket.get().get(), + cls.get().get(), + NLM_F_CREATE | NLM_F_EXCL); + + if (error != 0) { + if (error == -NLE_EXIST) { + return false; + } + return Error( + "Failed to add a queueing class to the link: " + + std::string(nl_geterror(error))); + } + + return true; +} + + +// Updates an existing queueing class on the link. Returns false if +// a queueing class attached to the parent does not exist. We use +// template here so that it works for any type of queueing class. +template <typename Config> +Try<bool> update( + const std::string& _link, + const Class<Config>& config) +{ + Result<Netlink<struct rtnl_link>> link = link::internal::get(_link); + if (link.isError()) { + return Error(link.error()); + } else if (link.isNone()) { + return Error("Link '" + _link + "' was not found"); + } + + Try<Netlink<struct rtnl_class>> cls = encode(link.get(), config); + if (cls.isError()) { + return Error("Failed to encode the queueing class: " + cls.error()); + } + + Try<Netlink<struct nl_sock>> socket = routing::socket(); + if (socket.isError()) { + return Error(socket.error()); + } + + // No flags are passed so an error will be returned if the class + // does not exist. + int error = rtnl_class_add( + socket.get().get(), + cls.get().get(), + 0); + + if (error != 0) { + if (error == -NLE_OBJ_NOTFOUND) { + return false; + } + return Error( + "Failed to add a queueing class to the link: " + + std::string(nl_geterror(error))); + } + + return true; +} + +// Returns the class config for a queueing class on the link. Returns +// None() if the link or class does not exist or an error if the +// result cannot be determined. +template <typename Config> +Result<Config> getConfig( + const std::string& _link, + const Handle& classid, + const std::string& kind) +{ + Result<Netlink<struct rtnl_link>> link = link::internal::get(_link); + if (link.isError()) { + return Error(link.error()); + } else if (link.isNone()) { + return None(); + } + + Result<Netlink<struct rtnl_class>> cls = getClass(link.get(), classid, kind); + if (cls.isError()) { + return Error(cls.error()); + } else if (cls.isNone()) { + return None(); + } + + return decode<Config>(cls.get()); +} + + +// Removes the specified queueing class with the given class ID that +// matches the specified queueing class kind on the link. Returns +// false if such a queueing class is not found. +inline Try<bool> remove( + const std::string& _link, + const Handle& classid, + const std::string& kind) +{ + Result<Netlink<struct rtnl_link>> link = link::internal::get(_link); + if (link.isError()) { + return Error(link.error()); + } else if (link.isNone()) { + return false; + } + + Result<Netlink<struct rtnl_class>> cls = getClass(link.get(), classid, kind); + if (cls.isError()) { + return Error(cls.error()); + } else if (cls.isNone()) { + return false; + } + + Try<Netlink<struct nl_sock>> socket = routing::socket(); + if (socket.isError()) { + return Error(socket.error()); + } + + int error = rtnl_class_delete(socket.get().get(), cls.get().get()); + if (error != 0) { + return Error(std::string(nl_geterror(error))); + } + + return true; +} + +} // namespace cls { // Returns the set of common Traffic Control statistics for the // queueing discipline on the link, None() if the link or qdisc does
