This is an automated email from the ASF dual-hosted git repository. gilbert pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 960497728be1d06724b077f73a978d1c798705b7 Author: Andrei Budnik <[email protected]> AuthorDate: Thu Jul 18 09:10:47 2019 -0700 Added `/containerizer/debug` endpoint. This patch introduces an agent's `/containerizer/debug` endpoint, which exposes the debug info related to Mesos containerizer. Currently, this endpoint returns a list of pending futures related to isolators or containerizer launcher. This endpoint is experimental, and the format of its output may change over time. Review: https://reviews.apache.org/r/70892/ --- docs/authorization.md | 1 + src/common/authorization.cpp | 1 + src/local/local.cpp | 1 + src/slave/http.cpp | 96 ++++++++++++++++++++++++++++++++++++++++++++ src/slave/http.hpp | 9 +++++ src/slave/main.cpp | 1 + src/slave/slave.cpp | 10 +++++ src/slave/slave.hpp | 3 ++ src/tests/cluster.cpp | 2 + src/tests/mock_slave.cpp | 2 + src/tests/mock_slave.hpp | 1 + 11 files changed, 127 insertions(+) diff --git a/docs/authorization.md b/docs/authorization.md index 91dc03b..698e485 100644 --- a/docs/authorization.md +++ b/docs/authorization.md @@ -366,6 +366,7 @@ The `get_endpoints` action covers: * `/logging/toggle` * `/metrics/snapshot` * `/slave(id)/containers` +* `/slave(id)/containerizer/debug` * `/slave(id)/monitor/statistics` ### Examples diff --git a/src/common/authorization.cpp b/src/common/authorization.cpp index 8948721..fa71b0e 100644 --- a/src/common/authorization.cpp +++ b/src/common/authorization.cpp @@ -46,6 +46,7 @@ namespace authorization { // action `GET_ENDPOINTS_WITH_PATH`. hashset<string> AUTHORIZABLE_ENDPOINTS{ "/containers", + "/containerizer/debug", "/files/debug", "/logging/toggle", "/metrics/snapshot", diff --git a/src/local/local.cpp b/src/local/local.cpp index 6ac6b02..68dc9fb 100644 --- a/src/local/local.cpp +++ b/src/local/local.cpp @@ -534,6 +534,7 @@ PID<Master> launch(const Flags& flags, Allocator* _allocator) qosControllers->back(), secretGenerators->back(), nullptr, + nullptr, authorizer_); // Same authorizer as master. slaves[containerizer.get()] = slave; diff --git a/src/slave/http.cpp b/src/slave/http.cpp index 321dca7..d9f113d 100644 --- a/src/slave/http.cpp +++ b/src/slave/http.cpp @@ -59,6 +59,7 @@ #include "common/authorization.hpp" #include "common/build.hpp" +#include "common/future_tracker.hpp" #include "common/http.hpp" #include "common/recordio.hpp" #include "common/resources_utils.hpp" @@ -71,6 +72,7 @@ #include "resource_provider/local.hpp" +#include "slave/constants.hpp" #include "slave/http.hpp" #include "slave/slave.hpp" #include "slave/validation.hpp" @@ -2350,6 +2352,100 @@ Future<JSON::Array> Http::__containers( } +string Http::CONTAINERIZER_DEBUG_HELP() +{ + return HELP( + TLDR( + "Retrieve debug information for the Mesos containerizer."), + DESCRIPTION( + "Returns a list of pending operations related to Mesos", + "containerizer. This endpoint can help investigating", + "container stuck issues.", + "", + "**Note**: There is no fixed schema for a pending operation.", + "Thereby, the output of this endpoint should not be used by", + "automated tools.", + "", + "Example (**Note**: this is not exhaustive):", + "", + "```", + "{", + " \"pending\":[", + " {", + " \"operation\":\"network/cni::attach\",", + " \"args\":{", + " \"containerId\":\"container\"", + " }", + " }", + " ]", + "}", + "```"), + AUTHENTICATION(true)); +} + + +Future<Response> Http::containerizerDebug( + const Request& request, + const Option<Principal>& principal) const +{ + // TODO(a10gupta): Remove check for enabled + // authorization as part of MESOS-5346. + if (request.method != "GET" && slave->authorizer.isSome()) { + return MethodNotAllowed({"GET"}, request.method); + } + + Try<string> endpoint = extractEndpoint(request.url); + if (endpoint.isError()) { + return Failure("Failed to extract endpoint: " + endpoint.error()); + } + + return authorizeEndpoint( + endpoint.get(), + request.method, + slave->authorizer, + principal) + .then(defer( + slave->self(), + [this, request](bool authorized) -> Future<Response> { + if (!authorized) { + return Forbidden(); + } + + return _containerizerDebug() + .then([request](const JSON::Object& result) -> Response { + return process::http::OK(result, request.url.query.get("jsonp")); + }); + })); +} + + +Future<JSON::Object> Http::_containerizerDebug() const +{ + return slave->futureTracker->pendingFutures().then( + defer(slave->self(), [](const vector<FutureMetadata>& pending) { + JSON::Object result; + + JSON::Array futures; + foreach (const FutureMetadata& metadata, pending) { + if (metadata.component != COMPONENT_NAME_CONTAINERIZER) { + continue; + } + + JSON::Object args; + foreachpair (const string& key, const string& value, metadata.args) { + args.values[key] = JSON::String(value); + } + + futures.values.emplace_back(JSON::Object{ + {"operation", JSON::String(metadata.operation)}, {"args", args}}); + } + result.values["pending"] = std::move(futures); + + return result; + })); +} + + Future<Response> Http::pruneImages( const agent::Call& call, ContentType acceptType, diff --git a/src/slave/http.hpp b/src/slave/http.hpp index b8c83f1..0afdad9 100644 --- a/src/slave/http.hpp +++ b/src/slave/http.hpp @@ -80,6 +80,11 @@ public: const process::http::Request& request, const Option<process::http::authentication::Principal>& principal) const; + // /slave/containerizer/debug + process::Future<process::http::Response> containerizerDebug( + const process::http::Request& request, + const Option<process::http::authentication::Principal>& principal) const; + static std::string API_HELP(); static std::string EXECUTOR_HELP(); static std::string RESOURCE_PROVIDER_HELP(); @@ -88,6 +93,7 @@ public: static std::string STATE_HELP(); static std::string STATISTICS_HELP(); static std::string CONTAINERS_HELP(); + static std::string CONTAINERIZER_DEBUG_HELP(); private: JSON::Object _flags() const; @@ -120,6 +126,9 @@ private: bool showNestedContainers, bool showStandaloneContainers) const; + // Continuation for `/containerizer/debug` endpoint + process::Future<JSON::Object> _containerizerDebug() const; + // Helper routines for endpoint authorization. Try<std::string> extractEndpoint(const process::http::URL& url) const; diff --git a/src/slave/main.cpp b/src/slave/main.cpp index c974ba0..fd58637 100644 --- a/src/slave/main.cpp +++ b/src/slave/main.cpp @@ -619,6 +619,7 @@ int main(int argc, char** argv) qosController.get(), secretGenerator, volumeGidManager, + futureTracker.get(), authorizer_); process::spawn(slave); diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp index 0e7e4d4..9c14784 100644 --- a/src/slave/slave.cpp +++ b/src/slave/slave.cpp @@ -198,6 +198,7 @@ Slave::Slave(const string& id, QoSController* _qosController, SecretGenerator* _secretGenerator, VolumeGidManager* _volumeGidManager, + PendingFutureTracker* _futureTracker, const Option<Authorizer*>& _authorizer) : ProcessBase(id), state(RECOVERING), @@ -228,6 +229,7 @@ Slave::Slave(const string& id, qosController(_qosController), secretGenerator(_secretGenerator), volumeGidManager(_volumeGidManager), + futureTracker(_futureTracker), authorizer(_authorizer), resourceVersion(protobuf::createUUID()) {} @@ -836,6 +838,14 @@ void Slave::initialize() logResponse(request, response); }); }); + route("/containerizer/debug", + READONLY_HTTP_AUTHENTICATION_REALM, + Http::CONTAINERIZER_DEBUG_HELP(), + [this](const http::Request& request, + const Option<Principal>& principal) { + logRequest(request); + return http.containerizerDebug(request, principal); + }); // TODO(tillt): Use generalized lambda capture once we adopt C++14. Option<Authorizer*> _authorizer = authorizer; diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp index 58a5608..556d8ea 100644 --- a/src/slave/slave.hpp +++ b/src/slave/slave.hpp @@ -129,6 +129,7 @@ public: mesos::slave::QoSController* qosController, mesos::SecretGenerator* secretGenerator, VolumeGidManager* volumeGidManager, + PendingFutureTracker* futureTracker, const Option<Authorizer*>& authorizer); ~Slave() override; @@ -868,6 +869,8 @@ private: VolumeGidManager* volumeGidManager; + PendingFutureTracker* futureTracker; + const Option<Authorizer*> authorizer; // The most recent estimate of the total amount of oversubscribed diff --git a/src/tests/cluster.cpp b/src/tests/cluster.cpp index 9f180cc..1646516 100644 --- a/src/tests/cluster.cpp +++ b/src/tests/cluster.cpp @@ -621,6 +621,7 @@ Try<process::Owned<Slave>> Slave::create( qosController.getOrElse(slave->qosController.get()), secretGenerator.getOrElse(slave->secretGenerator.get()), volumeGidManager, + futureTracker.get(), authorizer)); } else { slave->slave.reset(new slave::Slave( @@ -635,6 +636,7 @@ Try<process::Owned<Slave>> Slave::create( qosController.getOrElse(slave->qosController.get()), secretGenerator.getOrElse(slave->secretGenerator.get()), volumeGidManager, + futureTracker.get(), authorizer)); } diff --git a/src/tests/mock_slave.cpp b/src/tests/mock_slave.cpp index dd458e3..71be957 100644 --- a/src/tests/mock_slave.cpp +++ b/src/tests/mock_slave.cpp @@ -103,6 +103,7 @@ MockSlave::MockSlave( QoSController* qosController, SecretGenerator* secretGenerator, VolumeGidManager* volumeGidManager, + PendingFutureTracker* futureTracker, const Option<Authorizer*>& authorizer) // It is necessary to explicitly call `ProcessBase` constructor here even // though the direct parent `Slave` already does this. This is because @@ -122,6 +123,7 @@ MockSlave::MockSlave( qosController, secretGenerator, volumeGidManager, + futureTracker, authorizer) { // Set up default behaviors, calling the original methods. diff --git a/src/tests/mock_slave.hpp b/src/tests/mock_slave.hpp index c057b40..eb31a0f 100644 --- a/src/tests/mock_slave.hpp +++ b/src/tests/mock_slave.hpp @@ -100,6 +100,7 @@ public: mesos::slave::QoSController* qosController, SecretGenerator* secretGenerator, slave::VolumeGidManager* volumeGidManager, + PendingFutureTracker* futureTracker, const Option<Authorizer*>& authorizer); MOCK_METHOD6(___run, void(
