szaszm commented on code in PR #1503: URL: https://github.com/apache/nifi-minifi-cpp/pull/1503#discussion_r1159269222
########## controller/Controller.cpp: ########## @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "Controller.h" + +#include <utility> + +#include "io/BufferStream.h" +#include "c2/C2Payload.h" + +namespace org::apache::nifi::minifi::controller { + +bool sendSingleCommand(std::unique_ptr<io::Socket> socket, uint8_t op, const std::string& value) { + if (socket->initialize() < 0) { + return false; + } + io::BufferStream stream; + stream.write(&op, 1); + stream.write(value); + return socket->write(stream.getBuffer()) == stream.size(); +} + +bool stopComponent(std::unique_ptr<io::Socket> socket, const std::string& component) { + return sendSingleCommand(std::move(socket), c2::Operation::STOP, component); +} + +bool startComponent(std::unique_ptr<io::Socket> socket, const std::string& component) { + return sendSingleCommand(std::move(socket), c2::Operation::START, component); +} + +bool clearConnection(std::unique_ptr<io::Socket> socket, const std::string& connection) { + return sendSingleCommand(std::move(socket), c2::Operation::CLEAR, connection); +} + +int updateFlow(std::unique_ptr<io::Socket> socket, std::ostream &out, const std::string& file) { + if (socket->initialize() < 0) { + return -1; + } + uint8_t op = c2::Operation::UPDATE; + io::BufferStream stream; + stream.write(&op, 1); + stream.write("flow"); + stream.write(file); + if (io::isError(socket->write(stream.getBuffer()))) { + return -1; + } + // read the response + uint8_t resp = 0; + socket->read(resp); + if (resp == c2::Operation::DESCRIBE) { + uint16_t connections = 0; + socket->read(connections); + out << connections << " are full" << std::endl; + for (int i = 0; i < connections; i++) { + std::string fullcomponent; + socket->read(fullcomponent); + out << fullcomponent << " is full" << std::endl; + } + } + return 0; +} + +int getFullConnections(std::unique_ptr<io::Socket> socket, std::ostream &out) { + if (socket->initialize() < 0) { + return -1; + } + uint8_t op = c2::Operation::DESCRIBE; + io::BufferStream stream; + stream.write(&op, 1); + stream.write("getfull"); + if (io::isError(socket->write(stream.getBuffer()))) { + return -1; + } + // read the response + uint8_t resp = 0; + socket->read(resp); + if (resp == c2::Operation::DESCRIBE) { + uint16_t connections = 0; + socket->read(connections); + out << connections << " are full" << std::endl; + for (int i = 0; i < connections; i++) { + std::string fullcomponent; + socket->read(fullcomponent); + out << fullcomponent << " is full" << std::endl; + } + } + return 0; +} + +int getConnectionSize(std::unique_ptr<io::Socket> socket, std::ostream &out, const std::string& connection) { + if (socket->initialize() < 0) { + return -1; + } + uint8_t op = c2::Operation::DESCRIBE; + io::BufferStream stream; + stream.write(&op, 1); + stream.write("queue"); + stream.write(connection); + if (io::isError(socket->write(stream.getBuffer()))) { + return -1; + } + // read the response + uint8_t resp = 0; + socket->read(resp); + if (resp == c2::Operation::DESCRIBE) { + std::string size; + socket->read(size); + out << "Size/Max of " << connection << " " << size << std::endl; + } + return 0; +} + +int listComponents(std::unique_ptr<io::Socket> socket, std::ostream &out, bool show_header) { + if (socket->initialize() < 0) { + return -1; + } + io::BufferStream stream; + uint8_t op = c2::Operation::DESCRIBE; + stream.write(&op, 1); + stream.write("components"); + if (io::isError(socket->write(stream.getBuffer()))) { + return -1; + } + uint16_t responses = 0; + socket->read(op); + socket->read(responses); + if (show_header) + out << "Components:" << std::endl; + + for (int i = 0; i < responses; i++) { + std::string name; + socket->read(name, false); + std::string status; + socket->read(status, false); + out << name << ", running: " << status << std::endl; + } + return 0; +} + +int listConnections(std::unique_ptr<io::Socket> socket, std::ostream &out, bool show_header) { + if (socket->initialize() < 0) { + return -1; + } + io::BufferStream stream; + uint8_t op = c2::Operation::DESCRIBE; + stream.write(&op, 1); + stream.write("connections"); + if (io::isError(socket->write(stream.getBuffer()))) { + return -1; + } + uint16_t responses = 0; + socket->read(op); + socket->read(responses); + if (show_header) + out << "Connection Names:" << std::endl; + + for (int i = 0; i < responses; i++) { + std::string name; + socket->read(name, false); + out << name << std::endl; + } + return 0; +} + +int printManifest(std::unique_ptr<io::Socket> socket, std::ostream &out) { + if (socket->initialize() < 0) { + return -1; + } + io::BufferStream stream; + uint8_t op = c2::Operation::DESCRIBE; + stream.write(&op, 1); + stream.write("manifest"); + if (io::isError(socket->write(stream.getBuffer()))) { + return -1; + } + socket->read(op); + std::string manifest; + socket->read(manifest, true); + out << manifest << std::endl; + return 0; +} + +int getJstacks(std::unique_ptr<io::Socket> socket, std::ostream &out) { Review Comment: Should we drop this? Since it doesn't work AFAIK, and is removed from the docs. ########## controller/MiNiFiController.cpp: ########## @@ -75,33 +80,60 @@ int main(int argc, char **argv) { secure_context = std::make_shared<minifi::controllers::SSLContextService>("ControllerSocketProtocolSSL", configuration); secure_context->onEnable(); } + } else { + secure_context->onEnable(); } + return secure_context; +} - std::string value; +int main(int argc, char **argv) { + const auto logger = minifi::core::logging::LoggerConfiguration::getConfiguration().getLogger("controller"); + const std::string minifi_home = determineMinifiHome(logger); + if (minifi_home.empty()) { + // determineMinifiHome already logged everything we need + return -1; + } + + const auto configuration = std::make_shared<minifi::Configure>(); + configuration->setHome(minifi_home); + configuration->loadConfigureFile(DEFAULT_NIFI_PROPERTIES_FILE); + + const auto log_properties = std::make_shared<minifi::core::logging::LoggerProperties>(); + log_properties->setHome(minifi_home); + log_properties->loadConfigureFile(DEFAULT_LOG_PROPERTIES_FILE); + minifi::core::logging::LoggerConfiguration::getConfiguration().initialize(log_properties); + + std::shared_ptr<minifi::controllers::SSLContextService> secure_context; + try { + secure_context = getSSLContextService(configuration); + } catch(const minifi::Exception& ex) { + logger->log_error(ex.what()); + exit(1); + } auto stream_factory_ = minifi::io::StreamFactory::getInstance(configuration); std::string host = "localhost"; - std::string portStr; - std::string caCert; + std::string port_str; + std::string ca_cert; int port = -1; cxxopts::Options options("MiNiFiController", "MiNiFi local agent controller"); options.positional_help("[optional args]").show_positional_help(); - options.add_options() //NOLINT - ("h,help", "Shows Help") //NOLINT - ("host", "Specifies connecting host name", cxxopts::value<std::string>()) //NOLINT - ("port", "Specifies connecting host port", cxxopts::value<int>()) //NOLINT - ("stop", "Shuts down the provided component", cxxopts::value<std::vector<std::string>>()) //NOLINT - ("start", "Starts provided component", cxxopts::value<std::vector<std::string>>()) //NOLINT - ("l,list", "Provides a list of connections or processors", cxxopts::value<std::string>()) //NOLINT - ("c,clear", "Clears the associated connection queue", cxxopts::value<std::vector<std::string>>()) //NOLINT - ("getsize", "Reports the size of the associated connection queue", cxxopts::value<std::vector<std::string>>()) //NOLINT - ("updateflow", "Updates the flow of the agent using the provided flow file", cxxopts::value<std::string>()) //NOLINT - ("getfull", "Reports a list of full connections") //NOLINT - ("jstack", "Returns backtraces from the agent") //NOLINT - ("manifest", "Generates a manifest for the current binary") //NOLINT + options.add_options() + ("h,help", "Shows Help") + ("host", "Specifies connecting host name", cxxopts::value<std::string>()) + ("port", "Specifies connecting host port", cxxopts::value<int>()) + ("stop", "Shuts down the provided component", cxxopts::value<std::vector<std::string>>()) + ("start", "Starts provided component", cxxopts::value<std::vector<std::string>>()) + ("l,list", "Provides a list of connections or processors", cxxopts::value<std::string>()) + ("c,clear", "Clears the associated connection queue", cxxopts::value<std::vector<std::string>>()) + ("getsize", "Reports the size of the associated connection queue", cxxopts::value<std::vector<std::string>>()) + ("updateflow", "Updates the flow of the agent using the provided flow file", cxxopts::value<std::string>()) + ("getfull", "Reports a list of full connections") + ("jstack", "Returns backtraces from the agent") + ("manifest", "Generates a manifest for the current binary") Review Comment: The indentation on these lines is supposed to be continuation (4 spaces), since they are part of the same expression. ########## controller/tests/ControllerTests.cpp: ########## @@ -0,0 +1,545 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <vector> +#include <memory> +#include <utility> +#include <string> +#include <filesystem> +#include "range/v3/algorithm/find.hpp" + +#include "TestBase.h" +#include "Catch.h" +#include "io/ClientSocket.h" +#include "core/Processor.h" +#include "Controller.h" +#include "c2/ControllerSocketProtocol.h" +#include "utils/IntegrationTestUtils.h" +#include "c2/ControllerSocketMetricsPublisher.h" +#include "core/controller/ControllerServiceProvider.h" +#include "controllers/SSLContextService.h" +#include "utils/StringUtils.h" +#include "state/UpdateController.h" + +using namespace std::literals::chrono_literals; + +namespace org::apache::nifi::minifi::test { + +class TestStateController : public minifi::state::StateController { + public: + TestStateController() + : is_running(false) { + } + + std::string getComponentName() const override { + return "TestStateController"; + } + + minifi::utils::Identifier getComponentUUID() const override { + static auto dummyUUID = minifi::utils::Identifier::parse("12345678-1234-1234-1234-123456789abc").value(); + return dummyUUID; + } + + int16_t start() override { + is_running = true; + return 0; + } + + int16_t stop() override { + is_running = false; + return 0; + } + + bool isRunning() const override { + return is_running; + } + + int16_t pause() override { + return 0; + } + + int16_t resume() override { + return 0; + } + + std::atomic<bool> is_running; +}; + +class TestBackTrace : public BackTrace { + public: + using BackTrace::BackTrace; + void addTraceLines(uint32_t line_count) { + for (uint32_t i = 1; i <= line_count; ++i) { + addLine("bt line " + std::to_string(i) + " for " + getName()); + } + } +}; + +class TestUpdateSink : public minifi::state::StateMonitor { + public: + explicit TestUpdateSink(std::shared_ptr<StateController> controller) + : is_running(true), + clear_calls(0), + controller(std::move(controller)), + update_calls(0) { + } + + void executeOnComponent(const std::string&, std::function<void(minifi::state::StateController&)> func) override { + func(*controller); + } + + void executeOnAllComponents(std::function<void(minifi::state::StateController&)> func) override { + func(*controller); + } + + std::string getComponentName() const override { + return "TestUpdateSink"; + } + + minifi::utils::Identifier getComponentUUID() const override { + static auto dummyUUID = minifi::utils::Identifier::parse("12345678-1234-1234-1234-123456789abc").value(); + return dummyUUID; + } + + int16_t start() override { + is_running = true; + return 0; + } + + int16_t stop() override { + is_running = false; + return 0; + } + + bool isRunning() const override { + return is_running; + } + + int16_t pause() override { + return 0; + } + + int16_t resume() override { + return 0; + } + std::vector<BackTrace> getTraces() override { + std::vector<BackTrace> traces; + TestBackTrace trace1("trace1"); + trace1.addTraceLines(2); + traces.push_back(trace1); + TestBackTrace trace2("trace2"); + trace2.addTraceLines(3); + traces.push_back(trace2); + return traces; + } + + int16_t drainRepositories() override { + return 0; + } + + std::map<std::string, std::unique_ptr<minifi::io::InputStream>> getDebugInfo() override { + return {}; + } + + int16_t clearConnection(const std::string& /*connection*/) override { + clear_calls++; + return 0; + } + + std::vector<std::string> getSupportedConfigurationFormats() const override { + return {}; + } + + int16_t applyUpdate(const std::string& /*source*/, const std::string& /*configuration*/, bool /*persist*/ = false, const std::optional<std::string>& /*flow_id*/ = std::nullopt) override { + update_calls++; + return 0; + } + + int16_t applyUpdate(const std::string& /*source*/, const std::shared_ptr<minifi::state::Update>& /*updateController*/) override { + update_calls++; + return 0; + } + + uint64_t getUptime() override { + return 8765309; + } + + std::atomic<bool> is_running; + std::atomic<uint32_t> clear_calls; + std::shared_ptr<StateController> controller; + std::atomic<uint32_t> update_calls; +}; + +class TestControllerSocketReporter : public c2::ControllerSocketReporter { + std::unordered_map<std::string, ControllerSocketReporter::QueueSize> getQueueSizes() override { + return { + {"con1", {1, 2}}, + {"con2", {3, 3}} + }; + } + + std::unordered_set<std::string> getFullConnections() override { + return {"con2"}; + } + + std::unordered_set<std::string> getConnections() override { + return {"con1", "con2"}; + } + + std::string getAgentManifest() override { + return "testAgentManifest"; + } +}; + +class TestControllerServiceProvider : public core::controller::ControllerServiceProvider { + public: + explicit TestControllerServiceProvider(std::shared_ptr<controllers::SSLContextService> ssl_context_service) + : core::controller::ControllerServiceProvider("TestControllerServiceProvider"), + ssl_context_service_(std::move(ssl_context_service)) { + } + std::shared_ptr<core::controller::ControllerService> getControllerService(const std::string&) const override { + return is_ssl_ ? ssl_context_service_ : nullptr; + } + + std::shared_ptr<core::controller::ControllerServiceNode> createControllerService(const std::string&, const std::string&, const std::string&, bool) override { + return nullptr; + } + void clearControllerServices() override { + } + void enableAllControllerServices() override { + } + void disableAllControllerServices() override { + } + + void setSsl() { + is_ssl_ = true; + } + + private: + bool is_ssl_{}; + std::shared_ptr<controllers::SSLContextService> ssl_context_service_; +}; + +class ControllerTestFixture { + public: + enum class ConnectionType { + UNSECURE, + SSL_FROM_SERVICE_PROVIDER, + SSL_FROM_CONFIGURATION + }; + + ControllerTestFixture() + : configuration_(std::make_shared<minifi::Configure>()), + controller_(std::make_shared<TestStateController>()), + update_sink_(std::make_unique<TestUpdateSink>(controller_)), + stream_factory_(minifi::io::StreamFactory::getInstance(configuration_)) { + configuration_->set(minifi::Configure::controller_socket_host, "localhost"); + configuration_->set(minifi::Configure::controller_socket_port, "9997"); + configuration_->set(minifi::Configure::nifi_security_client_certificate, (minifi::utils::file::FileUtils::get_executable_dir() / "resources" / "minifi-cpp-flow.crt").string()); + configuration_->set(minifi::Configure::nifi_security_client_private_key, (minifi::utils::file::FileUtils::get_executable_dir() / "resources" / "minifi-cpp-flow.key").string()); + configuration_->set(minifi::Configure::nifi_security_client_pass_phrase, "abcdefgh"); + configuration_->set(minifi::Configure::nifi_security_client_ca_certificate, (minifi::utils::file::FileUtils::get_executable_dir() / "resources" / "root-ca.pem").string()); + configuration_->set(minifi::Configure::controller_ssl_context_service, "SSLContextService"); + ssl_context_service_ = std::make_shared<controllers::SSLContextService>("SSLContextService", configuration_); + ssl_context_service_->onEnable(); + controller_service_provider_ = std::make_unique<TestControllerServiceProvider>(ssl_context_service_); + } + + void initalizeControllerSocket(const std::shared_ptr<c2::ControllerSocketReporter>& reporter = nullptr) { + if (connection_type_ == ConnectionType::SSL_FROM_CONFIGURATION) { + configuration_->set(minifi::Configure::nifi_remote_input_secure, "true"); + } + controller_socket_protocol_ = std::make_unique<minifi::c2::ControllerSocketProtocol>(*controller_service_provider_, *update_sink_, configuration_, reporter); + if (connection_type_ == ConnectionType::SSL_FROM_SERVICE_PROVIDER) { + controller_service_provider_->setSsl(); + } + controller_socket_protocol_->initialize(); + } + + std::unique_ptr<minifi::io::Socket> createSocket() { + if (connection_type_ == ConnectionType::UNSECURE) { + return stream_factory_->createSocket("localhost", 9997); + } else { + return stream_factory_->createSecureSocket("localhost", 9997, ssl_context_service_); + } + } + + void setConnectionType(ConnectionType connection_type) { + connection_type_ = connection_type; + } + + protected: + ConnectionType connection_type_ = ConnectionType::UNSECURE; + std::shared_ptr<minifi::Configure> configuration_; + std::shared_ptr<TestStateController> controller_; + std::unique_ptr<TestUpdateSink> update_sink_; + std::shared_ptr<minifi::io::StreamFactory> stream_factory_; + std::unique_ptr<minifi::c2::ControllerSocketProtocol> controller_socket_protocol_; + std::shared_ptr<controllers::SSLContextService> ssl_context_service_; + std::unique_ptr<TestControllerServiceProvider> controller_service_provider_; +}; + +#ifdef WIN32 +TEST_CASE("TestWindows", "[controllerTests]") { + std::cout << "Controller Tests are not supported on windows"; +} +#else +TEST_CASE_METHOD(ControllerTestFixture, "Test listComponents", "[controllerTests]") { + SECTION("With SSL from service provider") { + setConnectionType(ControllerTestFixture::ConnectionType::SSL_FROM_SERVICE_PROVIDER); + } + + SECTION("With SSL from properties") { + setConnectionType(ControllerTestFixture::ConnectionType::SSL_FROM_CONFIGURATION); + } + + SECTION("Without SSL") { + setConnectionType(ControllerTestFixture::ConnectionType::UNSECURE); + } + + initalizeControllerSocket(); + + auto socket = createSocket(); + + minifi::controller::startComponent(std::move(socket), "TestStateController"); + + using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; + REQUIRE(verifyEventHappenedInPollTime(500ms, [&] { return controller_->isRunning(); }, 20ms)); + + socket = createSocket(); + + minifi::controller::stopComponent(std::move(socket), "TestStateController"); + + REQUIRE(verifyEventHappenedInPollTime(500ms, [&] { return !controller_->isRunning(); }, 20ms)); + + socket = createSocket(); + std::stringstream ss; + minifi::controller::listComponents(std::move(socket), ss); + + REQUIRE(ss.str() == "Components:\nTestStateController, running: false\n"); +} + +TEST_CASE_METHOD(ControllerTestFixture, "TestClear", "[controllerTests]") { + SECTION("With SSL from service provider") { + setConnectionType(ControllerTestFixture::ConnectionType::SSL_FROM_SERVICE_PROVIDER); + } + + SECTION("With SSL from properties") { + setConnectionType(ControllerTestFixture::ConnectionType::SSL_FROM_CONFIGURATION); + } + + SECTION("Without SSL") { + setConnectionType(ControllerTestFixture::ConnectionType::UNSECURE); + } + + initalizeControllerSocket(); + + auto socket = createSocket(); + + minifi::controller::startComponent(std::move(socket), "TestStateController"); + + using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; + REQUIRE(verifyEventHappenedInPollTime(500ms, [&] { return controller_->isRunning(); }, 20ms)); + + socket = createSocket(); + + minifi::controller::clearConnection(std::move(socket), "connection"); + + socket = createSocket(); + + minifi::controller::clearConnection(std::move(socket), "connection"); + + socket = createSocket(); + + minifi::controller::clearConnection(std::move(socket), "connection"); + + REQUIRE(verifyEventHappenedInPollTime(500ms, [&] { return 3 == update_sink_->clear_calls; }, 20ms)); +} + +TEST_CASE_METHOD(ControllerTestFixture, "TestUpdate", "[controllerTests]") { + SECTION("With SSL from service provider") { + setConnectionType(ControllerTestFixture::ConnectionType::SSL_FROM_SERVICE_PROVIDER); + } + + SECTION("With SSL from properties") { + setConnectionType(ControllerTestFixture::ConnectionType::SSL_FROM_CONFIGURATION); + } + + SECTION("Without SSL") { + setConnectionType(ControllerTestFixture::ConnectionType::UNSECURE); + } + + initalizeControllerSocket(); + + auto socket = createSocket(); + + minifi::controller::startComponent(std::move(socket), "TestStateController"); + + using org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime; + REQUIRE(verifyEventHappenedInPollTime(500ms, [&] { return controller_->isRunning(); }, 20ms)); + + std::stringstream ss; + + socket = createSocket(); Review Comment: nitpick, but I would restrict the scope of `socket` instead of reassigning. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
