This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 346f74605fd65fdb9d148e34a467862c38ec03cb Author: Adam Debreceni <[email protected]> AuthorDate: Mon Aug 22 20:37:25 2022 +0200 MINIFICPP-1908 Set manifest description for script processors Closes #1390 Signed-off-by: Marton Szasz <[email protected]> --- extensions/script/tests/CMakeLists.txt | 2 +- extensions/script/tests/PythonManifestTests.cpp | 142 +++++++++++++++++++++ .../include/core/state/nodes/AgentInformation.h | 10 +- .../http-curl/tests => libminifi/test}/EmptyFlow.h | 0 libminifi/test/flow-tests/TestControllerWithFlow.h | 25 ++-- 5 files changed, 162 insertions(+), 17 deletions(-) diff --git a/extensions/script/tests/CMakeLists.txt b/extensions/script/tests/CMakeLists.txt index 1442241bd..e980638cd 100644 --- a/extensions/script/tests/CMakeLists.txt +++ b/extensions/script/tests/CMakeLists.txt @@ -18,7 +18,7 @@ # if (NOT DISABLE_PYTHON_SCRIPTING) - file(GLOB EXECUTESCRIPT_PYTHON_TESTS "TestExecuteScriptProcessorWithPythonScript.cpp" "PythonScriptEngineTests.cpp") + file(GLOB EXECUTESCRIPT_PYTHON_TESTS "TestExecuteScriptProcessorWithPythonScript.cpp" "PythonScriptEngineTests.cpp" "PythonManifestTests.cpp") file(GLOB EXECUTEPYTHONPROCESSOR_UNIT_TESTS "ExecutePythonProcessorTests.cpp") file(GLOB PY_SOURCES "python/*.cpp") find_package(PythonLibs 3.5) diff --git a/extensions/script/tests/PythonManifestTests.cpp b/extensions/script/tests/PythonManifestTests.cpp new file mode 100644 index 000000000..2103b7b2f --- /dev/null +++ b/extensions/script/tests/PythonManifestTests.cpp @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define CUSTOM_EXTENSION_INIT + +#undef NDEBUG +#include <fstream> + +#include "TestBase.h" +#include "Catch.h" +#include "flow-tests/TestControllerWithFlow.h" +#include "EmptyFlow.h" + +using minifi::state::response::SerializedResponseNode; + +template<typename F> +const SerializedResponseNode* findNode(const std::vector<SerializedResponseNode>& nodes, F&& filter) { + for (auto& node : nodes) { + if (filter(node)) return &node; + } + return nullptr; +} + +const SerializedResponseNode& getNode(const std::vector<SerializedResponseNode>& nodes, std::string_view name) { + for (auto& node : nodes) { + if (node.name == name) return node; + } + assert(false); +} + +TEST_CASE("Python processor's description is part of the manifest") { + TestControllerWithFlow controller(empty_flow, false /* DEFER FLOW SETUP */); + + auto python_dir = std::filesystem::path(controller.configuration_->getHome()) / "minifi-python"; + utils::file::create_dir(python_dir); + std::ofstream{python_dir / "MyPyProc.py"} << + "def describe(proc):\n" + " proc.setDescription('An amazing processor')\n"; + + std::ofstream{python_dir / "MyPyProc2.py"} << + "def describe(proc):\n" + " proc.setDescription('Another amazing processor')\n" + " proc.setSupportsDynamicProperties()\n" + " proc.addProperty('Prop1', 'A great property', 'banana', True, False)\n"; + + controller.configuration_->set(minifi::Configuration::nifi_python_processor_dir, python_dir); + controller.configuration_->set(minifi::Configuration::nifi_extension_path, "*minifi-script*"); + + core::extension::ExtensionManager::get().initialize(controller.configuration_); + + controller.setupFlow(); + + auto agent_info = controller.controller_->getAgentManifest(); + + auto& manifest = getNode(agent_info.serialized_nodes, "agentManifest"); + + auto findPythonProcessor = [&] (const std::string& name) { + // each python file gets its own bundle + auto* python_bundle = findNode(manifest.children, [&] (auto& child) { + return child.name == "bundles" && findNode(child.children, [&] (auto& bundle_child) { + return bundle_child.name == "artifact" && bundle_child.value == name + ".py"; + }); + }); + REQUIRE(python_bundle); + + auto& py_processors = getNode(getNode(python_bundle->children, "componentManifest").children, "processors"); + + // single processor in each bundle + REQUIRE(py_processors.children.size() == 1); + + return findNode(py_processors.children, [&] (auto& child) {return utils::StringUtils::endsWith(child.name, name);}); + }; + + { + auto* MyPyProc = findPythonProcessor("MyPyProc"); + REQUIRE(MyPyProc); + + REQUIRE(getNode(MyPyProc->children, "inputRequirement").value == "INPUT_ALLOWED"); + REQUIRE(getNode(MyPyProc->children, "isSingleThreaded").value == true); + REQUIRE(getNode(MyPyProc->children, "typeDescription").value == "An amazing processor"); + REQUIRE(getNode(MyPyProc->children, "supportsDynamicRelationships").value == false); + REQUIRE(getNode(MyPyProc->children, "supportsDynamicProperties").value == false); + REQUIRE(getNode(MyPyProc->children, "type").value == "org.apache.nifi.minifi.processors.MyPyProc"); + + auto& rels = getNode(MyPyProc->children, "supportedRelationships").children; + REQUIRE(rels.size() == 2); + + auto* success = findNode(rels, [] (auto& rel) {return getNode(rel.children, "name").value == "success";}); + REQUIRE(success); + REQUIRE(getNode(success->children, "description").value == "Script succeeds"); + + auto* failure = findNode(rels, [] (auto& rel) {return getNode(rel.children, "name").value == "failure";}); + REQUIRE(failure); + REQUIRE(getNode(failure->children, "description").value == "Script fails"); + } + + + { + auto* MyPyProc2 = findPythonProcessor("MyPyProc2"); + REQUIRE(MyPyProc2); + + REQUIRE(getNode(MyPyProc2->children, "inputRequirement").value == "INPUT_ALLOWED"); + REQUIRE(getNode(MyPyProc2->children, "isSingleThreaded").value == true); + REQUIRE(getNode(MyPyProc2->children, "typeDescription").value == "Another amazing processor"); + REQUIRE(getNode(MyPyProc2->children, "supportsDynamicRelationships").value == false); + REQUIRE(getNode(MyPyProc2->children, "supportsDynamicProperties").value == true); + REQUIRE(getNode(MyPyProc2->children, "type").value == "org.apache.nifi.minifi.processors.MyPyProc2"); + + auto& properties = getNode(MyPyProc2->children, "propertyDescriptors").children; + REQUIRE(properties.size() == 1); + REQUIRE(properties[0].name == "Prop1"); + REQUIRE(getNode(properties[0].children, "name").value == "Prop1"); + REQUIRE(getNode(properties[0].children, "required").value == true); + REQUIRE(getNode(properties[0].children, "expressionLanguageScope").value == "NONE"); + REQUIRE(getNode(properties[0].children, "defaultValue").value == "banana"); + + auto& rels = getNode(MyPyProc2->children, "supportedRelationships").children; + REQUIRE(rels.size() == 2); + + auto* success = findNode(rels, [] (auto& rel) {return getNode(rel.children, "name").value == "success";}); + REQUIRE(success); + REQUIRE(getNode(success->children, "description").value == "Script succeeds"); + + auto* failure = findNode(rels, [] (auto& rel) {return getNode(rel.children, "name").value == "failure";}); + REQUIRE(failure); + REQUIRE(getNode(failure->children, "description").value == "Script fails"); + } +} diff --git a/libminifi/include/core/state/nodes/AgentInformation.h b/libminifi/include/core/state/nodes/AgentInformation.h index 7c7408061..42515d120 100644 --- a/libminifi/include/core/state/nodes/AgentInformation.h +++ b/libminifi/include/core/state/nodes/AgentInformation.h @@ -285,18 +285,12 @@ class ComponentManifest : public DeviceInformation { size_t nameLength = group.full_name_.length() - lastOfIdx; processorName = group.full_name_.substr(lastOfIdx, nameLength); } - std::string description; - bool foundDescription = AgentDocs::getDescription(processorName, description); - if (!foundDescription) { - foundDescription = AgentDocs::getDescription(group.full_name_, description); - } - - if (foundDescription) { + { SerializedResponseNode proc_desc; proc_desc.name = "typeDescription"; - proc_desc.value = description; + proc_desc.value = group.description_; desc.children.push_back(proc_desc); } diff --git a/extensions/http-curl/tests/EmptyFlow.h b/libminifi/test/EmptyFlow.h similarity index 100% rename from extensions/http-curl/tests/EmptyFlow.h rename to libminifi/test/EmptyFlow.h diff --git a/libminifi/test/flow-tests/TestControllerWithFlow.h b/libminifi/test/flow-tests/TestControllerWithFlow.h index bea98c71d..b87c76cb0 100644 --- a/libminifi/test/flow-tests/TestControllerWithFlow.h +++ b/libminifi/test/flow-tests/TestControllerWithFlow.h @@ -29,7 +29,7 @@ class TestControllerWithFlow: public TestController{ public: - explicit TestControllerWithFlow(const char* yamlConfigContent) { + explicit TestControllerWithFlow(const char* yamlConfigContent, bool setup_flow = true) { LogTestController::getInstance().setTrace<minifi::processors::TestProcessor>(); LogTestController::getInstance().setTrace<minifi::processors::TestFlowFileGenerator>(); LogTestController::getInstance().setTrace<minifi::Connection>(); @@ -40,22 +40,29 @@ class TestControllerWithFlow: public TestController{ LogTestController::getInstance().setTrace<minifi::TimerDrivenSchedulingAgent>(); LogTestController::getInstance().setTrace<minifi::EventDrivenSchedulingAgent>(); - std::string dir = createTempDirectory(); + home_ = createTempDirectory(); - std::string yamlPath = utils::file::FileUtils::concat_path(dir, "config.yml"); - std::ofstream{yamlPath} << yamlConfigContent; + yaml_path_ = home_ / "config.yml"; + std::ofstream{yaml_path_} << yamlConfigContent; configuration_ = std::make_shared<minifi::Configure>(); + configuration_->setHome(home_); + configuration_->set(minifi::Configure::nifi_flow_configuration_file, yaml_path_); + + if (setup_flow) { + setupFlow(); + } + } + + void setupFlow() { std::shared_ptr<core::Repository> prov_repo = std::make_shared<TestRepository>(); std::shared_ptr<core::Repository> ff_repo = std::make_shared<TestFlowRepository>(); std::shared_ptr<core::ContentRepository> content_repo = std::make_shared<core::repository::VolatileContentRepository>(); - configuration_->set(minifi::Configure::nifi_flow_configuration_file, yamlPath); - REQUIRE(content_repo->initialize(configuration_)); std::shared_ptr<minifi::io::StreamFactory> stream_factory = minifi::io::StreamFactory::getInstance(configuration_); - auto flow = std::make_unique<core::YamlConfiguration>(prov_repo, ff_repo, content_repo, stream_factory, configuration_, yamlPath); + auto flow = std::make_unique<core::YamlConfiguration>(prov_repo, ff_repo, content_repo, stream_factory, configuration_, yaml_path_); auto root = flow->getRoot(); root_ = root.get(); controller_ = std::make_shared<minifi::FlowController>( @@ -76,7 +83,9 @@ class TestControllerWithFlow: public TestController{ LogTestController::getInstance().reset(); } + std::filesystem::path home_; + std::filesystem::path yaml_path_; std::shared_ptr<minifi::Configure> configuration_; std::shared_ptr<minifi::FlowController> controller_; - core::ProcessGroup* root_; + core::ProcessGroup* root_{nullptr}; };
