This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 098dab167ef1bcff6d680d6d2d3a107f67068ba1 Author: Martin Zink <[email protected]> AuthorDate: Wed Dec 10 13:35:29 2025 +0100 MINIFICPP-2690 Simplify AgentDocs Closes #2070 Signed-off-by: Marton Szasz <[email protected]> --- CMakeLists.txt | 2 - core-framework/include/agent/agent_docs.h | 25 ++++----- core-framework/include/core/Resource.h | 3 +- extensions/python/PythonCreator.h | 8 ++- .../tests/unit/ManifestTests.cpp | 28 ++++++---- generateVersion.bat | 31 ++--------- generateVersion.sh | 27 +--------- libminifi/include/agent/build_description.h | 50 ------------------ .../include/core/state/nodes/AgentInformation.h | 37 +------------ libminifi/src/agent/JsonSchema.cpp | 35 ++++--------- libminifi/src/agent/agent_docs.cpp | 8 +-- libminifi/src/agent/build_description.cpp | 61 ---------------------- .../src/core/state/nodes/AgentInformation.cpp | 60 ++++++--------------- libminifi/src/minifi-c.cpp | 10 ++-- .../test/libtest/integration/HTTPHandlers.cpp | 14 ++--- libminifi/test/unit/ComponentManifestTests.cpp | 10 ++-- minifi-api/include/minifi-cpp/agent/agent_docs.h | 37 +++++++++---- .../include/minifi-cpp/agent/agent_version.h | 1 - .../include/minifi-cpp/agent/build_description.h | 49 ----------------- minifi_main/AgentDocs.cpp | 26 ++++----- minifi_main/AgentDocs.h | 7 +-- minifi_main/MiNiFiMain.cpp | 28 ++++------ 22 files changed, 139 insertions(+), 418 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 68de4ed07..595e570de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -426,7 +426,6 @@ if (WIN32) "${CMAKE_CXX_COMPILER}" "${CMAKE_CXX_COMPILER_VERSION}" "${CMAKE_CXX_FLAGS};${MINIFI_CPP_COMPILE_OPTIONS};${MINIFI_CPP_COMPILE_DEFINITIONS}" - \"${selected_extensions}\" "${BUILD_IDENTIFIER}" "${BUILD_REV}" COMMAND_ECHO STDOUT) @@ -439,7 +438,6 @@ else() "${CMAKE_CXX_COMPILER}" "${CMAKE_CXX_COMPILER_VERSION}" "${CMAKE_CXX_FLAGS};${MINIFI_CPP_COMPILE_OPTIONS};${MINIFI_CPP_COMPILE_DEFINITIONS}" - "${selected_extensions}" "${BUILD_IDENTIFIER}") endif() diff --git a/core-framework/include/agent/agent_docs.h b/core-framework/include/agent/agent_docs.h index 71149aca9..14544f2d6 100644 --- a/core-framework/include/agent/agent_docs.h +++ b/core-framework/include/agent/agent_docs.h @@ -60,13 +60,14 @@ std::string classNameWithDots() { } // namespace detail template<typename Class, ResourceType Type> -void AgentDocs::createClassDescription(const std::string& group, const std::string& name) { - Components& components = getMutableClassDescriptions()[group]; +void ClassDescriptionRegistry::createClassDescription(std::string bundle_name, std::string class_name, std::string version) { + const BundleIdentifier group_details{.name = std::move(bundle_name), .version = std::move(version)}; + auto& [processors, controller_services, parameter_providers, other_components] = getMutableClassDescriptions()[group_details]; if constexpr (Type == ResourceType::Processor) { - components.processors_.push_back(ClassDescription{ + processors.push_back(ClassDescription{ .type_ = Type, - .short_name_ = name, + .short_name_ = std::move(class_name), .full_name_ = detail::classNameWithDots<Class>(), .description_ = Class::Description, .class_properties_ = detail::toVector(Class::Properties), @@ -79,9 +80,9 @@ void AgentDocs::createClassDescription(const std::string& group, const std::stri .isSingleThreaded_ = Class::IsSingleThreaded }); } else if constexpr (Type == ResourceType::ControllerService) { - components.controller_services_.push_back(ClassDescription{ + controller_services.push_back(ClassDescription{ .type_ = Type, - .short_name_ = name, + .short_name_ = std::move(class_name), .full_name_ = detail::classNameWithDots<Class>(), .description_ = Class::Description, .class_properties_ = detail::toVector(Class::Properties), @@ -89,25 +90,25 @@ void AgentDocs::createClassDescription(const std::string& group, const std::stri .supports_dynamic_properties_ = Class::SupportsDynamicProperties }); } else if constexpr (Type == ResourceType::InternalResource) { - components.other_components_.push_back(ClassDescription{ + other_components.push_back(ClassDescription{ .type_ = Type, - .short_name_ = name, + .short_name_ = std::move(class_name), .full_name_ = detail::classNameWithDots<Class>(), .class_properties_ = detail::toVector(Class::Properties), .supports_dynamic_properties_ = Class::SupportsDynamicProperties, }); } else if constexpr (Type == ResourceType::ParameterProvider) { - components.parameter_providers_.push_back(ClassDescription{ + parameter_providers.push_back(ClassDescription{ .type_ = Type, - .short_name_ = name, + .short_name_ = std::move(class_name), .full_name_ = detail::classNameWithDots<Class>(), .description_ = Class::Description, .class_properties_ = detail::toVector(Class::Properties) }); } else if constexpr (Type == ResourceType::DescriptionOnly) { - components.other_components_.push_back(ClassDescription{ + other_components.push_back(ClassDescription{ .type_ = Type, - .short_name_ = name, + .short_name_ = std::move(class_name), .full_name_ = detail::classNameWithDots<Class>(), .description_ = Class::Description }); diff --git a/core-framework/include/core/Resource.h b/core-framework/include/core/Resource.h index 0b73c8ba9..a4dd1be87 100644 --- a/core-framework/include/core/Resource.h +++ b/core-framework/include/core/Resource.h @@ -31,6 +31,7 @@ #include "utils/Macro.h" #include "core/ProcessorFactoryImpl.h" #include "core/ObjectFactory.h" +#include "minifi-cpp/agent/agent_version.h" namespace org::apache::nifi::minifi::core { @@ -68,7 +69,7 @@ class StaticClassType { } } - minifi::AgentDocs::createClassDescription<Class, Type>(module_name, class_name); + minifi::ClassDescriptionRegistry::createClassDescription<Class, Type>(module_name, class_name, minifi::AgentBuild::VERSION); } ~StaticClassType() { diff --git a/extensions/python/PythonCreator.h b/extensions/python/PythonCreator.h index 17dbce119..6b64d9a52 100644 --- a/extensions/python/PythonCreator.h +++ b/extensions/python/PythonCreator.h @@ -32,7 +32,6 @@ #include "core/Resource.h" #include "core/logging/LoggerFactory.h" #include "minifi-cpp/agent/agent_version.h" -#include "minifi-cpp/agent/build_description.h" #include "range/v3/algorithm.hpp" #include "range/v3/view/filter.hpp" #include "utils/Environment.h" @@ -159,10 +158,9 @@ class PythonCreator : public minifi::core::CoreComponentImpl { } DummyProcessorDescriptor descriptor; processor->core::ProcessorImpl::initialize(descriptor); - minifi::BundleDetails details; - details.artifact = path.filename().string(); + minifi::BundleIdentifier details; + details.name = path.filename().string(); details.version = processor->getVersion() && !processor->getVersion()->empty() ? *processor->getVersion() : minifi::AgentBuild::VERSION; - details.group = "python"; minifi::ClassDescription description{ .type_ = ResourceType::Processor, @@ -175,7 +173,7 @@ class PythonCreator : public minifi::core::CoreComponentImpl { .inputRequirement_ = toString(processor->getInputRequirement()), .isSingleThreaded_ = processor->isSingleThreaded()}; - minifi::ExternalBuildDescription::addExternalComponent(details, description); + minifi::ClassDescriptionRegistry::getMutableClassDescriptions()[details].processors.push_back(description); } void configure(const std::vector<std::string> &pythonFiles) { diff --git a/extensions/standard-processors/tests/unit/ManifestTests.cpp b/extensions/standard-processors/tests/unit/ManifestTests.cpp index 638ea37f2..4cbf93873 100644 --- a/extensions/standard-processors/tests/unit/ManifestTests.cpp +++ b/extensions/standard-processors/tests/unit/ManifestTests.cpp @@ -19,16 +19,19 @@ #define EXTENSION_LIST "*minifi-system*, *minifi-standard-processors*" // NOLINT(cppcoreguidelines-macro-usage) #include <memory> -#include "core/state/nodes/DeviceInformation.h" + #include "core/state/nodes/AgentInformation.h" -#include "unit/TestBase.h" -#include "unit/Catch.h" +#include "core/state/nodes/DeviceInformation.h" +#include "minifi-cpp/agent/agent_version.h" #include "range/v3/algorithm/contains.hpp" #include "range/v3/algorithm/find_if.hpp" +#include "unit/Catch.h" +#include "unit/TestBase.h" TEST_CASE("Test Required", "[required]") { - minifi::state::response::ComponentManifest manifest("minifi-standard-processors"); - auto serialized = manifest.serialize(); + const auto standard_processor_bundle_id = minifi::BundleIdentifier{.name = "minifi-standard-processors", .version = minifi::AgentBuild::VERSION}; + const auto standard_processors_components = minifi::ClassDescriptionRegistry::getClassDescriptions().at(standard_processor_bundle_id); + auto serialized = minifi::state::response::serializeComponentManifest(standard_processors_components); REQUIRE_FALSE(serialized.empty()); const auto &resp = serialized[0]; REQUIRE_FALSE(resp.children.empty()); @@ -62,8 +65,9 @@ TEST_CASE("Test Required", "[required]") { } TEST_CASE("Test Valid Regex", "[validRegex]") { - minifi::state::response::ComponentManifest manifest("minifi-standard-processors"); - auto serialized = manifest.serialize(); + const auto standard_processor_bundle_id = minifi::BundleIdentifier{.name = "minifi-standard-processors", .version = minifi::AgentBuild::VERSION}; + const auto standard_processors_components = minifi::ClassDescriptionRegistry::getClassDescriptions().at(standard_processor_bundle_id); + auto serialized = minifi::state::response::serializeComponentManifest(standard_processors_components); REQUIRE_FALSE(serialized.empty()); const auto &resp = serialized[0]; REQUIRE_FALSE(resp.children.empty()); @@ -81,8 +85,9 @@ TEST_CASE("Test Valid Regex", "[validRegex]") { } TEST_CASE("Test Relationships", "[rel1]") { - minifi::state::response::ComponentManifest manifest("minifi-standard-processors"); - auto serialized = manifest.serialize(); + const auto standard_processor_bundle_id = minifi::BundleIdentifier{.name = "minifi-standard-processors", .version = minifi::AgentBuild::VERSION}; + const auto standard_processors_components = minifi::ClassDescriptionRegistry::getClassDescriptions().at(standard_processor_bundle_id); + auto serialized = minifi::state::response::serializeComponentManifest(standard_processors_components); REQUIRE_FALSE(serialized.empty()); const auto &resp = serialized[0]; REQUIRE_FALSE(resp.children.empty()); @@ -120,8 +125,9 @@ TEST_CASE("Test Relationships", "[rel1]") { } TEST_CASE("Test Dependent", "[dependent]") { - minifi::state::response::ComponentManifest manifest("minifi-standard-processors"); - auto serialized = manifest.serialize(); + const auto standard_processor_bundle_id = minifi::BundleIdentifier{.name = "minifi-standard-processors", .version = minifi::AgentBuild::VERSION}; + const auto standard_processors_components = minifi::ClassDescriptionRegistry::getClassDescriptions().at(standard_processor_bundle_id); + auto serialized = minifi::state::response::serializeComponentManifest(standard_processors_components); REQUIRE_FALSE(serialized.empty()); const auto &resp = serialized[0]; REQUIRE_FALSE(resp.children.empty()); diff --git a/generateVersion.bat b/generateVersion.bat index 0f46da4e9..cf9bfb76e 100644 --- a/generateVersion.bat +++ b/generateVersion.bat @@ -26,12 +26,9 @@ set compiler_version=%5 set compiler_version=%compiler_version:"=% set flags=%6 set flags=%flags:"=% -set extensions=%7 -set extensions=%extensions:"=% -set extensions=%extensions:\=% -set buildident=%8 +set buildident=%7 set buildident=%buildident:"=% -set buildrev=%9 +set buildrev=%8 set buildrev=%buildrev:"=% @@ -39,14 +36,8 @@ call :GetUnixTime builddate set /a count=0 -for %%i in (%extensions%) do ( - set /a count+=1 - set "extension!count!=%%i" -) ( - echo #include ^<string^> - echo #include ^<vector^> echo #include "minifi-cpp/agent/agent_version.h" echo namespace org::apache::nifi::minifi { @@ -58,25 +49,9 @@ for %%i in (%extensions%) do ( echo const char* const AgentBuild::COMPILER = "%compiler%"; echo const char* const AgentBuild::COMPILER_VERSION = "%compiler_version%"; echo const char* const AgentBuild::COMPILER_FLAGS = R"(%flags%)"; - echo std^:^:vector^<std^:^:string^> AgentBuild::getExtensions^(^) { - echo static std^:^:vector^<std^:^:string^> extensions; - echo if ^(extensions.empty^(^)^){ - ) > "%out_dir%/agent_version.cpp" - - for /l %%i in (1,1,%count%) do ( - ( - echo extensions.push_back^("!extension%%i!"^); - )>> "%out_dir%/agent_version.cpp" - ) - - -( - echo extensions.push_back^("minifi-system"^); - echo } - echo return extensions; - echo } echo } // namespace org::apache::nifi::minifi + ) >> "%out_dir%/agent_version.cpp" goto :EOF diff --git a/generateVersion.sh b/generateVersion.sh index 0977e3553..5c614240a 100755 --- a/generateVersion.sh +++ b/generateVersion.sh @@ -22,8 +22,7 @@ out_dir=$3 compiler=$4 compiler_version=$5 flags=$6 -extensions=$7 -buildident=$8 +buildident=$7 date=$(date +%s) @@ -33,10 +32,6 @@ else buildrev="Unknown" fi -IFS=';' read -r -a extensions_array <<< "$extensions" - -extension_list="${extension_list} } " - cat >"$out_dir/agent_version.cpp" <<EOF /** * @@ -56,8 +51,6 @@ cat >"$out_dir/agent_version.cpp" <<EOF * limitations under the License. */ -#include <string> -#include <vector> #include "minifi-cpp/agent/agent_version.h" namespace org::apache::nifi::minifi { @@ -70,24 +63,6 @@ const char* const AgentBuild::COMPILER = "$compiler"; const char* const AgentBuild::COMPILER_VERSION = "$compiler_version"; const char* const AgentBuild::COMPILER_FLAGS = R"($flags)"; -std::vector<std::string> AgentBuild::getExtensions() { - static std::vector<std::string> extensions; - if (extensions.empty()) { -EOF - -for EXTENSION in "${extensions_array[@]}" -do -cat <<EOF >> "$out_dir/agent_version.cpp" - extensions.push_back("${EXTENSION}"); -EOF -done - -cat <<EOF >> "$out_dir/agent_version.cpp" - extensions.push_back("minifi-system"); - } - return extensions; -} - } // namespace org::apache::nifi::minifi EOF diff --git a/libminifi/include/agent/build_description.h b/libminifi/include/agent/build_description.h deleted file mode 100644 index 85d96741f..000000000 --- a/libminifi/include/agent/build_description.h +++ /dev/null @@ -1,50 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include <map> -#include <memory> -#include <string> -#include <utility> -#include <vector> -#include "agent/agent_docs.h" -#include "core/ClassLoader.h" -#include "core/expect.h" -#include "minifi-cpp/core/Property.h" -#include "core/Relationship.h" -#include "core/Processor.h" -#include "minifi-cpp/core/Annotation.h" -#include "io/validation.h" -#include "minifi-cpp/agent/build_description.h" - -namespace org::apache::nifi::minifi { - -class BuildDescription { - public: - Components getClassDescriptions(const std::string& group = "minifi-system") { - if (class_mappings_[group].empty() && AgentDocs::getClassDescriptions().contains(group)) { - class_mappings_[group] = AgentDocs::getClassDescriptions().at(group); - } - return class_mappings_[group]; - } - - private: - std::map<std::string, Components> class_mappings_; -}; - -} // namespace org::apache::nifi::minifi diff --git a/libminifi/include/core/state/nodes/AgentInformation.h b/libminifi/include/core/state/nodes/AgentInformation.h index 37788ed0c..d76846b8e 100644 --- a/libminifi/include/core/state/nodes/AgentInformation.h +++ b/libminifi/include/core/state/nodes/AgentInformation.h @@ -23,7 +23,6 @@ #include <map> #include "agent/agent_docs.h" -#include "agent/build_description.h" #include "minifi-cpp/core/state/nodes/MetricsBase.h" #include "core/state/nodes/StateMonitor.h" #include "utils/ProcessCpuUsageTracker.h" @@ -37,41 +36,7 @@ namespace org::apache::nifi::minifi::state::response { #define GROUP_STR "org.apache.nifi.minifi" -class ComponentManifest : public DeviceInformation { - public: - ComponentManifest(std::string_view name, const utils::Identifier& uuid) - : DeviceInformation(name, uuid) { - } - - explicit ComponentManifest(std::string_view name) - : DeviceInformation(name) { - } - - std::string getName() const override { - return CoreComponentImpl::getName(); - } - - std::vector<SerializedResponseNode> serialize() override; - - protected: - static void serializeClassDescription(const std::vector<ClassDescription>& descriptions, const std::string& name, SerializedResponseNode& response); - - private: - BuildDescription build_description_; -}; - -class ExternalManifest : public ComponentManifest { - public: - ExternalManifest(std::string_view name, const utils::Identifier& uuid) - : ComponentManifest(name, uuid) { - } - - explicit ExternalManifest(std::string_view name) - : ComponentManifest(name) { - } - - std::vector<SerializedResponseNode> serialize() override; -}; +std::vector<SerializedResponseNode> serializeComponentManifest(const Components& components); class Bundles : public DeviceInformation { public: diff --git a/libminifi/src/agent/JsonSchema.cpp b/libminifi/src/agent/JsonSchema.cpp index d0fd90d9c..58d67df5d 100644 --- a/libminifi/src/agent/JsonSchema.cpp +++ b/libminifi/src/agent/JsonSchema.cpp @@ -17,21 +17,21 @@ #include "agent/JsonSchema.h" +#include <ranges> #include <string> #include <unordered_map> #include <vector> -#include "minifi-cpp/agent/agent_version.h" -#include "agent/build_description.h" -#include "rapidjson/document.h" -#include "rapidjson/prettywriter.h" #include "RemoteProcessGroupPort.h" +#include "minifi-cpp/agent/agent_docs.h" +#include "minifi-cpp/agent/agent_version.h" #include "minifi-cpp/utils/gsl.h" - +#include "range/v3/range/conversion.hpp" #include "range/v3/view/filter.hpp" -#include "range/v3/view/transform.hpp" #include "range/v3/view/join.hpp" -#include "range/v3/range/conversion.hpp" +#include "range/v3/view/transform.hpp" +#include "rapidjson/document.h" +#include "rapidjson/prettywriter.h" namespace org::apache::nifi::minifi::docs { @@ -441,26 +441,11 @@ std::string generateJsonSchema() { controller_services.push_back(std::move(schema).str()); }; - const auto& descriptions = AgentDocs::getClassDescriptions(); - for (const std::string& group : AgentBuild::getExtensions()) { - auto it = descriptions.find(group); - if (it == descriptions.end()) { - continue; - } - for (const auto& proc : it->second.processors_) { - putProcSchema(proc); - } - for (const auto& service : it->second.controller_services_) { - putControllerService(service); - } - } - - for (const auto& bundle : ExternalBuildDescription::getExternalGroups()) { - auto description = ExternalBuildDescription::getClassDescriptions(bundle.artifact); - for (const auto& proc : description.processors_) { + for (const auto& [bundle_id, components]: ClassDescriptionRegistry::getClassDescriptions()) { + for (const auto& proc : components.processors) { putProcSchema(proc); } - for (const auto& service : description.controller_services_) { + for (const auto& service : components.controller_services) { putControllerService(service); } } diff --git a/libminifi/src/agent/agent_docs.cpp b/libminifi/src/agent/agent_docs.cpp index 9f6b58624..0aeb58487 100644 --- a/libminifi/src/agent/agent_docs.cpp +++ b/libminifi/src/agent/agent_docs.cpp @@ -20,17 +20,17 @@ namespace org::apache::nifi::minifi { namespace { -std::map<std::string, Components>& getAgentDocsClassMappings() { - static std::map<std::string, Components> mappings; +std::map<BundleIdentifier, Components>& getAgentDocsClassMappings() { + static std::map<BundleIdentifier, Components> mappings; return mappings; } } // namespace -const std::map<std::string, Components>& AgentDocs::getClassDescriptions() { +const std::map<BundleIdentifier, Components>& ClassDescriptionRegistry::getClassDescriptions() { return getAgentDocsClassMappings(); } -std::map<std::string, Components>& AgentDocs::getMutableClassDescriptions() { +std::map<BundleIdentifier, Components>& ClassDescriptionRegistry::getMutableClassDescriptions() { return getAgentDocsClassMappings(); } diff --git a/libminifi/src/agent/build_description.cpp b/libminifi/src/agent/build_description.cpp deleted file mode 100644 index 6e8aede38..000000000 --- a/libminifi/src/agent/build_description.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "agent/build_description.h" - -namespace org::apache::nifi::minifi { - -std::vector<BundleDetails>& ExternalBuildDescription::getExternal() { - static std::vector<BundleDetails> external_groups; - return external_groups; -} - -std::map<std::string, Components>& ExternalBuildDescription::getExternalMappings() { - static std::map<std::string, Components> external_mappings; - return external_mappings; -} - -void ExternalBuildDescription::addExternalComponent(const BundleDetails& details, const ClassDescription& description) { - bool found = false; - for (const auto &d : getExternal()) { - if (d.artifact == details.artifact) { - found = true; - break; - } - } - if (!found) { - getExternal().push_back(details); - } - if (description.type_ == ResourceType::Processor) { - getExternalMappings()[details.artifact].processors_.push_back(description); - } else if (description.type_ == ResourceType::ControllerService) { - getExternalMappings()[details.artifact].controller_services_.push_back(description); - } else { - getExternalMappings()[details.artifact].other_components_.push_back(description); - } -} - -Components ExternalBuildDescription::getClassDescriptions(const std::string &group) { - return getExternalMappings()[group]; -} - -std::vector<BundleDetails> ExternalBuildDescription::getExternalGroups() { - return getExternal(); -} - -} // namespace org::apache::nifi::minifi diff --git a/libminifi/src/core/state/nodes/AgentInformation.cpp b/libminifi/src/core/state/nodes/AgentInformation.cpp index 92952ae81..498dada87 100644 --- a/libminifi/src/core/state/nodes/AgentInformation.cpp +++ b/libminifi/src/core/state/nodes/AgentInformation.cpp @@ -29,18 +29,8 @@ namespace org::apache::nifi::minifi::state::response { utils::ProcessCpuUsageTracker AgentStatus::cpu_load_tracker_; std::mutex AgentStatus::cpu_load_tracker_mutex_; -std::vector<SerializedResponseNode> ComponentManifest::serialize() { - std::vector<SerializedResponseNode> serialized; - SerializedResponseNode resp; - resp.name = "componentManifest"; - struct Components group = build_description_.getClassDescriptions(getName()); - serializeClassDescription(group.processors_, "processors", resp); - serializeClassDescription(group.controller_services_, "controllerServices", resp); - serialized.push_back(resp); - return serialized; -} - -void ComponentManifest::serializeClassDescription(const std::vector<ClassDescription>& descriptions, const std::string& name, SerializedResponseNode& response) { +namespace { +void serializeClassDescription(const std::vector<ClassDescription>& descriptions, const std::string& name, SerializedResponseNode& response) { if (descriptions.empty()) { return; } @@ -145,58 +135,38 @@ void ComponentManifest::serializeClassDescription(const std::vector<ClassDescrip } response.children.push_back(type); } +} // namespace -std::vector<SerializedResponseNode> ExternalManifest::serialize() { +std::vector<SerializedResponseNode> serializeComponentManifest(const Components& components) { std::vector<SerializedResponseNode> serialized; SerializedResponseNode resp; resp.name = "componentManifest"; - struct Components group = ExternalBuildDescription::getClassDescriptions(getName()); - serializeClassDescription(group.processors_, "processors", resp); - serializeClassDescription(group.controller_services_, "controllerServices", resp); + serializeClassDescription(components.processors, "processors", resp); + serializeClassDescription(components.controller_services, "controllerServices", resp); serialized.push_back(resp); return serialized; } std::vector<SerializedResponseNode> Bundles::serialize() { std::vector<SerializedResponseNode> serialized; - for (const auto& group : AgentBuild::getExtensions()) { - ComponentManifest component_manifest(group); - const auto components = component_manifest.serialize(); - gsl_Expects(components.size() == 1); - if (components[0].children.empty()) { + for (const auto& [bundle, components] : ClassDescriptionRegistry::getClassDescriptions()) { + const auto serialized_components = serializeComponentManifest(components); + gsl_Expects(serialized_components.size() == 1); + if (serialized_components[0].children.empty()) { continue; } - SerializedResponseNode bundle { + SerializedResponseNode serialized_bundle { .name = "bundles", .children = { - components[0], + serialized_components[0], {.name = "group", .value = GROUP_STR}, - {.name = "artifact", .value = group}, - {.name = "version", .value = AgentBuild::VERSION}, + {.name = "artifact", .value = bundle.name}, + {.name = "version", .value = bundle.version}, } }; - serialized.push_back(bundle); - } - - // let's provide our external manifests. - for (const auto& group : ExternalBuildDescription::getExternalGroups()) { - SerializedResponseNode bundle { - .name = "bundles", - .children = { - {.name = "group", .value = group.group}, - {.name = "artifact", .value = group.artifact}, - {.name = "version", .value = group.version}, - } - }; - - ExternalManifest compMan(group.artifact); - // serialize the component information. - for (const auto& component : compMan.serialize()) { - bundle.children.push_back(component); - } - serialized.push_back(bundle); + serialized.push_back(serialized_bundle); } return serialized; diff --git a/libminifi/src/minifi-c.cpp b/libminifi/src/minifi-c.cpp index 815998356..f14847363 100644 --- a/libminifi/src/minifi-c.cpp +++ b/libminifi/src/minifi-c.cpp @@ -36,7 +36,6 @@ #include "minifi-cpp/Exception.h" #include "core/extension/ExtensionManager.h" #include "utils/PropertyErrors.h" -#include "minifi-cpp/agent/build_description.h" #include "utils/CProcessor.h" namespace minifi = org::apache::nifi::minifi; @@ -229,18 +228,17 @@ extern "C" { MinifiExtension* MinifiCreateExtension(MinifiStringView /*api_version*/, const MinifiExtensionCreateInfo* extension_create_info) { gsl_Assert(extension_create_info); auto extension_name = toString(extension_create_info->name); - minifi::BundleDetails bundle{ - .artifact = extension_name, - .group = "org.apache.nifi.minifi", + minifi::BundleIdentifier bundle{ + .name = extension_name, .version = toString(extension_create_info->version) }; + auto& bundle_components = minifi::ClassDescriptionRegistry::getMutableClassDescriptions()[bundle]; for (size_t proc_idx = 0; proc_idx < extension_create_info->processors_count; ++proc_idx) { minifi::utils::useCProcessorClassDescription(extension_create_info->processors_ptr[proc_idx], [&] (const auto& description, const auto& c_class_description) { - minifi::ExternalBuildDescription::addExternalComponent(bundle, description); - minifi::core::ClassLoader::getDefaultClassLoader().getClassLoader(extension_name).registerClass( c_class_description.name, std::make_unique<CProcessorFactory>(extension_name, toString(extension_create_info->processors_ptr[proc_idx].full_name), c_class_description)); + bundle_components.processors.emplace_back(description); }); } return reinterpret_cast<MinifiExtension*>(new org::apache::nifi::minifi::core::extension::Extension::Info{ diff --git a/libminifi/test/libtest/integration/HTTPHandlers.cpp b/libminifi/test/libtest/integration/HTTPHandlers.cpp index 1ad637b15..5236d96fd 100644 --- a/libminifi/test/libtest/integration/HTTPHandlers.cpp +++ b/libminifi/test/libtest/integration/HTTPHandlers.cpp @@ -20,17 +20,18 @@ #include <algorithm> #include "CivetStream.h" -#include "io/CRCStream.h" #include "io/BufferStream.h" -#include "rapidjson/error/en.h" +#include "io/CRCStream.h" +#include "minifi-cpp/agent/agent_docs.h" +#include "minifi-cpp/agent/agent_version.h" #include "minifi-cpp/utils/gsl.h" -#include "agent/build_description.h" #include "range/v3/algorithm/contains.hpp" #include "range/v3/view/filter.hpp" #include "range/v3/view/view.hpp" -#include "utils/net/DNS.h" +#include "rapidjson/error/en.h" #include "sitetosite/HttpSiteToSiteClient.h" #include "utils/StringUtils.h" +#include "utils/net/DNS.h" namespace org::apache::nifi::minifi::test { @@ -313,8 +314,9 @@ void HeartbeatHandler::verifyJsonHasAgentManifest(const rapidjson::Document& roo classes.push_back(proc["type"].GetString()); } - auto group = minifi::BuildDescription{}.getClassDescriptions(str); - for (const auto& proc : group.processors_) { + const auto bundle_details = BundleIdentifier{.name = str, .version = AgentBuild::VERSION}; + const auto group = minifi::ClassDescriptionRegistry::getClassDescriptions().at(bundle_details); + for (const auto& proc : group.processors) { REQUIRE(std::find(classes.begin(), classes.end(), proc.full_name_) != std::end(classes)); (void)proc; found = true; diff --git a/libminifi/test/unit/ComponentManifestTests.cpp b/libminifi/test/unit/ComponentManifestTests.cpp index dd350f35b..a02bc87e6 100644 --- a/libminifi/test/unit/ComponentManifestTests.cpp +++ b/libminifi/test/unit/ComponentManifestTests.cpp @@ -85,8 +85,9 @@ REGISTER_RESOURCE(ExampleProcessor, Processor); } // namespace test::apple TEST_CASE("Manifest indicates property type requirement") { - minifi::state::response::ComponentManifest manifest("minifi-system"); - auto nodes = manifest.serialize(); + const auto system_bundle_id = minifi::BundleIdentifier{.name = "minifi-system", .version = minifi::AgentBuild::VERSION}; + const auto system_components = minifi::ClassDescriptionRegistry::getClassDescriptions().at(system_bundle_id); + auto nodes = minifi::state::response::serializeComponentManifest(system_components); REQUIRE(nodes.size() == 1); REQUIRE(nodes.at(0).name == "componentManifest"); @@ -118,8 +119,9 @@ TEST_CASE("Manifest indicates property type requirement") { TEST_CASE("Processors do not get instantiated during manifest creation") { LogTestController::getInstance().setDebug<core::Processor>(); - minifi::state::response::ComponentManifest manifest("minifi-system"); - manifest.serialize(); + const auto system_bundle_id = minifi::BundleIdentifier{.name = "minifi-system", .version = minifi::AgentBuild::VERSION}; + const auto system_components = minifi::ClassDescriptionRegistry::getClassDescriptions().at(system_bundle_id); + minifi::state::response::serializeComponentManifest(system_components); CHECK_FALSE(LogTestController::getInstance().contains("Processor ExampleProcessor created")); } diff --git a/minifi-api/include/minifi-cpp/agent/agent_docs.h b/minifi-api/include/minifi-cpp/agent/agent_docs.h index 833288902..912377d39 100644 --- a/minifi-api/include/minifi-cpp/agent/agent_docs.h +++ b/minifi-api/include/minifi-cpp/agent/agent_docs.h @@ -27,6 +27,7 @@ #include "minifi-cpp/core/OutputAttribute.h" #include "minifi-cpp/core/Property.h" #include "minifi-cpp/core/Relationship.h" +#include "utils/Hash.h" namespace org::apache::nifi::minifi { @@ -51,23 +52,41 @@ struct ClassDescription { }; struct Components { - std::vector<ClassDescription> processors_; - std::vector<ClassDescription> controller_services_; - std::vector<ClassDescription> parameter_providers_; - std::vector<ClassDescription> other_components_; + std::vector<ClassDescription> processors; + std::vector<ClassDescription> controller_services; + std::vector<ClassDescription> parameter_providers; + std::vector<ClassDescription> other_components; [[nodiscard]] bool empty() const noexcept { - return processors_.empty() && controller_services_.empty() && parameter_providers_.empty() && other_components_.empty(); + return processors.empty() && controller_services.empty() && parameter_providers.empty() && other_components.empty(); } }; -class AgentDocs { +struct BundleIdentifier { + std::string name; + std::string version; + + auto operator<=>(const BundleIdentifier& rhs) const = default; +}; + +class ClassDescriptionRegistry { public: - static const std::map<std::string, Components>& getClassDescriptions(); - static std::map<std::string, Components>& getMutableClassDescriptions(); + static const std::map<minifi::BundleIdentifier, Components>& getClassDescriptions(); + static std::map<minifi::BundleIdentifier, Components>& getMutableClassDescriptions(); template<typename Class, ResourceType Type> - static void createClassDescription(const std::string& group, const std::string& name); + static void createClassDescription(std::string bundle_name, std::string class_name, std::string version); }; } // namespace org::apache::nifi::minifi + +template<> +struct std::hash<org::apache::nifi::minifi::BundleIdentifier> { + size_t operator()(const org::apache::nifi::minifi::BundleIdentifier& bundle_details) const noexcept { + size_t hash_value{0}; + hash_value = org::apache::nifi::minifi::utils::hash_combine(hash_value, std::hash<std::string>{}(bundle_details.name)); + hash_value = org::apache::nifi::minifi::utils::hash_combine(hash_value, std::hash<std::string>{}(bundle_details.version)); + + return hash_value; + } +}; diff --git a/minifi-api/include/minifi-cpp/agent/agent_version.h b/minifi-api/include/minifi-cpp/agent/agent_version.h index 0562a5b38..84826140f 100644 --- a/minifi-api/include/minifi-cpp/agent/agent_version.h +++ b/minifi-api/include/minifi-cpp/agent/agent_version.h @@ -37,7 +37,6 @@ class AgentBuild { MINIFIAPI static const char* const COMPILER; MINIFIAPI static const char* const COMPILER_VERSION; MINIFIAPI static const char* const COMPILER_FLAGS; - static std::vector<std::string> getExtensions(); }; } // namespace namespace org::apache::nifi::minifi diff --git a/minifi-api/include/minifi-cpp/agent/build_description.h b/minifi-api/include/minifi-cpp/agent/build_description.h deleted file mode 100644 index e85f895cb..000000000 --- a/minifi-api/include/minifi-cpp/agent/build_description.h +++ /dev/null @@ -1,49 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include <map> -#include <memory> -#include <string> -#include <utility> -#include <vector> -#include "agent_docs.h" - -namespace org::apache::nifi::minifi { - -struct BundleDetails { - std::string artifact; - std::string group; - std::string version; -}; - -class ExternalBuildDescription { - private: - static std::vector<BundleDetails> &getExternal(); - - static std::map<std::string, Components> &getExternalMappings(); - - public: - static void addExternalComponent(const BundleDetails& details, const ClassDescription& description); - - static Components getClassDescriptions(const std::string &group); - - static std::vector<BundleDetails> getExternalGroups(); -}; - -} // namespace org::apache::nifi::minifi diff --git a/minifi_main/AgentDocs.cpp b/minifi_main/AgentDocs.cpp index 26983d028..ff37268c9 100644 --- a/minifi_main/AgentDocs.cpp +++ b/minifi_main/AgentDocs.cpp @@ -19,21 +19,21 @@ #include <algorithm> #include <iostream> +#include <ranges> #include <string> #include <utility> #include <vector> -#include "range/v3/algorithm/lexicographical_compare.hpp" -#include "range/v3/range/conversion.hpp" -#include "range/v3/view/transform.hpp" -#include "range/v3/view/join.hpp" - +#include "TableFormatter.h" #include "agent/agent_docs.h" -#include "minifi-cpp/agent/agent_version.h" #include "core/Core.h" -#include "minifi-cpp/core/PropertyValidator.h" #include "core/Relationship.h" -#include "TableFormatter.h" +#include "minifi-cpp/agent/agent_version.h" +#include "minifi-cpp/core/PropertyValidator.h" +#include "range/v3/algorithm/lexicographical_compare.hpp" +#include "range/v3/range/conversion.hpp" +#include "range/v3/view/join.hpp" +#include "range/v3/view/transform.hpp" #include "utils/StringUtils.h" namespace { @@ -187,15 +187,15 @@ void AgentDocs::generate(const std::filesystem::path& docs_dir) { std::vector<std::pair<std::string, minifi::ClassDescription>> controller_services; std::vector<std::pair<std::string, minifi::ClassDescription>> processors; std::vector<std::pair<std::string, minifi::ClassDescription>> parameter_providers; - for (const auto &group : minifi::AgentBuild::getExtensions()) { - struct Components descriptions = build_description_.getClassDescriptions(group); - for (const auto &controller_service_description : descriptions.controller_services_) { + + for (const auto& [bundle_id, components]: ClassDescriptionRegistry::getClassDescriptions()) { + for (const auto &controller_service_description : components.controller_services) { controller_services.emplace_back(extractClassName(controller_service_description.full_name_), controller_service_description); } - for (const auto &processor_description : descriptions.processors_) { + for (const auto &processor_description : components.processors) { processors.emplace_back(extractClassName(processor_description.full_name_), processor_description); } - for (const auto& parameter_provider_description : descriptions.parameter_providers_) { + for (const auto& parameter_provider_description : components.parameter_providers) { parameter_providers.emplace_back(extractClassName(parameter_provider_description.full_name_), parameter_provider_description); } } diff --git a/minifi_main/AgentDocs.h b/minifi_main/AgentDocs.h index 276aef763..56cdc9952 100644 --- a/minifi_main/AgentDocs.h +++ b/minifi_main/AgentDocs.h @@ -17,16 +17,13 @@ #pragma once #include <iostream> -#include "agent/build_description.h" +#include <filesystem> namespace org::apache::nifi::minifi::docs { class AgentDocs { public: - void generate(const std::filesystem::path& docs_dir); - - private: - BuildDescription build_description_; + static void generate(const std::filesystem::path& docs_dir); }; } // namespace org::apache::nifi::minifi::docs diff --git a/minifi_main/MiNiFiMain.cpp b/minifi_main/MiNiFiMain.cpp index b34408576..d6d32a50c 100644 --- a/minifi_main/MiNiFiMain.cpp +++ b/minifi_main/MiNiFiMain.cpp @@ -114,23 +114,13 @@ void sigHandler(int signal) { } } -void dumpDocs(const std::shared_ptr<minifi::Configure>& configuration, const std::string& dir) { - const auto pythoncreator = core::ClassLoader::getDefaultClassLoader().instantiate("PythonCreator", "PythonCreator"); - if (nullptr != pythoncreator) { - pythoncreator->configure(configuration); - } - +void dumpDocs(const std::string& dir) { minifi::docs::AgentDocs docsCreator; docsCreator.generate(dir); } -void writeJsonSchema(const std::shared_ptr<minifi::Configure>& configuration, std::ostream& out) { - const auto pythoncreator = core::ClassLoader::getDefaultClassLoader().instantiate("PythonCreator", "PythonCreator"); - if (nullptr != pythoncreator) { - pythoncreator->configure(configuration); - } - +void writeJsonSchema(std::ostream& out) { out << minifi::docs::generateJsonSchema() << '\n'; } @@ -147,7 +137,7 @@ void overridePropertiesFromCommandLine(const argparse::ArgumentParser& parser, c } } -[[nodiscard]] std::optional<int /* exit code */> dumpDocsIfRequested(const argparse::ArgumentParser& parser, const std::shared_ptr<minifi::Configure>& configure) { +[[nodiscard]] std::optional<int /* exit code */> dumpDocsIfRequested(const argparse::ArgumentParser& parser) { if (!parser.is_used("--docs")) { return std::nullopt; // don't exit } @@ -158,17 +148,17 @@ void overridePropertiesFromCommandLine(const argparse::ArgumentParser& parser, c } std::cout << "Dumping docs to " << docs_params[0] << std::endl; - dumpDocs(configure, docs_params[0]); + dumpDocs(docs_params[0]); return 0; } -[[nodiscard]] std::optional<int /* exit code */> writeSchemaIfRequested(const argparse::ArgumentParser& parser, const std::shared_ptr<minifi::Configure>& configure) { +[[nodiscard]] std::optional<int /* exit code */> writeSchemaIfRequested(const argparse::ArgumentParser& parser) { if (!parser.is_used("--schema")) { return std::nullopt; } const auto& schema_path = parser.get("--schema"); if (schema_path == "-") { - writeJsonSchema(configure, std::cout); + writeJsonSchema(std::cout); return 0; } @@ -180,7 +170,7 @@ void overridePropertiesFromCommandLine(const argparse::ArgumentParser& parser, c << "\n"; return 1; } - writeJsonSchema(configure, schema_file); + writeJsonSchema(schema_file); } return 0; } @@ -335,8 +325,8 @@ int main(int argc, char **argv) { minifi::core::extension::ExtensionManager extension_manager(configure); - if (const auto maybe_exit_code = dumpDocsIfRequested(argument_parser, configure)) { return *maybe_exit_code; } - if (const auto maybe_exit_code = writeSchemaIfRequested(argument_parser, configure)) { return *maybe_exit_code; } + if (const auto maybe_exit_code = dumpDocsIfRequested(argument_parser)) { return *maybe_exit_code; } + if (const auto maybe_exit_code = writeSchemaIfRequested(argument_parser)) { return *maybe_exit_code; } std::chrono::milliseconds stop_wait_time = configure->get(minifi::Configure::nifi_graceful_shutdown_seconds) | utils::andThen(utils::timeutils::StringToDuration<std::chrono::milliseconds>)
