This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 85fb25d5531974786dfaae1d24b8e0f31fdb1881 Author: Martin Zink <[email protected]> AuthorDate: Thu Nov 20 14:33:23 2025 +0100 MINIFICPP-2676 Variable Registry refactor Closes #2063 Signed-off-by: Marton Szasz <[email protected]> --- EXPRESSIONS.md | 10 +++ core-framework/include/core/VariableRegistry.h | 60 ++++++--------- libminifi/src/Configuration.cpp | 4 +- libminifi/src/expression-language/Expression.cpp | 17 ++-- libminifi/test/unit/ExpressionLanguageTests.cpp | 26 +++++++ libminifi/test/unit/ProcessContextExprTests.cpp | 31 ++++++++ libminifi/test/unit/VariableRegistryTests.cpp | 90 ++++++++++++++++++++++ .../include/minifi-cpp/core/VariableRegistry.h | 2 +- .../include/minifi-cpp/properties/Configuration.h | 4 + 9 files changed, 198 insertions(+), 46 deletions(-) diff --git a/EXPRESSIONS.md b/EXPRESSIONS.md index db2333d46..fbb718d0f 100644 --- a/EXPRESSIONS.md +++ b/EXPRESSIONS.md @@ -146,6 +146,16 @@ cannot, however, use `${file name:equals(${uuid})}`, because this results in file and name being interpreted as different tokens, rather than a single token, filename. +### Variable Registry + +You can also access custom properties that are defined in the `minifi.properties` configuration file. + +#### Refining access +If `minifi.variable.registry.whitelist` is defined in the properties file (as a comma separated list), only these explicitly listed properties will be available. +You can also restrict access via `minifi.variable.registry.blacklist` (also a comma separated list) any property listed here won't be accessible. +Furthermore, any property with `password` in its name will be inaccessible. + + ## Supported Features ### Boolean Logic diff --git a/core-framework/include/core/VariableRegistry.h b/core-framework/include/core/VariableRegistry.h index f4d15491a..074fa3d70 100644 --- a/core-framework/include/core/VariableRegistry.h +++ b/core-framework/include/core/VariableRegistry.h @@ -17,12 +17,16 @@ */ #pragma once +#include <map> #include <memory> +#include <ranges> #include <string> -#include <map> + +#include "minifi-cpp/core/VariableRegistry.h" #include "minifi-cpp/properties/Configure.h" +#include "utils/OptionalUtils.h" #include "utils/StringUtils.h" -#include "minifi-cpp/core/VariableRegistry.h" +#include "range/v3/algorithm/contains.hpp" namespace org::apache::nifi::minifi::core { @@ -40,53 +44,37 @@ class VariableRegistryImpl : public virtual VariableRegistry { ~VariableRegistryImpl() override = default; - bool getConfigurationProperty(const std::string &property, std::string &value) const override { - auto prop = variable_registry_.find(property); - if (prop != variable_registry_.end()) { - value = prop->second; - return true; + [[nodiscard]] std::optional<std::string> getConfigurationProperty(const std::string_view key) const override { + const auto it = variable_registry_.find(key); + if (it == variable_registry_.end()) { + return std::nullopt; } - return false; + return it->second; } protected: void loadVariableRegistry() { - std::string registry_values; - - auto options = configuration_->getConfiguredKeys(); - std::string white_list_opt = "minifi.variable.registry.whitelist", white_list; - std::string black_list_opt = "minifi.variable.registry.blacklist", black_list; + gsl_Assert(configuration_); + auto options = configuration_->get(Configuration::minifi_variable_registry_whitelist) + .transform([](std::string&& list) { return utils::string::split(std::move(list), ","); }) + .value_or(configuration_->getConfiguredKeys()); - // only allow those in the white liset - if (configuration_->get(white_list_opt, white_list)) { - options = utils::string::split(white_list, ","); - } - - for (const auto &opt : options) { - if (opt.find("password") != std::string::npos) - options.erase(std::remove(options.begin(), options.end(), opt), options.end()); - } + const auto black_listed_options = configuration_->get(Configuration::minifi_variable_registry_blacklist) + .transform([](std::string&& list) { return utils::string::split(std::move(list), ","); }); - // even if a white list is configured, remove the black listed fields - - if (configuration_->get(black_list_opt, black_list)) { - auto bl_opts = utils::string::split(black_list, ","); - for (const auto &opt : bl_opts) { - options.erase(std::remove(options.begin(), options.end(), opt), options.end()); - } - } + auto not_password = [](const std::string& s) { return !s.contains("password"); }; + auto not_blacklisted = [&black_listed_options](const std::string& s) { return !(black_listed_options && ranges::contains(*black_listed_options, s)); }; - for (const auto &opt : options) { - std::string value; - if (configuration_->get(opt, value)) { - variable_registry_[opt] = value; + for (const auto& option : options | std::views::filter(not_password) | std::views::filter(not_blacklisted)) { + if (const auto val = configuration_->get(option)) { + variable_registry_[option] = *val; } } } - std::map<std::string, std::string> variable_registry_; + std::map<std::string, std::string, std::less<>> variable_registry_; - std::shared_ptr<minifi::Configure> configuration_; + std::shared_ptr<Configure> configuration_; }; } // namespace org::apache::nifi::minifi::core diff --git a/libminifi/src/Configuration.cpp b/libminifi/src/Configuration.cpp index cab4ed8e5..191705fac 100644 --- a/libminifi/src/Configuration.cpp +++ b/libminifi/src/Configuration.cpp @@ -149,7 +149,9 @@ const std::unordered_map<std::string_view, gsl::not_null<const core::PropertyVal {Configuration::nifi_python_virtualenv_directory, gsl::make_not_null(&core::StandardPropertyValidators::ALWAYS_VALID_VALIDATOR)}, {Configuration::nifi_python_env_setup_binary, gsl::make_not_null(&core::StandardPropertyValidators::ALWAYS_VALID_VALIDATOR)}, {Configuration::nifi_python_install_packages_automatically, gsl::make_not_null(&core::StandardPropertyValidators::BOOLEAN_VALIDATOR)}, - {Configuration::nifi_openssl_fips_support_enable, gsl::make_not_null(&core::StandardPropertyValidators::BOOLEAN_VALIDATOR)} + {Configuration::nifi_openssl_fips_support_enable, gsl::make_not_null(&core::StandardPropertyValidators::BOOLEAN_VALIDATOR)}, + {Configuration::minifi_variable_registry_whitelist, gsl::make_not_null(&core::StandardPropertyValidators::ALWAYS_VALID_VALIDATOR)}, + {Configuration::minifi_variable_registry_blacklist, gsl::make_not_null(&core::StandardPropertyValidators::ALWAYS_VALID_VALIDATOR)} }; const std::array<const char*, 2> Configuration::DEFAULT_SENSITIVE_PROPERTIES = {Configuration::nifi_security_client_pass_phrase, diff --git a/libminifi/src/expression-language/Expression.cpp b/libminifi/src/expression-language/Expression.cpp index ae2d17f60..f8de01751 100644 --- a/libminifi/src/expression-language/Expression.cpp +++ b/libminifi/src/expression-language/Expression.cpp @@ -92,16 +92,17 @@ Expression make_dynamic(const std::function<Value(const Parameters ¶ms, cons Expression make_dynamic_attr(const std::string &attribute_id) { return make_dynamic([attribute_id](const Parameters ¶ms, const std::vector<Expression>& /*sub_exprs*/) -> Value { - std::string result; - const auto cur_flow_file = params.flow_file; - if (cur_flow_file && cur_flow_file->getAttribute(attribute_id, result)) { - return Value(result); - } else { - auto registry = params.registry_; - if (registry && registry->getConfigurationProperty(attribute_id , result)) { - return Value(result); + if (params.flow_file) { + if (auto result = params.flow_file->getAttribute(attribute_id)) { + return Value(std::move(*result)); } } + if (params.registry_) { + if (auto result = params.registry_->getConfigurationProperty(attribute_id)) { + return Value(std::move(*result)); + } + } + return {}; }); } diff --git a/libminifi/test/unit/ExpressionLanguageTests.cpp b/libminifi/test/unit/ExpressionLanguageTests.cpp index f05bbdd91..bf0534e28 100644 --- a/libminifi/test/unit/ExpressionLanguageTests.cpp +++ b/libminifi/test/unit/ExpressionLanguageTests.cpp @@ -34,6 +34,7 @@ #include "expression-language/Expression.h" #include "minifi-cpp/core/FlowFile.h" #include "minifi-cpp/utils/gsl.h" +#include "core/VariableRegistry.h" #include "unit/TestBase.h" #include "unit/Catch.h" #include "catch2/catch_approx.hpp" @@ -1654,3 +1655,28 @@ TEST_CASE("resolve_user_id_test", "[resolve_user_id tests]") { REQUIRE(expr(expression::Parameters{flow_file_a.get()}).asString().empty()); } } + +TEST_CASE("variable registry test") { + const std::shared_ptr<minifi::Configure> configuration = std::make_shared<minifi::ConfigureImpl>(); + configuration->set("foo", "foo_val"); + configuration->set("minifi.variable.registry.blacklist", "foo"); + configuration->set("bar", "bar_val"); + configuration->set("baz", "baz_val"); + const auto variable_registry = minifi::core::VariableRegistryImpl(configuration); + const auto flow_file = std::make_shared<core::FlowFileImpl>(); + flow_file->setAttribute("baz", "ff_baz"); + + { + const auto foo_expr = expression::compile("${foo}"); + CHECK(foo_expr(expression::Parameters{&variable_registry, flow_file.get()}).asString().empty()); + } + + { + const auto bar_expr = expression::compile("${bar}"); + CHECK(bar_expr(expression::Parameters{&variable_registry, flow_file.get()}).asString() == "bar_val"); + } + { + const auto baz_expr = expression::compile("${baz}"); + CHECK(baz_expr(expression::Parameters{&variable_registry, flow_file.get()}).asString() == "ff_baz"); + } +} diff --git a/libminifi/test/unit/ProcessContextExprTests.cpp b/libminifi/test/unit/ProcessContextExprTests.cpp index 08d38479f..a8cd6a117 100644 --- a/libminifi/test/unit/ProcessContextExprTests.cpp +++ b/libminifi/test/unit/ProcessContextExprTests.cpp @@ -100,6 +100,37 @@ TEST_CASE("ProcessContextExpr can update existing processor properties", "[setPr } } +TEST_CASE("Expression language via variable registry") { + const std::shared_ptr<minifi::Configure> configuration = std::make_shared<minifi::ConfigureImpl>(); + configuration->set("foo", "foo_val"); + configuration->set("bar", "bar_val"); + configuration->set("minifi.variable.registry.blacklist", "foo"); + + TestController test_controller; + std::shared_ptr<TestPlan> test_plan = test_controller.createPlan(configuration); + + [[maybe_unused]] minifi::core::Processor* dummy_processor = test_plan->addProcessor("DummyProcessContextExprProcessor", "dummy_processor"); + std::shared_ptr<minifi::core::ProcessContext> context = [test_plan] { test_plan->runNextProcessor(); return test_plan->getCurrentContext(); }(); + REQUIRE(dynamic_pointer_cast<minifi::core::ProcessContextImpl>(context) != nullptr); + + { + REQUIRE(context->setProperty(minifi::DummyProcessContextExprProcessor::ExpressionLanguageProperty, "${bar}")); + CHECK(context->getProperty(minifi::DummyProcessContextExprProcessor::ExpressionLanguageProperty, nullptr) == "bar_val"); + } + { + REQUIRE(context->setProperty(minifi::DummyProcessContextExprProcessor::ExpressionLanguageProperty, "${foo}")); + CHECK(context->getProperty(minifi::DummyProcessContextExprProcessor::ExpressionLanguageProperty, nullptr) == ""); + } + { + REQUIRE(context->setProperty(minifi::DummyProcessContextExprProcessor::SimpleProperty, "${bar}")); + CHECK(context->getProperty(minifi::DummyProcessContextExprProcessor::SimpleProperty, nullptr) == "${bar}"); + } + { + REQUIRE(context->setProperty(minifi::DummyProcessContextExprProcessor::SimpleProperty, "${foo}")); + CHECK(context->getProperty(minifi::DummyProcessContextExprProcessor::SimpleProperty, nullptr) == "${foo}"); + } +} + TEST_CASE("ProcessContextExpr can use expression language in dynamic properties", "[getDynamicProperty][getDynamicProperties]") { TestController test_controller; std::shared_ptr<TestPlan> test_plan = test_controller.createPlan(); diff --git a/libminifi/test/unit/VariableRegistryTests.cpp b/libminifi/test/unit/VariableRegistryTests.cpp new file mode 100644 index 000000000..ba23176c5 --- /dev/null +++ b/libminifi/test/unit/VariableRegistryTests.cpp @@ -0,0 +1,90 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "properties/Configure.h" +#include "core/VariableRegistry.h" +#include "unit/Catch.h" + +TEST_CASE("VariableRegistry default test") { + namespace minifi = org::apache::nifi::minifi; + const std::shared_ptr<minifi::Configure> configuration = std::make_shared<minifi::ConfigureImpl>(); + configuration->set("foo", "foo_val"); + configuration->set("bar", "bar_val"); + configuration->set("foo_password", "secret password"); + const auto variable_registry = minifi::core::VariableRegistryImpl(configuration); + CHECK(variable_registry.getConfigurationProperty("foo") == "foo_val"); + CHECK(variable_registry.getConfigurationProperty("bar") == "bar_val"); + CHECK_FALSE(variable_registry.getConfigurationProperty("foo_password")); // passwords are blacklisted +} + +TEST_CASE("VariableRegistry whitelist test") { + namespace minifi = org::apache::nifi::minifi; + const std::shared_ptr<minifi::Configure> configuration = std::make_shared<minifi::ConfigureImpl>(); + configuration->set("foo", "foo_val"); + configuration->set("minifi.variable.registry.whitelist", "foo,foo_password"); + configuration->set("bar", "bar_val"); + configuration->set("foo_password", "secret password"); + const auto variable_registry = minifi::core::VariableRegistryImpl(configuration); + CHECK(variable_registry.getConfigurationProperty("foo") == "foo_val"); + CHECK_FALSE(variable_registry.getConfigurationProperty("bar")); // not whitelisted + CHECK_FALSE(variable_registry.getConfigurationProperty("foo_password")); // passwords are blacklisted +} + +TEST_CASE("VariableRegistry blacklist test") { + namespace minifi = org::apache::nifi::minifi; + const std::shared_ptr<minifi::Configure> configuration = std::make_shared<minifi::ConfigureImpl>(); + configuration->set("foo", "foo_val"); + configuration->set("minifi.variable.registry.blacklist", "foo,foo_password"); + configuration->set("bar", "bar_val"); + configuration->set("foo_password", "secret password"); + const auto variable_registry = minifi::core::VariableRegistryImpl(configuration); + CHECK(variable_registry.getConfigurationProperty("bar") == "bar_val"); + CHECK_FALSE(variable_registry.getConfigurationProperty("foo")); // blacklisted + CHECK_FALSE(variable_registry.getConfigurationProperty("foo_password")); // passwords are blacklisted +} + +TEST_CASE("VariableRegistry whitelist and blacklist test") { + namespace minifi = org::apache::nifi::minifi; + const std::shared_ptr<minifi::Configure> configuration = std::make_shared<minifi::ConfigureImpl>(); + configuration->set("foo", "foo_val"); + configuration->set("minifi.variable.registry.whitelist", "foo,foo_password,baz"); + configuration->set("minifi.variable.registry.blacklist", "foo"); + configuration->set("bar", "bar_val"); + configuration->set("baz", "baz_val"); + configuration->set("foo_password", "secret password"); + const auto variable_registry = minifi::core::VariableRegistryImpl(configuration); + CHECK_FALSE(variable_registry.getConfigurationProperty("foo")); // whitelisted but also blacklisted + CHECK_FALSE(variable_registry.getConfigurationProperty("bar")); // not whitelisted + CHECK(variable_registry.getConfigurationProperty("baz") == "baz_val"); // whitelisted + CHECK_FALSE(variable_registry.getConfigurationProperty("foo_password")); // passwords are blacklisted +} + +TEST_CASE("Explicit empty whitelist") { + namespace minifi = org::apache::nifi::minifi; + const std::shared_ptr<minifi::Configure> configuration = std::make_shared<minifi::ConfigureImpl>(); + configuration->set("minifi.variable.registry.whitelist", ""); + configuration->set("foo", "foo_val"); + configuration->set("bar", "bar_val"); + configuration->set("baz", "baz_val"); + configuration->set("foo_password", "secret password"); + const auto variable_registry = minifi::core::VariableRegistryImpl(configuration); + CHECK_FALSE(variable_registry.getConfigurationProperty("foo")); // empty whitelist + CHECK_FALSE(variable_registry.getConfigurationProperty("bar")); // empty whitelist + CHECK_FALSE(variable_registry.getConfigurationProperty("baz")); // empty whitelist + CHECK_FALSE(variable_registry.getConfigurationProperty("foo_password")); // empty whitelist (and also passwords are blacklisted) +} diff --git a/minifi-api/include/minifi-cpp/core/VariableRegistry.h b/minifi-api/include/minifi-cpp/core/VariableRegistry.h index 1e05af505..8c35e5149 100644 --- a/minifi-api/include/minifi-cpp/core/VariableRegistry.h +++ b/minifi-api/include/minifi-cpp/core/VariableRegistry.h @@ -31,7 +31,7 @@ namespace org::apache::nifi::minifi::core { class VariableRegistry { public: virtual ~VariableRegistry() = default; - virtual bool getConfigurationProperty(const std::string &property, std::string &value) const = 0; + [[nodiscard]] virtual std::optional<std::string> getConfigurationProperty(std::string_view key) const = 0; }; } // namespace org::apache::nifi::minifi::core diff --git a/minifi-api/include/minifi-cpp/properties/Configuration.h b/minifi-api/include/minifi-cpp/properties/Configuration.h index 070bc570e..3fccff2c2 100644 --- a/minifi-api/include/minifi-cpp/properties/Configuration.h +++ b/minifi-api/include/minifi-cpp/properties/Configuration.h @@ -194,6 +194,10 @@ class Configuration : public virtual Properties { static constexpr const char *nifi_openssl_fips_support_enable = "nifi.openssl.fips.support.enable"; + // minifi variable registry options + static constexpr const char *minifi_variable_registry_whitelist = "minifi.variable.registry.whitelist"; + static constexpr const char *minifi_variable_registry_blacklist = "minifi.variable.registry.blacklist"; + MINIFIAPI static const std::unordered_map<std::string_view, gsl::not_null<const core::PropertyValidator*>> CONFIGURATION_PROPERTIES; MINIFIAPI static const std::array<const char*, 2> DEFAULT_SENSITIVE_PROPERTIES;
