This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit b95e584263bf5117afa9b1bebe0f2948c030c3d0 Author: Ferenc Gerlits <[email protected]> AuthorDate: Thu Jan 22 15:30:25 2026 +0100 MINIFICPP-2342 Do not overwrite config files during an upgrade Currently, MiNiFi modifies the installed config files (minifi.properties and config.yml) while it runs. This confuses the Windows installer, so changes to these config files get lost when MiNiFi is upgraded. After this change, the minifi.properties, minifi-log.properties and minifi-uid.properties files are no longer modified by MiNiFi at runtime, so they can be safely replaced by new versions during upgrade. All changes to the settings should be put into new files in the minifi.properties.d (minifi-log.properties.d, minifi-uid.properties.d) directory; these new files will not be touched by the upgrade. The config.yml file will no longer be installed as part of MiNiFi. If C2 is enabled, config.yml will be fetched from the C2 server; otherwise, MiNiFi will create a new file with an empty flow, and the user can edit this. Either way, config.yml will not be touched by an upgrade. Closes #2069 Signed-off-by: Marton Szasz <[email protected]> --- README.md | 16 +- conf/CMakeLists.txt | 8 - conf/config.yml.in | 21 --- conf/minifi-log.properties.in | 5 + conf/minifi-uid.properties.in | 5 + conf/minifi.properties.in | 5 + core-framework/include/utils/file/FileUtils.h | 6 + .../cluster/containers/MinifiContainer.py | 3 - .../tests/unit/ConfigurationTests.cpp | 203 +++++++++++++++------ libminifi/include/core/logging/LoggerProperties.h | 2 +- libminifi/include/properties/Configuration.h | 2 +- libminifi/include/properties/Properties.h | 20 +- libminifi/include/utils/ChecksumCalculator.h | 11 +- libminifi/src/core/FlowConfiguration.cpp | 19 +- .../core/state/nodes/ConfigurationChecksums.cpp | 2 +- libminifi/src/properties/Properties.cpp | 139 ++++++++++---- libminifi/src/utils/ChecksumCalculator.cpp | 63 ++++--- .../test/integration/C2PropertiesUpdateTests.cpp | 32 ++-- libminifi/test/unit/ChecksumCalculatorTests.cpp | 34 +++- .../test/unit/ConfigurationChecksumsTests.cpp | 6 +- libminifi/test/unit/IdTests.cpp | 28 +-- minifi_main/MiNiFiMain.cpp | 2 +- packaging/msi/WixWin.wsi.in | 60 +++--- packaging/rpm/expected-rpm-contents.in | 1 - 24 files changed, 440 insertions(+), 253 deletions(-) diff --git a/README.md b/README.md index 31d067ffb..1d0798b69 100644 --- a/README.md +++ b/README.md @@ -457,8 +457,22 @@ The performance tests can similarly be enabled. To execute them and see their ou $ ctest --verbose -L performance ``` + ### Configuring -The 'conf' directory in the installation root contains a template config.yml document, minifi.properties, and minifi-log.properties. Please see our [Configuration document](CONFIGURE.md) for details on how to configure agents. +The 'conf' directory in the installation root contains all configuration files. + +The files conf/minifi.properties, conf/minifi-log.properties and conf/minifi-uid.properties contain key-value pair configuration settings; +these are the default settings supplied by the latest MiNiFi version. If you would like to modify these, you should create a corresponding +.d directory (e.g. conf/minifi.properties.d) and put your settings in a new file inside this directory. These files are read and applied +in lexicographic order, after the default settings file. +The Windows installer creates a conf/minifi.properties.d/10_installer.properties file, which contains C2 connection settings. +If C2 is enabled and settings are added/modified from the C2 server, these will be saved in conf/minifi.properties.d/90_c2.properties. + +The conf/config.yml file contains the flow definition (i.e. the layout of processors, controller services etc). When you start MiNiFi for +the first time, the flow will be fetched from the C2 server (if available), or a file containing an empty flow will be created by MiNiFi. + +Please see our [Configuration document](CONFIGURE.md) for details on how to configure agents. + ### Installing as a service diff --git a/conf/CMakeLists.txt b/conf/CMakeLists.txt index 309b5453b..dcdcb80a1 100644 --- a/conf/CMakeLists.txt +++ b/conf/CMakeLists.txt @@ -41,12 +41,6 @@ else() message(FATAL_ERROR "Invalid MINIFI_PACKAGING_TYPE") endif() -configure_file( - config.yml.in - ${CMAKE_BINARY_DIR}/conf/config.yml - @ONLY -) - configure_file( minifi.properties.in ${CMAKE_BINARY_DIR}/conf/minifi.properties @@ -75,7 +69,6 @@ if (MINIFI_PACKAGING_TYPE STREQUAL "RPM") ${CMAKE_BINARY_DIR}/conf/minifi.properties ${CMAKE_BINARY_DIR}/conf/minifi-log.properties ${CMAKE_BINARY_DIR}/conf/minifi-uid.properties - ${CMAKE_BINARY_DIR}/conf/config.yml DESTINATION /${CMAKE_INSTALL_SYSCONFDIR}/${PROJECT_NAME} COMPONENT bin) elseif (MINIFI_PACKAGING_TYPE STREQUAL "TGZ") @@ -83,7 +76,6 @@ elseif (MINIFI_PACKAGING_TYPE STREQUAL "TGZ") ${CMAKE_BINARY_DIR}/conf/minifi.properties ${CMAKE_BINARY_DIR}/conf/minifi-log.properties ${CMAKE_BINARY_DIR}/conf/minifi-uid.properties - ${CMAKE_BINARY_DIR}/conf/config.yml DESTINATION conf COMPONENT bin) else() diff --git a/conf/config.yml.in b/conf/config.yml.in deleted file mode 100644 index 0bf85fc3d..000000000 --- a/conf/config.yml.in +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -Flow Controller: - name: MiNiFi Flow -Processors: [] -Connections: [] -Remote Processing Groups: [] -Provenance Reporting: diff --git a/conf/minifi-log.properties.in b/conf/minifi-log.properties.in index 9d9946715..3e9930fb4 100644 --- a/conf/minifi-log.properties.in +++ b/conf/minifi-log.properties.in @@ -13,6 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +### NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files ### +### in the conf/minifi-log.properties.d directory instead. These custom settings files should have ### +### a name ending with ".properties", and they are applied in lexicographic order, after this file. ### +### Example filename: conf/minifi-log.properties.d/90-log-to-stderr.properties ### + #More verbose pattern by default #Format details at https://github.com/gabime/spdlog/wiki/3.-Custom-formatting spdlog.pattern=[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v diff --git a/conf/minifi-uid.properties.in b/conf/minifi-uid.properties.in index fb49014d2..e07ea6837 100644 --- a/conf/minifi-uid.properties.in +++ b/conf/minifi-uid.properties.in @@ -13,6 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +### NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files ### +### in the conf/minifi-uid.properties.d directory instead. These custom settings files should have ### +### a name ending with ".properties", and they are applied in lexicographic order, after this file. ### +### Example filename: conf/minifi-uid.properties.d/10-use-random.properties ### + # Implementation for uid generation. # Valid values: # time - use uuid_generate_time diff --git a/conf/minifi.properties.in b/conf/minifi.properties.in index 7eceafc60..7ae002fbc 100644 --- a/conf/minifi.properties.in +++ b/conf/minifi.properties.in @@ -13,6 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +### NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files ### +### in the conf/minifi.properties.d directory instead. These custom settings files should have ### +### a name ending with ".properties", and they are applied in lexicographic order, after this file. ### +### Example filenames: conf/minifi.properties.d/10-c2.properties, conf/minifi.properties.d/90-repository-tweaks.properties ### + # Core Properties # nifi.flow.configuration.file=@MINIFI_PATH_FLOW_CONFIG@ nifi.administrative.yield.duration=30 sec diff --git a/core-framework/include/utils/file/FileUtils.h b/core-framework/include/utils/file/FileUtils.h index 06b1cc2bd..07d5d610f 100644 --- a/core-framework/include/utils/file/FileUtils.h +++ b/core-framework/include/utils/file/FileUtils.h @@ -242,6 +242,12 @@ inline int copy_file(const std::filesystem::path& path_from, const std::filesyst return 0; } +inline bool move_file(const std::filesystem::path& source_path, const std::filesystem::path& dest_path) { + std::error_code ec; + std::filesystem::rename(source_path, dest_path, ec); + return !ec; +} + inline void addFilesMatchingExtension(const std::shared_ptr<core::logging::Logger> &logger, const std::filesystem::path& originalPath, const std::filesystem::path& extension, diff --git a/docker/test/integration/cluster/containers/MinifiContainer.py b/docker/test/integration/cluster/containers/MinifiContainer.py index 387a3bd0d..62c5564de 100644 --- a/docker/test/integration/cluster/containers/MinifiContainer.py +++ b/docker/test/integration/cluster/containers/MinifiContainer.py @@ -238,9 +238,6 @@ class MinifiContainer(FlowContainer): else: image = 'apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION - if self.options.use_flow_config_from_url: - self.command = ["/bin/sh", "-c", "rm " + MinifiContainer.MINIFI_LOCATIONS.config_path + " && " + MinifiContainer.MINIFI_LOCATIONS.run_minifi_cmd] - ports = {} if self.options.enable_prometheus or self.options.enable_prometheus_with_ssl: ports = {'9936/tcp': 9936} diff --git a/extensions/standard-processors/tests/unit/ConfigurationTests.cpp b/extensions/standard-processors/tests/unit/ConfigurationTests.cpp index bc4db378a..f8b078e42 100644 --- a/extensions/standard-processors/tests/unit/ConfigurationTests.cpp +++ b/extensions/standard-processors/tests/unit/ConfigurationTests.cpp @@ -15,12 +15,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include <unordered_set> + #include "unit/TestBase.h" #include "unit/Catch.h" - #include "properties/Configuration.h" #include "utils/Environment.h" +namespace { +bool settingsInFileAreAsExpected(const std::filesystem::path& file_name, const std::unordered_set<std::string>& expected_contents) { + std::unordered_set<std::string> actual_contents; + std::ifstream file{file_name}; + if (file.is_open()) { + for (std::string line; std::getline(file, line); ) { + actual_contents.insert(line); + } + } + return expected_contents == actual_contents; +} +} // namespace + namespace org::apache::nifi::minifi::test { TEST_CASE("Configuration can merge lists of property names", "[mergeProperties]") { @@ -59,41 +73,37 @@ TEST_CASE("Configuration can fix misconfigured timeperiod<->integer validated pr LogTestController::getInstance().setInfo<minifi::Properties>(); TestController test_controller; - auto properties_path = test_controller.createTempDirectory() / "test.properties"; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; - std::ofstream{properties_path} + std::ofstream{original_properties_path} << "nifi.c2.agent.heartbeat.period=1min\n" << "nifi.administrative.yield.duration=30000\n"; - auto properties_file_time_after_creation = std::filesystem::last_write_time(properties_path); + auto properties_file_time_after_creation = std::filesystem::last_write_time(original_properties_path); const std::shared_ptr<minifi::Configure> configure = std::make_shared<minifi::ConfigureImpl>(); - configure->loadConfigureFile(properties_path); + configure->loadConfigureFile(original_properties_path); CHECK(configure->get("nifi.c2.agent.heartbeat.period") == "60000"); CHECK(configure->get("nifi.administrative.yield.duration") == "30000 ms"); { - CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "nifi.c2.agent.heartbeat.period=1min"); - CHECK(second_line == "nifi.administrative.yield.duration=30000"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(settingsInFileAreAsExpected(original_properties_path, {"nifi.c2.agent.heartbeat.period=1min", "nifi.administrative.yield.duration=30000"})); } CHECK(configure->commitChanges()); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "nifi.c2.agent.heartbeat.period=60000"); - CHECK(second_line == "nifi.administrative.yield.duration=30000 ms"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"nifi.c2.agent.heartbeat.period=60000", "nifi.administrative.yield.duration=30000 ms"})); } + + const std::shared_ptr<minifi::Configure> configure_reread = std::make_shared<minifi::ConfigureImpl>(); + configure_reread->loadConfigureFile(original_properties_path); + CHECK(configure_reread->get("nifi.c2.agent.heartbeat.period") == "60000"); + CHECK(configure_reread->get("nifi.administrative.yield.duration") == "30000 ms"); } TEST_CASE("Configuration can fix misconfigured datasize<->integer validated properties") { @@ -101,80 +111,159 @@ TEST_CASE("Configuration can fix misconfigured datasize<->integer validated prop LogTestController::getInstance().setInfo<minifi::Properties>(); TestController test_controller; - auto properties_path = test_controller.createTempDirectory() / "test.properties"; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; { - std::ofstream properties_file(properties_path); + std::ofstream properties_file(original_properties_path); properties_file << "appender.rolling.max_file_size=6000" << std::endl; properties_file.close(); } - auto properties_file_time_after_creation = std::filesystem::last_write_time(properties_path); + auto properties_file_time_after_creation = std::filesystem::last_write_time(original_properties_path); const std::shared_ptr<minifi::Configure> configure = std::make_shared<minifi::ConfigureImpl>(); - - configure->loadConfigureFile(properties_path, "nifi.log."); + configure->loadConfigureFile(original_properties_path, "nifi.log."); CHECK(configure->get("appender.rolling.max_file_size") == "6000 B"); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(first_line == "appender.rolling.max_file_size=6000"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(settingsInFileAreAsExpected(original_properties_path, {"appender.rolling.max_file_size=6000"})); } CHECK(configure->commitChanges()); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(first_line == "appender.rolling.max_file_size=6000 B"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"appender.rolling.max_file_size=6000 B"})); } -} + const std::shared_ptr<minifi::Configure> configure_reread = std::make_shared<minifi::ConfigureImpl>(); + configure_reread->loadConfigureFile(original_properties_path, "nifi.log."); + CHECK(configure_reread->get("appender.rolling.max_file_size") == "6000 B"); +} TEST_CASE("Configuration can fix misconfigured validated properties within environmental variables") { LogTestController::getInstance().setInfo<minifi::Configure>(); LogTestController::getInstance().setInfo<minifi::Properties>(); + TestController test_controller; - auto properties_path = test_controller.createTempDirectory() / "test.properties"; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; CHECK(minifi::utils::Environment::setEnvironmentVariable("SOME_VARIABLE", "4000")); - std::ofstream{properties_path} + std::ofstream{original_properties_path} << "compression.cached.log.max.size=${SOME_VARIABLE}\n" << "compression.compressed.log.max.size=3000\n"; - auto properties_file_time_after_creation = std::filesystem::last_write_time(properties_path); + auto properties_file_time_after_creation = std::filesystem::last_write_time(original_properties_path); const std::shared_ptr<minifi::Configure> configure = std::make_shared<minifi::ConfigureImpl>(); - configure->loadConfigureFile(properties_path, "nifi.log."); + configure->loadConfigureFile(original_properties_path, "nifi.log."); CHECK(configure->get("compression.cached.log.max.size") == "4000 B"); CHECK(configure->get("compression.compressed.log.max.size") == "3000 B"); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "compression.cached.log.max.size=${SOME_VARIABLE}"); - CHECK(second_line == "compression.compressed.log.max.size=3000"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(settingsInFileAreAsExpected(original_properties_path, {"compression.cached.log.max.size=${SOME_VARIABLE}", "compression.compressed.log.max.size=3000"})); } CHECK(configure->commitChanges()); { - CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(properties_path)); - std::ifstream properties_file(properties_path); - std::string first_line; - std::string second_line; - CHECK(std::getline(properties_file, first_line)); - CHECK(std::getline(properties_file, second_line)); - CHECK(first_line == "compression.cached.log.max.size=${SOME_VARIABLE}"); - CHECK(second_line == "compression.compressed.log.max.size=3000 B"); + CHECK(properties_file_time_after_creation == std::filesystem::last_write_time(original_properties_path)); + CHECK(properties_file_time_after_creation <= std::filesystem::last_write_time(updated_properties_path)); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"compression.compressed.log.max.size=3000 B"})); } + + const std::shared_ptr<minifi::Configure> configure_reread = std::make_shared<minifi::ConfigureImpl>(); + configure_reread->loadConfigureFile(original_properties_path, "nifi.log."); + CHECK(configure_reread->get("compression.cached.log.max.size") == "4000 B"); + CHECK(configure_reread->get("compression.compressed.log.max.size") == "3000 B"); +} + +TEST_CASE("Committing changes to a configuration creates a backup file") { + LogTestController::getInstance().setInfo<minifi::Configure>(); + LogTestController::getInstance().setInfo<minifi::Properties>(); + + TestController test_controller; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; + const auto backup_properties_path = [&]() { + auto path = updated_properties_path; + path += ".bak"; + return path; + }(); + + std::ofstream{original_properties_path} + << "number.of.lions=7\n" + << "number.of.elephants=12\n" + << "number.of.giraffes=30\n"; + + const std::shared_ptr<minifi::Configure> configure = std::make_shared<minifi::ConfigureImpl>(); + configure->loadConfigureFile(original_properties_path); + + CHECK(configure->get("number.of.lions") == "7"); + CHECK(configure->get("number.of.elephants") == "12"); + CHECK(configure->get("number.of.giraffes") == "30"); + + configure->set("number.of.lions", "8"); + CHECK(configure->commitChanges()); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"number.of.lions=8"})); + CHECK_FALSE(std::filesystem::exists(backup_properties_path)); + + const std::shared_ptr<minifi::Configure> configure_2 = std::make_shared<minifi::ConfigureImpl>(); + configure_2->loadConfigureFile(original_properties_path); + CHECK(configure_2->get("number.of.lions") == "8"); + CHECK(configure_2->get("number.of.elephants") == "12"); + CHECK(configure_2->get("number.of.giraffes") == "30"); + + configure->set("number.of.giraffes", "29"); + CHECK(configure->commitChanges()); + CHECK(settingsInFileAreAsExpected(updated_properties_path, {"number.of.lions=8", "number.of.giraffes=29"})); + CHECK(settingsInFileAreAsExpected(backup_properties_path, {"number.of.lions=8"})); + + const std::shared_ptr<minifi::Configure> configure_3 = std::make_shared<minifi::ConfigureImpl>(); + configure_3->loadConfigureFile(original_properties_path); + CHECK(configure_3->get("number.of.lions") == "8"); + CHECK(configure_3->get("number.of.elephants") == "12"); + CHECK(configure_3->get("number.of.giraffes") == "29"); +} + +TEST_CASE("Backup file are skipped when reading config files") { + LogTestController::getInstance().setInfo<minifi::Configure>(); + LogTestController::getInstance().setInfo<minifi::Properties>(); + + TestController test_controller; + const auto conf_directory = test_controller.createTempDirectory(); + const auto original_properties_path = conf_directory / "test.properties"; + const auto updated_properties_path = conf_directory / "test.properties.d" / PropertiesImpl::C2PropertiesFileName; + const auto backup_properties_path = [&]() { + auto path = updated_properties_path; + path += ".bak"; + return path; + }(); + + std::ofstream{original_properties_path} + << "number.of.lions=7\n" + << "number.of.elephants=12\n"; + + utils::file::create_dir(updated_properties_path.parent_path()); + std::ofstream{updated_properties_path} + << "number.of.lions=8\n"; + + std::ofstream{backup_properties_path} + << "number.of.elephants=20\n" + << "number.of.giraffes=30\n"; + + const std::shared_ptr<minifi::Configure> configure = std::make_shared<minifi::ConfigureImpl>(); + configure->loadConfigureFile(original_properties_path); + + CHECK(configure->get("number.of.lions") == "8"); + CHECK(configure->get("number.of.elephants") == "12"); + CHECK_FALSE(configure->get("number.of.giraffes")); } } // namespace org::apache::nifi::minifi::test diff --git a/libminifi/include/core/logging/LoggerProperties.h b/libminifi/include/core/logging/LoggerProperties.h index 90098d447..c776216cb 100644 --- a/libminifi/include/core/logging/LoggerProperties.h +++ b/libminifi/include/core/logging/LoggerProperties.h @@ -32,7 +32,7 @@ namespace org::apache::nifi::minifi::core::logging { class LoggerProperties : public PropertiesImpl { public: explicit LoggerProperties(std::filesystem::path default_log_dir) - : PropertiesImpl("Logger properties"), + : PropertiesImpl(PersistTo::MultipleFiles, "Logger properties"), default_log_dir_(std::move(default_log_dir)) { } /** diff --git a/libminifi/include/properties/Configuration.h b/libminifi/include/properties/Configuration.h index 547aeb35d..0dbdb434f 100644 --- a/libminifi/include/properties/Configuration.h +++ b/libminifi/include/properties/Configuration.h @@ -34,7 +34,7 @@ class PropertyValidator; class ConfigurationImpl : public PropertiesImpl, public virtual Configuration { public: - ConfigurationImpl() : PropertiesImpl("MiNiFi configuration") {} + ConfigurationImpl() : PropertiesImpl(PersistTo::MultipleFiles, "MiNiFi configuration") {} }; } // namespace org::apache::nifi::minifi diff --git a/libminifi/include/properties/Properties.h b/libminifi/include/properties/Properties.h index f103d0565..4f94f6401 100644 --- a/libminifi/include/properties/Properties.h +++ b/libminifi/include/properties/Properties.h @@ -41,10 +41,14 @@ class PropertiesImpl : public virtual Properties { }; public: - explicit PropertiesImpl(std::string name = ""); + enum class PersistTo { SingleFile, MultipleFiles }; + + explicit PropertiesImpl(PersistTo persist_to, std::string name = ""); ~PropertiesImpl() override = default; + static constexpr std::string_view C2PropertiesFileName = "90_c2.properties"; + const std::string& getName() const override { return name_; } @@ -104,7 +108,6 @@ class PropertiesImpl : public virtual Properties { void loadConfigureFile(const std::filesystem::path& configuration_file, std::string_view prefix = "") override; - std::vector<std::string> getConfiguredKeys() const override { std::vector<std::string> keys; for (auto &property : properties_) { @@ -122,18 +125,17 @@ class PropertiesImpl : public virtual Properties { std::map<std::string, std::string> getProperties() const override; private: - std::map<std::string, PropertyValue> properties_; + std::filesystem::path extraPropertiesFilesDirName() const; + void setPropertiesFromFile(const std::filesystem::path& properties_file, std::string_view prefix); + std::map<std::string, PropertyValue> properties_; bool dirty_{false}; - - std::filesystem::path properties_file_; - + std::filesystem::path base_properties_file_; + std::vector<std::filesystem::path> properties_files_; utils::ChecksumCalculator checksum_calculator_; - - // Mutex for protection mutable std::mutex mutex_; std::shared_ptr<core::logging::Logger> logger_; - + PersistTo persist_to_; std::string name_; }; diff --git a/libminifi/include/utils/ChecksumCalculator.h b/libminifi/include/utils/ChecksumCalculator.h index 8cc1a4ed4..c11931d58 100644 --- a/libminifi/include/utils/ChecksumCalculator.h +++ b/libminifi/include/utils/ChecksumCalculator.h @@ -20,7 +20,7 @@ #include <filesystem> #include <optional> #include <string> -#include <utility> +#include <vector> namespace org::apache::nifi::minifi::utils { @@ -29,16 +29,15 @@ class ChecksumCalculator { static constexpr const char* CHECKSUM_TYPE = "SHA256"; static constexpr size_t LENGTH_OF_HASH_IN_BYTES = 32; - void setFileLocation(const std::filesystem::path& file_location); - [[nodiscard]] std::filesystem::path getFileName() const; + void setFileLocations(std::vector<std::filesystem::path> file_locations); + [[nodiscard]] std::filesystem::path getFileNameOfFirstFileLocation() const; std::string getChecksum(); void invalidateChecksum(); private: - static std::string computeChecksum(const std::filesystem::path& file_location); + static std::string computeChecksum(const std::vector<std::filesystem::path>& file_locations); - std::optional<std::filesystem::path> file_location_; - std::optional<std::filesystem::path> file_name_; + std::vector<std::filesystem::path> file_locations_; std::optional<std::string> checksum_; }; diff --git a/libminifi/src/core/FlowConfiguration.cpp b/libminifi/src/core/FlowConfiguration.cpp index 2e7b56484..11f58bd10 100644 --- a/libminifi/src/core/FlowConfiguration.cpp +++ b/libminifi/src/core/FlowConfiguration.cpp @@ -29,6 +29,19 @@ #include "minifi-cpp/SwapManager.h" #include "Connection.h" +namespace { +void createDefaultFlowConfigFile(const std::filesystem::path& path) { + std::ofstream ostream(path); + ostream.exceptions(std::ofstream::failbit | std::ofstream::badbit); + ostream << "Flow Controller:\n" + " name: MiNiFi Flow\n" + "Processors: []\n" + "Connections: []\n" + "Remote Processing Groups: []\n" + "Provenance Reporting:\n"; +} +} // namespace + namespace org::apache::nifi::minifi::core { FlowConfiguration::FlowConfiguration(ConfigurationContext ctx) @@ -52,12 +65,16 @@ FlowConfiguration::FlowConfiguration(ConfigurationContext ctx) if (!ctx.path) { logger_->log_error("Configuration path is not specified."); } else { + const bool c2_enabled = configuration_->get(Configure::nifi_c2_enable).and_then(&utils::string::toBool).value_or(false); + if (!c2_enabled && !ctx.path->empty() && !std::filesystem::exists(*ctx.path)) { + createDefaultFlowConfigFile(*ctx.path); + } config_path_ = utils::file::canonicalize(*ctx.path); if (!config_path_) { logger_->log_error("Couldn't find config file \"{}\".", ctx.path->string()); config_path_ = ctx.path; } - checksum_calculator_.setFileLocation(*config_path_); + checksum_calculator_.setFileLocations(std::vector{*config_path_}); } } diff --git a/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp b/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp index c2181e7fc..f9b965c10 100644 --- a/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp +++ b/libminifi/src/core/state/nodes/ConfigurationChecksums.cpp @@ -32,7 +32,7 @@ std::vector<SerializedResponseNode> ConfigurationChecksums::serialize() { for (auto checksum_calculator : checksum_calculators_) { SerializedResponseNode file_checksum_node; - file_checksum_node.name = checksum_calculator->getFileName().string(); + file_checksum_node.name = checksum_calculator->getFileNameOfFirstFileLocation().string(); file_checksum_node.value = checksum_calculator->getChecksum(); checksums_node.children.push_back(file_checksum_node); } diff --git a/libminifi/src/properties/Properties.cpp b/libminifi/src/properties/Properties.cpp index b60e1499d..1624a8557 100644 --- a/libminifi/src/properties/Properties.cpp +++ b/libminifi/src/properties/Properties.cpp @@ -18,19 +18,20 @@ #include "properties/Properties.h" #include <fstream> +#include <ranges> #include <string> #include "core/logging/LoggerConfiguration.h" #include "properties/Configuration.h" #include "properties/PropertiesFile.h" -#include "range/v3/algorithm/all_of.hpp" #include "utils/StringUtils.h" #include "utils/file/FileUtils.h" namespace org::apache::nifi::minifi { -PropertiesImpl::PropertiesImpl(std::string name) +PropertiesImpl::PropertiesImpl(PersistTo persist_to, std::string name) : logger_(core::logging::LoggerFactory<Properties>::getLogger()), + persist_to_(persist_to), name_(std::move(name)) { } @@ -72,12 +73,12 @@ const core::PropertyValidator* getValidator(const std::string& lookup_value) { // isdigit requires unsigned chars as input bool allDigits(const std::string& value) { - return ranges::all_of(value, [](const unsigned char c){ return ::isdigit(c); }); + return std::ranges::all_of(value, [](const unsigned char c){ return ::isdigit(c); }); } // isdigit requires unsigned chars as input bool allDigitsOrSpaces(const std::string& value) { - return ranges::all_of(value, [](const unsigned char c) { return std::isdigit(c) || std::isspace(c);}); + return std::ranges::all_of(value, [](const unsigned char c) { return std::isdigit(c) || std::isspace(c);}); } std::optional<std::string> ensureTimePeriodValidatedPropertyHasExplicitUnit(const core::PropertyValidator* const validator, const std::string& value) { @@ -157,8 +158,41 @@ void fixValidatedProperty(const std::string& property_name, needs_to_persist_new_value = false; } } + +auto getExtraPropertiesFileNames(const std::filesystem::path& extra_properties_files_dir, const std::shared_ptr<core::logging::Logger>& logger) { + std::vector<std::filesystem::path> extra_properties_file_names; + if (utils::file::exists(extra_properties_files_dir) && utils::file::is_directory(extra_properties_files_dir)) { + utils::file::list_dir(extra_properties_files_dir, [&](const std::filesystem::path&, const std::filesystem::path& file_name) { + if (file_name.string().ends_with(".properties")) { + extra_properties_file_names.push_back(file_name); + } + return true; + }, logger, /* recursive = */ false); + } + std::ranges::sort(extra_properties_file_names); + return extra_properties_file_names; +} + +void updateChangedPropertiesInPropertiesFile(minifi::PropertiesFile& current_content, const auto& properties) { + for (const auto& prop : properties) { + if (!prop.second.need_to_persist_new_value) { + continue; + } + if (current_content.hasValue(prop.first)) { + current_content.update(prop.first, prop.second.persisted_value); + } else { + current_content.append(prop.first, prop.second.persisted_value); + } + } +} } // namespace +std::filesystem::path PropertiesImpl::extraPropertiesFilesDirName() const { + auto extra_properties_files_dir = base_properties_file_; + extra_properties_files_dir += ".d"; + return extra_properties_files_dir; +} + void PropertiesImpl::loadConfigureFile(const std::filesystem::path& configuration_file, std::string_view prefix) { std::lock_guard<std::mutex> lock(mutex_); if (configuration_file.empty()) { @@ -173,23 +207,42 @@ void PropertiesImpl::loadConfigureFile(const std::filesystem::path& configuratio } std::error_code ec; - properties_file_ = std::filesystem::canonical(configuration_file, ec); + base_properties_file_ = std::filesystem::canonical(configuration_file, ec); if (ec.value() != 0) { logger_->log_warn("Configuration file '{}' does not exist, and it could not be created", configuration_file); return; } + properties_files_ = { base_properties_file_ }; + const auto extra_properties_files_dir = extraPropertiesFilesDirName(); + const auto extra_properties_file_names = getExtraPropertiesFileNames(extra_properties_files_dir, logger_); + for (const auto& file_name : extra_properties_file_names) { + properties_files_.push_back(extra_properties_files_dir / file_name); + } + logger_->log_info("Using configuration file to load configuration for {} from {} (located at {})", - getName().c_str(), configuration_file.string(), properties_file_.string()); + getName().c_str(), configuration_file.string(), base_properties_file_.string()); + if (!extra_properties_file_names.empty()) { + auto list_of_files = utils::string::join(", ", extra_properties_file_names, [](const auto& path) { return path.string(); }); + logger_->log_info("Also reading configuration from files {} in {}", list_of_files, extra_properties_files_dir.string()); + } + + properties_.clear(); + dirty_ = false; + for (const auto& properties_file : properties_files_) { + setPropertiesFromFile(properties_file, prefix); + } + + checksum_calculator_.setFileLocations(properties_files_); +} - std::ifstream file(properties_file_, std::ifstream::in); +void PropertiesImpl::setPropertiesFromFile(const std::filesystem::path& properties_file, std::string_view prefix) { + std::ifstream file(properties_file, std::ifstream::in); if (!file.good()) { - logger_->log_error("load configure file failed {}", properties_file_); + logger_->log_error("load configure file failed {}", properties_file); return; } - properties_.clear(); - dirty_ = false; for (const auto& line : PropertiesFile{file}) { auto key = line.getKey(); auto persisted_value = line.getValue(); @@ -199,59 +252,65 @@ void PropertiesImpl::loadConfigureFile(const std::filesystem::path& configuratio dirty_ = dirty_ || need_to_persist_new_value; properties_[key] = {persisted_value, value, need_to_persist_new_value}; } - checksum_calculator_.setFileLocation(properties_file_); } std::filesystem::path PropertiesImpl::getFilePath() const { std::lock_guard<std::mutex> lock(mutex_); - return properties_file_; + return base_properties_file_; } bool PropertiesImpl::commitChanges() { std::lock_guard<std::mutex> lock(mutex_); if (!dirty_) { - logger_->log_info("Attempt to persist, but properties are not updated"); + logger_->log_debug("commitChanges() called, but properties have not changed, nothing to do"); return true; } - std::ifstream file(properties_file_, std::ifstream::in); + const auto output_file = (persist_to_ == PersistTo::SingleFile ? base_properties_file_ : extraPropertiesFilesDirName() / C2PropertiesFileName); + if (!std::filesystem::exists(output_file)) { + logger_->log_debug("Configuration file {} does not exist yet, creating it", output_file); + utils::file::create_dir(output_file.parent_path(), /* recursive = */ true); + std::ofstream file{output_file}; + } + + std::ifstream file(output_file, std::ifstream::in); if (!file) { - logger_->log_error("load configure file failed {}", properties_file_); + logger_->log_error("Failed to load configuration file {}", output_file); return false; } - - auto new_file = properties_file_; - new_file += ".new"; - PropertiesFile current_content{file}; - for (const auto& prop : properties_) { - if (!prop.second.need_to_persist_new_value) { - continue; - } - if (current_content.hasValue(prop.first)) { - current_content.update(prop.first, prop.second.persisted_value); - } else { - current_content.append(prop.first, prop.second.persisted_value); - } - } + file.close(); + + updateChangedPropertiesInPropertiesFile(current_content, properties_); + auto new_file = output_file; + new_file += ".new"; try { current_content.writeTo(new_file); } catch (const std::exception&) { - logger_->log_error("Could not update {}", properties_file_); + logger_->log_error("Could not write to {}", new_file); return false; } - auto backup = properties_file_; - backup += ".bak"; - if (utils::file::FileUtils::copy_file(properties_file_, backup) == 0 && utils::file::FileUtils::copy_file(new_file, properties_file_) == 0) { - logger_->log_info("Persisted {}", properties_file_); - checksum_calculator_.invalidateChecksum(); - dirty_ = false; - return true; + std::error_code ec; + const auto existing_file_size = std::filesystem::file_size(output_file, ec); + if (ec || existing_file_size == 0) { + if (!utils::file::move_file(new_file, output_file)) { + logger_->log_error("Could not create minifi properties file {}", output_file); + return false; + } + } else { + auto backup = output_file; + backup += ".bak"; + if (!utils::file::move_file(output_file, backup) || !utils::file::move_file(new_file, output_file)) { + logger_->log_error("Could not update minifi properties file {}", output_file); + return false; + } } - logger_->log_error("Could not update {}", properties_file_); - return false; + logger_->log_info("Persisted {}", output_file); + checksum_calculator_.invalidateChecksum(); + dirty_ = false; + return true; } std::map<std::string, std::string> PropertiesImpl::getProperties() const { @@ -264,7 +323,7 @@ std::map<std::string, std::string> PropertiesImpl::getProperties() const { } std::shared_ptr<Properties> Properties::create() { - return std::make_shared<PropertiesImpl>(); + return std::make_shared<PropertiesImpl>(PropertiesImpl::PersistTo::SingleFile); } } // namespace org::apache::nifi::minifi diff --git a/libminifi/src/utils/ChecksumCalculator.cpp b/libminifi/src/utils/ChecksumCalculator.cpp index 14e1f831d..ae32e8ee4 100644 --- a/libminifi/src/utils/ChecksumCalculator.cpp +++ b/libminifi/src/utils/ChecksumCalculator.cpp @@ -21,7 +21,6 @@ #include <fstream> #include "sodium/crypto_hash_sha256.h" -#include "utils/file/FileUtils.h" #include "utils/StringUtils.h" #include "properties/Configuration.h" @@ -29,51 +28,59 @@ namespace { const std::string AGENT_IDENTIFIER_KEY = std::string(org::apache::nifi::minifi::Configuration::nifi_c2_agent_identifier) + "="; +namespace utils = org::apache::nifi::minifi::utils; + +void addFileToChecksum(const std::filesystem::path& file_path, crypto_hash_sha256_state& state) { + std::ifstream input_file{file_path, std::ios::in | std::ios::binary}; + if (!input_file.is_open()) { + throw std::runtime_error(utils::string::join_pack("Could not open config file '", file_path.string(), "' to compute the checksum: ", std::strerror(errno))); + } + + std::string line; + while (std::getline(input_file, line)) { + // skip lines containing the agent identifier, so agents in the same class will have the same checksum + if (line.starts_with(AGENT_IDENTIFIER_KEY)) { + continue; + } + if (!input_file.eof()) { // eof() means we have just read the last line, which was not terminated by a newline + line.append("\n"); + } + crypto_hash_sha256_update(&state, reinterpret_cast<const unsigned char*>(line.data()), line.size()); + } + if (input_file.bad()) { + throw std::runtime_error(utils::string::join_pack("Error reading config file '", file_path.string(), "' while computing the checksum: ", std::strerror(errno))); + } +} + } // namespace namespace org::apache::nifi::minifi::utils { -void ChecksumCalculator::setFileLocation(const std::filesystem::path& file_location) { - file_location_ = file_location; - file_name_ = file_location.filename(); +void ChecksumCalculator::setFileLocations(std::vector<std::filesystem::path> file_locations) { + gsl_Expects(!file_locations.empty()); + file_locations_ = std::move(file_locations); invalidateChecksum(); } -std::filesystem::path ChecksumCalculator::getFileName() const { - gsl_Expects(file_name_); - return *file_name_; +std::filesystem::path ChecksumCalculator::getFileNameOfFirstFileLocation() const { + gsl_Expects(!file_locations_.empty()); + return file_locations_.front().filename(); } std::string ChecksumCalculator::getChecksum() { - gsl_Expects(file_location_); + gsl_Expects(!file_locations_.empty()); if (!checksum_) { - checksum_ = computeChecksum(*file_location_); + checksum_ = computeChecksum(file_locations_); } return *checksum_; } -std::string ChecksumCalculator::computeChecksum(const std::filesystem::path& file_location) { - std::ifstream input_file{file_location, std::ios::in | std::ios::binary}; - if (!input_file.is_open()) { - throw std::runtime_error(string::join_pack("Could not open config file '", file_location.string(), "' to compute the checksum: ", std::strerror(errno))); - } - +std::string ChecksumCalculator::computeChecksum(const std::vector<std::filesystem::path>& file_locations) { crypto_hash_sha256_state state; crypto_hash_sha256_init(&state); - std::string line; - while (std::getline(input_file, line)) { - // skip lines containing the agent identifier, so agents in the same class will have the same checksum - if (string::startsWith(line, AGENT_IDENTIFIER_KEY)) { - continue; - } - if (!input_file.eof()) { // eof() means we have just read the last line, which was not terminated by a newline - line.append("\n"); - } - crypto_hash_sha256_update(&state, reinterpret_cast<const unsigned char*>(line.data()), line.size()); - } - if (input_file.bad()) { - throw std::runtime_error(string::join_pack("Error reading config file '", file_location.string(), "' while computing the checksum: ", std::strerror(errno))); + for (const auto& file_location : file_locations) { + addFileToChecksum(file_location, state); } std::array<unsigned char, LENGTH_OF_HASH_IN_BYTES> hash{}; diff --git a/libminifi/test/integration/C2PropertiesUpdateTests.cpp b/libminifi/test/integration/C2PropertiesUpdateTests.cpp index b762a2c86..e6cf2389d 100644 --- a/libminifi/test/integration/C2PropertiesUpdateTests.cpp +++ b/libminifi/test/integration/C2PropertiesUpdateTests.cpp @@ -155,9 +155,9 @@ TEST_CASE("C2PropertiesUpdateTests", "[c2test]") { logger2->log_debug("DummyClass2::before"); logger3->log_debug("DummyClass3::before"); - REQUIRE(!log_test_controller->contains("DummyClass1::before", 0s)); - REQUIRE(!log_test_controller->contains("DummyClass2::before", 0s)); - REQUIRE(!log_test_controller->contains("DummyClass3::before", 0s)); + CHECK(!log_test_controller->contains("DummyClass1::before", 0s)); + CHECK(!log_test_controller->contains("DummyClass2::before", 0s)); + CHECK(!log_test_controller->contains("DummyClass3::before", 0s)); } // On msvc, the passed lambda can't capture a reference to the object under construction, so we need to late-init harness. @@ -194,23 +194,25 @@ TEST_CASE("C2PropertiesUpdateTests", "[c2test]") { logger2->log_debug("DummyClass2::after"); // this should still not log logger3->log_debug("DummyClass3::after"); } - REQUIRE(log_test_controller->contains("DummyClass1::after", 0s)); - REQUIRE(!log_test_controller->contains("DummyClass2::after", 0s)); - REQUIRE(log_test_controller->contains("DummyClass3::after", 0s)); + CHECK(log_test_controller->contains("DummyClass1::after", 0s)); + CHECK_FALSE(log_test_controller->contains("DummyClass2::after", 0s)); + CHECK(log_test_controller->contains("DummyClass3::after", 0s)); { - minifi::PropertiesFile minifi_properties(std::ifstream{home_dir / "conf/minifi.properties"}); - REQUIRE(!minifi_properties.hasValue("nifi.dummy.property")); - REQUIRE(minifi_properties.getValue("nifi.property.one") == "bush"); - REQUIRE(minifi_properties.getValue("nifi.property.two") == "ring"); - REQUIRE(!minifi_properties.hasValue(minifi::Configuration::nifi_c2_rest_heartbeat_minimize_updates)); - REQUIRE(minifi_properties.getValue(minifi::Configuration::minifi_disk_space_watchdog_enable) == "true"); + const std::shared_ptr<minifi::Configure> minifi_properties = std::make_shared<minifi::ConfigureImpl>(); + minifi_properties->loadConfigureFile(home_dir / "conf" / "minifi.properties"); + CHECK_FALSE(minifi_properties->get("nifi.dummy.property")); + CHECK(minifi_properties->get("nifi.property.one") == "bush"); + CHECK(minifi_properties->get("nifi.property.two") == "ring"); + CHECK_FALSE(minifi_properties->get(minifi::Configuration::nifi_c2_rest_heartbeat_minimize_updates)); + CHECK(minifi_properties->get(minifi::Configuration::minifi_disk_space_watchdog_enable) == "true"); } { - minifi::PropertiesFile minifi_log_properties(std::ifstream{home_dir / "conf/minifi-log.properties"}); - REQUIRE(!minifi_log_properties.hasValue("logger.org::apache::nifi::minifi::test::dummy")); - REQUIRE(minifi_log_properties.getValue("logger.org::apache::nifi::minifi::test::DummyClass1") == "DEBUG,ostream"); + const std::shared_ptr<minifi::Configure> minifi_log_properties = std::make_shared<minifi::ConfigureImpl>(); + minifi_log_properties->loadConfigureFile(home_dir / "conf" / "minifi-log.properties"); + CHECK_FALSE(minifi_log_properties->get("logger.org::apache::nifi::minifi::test::dummy")); + CHECK(minifi_log_properties->get("logger.org::apache::nifi::minifi::test::DummyClass1") == "DEBUG,ostream"); } }); diff --git a/libminifi/test/unit/ChecksumCalculatorTests.cpp b/libminifi/test/unit/ChecksumCalculatorTests.cpp index fabecddfe..82623814c 100644 --- a/libminifi/test/unit/ChecksumCalculatorTests.cpp +++ b/libminifi/test/unit/ChecksumCalculatorTests.cpp @@ -37,17 +37,33 @@ TEST_CASE("ChecksumCalculator can calculate the checksum, which is equal to sha2 REQUIRE(size_t{utils::ChecksumCalculator::LENGTH_OF_HASH_IN_BYTES} == size_t{32}); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_ONE_LINE_OF_TEXT); } +TEST_CASE("The input of ChecksumCalculator can be in multiple files", "[ChecksumCalculator]") { + TestController test_controller; + const auto test_dir = test_controller.createTempDirectory(); + const auto single_file_location = minifi::test::utils::putFileToDir(test_dir, "single.txt", "one line of text\nsecond line of text\n"); + const auto multiple_1_file_location = minifi::test::utils::putFileToDir(test_dir, "multiple_1.txt", "one line of text\n"); + const auto multiple_2_file_location = minifi::test::utils::putFileToDir(test_dir, "multiple_2.txt", "second line of text\n"); + + utils::ChecksumCalculator checksum_calculator_single; + checksum_calculator_single.setFileLocations(std::vector{single_file_location}); + + utils::ChecksumCalculator checksum_calculator_multiple; + checksum_calculator_multiple.setFileLocations(std::vector{multiple_1_file_location, multiple_2_file_location}); + + CHECK(checksum_calculator_single.getChecksum() == checksum_calculator_multiple.getChecksum()); +} + TEST_CASE("On Windows text files, the checksum calculated is also the same as sha256sum", "[ChecksumCalculator]") { TestController test_controller; auto test_dir = test_controller.createTempDirectory(); auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\r\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == "94fc46c62ef6cc5b45cbad9fd53116cfb15a80960a9b311c1c27e5b5265ad4b4"); } @@ -57,7 +73,7 @@ TEST_CASE("The checksum can be reset and recomputed", "[ChecksumCalculator]") { auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_ONE_LINE_OF_TEXT); std::ofstream append_to_file(file_location, std::ios::binary | std::ios::app); @@ -76,11 +92,11 @@ TEST_CASE("If the file location is updated, the checksum will be recomputed", "[ auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_ONE_LINE_OF_TEXT); auto other_file_location = minifi::test::utils::putFileToDir(test_dir, "long.txt", "one line of text\nanother line of text\n"); - checksum_calculator.setFileLocation(other_file_location); + checksum_calculator.setFileLocations(std::vector{other_file_location}); REQUIRE(checksum_calculator.getChecksum() == CHECKSUM_FOR_TWO_LINES_OF_TEXT); } @@ -92,7 +108,7 @@ TEST_CASE("Checksums can be computed for binary (eg. encrypted) files, too", "[C auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", binary_data); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); REQUIRE(checksum_calculator.getChecksum() == "bdec77160c394c067419735de757e4daa1c4679ea45e82a33fa8f706eed87709"); } @@ -109,16 +125,16 @@ TEST_CASE("The agent identifier is excluded from the checksum", "[ChecksumCalcul "nifi.c2.agent.heartbeat.period=10 sec\n"); utils::ChecksumCalculator checksum_calculator_1; - checksum_calculator_1.setFileLocation(file_location_1); + checksum_calculator_1.setFileLocations(std::vector{file_location_1}); utils::ChecksumCalculator checksum_calculator_2; - checksum_calculator_2.setFileLocation(file_location_2); + checksum_calculator_2.setFileLocations(std::vector{file_location_2}); REQUIRE(checksum_calculator_1.getChecksum() == checksum_calculator_2.getChecksum()); } TEST_CASE("ChecksumCalculator::getChecksum will throw if the file does not exist", "[ChecksumCalculator]") { utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation("/this/file/does/not/exist/84a77fd9-16b3-49d2-aead-a1f9e58e530d"); + checksum_calculator.setFileLocations(std::vector{std::filesystem::path{"/this/file/does/not/exist/84a77fd9-16b3-49d2-aead-a1f9e58e530d"}}); REQUIRE_THROWS(checksum_calculator.getChecksum()); } diff --git a/libminifi/test/unit/ConfigurationChecksumsTests.cpp b/libminifi/test/unit/ConfigurationChecksumsTests.cpp index 6da67a66e..956907ff2 100644 --- a/libminifi/test/unit/ConfigurationChecksumsTests.cpp +++ b/libminifi/test/unit/ConfigurationChecksumsTests.cpp @@ -39,7 +39,7 @@ TEST_CASE("If one checksum calculator is added, we get a node with one child", " auto file_location = minifi::test::utils::putFileToDir(test_dir, "simple.txt", "one line of text\n"); utils::ChecksumCalculator checksum_calculator; - checksum_calculator.setFileLocation(file_location); + checksum_calculator.setFileLocations(std::vector{file_location}); ConfigurationChecksums configuration_checksums; configuration_checksums.addChecksumCalculator(checksum_calculator); @@ -62,9 +62,9 @@ TEST_CASE("If two checksum calculators are added, we get a node with two childre auto file_location_2 = minifi::test::utils::putFileToDir(test_dir, "second.txt", "this is the second file\n"); utils::ChecksumCalculator checksum_calculator_1; - checksum_calculator_1.setFileLocation(file_location_1); + checksum_calculator_1.setFileLocations(std::vector{file_location_1}); utils::ChecksumCalculator checksum_calculator_2; - checksum_calculator_2.setFileLocation(file_location_2); + checksum_calculator_2.setFileLocations(std::vector{file_location_2}); ConfigurationChecksums configuration_checksums; configuration_checksums.addChecksumCalculator(checksum_calculator_1); diff --git a/libminifi/test/unit/IdTests.cpp b/libminifi/test/unit/IdTests.cpp index 048072b5d..aac8ef0fb 100644 --- a/libminifi/test/unit/IdTests.cpp +++ b/libminifi/test/unit/IdTests.cpp @@ -39,7 +39,7 @@ TEST_CASE("Test default is time", "[id]") { LogTestController::getInstance().setDebug<utils::IdGenerator>(); std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); - generator->initialize(std::make_shared<minifi::PropertiesImpl>()); + generator->initialize(std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties")); REQUIRE(true == LogTestController::getInstance().contains("Using uuid_generate_time implementation for uids.")); LogTestController::getInstance().reset(); @@ -49,7 +49,7 @@ TEST_CASE("Test time", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "TiMe"); std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); @@ -69,7 +69,7 @@ TEST_CASE("Test Generate Move", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "TiMe"); std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); @@ -86,7 +86,7 @@ TEST_CASE("Test random", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "RaNDoM"); std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); @@ -106,7 +106,7 @@ TEST_CASE("Test uuid_default", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "UUID_default"); std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); @@ -120,7 +120,7 @@ TEST_CASE("Test invalid", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "InVaLiD"); std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); @@ -134,7 +134,7 @@ TEST_CASE("Test parse", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "time"); std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); @@ -160,7 +160,7 @@ TEST_CASE("Test parse invalid", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "time"); std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); @@ -187,7 +187,7 @@ TEST_CASE("Test to_string", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "time"); std::shared_ptr<utils::IdGenerator> generator = utils::IdGenerator::getIdGenerator(); @@ -229,7 +229,7 @@ TEST_CASE("Test Hex Device Segment 16 bits correct digits", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "minifi_uid"); id_props->set("uid.minifi.device.segment", "09aF"); @@ -255,7 +255,7 @@ TEST_CASE("Test Hex Device Segment 16 bits too many digits", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "minifi_uid"); id_props->set("uid.minifi.device.segment", "09aFee"); @@ -283,7 +283,7 @@ TEST_CASE("Test Hex Device Segment 18 bits", "[id]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); id_props->set("uid.implementation", "minifi_uid"); id_props->set("uid.minifi.device.segment.bits", "18"); id_props->set("uid.minifi.device.segment", "09aF8"); @@ -317,7 +317,7 @@ TEST_CASE("Collision", "[collision]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); SECTION("random") { id_props->set("uid.implementation", "random"); } @@ -357,7 +357,7 @@ TEST_CASE("Speed", "[speed]") { TestController test_controller; LogTestController::getInstance().setDebug<utils::IdGenerator>(); - std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(); + std::shared_ptr<minifi::Properties> id_props = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); std::string implementation; SECTION("random") { implementation = "random"; diff --git a/minifi_main/MiNiFiMain.cpp b/minifi_main/MiNiFiMain.cpp index d6d32a50c..f5c852cf1 100644 --- a/minifi_main/MiNiFiMain.cpp +++ b/minifi_main/MiNiFiMain.cpp @@ -306,7 +306,7 @@ int main(int argc, char **argv) { logger_configuration.initialize(log_properties); - std::shared_ptr<minifi::Properties> uid_properties = std::make_shared<minifi::PropertiesImpl>("UID properties"); + std::shared_ptr<minifi::Properties> uid_properties = std::make_shared<minifi::PropertiesImpl>(minifi::PropertiesImpl::PersistTo::MultipleFiles, "UID properties"); uid_properties->loadConfigureFile(locations->uid_properties_path_); utils::IdGenerator::getIdGenerator()->initialize(uid_properties); diff --git a/packaging/msi/WixWin.wsi.in b/packaging/msi/WixWin.wsi.in index e294b547d..9c912f604 100644 --- a/packaging/msi/WixWin.wsi.in +++ b/packaging/msi/WixWin.wsi.in @@ -58,10 +58,9 @@ ${WIX_EXTRA_FEATURES} <Feature Id="InstallConf" Title="Apache NiFi MiNiFi C++ Configuration" AllowAdvertise="yes" Level="1"> <ComponentRef Id="LOGPROP"/> <ComponentRef Id="UIDPROP"/> - <ComponentRef Id="CONFIGFILE"/> - <ComponentRef Id="OPENSSLCONFCOMP"/> <ComponentRef Id="UpdateConfig"/> - <ComponentRef Id="UpdateConfigNotExist"/> + <ComponentRef Id="MINIFIPROP"/> + <ComponentRef Id="OPENSSLCONFCOMP"/> </Feature> <?ifdef INCLUDE_PYTHON_PROCESSORS?> @@ -263,16 +262,16 @@ ${WIX_EXTRA_FEATURES} </UI> - <Property Id="AGENT_CLASS" Value="Your Agent Class" /> - <Property Id="AGENT_IDENTIFIER" /> - <Property Id="AGENT_HEARTBEAT" Value="30 sec" /> - <Property Id="SERVER_PATH_BASE" Value="http://localhost:8181/api"; /> - <Property Id="SERVER_HEARTBEAT" Value="/c2-protocol/heartbeat" /> - <Property Id="SERVER_ACK" Value="/c2-protocol/acknowledge" /> - <Property Id="ENABLEC2" /> - <Property Id="AUTOSTART" Value="1" /> - <Property Id="SERVICEACCOUNT" Value="LocalSystem" /> - <Property Id="SERVICEACCOUNTPASSWORD" /> + <Property Id="AGENT_CLASS" Secure="yes" Value="Your Agent Class" /> + <Property Id="AGENT_IDENTIFIER" Secure="yes" /> + <Property Id="AGENT_HEARTBEAT" Secure="yes" Value="30 sec" /> + <Property Id="SERVER_PATH_BASE" Secure="yes" Value="http://localhost:8181/api"; /> + <Property Id="SERVER_HEARTBEAT" Secure="yes" Value="/c2-protocol/heartbeat" /> + <Property Id="SERVER_ACK" Secure="yes" Value="/c2-protocol/acknowledge" /> + <Property Id="ENABLEC2" Secure="yes" /> + <Property Id="AUTOSTART" Secure="yes" Value="1" /> + <Property Id="SERVICEACCOUNT" Secure="yes" Value="LocalSystem" /> + <Property Id="SERVICEACCOUNTPASSWORD" Secure="yes" /> <SetProperty Id="AGENT_IDENTIFIER" After="AppSearch" Value="[ComputerName]" Sequence="first" /> <SetProperty Id="ENABLEC2" After="AppSearch" Value="0" Sequence="first" > @@ -303,27 +302,22 @@ ${WIX_EXTRA_FEATURES} <File Id="UID" Source="conf/minifi-uid.properties" KeyPath="yes"/> </Component> - <Component Id="CONFIGFILE" Guid="87658309-0339-425c-8633-f54ffaaa4944"> - <File Id="CONFIG" Source="conf/config.yml" KeyPath="yes"/> - </Component> - - <Component Id="UpdateConfig" Guid="87658309-0339-425c-8633-f54ffaaa4945"> - - <File Source="conf/minifi.properties" Id="minifiprops" KeyPath="yes" /> - - <IniFile Id="ConfigFileA" Action="addLine" Name="minifi.properties" Directory="CONFIGDIR" Section="c2props" Key="nifi.c2.agent.class" Value="[AGENT_CLASS]" /> - <IniFile Id="ConfigFileI" Action="addLine" Name="minifi.properties" Directory="CONFIGDIR" Section="c2props" Key="nifi.c2.agent.identifier" Value="[AGENT_IDENTIFIER]" /> - <IniFile Id="ConfigFileE" Action="addLine" Name="minifi.properties" Directory="CONFIGDIR" Section="c2props" Key="nifi.c2.enable" Value="true" /> - <IniFile Id="ConfigFileT" Action="addLine" Name="minifi.properties" Directory="CONFIGDIR" Section="c2props" Key="nifi.c2.agent.heartbeat.period" Value="[AGENT_HEARTBEAT]" /> - <IniFile Id="ConfigFileS" Action="addLine" Name="minifi.properties" Directory="CONFIGDIR" Section="c2props" Key="nifi.c2.rest.path.base" Value="[SERVER_PATH_BASE]" /> - <IniFile Id="ConfigFileH" Action="addLine" Name="minifi.properties" Directory="CONFIGDIR" Section="c2props" Key="nifi.c2.rest.url" Value="[SERVER_HEARTBEAT]" /> - <IniFile Id="ConfigFileAck" Action="addLine" Name="minifi.properties" Directory="CONFIGDIR" Section="c2props" Key="nifi.c2.rest.url.ack" Value="[SERVER_ACK]" /> - <Condition><![CDATA[ENABLEC2="1"]]></Condition> - </Component> - <Component Id="UpdateConfigNotExist" Guid="87658309-0339-425c-8633-f54ffaaa4946"> + <Directory Id="MINIFIPROP.D" Name="minifi.properties.d"> + <Component Id="UpdateConfig" Guid="87658309-0339-425c-8633-f54ffaaa4945" Permanent="yes"> + <CreateFolder /> + <IniFile Id="ConfigFileA" Action="addLine" Name="10_installer.properties" Directory="MINIFIPROP.D" Section="c2props" Key="nifi.c2.agent.class" Value="[AGENT_CLASS]" /> + <IniFile Id="ConfigFileI" Action="addLine" Name="10_installer.properties" Directory="MINIFIPROP.D" Section="c2props" Key="nifi.c2.agent.identifier" Value="[AGENT_IDENTIFIER]" /> + <IniFile Id="ConfigFileE" Action="addLine" Name="10_installer.properties" Directory="MINIFIPROP.D" Section="c2props" Key="nifi.c2.enable" Value="true" /> + <IniFile Id="ConfigFileT" Action="addLine" Name="10_installer.properties" Directory="MINIFIPROP.D" Section="c2props" Key="nifi.c2.agent.heartbeat.period" Value="[AGENT_HEARTBEAT]" /> + <IniFile Id="ConfigFileS" Action="addLine" Name="10_installer.properties" Directory="MINIFIPROP.D" Section="c2props" Key="nifi.c2.rest.path.base" Value="[SERVER_PATH_BASE]" /> + <IniFile Id="ConfigFileH" Action="addLine" Name="10_installer.properties" Directory="MINIFIPROP.D" Section="c2props" Key="nifi.c2.rest.url" Value="[SERVER_HEARTBEAT]" /> + <IniFile Id="ConfigFileAck" Action="addLine" Name="10_installer.properties" Directory="MINIFIPROP.D" Section="c2props" Key="nifi.c2.rest.url.ack" Value="[SERVER_ACK]" /> + <Condition><![CDATA[ENABLEC2="1"]]></Condition> + </Component> + </Directory> - <File Source="conf/minifi.properties" Id="minifiprops2" KeyPath="yes" /> - <Condition><![CDATA[ENABLEC2<>"1"]]></Condition> + <Component Id="MINIFIPROP" Guid="87658309-0339-425c-8633-f54ffaaa4946"> + <File Id="PROPERTIES" Source="conf/minifi.properties" KeyPath="yes" /> </Component> </Directory> diff --git a/packaging/rpm/expected-rpm-contents.in b/packaging/rpm/expected-rpm-contents.in index cfdd8fa0b..b4eb8a37f 100644 --- a/packaging/rpm/expected-rpm-contents.in +++ b/packaging/rpm/expected-rpm-contents.in @@ -1,5 +1,4 @@ /etc/nifi-minifi-cpp -/etc/nifi-minifi-cpp/config.yml /etc/nifi-minifi-cpp/fips /etc/nifi-minifi-cpp/fips/openssl.cnf /etc/nifi-minifi-cpp/minifi-log.properties
