This is an automated email from the ASF dual-hosted git repository. fgerlits pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 8edef4414d42e3de4274c4234161fef54acf8900 Author: Gabor Gyimesi <[email protected]> AuthorDate: Mon Oct 27 15:00:05 2025 +0100 MINIFICPP-2649 Move expression language extension to libminifi Signed-off-by: Ferenc Gerlits <[email protected]> Closes #2047 --- .github/workflows/ci.yml | 3 - CMakeLists.txt | 3 +- bootstrap/system_dependency.py | 6 +- cmake/BuildTests.cmake | 1 + cmake/DockerConfig.cmake | 1 - .../ExpressionLanguage.cmake | 39 +-------- cmake/MiNiFiOptions.cmake | 1 - conanfile.py | 1 - .../include/core/ProcessContextBuilder.h | 78 ------------------ core-framework/src/core/ProcessContextBuilder.cpp | 67 --------------- extensions/aws/tests/CMakeLists.txt | 2 - extensions/azure/tests/CMakeLists.txt | 2 - .../ExpressionContextBuilder.cpp | 44 ---------- .../expression-language/ExpressionContextBuilder.h | 50 ------------ .../expression-language/ProcessContextExpr.cpp | 92 --------------------- .../expression-language/ProcessContextExpr.h | 54 ------------- .../expression-language/tests/CMakeLists.txt | 64 --------------- extensions/gcp/tests/CMakeLists.txt | 2 - extensions/gcp/tests/PutGCSObjectTests.cpp | 1 - .../libarchive/tests/CompressContentTests.cpp | 2 +- extensions/libarchive/tests/MergeFileTests.cpp | 2 +- extensions/rocksdb-repos/tests/RepoTests.cpp | 2 +- extensions/rocksdb-repos/tests/SwapTests.cpp | 2 +- extensions/sftp/processors/PutSFTP.cpp | 2 +- extensions/sftp/processors/SFTPProcessorBase.cpp | 2 +- extensions/sftp/tests/CMakeLists.txt | 3 +- extensions/sql/tests/CMakeLists.txt | 1 - .../processors/EvaluateJsonPath.cpp | 2 +- .../standard-processors/processors/SplitJson.cpp | 2 +- .../standard-processors/tests/CMakeLists.txt | 2 - .../ExpressionLanguageInDynamicPropertiesTests.cpp | 94 ++++++++++++++++++++++ .../tests/unit/GetFileTests.cpp | 2 +- .../tests/unit/ProcessorTests.cpp | 2 +- .../tests/unit}/RouteOnAttributeTests.cpp | 0 libminifi/CMakeLists.txt | 17 ++-- .../include/core/ProcessContextImpl.h | 5 ++ .../include}/expression-language/Driver.h | 2 +- .../include/expression-language}/Expression.h | 4 +- .../include}/expression-language/Parser.yy | 6 +- .../include}/expression-language/Scanner.ll | 2 +- .../include/expression-language}/Value.h | 0 libminifi/src/ThreadedSchedulingAgent.cpp | 10 +-- libminifi/src/c2/C2Agent.cpp | 2 +- libminifi/src/core/ClassLoader.cpp | 8 +- libminifi/src/core/ProcessContextBuilder.cpp | 24 ------ .../{ProcessContext.cpp => ProcessContextImpl.cpp} | 62 ++++++++++++-- .../src}/expression-language/Expression.cpp | 4 +- libminifi/test/flow-tests/SessionTests.cpp | 2 +- .../integration/UpdateAttributeIntegrationTest.cpp | 4 +- .../test/libtest/integration/IntegrationBase.cpp | 2 +- libminifi/test/libtest/unit/TestBase.cpp | 10 +-- .../test/persistence-tests/PersistenceTests.cpp | 2 +- .../test/unit}/ExpressionLanguageTests.cpp | 74 +---------------- .../test/unit}/ProcessContextExprTests.cpp | 9 +-- libminifi/test/unit/ProcessorConfigUtilsTests.cpp | 6 +- libminifi/test/unit/SchedulingAgentTests.cpp | 2 +- .../minifi-cpp/core/ProcessContextBuilder.h | 48 ----------- 57 files changed, 212 insertions(+), 724 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 321eb0e4e..7c2263b06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,6 @@ jobs: -DENABLE_COUCHBASE=ON -DENABLE_ELASTICSEARCH=ON -DENABLE_ENCRYPT_CONFIG=ON - -DENABLE_EXPRESSION_LANGUAGE=ON -DENABLE_GCP=ON -DENABLE_KUBERNETES=ON -DENABLE_LIBARCHIVE=ON @@ -135,7 +134,6 @@ jobs: -DENABLE_COVERAGE= -DENABLE_ELASTICSEARCH=ON -DENABLE_ENCRYPT_CONFIG=ON - -DENABLE_EXPRESSION_LANGUAGE=ON -DENABLE_GCP=ON -DENABLE_GRAFANA_LOKI=ON -DENABLE_KUBERNETES=ON @@ -239,7 +237,6 @@ jobs: -DENABLE_COUCHBASE=ON -DENABLE_ELASTICSEARCH=ON -DENABLE_ENCRYPT_CONFIG=ON - -DENABLE_EXPRESSION_LANGUAGE=ON -DENABLE_EXECUTE_PROCESS=ON -DENABLE_GCP=ON -DENABLE_GRAFANA_LOKI=ON diff --git a/CMakeLists.txt b/CMakeLists.txt index 5cf1c4e5f..77c1b74cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -756,7 +756,6 @@ set(EXTENSIONS_ENABLED_BY_DEFAULT ( minifi-azure minifi-civet-extensions minifi-elasticsearch - minifi-expression-language-extensions minifi-gcp minifi-grafana-loki minifi-archive-extensions @@ -852,7 +851,7 @@ if (MINIFI_ADVANCED_CODE_COVERAGE) setup_target_for_coverage_gcovr_html( NAME coverage EXCLUDE "build/*" "cmake/*" "minifi_main/*" "thirdparty/*" "libminifi/test/*" "encrypt-config/tests/*" "extensions/aws/tests/*" - "extensions/civetweb/tests/*" "extensions/elasticsearch/tests/*" "extensions/expression-language/tests/*" "extensions/gcp/tests/*" "extensions/grafana-loki/tests/*" + "extensions/civetweb/tests/*" "extensions/elasticsearch/tests/*" "extensions/gcp/tests/*" "extensions/grafana-loki/tests/*" "extensions/kubernetes/tests/*" "extensions/kafka/tests/*" "extensions/lua/tests/*" "extensions/mqtt/tests/*" "extensions/opencv/tests/*" "extensions/procfs/tests/*" "extensions/prometheus/tests/*" "extensions/script/tests/*" "extensions/sftp/tests/*" "extensions/splunk/tests/*" "extensions/standard-processors/tests/*" "extensions/systemd/tests/*" "extensions/test-processors/*" "controller/MiNiFiController.cpp" diff --git a/bootstrap/system_dependency.py b/bootstrap/system_dependency.py index 2a3b39386..cc97894f7 100644 --- a/bootstrap/system_dependency.py +++ b/bootstrap/system_dependency.py @@ -23,11 +23,7 @@ import platform def _create_system_dependencies(minifi_options: MinifiOptions) -> Dict[str, Set[str]]: - system_dependencies = {'patch': {'patch'}, 'make': {'make'}, 'perl': {'perl'}, 'git': {'git'}} - if minifi_options.is_enabled("ENABLE_EXPRESSION_LANGUAGE"): - system_dependencies['bison'] = {'bison'} - system_dependencies['flex'] = {'flex'} - system_dependencies['m4'] = {'m4'} + system_dependencies = {'patch': {'patch'}, 'make': {'make'}, 'perl': {'perl'}, 'git': {'git'}, 'bison': {'bison'}, 'flex': {'flex'}, 'm4': {'m4'}} if minifi_options.is_enabled("ENABLE_LIBARCHIVE"): system_dependencies['libarchive'] = {'libarchive'} if minifi_options.is_enabled("ENABLE_SQL"): diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake index 6254fe171..878628230 100644 --- a/cmake/BuildTests.cmake +++ b/cmake/BuildTests.cmake @@ -60,6 +60,7 @@ function(appendIncludes testName) target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/core/statemanagement/metrics") target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/core/parameter-providers") target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/io") + target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/expression-language") if(WIN32) target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/opsys/win") target_include_directories(${testName} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/opsys/win/io") diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake index 4e42ac3c0..b140ef724 100644 --- a/cmake/DockerConfig.cmake +++ b/cmake/DockerConfig.cmake @@ -71,7 +71,6 @@ add_custom_target( -DENABLE_TEST_PROCESSORS=OFF -DENABLE_ROCKSDB=ON -DENABLE_LIBARCHIVE=ON - -DENABLE_EXPRESSION_LANGUAGE=ON -DENABLE_LZMA=ON -DENABLE_BZIP2=ON -DCI_BUILD=${CI_BUILD}\" diff --git a/extensions/expression-language/CMakeLists.txt b/cmake/ExpressionLanguage.cmake similarity index 74% rename from extensions/expression-language/CMakeLists.txt rename to cmake/ExpressionLanguage.cmake index 9a28e0395..062663709 100644 --- a/extensions/expression-language/CMakeLists.txt +++ b/cmake/ExpressionLanguage.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,13 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# - -if (NOT ENABLE_EXPRESSION_LANGUAGE) - return() -endif() - -message("minifi-expression-language-extensions will depend on curl-external") if(WIN32) include(FetchContent) @@ -96,13 +88,13 @@ find_package(FLEX REQUIRED) bison_target( el-parser - ${CMAKE_CURRENT_SOURCE_DIR}/Parser.yy + ${CMAKE_SOURCE_DIR}/libminifi/include/expression-language/Parser.yy ${CMAKE_BINARY_DIR}/el-generated/Parser.cpp ) flex_target( el-scanner - ${CMAKE_CURRENT_SOURCE_DIR}/Scanner.ll + ${CMAKE_SOURCE_DIR}/libminifi/include/expression-language/Scanner.ll ${CMAKE_BINARY_DIR}/el-generated/Scanner.cpp COMPILE_FLAGS --c++ ) @@ -112,22 +104,6 @@ file(MAKE_DIRECTORY ${EL_GENERATED_INCLUDE_DIR}) add_flex_bison_dependency(el-scanner el-parser) -include_directories(./ ../../libminifi/include ../../libminifi/include/core) -include_directories(SYSTEM ../../thirdparty/) -include_directories(common) -include_directories(impl) - -if(WIN32) - include_directories(../../libminifi/opsys/win) - set(SOCKET_SOURCES "src/io/win/*.cpp") -else() - include_directories(../../libminifi/opsys/posix) - set(SOCKET_SOURCES "src/io/posix/*.cpp") -endif() - - -file(GLOB SOURCES "*.cpp") - if (NOT WIN32) set_source_files_properties(${BISON_el-parser_OUTPUTS} PROPERTIES COMPILE_FLAGS -Wno-error) set_source_files_properties(${FLEX_el-scanner_OUTPUTS} PROPERTIES COMPILE_FLAGS -Wno-error) @@ -135,14 +111,3 @@ else() set_source_files_properties(${BISON_el-parser_OUTPUTS} PROPERTIES COMPILE_FLAGS /wd4244) set_source_files_properties(${FLEX_el-scanner_OUTPUTS} PROPERTIES COMPILE_FLAGS /wd4244) endif() - -add_minifi_library(minifi-expression-language-extensions SHARED ${SOURCES} ${BISON_el-parser_OUTPUTS} ${FLEX_el-scanner_OUTPUTS}) - -target_include_directories(minifi-expression-language-extensions SYSTEM PRIVATE ${EL_GENERATED_INCLUDE_DIR}) -target_link_libraries(minifi-expression-language-extensions ${LIBMINIFI}) -target_link_libraries(minifi-expression-language-extensions RapidJSON) -if (BREW_FLEX_INCLUDE) - target_include_directories(minifi-expression-language-extensions SYSTEM PRIVATE ${BREW_FLEX_INCLUDE}) -endif() - -register_extension(minifi-expression-language-extensions "EXPRESSION LANGUAGE EXTENSIONS" EXPRESSION-LANGUAGE-EXTENSIONS "This enables NiFi expression language" "extensions/expression-language/tests") diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index ee5633269..da543b1e6 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -88,7 +88,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") endif() add_minifi_option(ENABLE_ALL "Enables all extensions" OFF) -add_minifi_option(ENABLE_EXPRESSION_LANGUAGE "Enables expression language." ON) add_minifi_option(ENABLE_CIVET "Enables CivetWeb components." ON) add_minifi_option(ENABLE_ROCKSDB "Enables the RocksDB extension." ON) add_minifi_option(ENABLE_LIBARCHIVE "Enables the lib archive extensions." ON) diff --git a/conanfile.py b/conanfile.py index 318a9f941..360c35066 100644 --- a/conanfile.py +++ b/conanfile.py @@ -42,7 +42,6 @@ class MiNiFiCppMain(ConanFile): tc.variables["SKIP_TESTS"] = "OFF" tc.variables["ENABLE_CIVET"] = "ON" - tc.variables["ENABLE_EXPRESSION_LANGUAGE"] = "ON" tc.variables["ENABLE_LIBARCHIVE"] = "OFF" tc.variables["ENABLE_AWS"] = "OFF" tc.variables["ENABLE_SQL"] = "OFF" diff --git a/core-framework/include/core/ProcessContextBuilder.h b/core-framework/include/core/ProcessContextBuilder.h deleted file mode 100644 index 3b6cbf517..000000000 --- a/core-framework/include/core/ProcessContextBuilder.h +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include <string> -#include <vector> -#include <queue> -#include <map> -#include <mutex> -#include <atomic> -#include <algorithm> -#include <memory> -#include "minifi-cpp/core/Property.h" -#include "core/Core.h" -#include "utils/Id.h" -#include "minifi-cpp/core/ContentRepository.h" -#include "core/logging/LoggerFactory.h" -#include "ProcessContext.h" -#include "minifi-cpp/core/FlowFile.h" -#include "minifi-cpp/core/ProcessContextBuilder.h" -#include "utils/GeneralUtils.h" - -namespace org::apache::nifi::minifi::core { -/** - * Could use instantiate<T> from core, which uses a simple compile time check to figure out if a destructor is defined - * and thus that will allow us to know if the context instance exists, but I like using the build because it allows us - * to eventually share the builder across different contexts and shares up the construction ever so slightly. - * - * While this incurs a tiny cost to look up, it allows us to have a replaceable builder that erases the type we are - * constructing. - */ -class ProcessContextBuilderImpl : public core::CoreComponentImpl, public virtual ProcessContextBuilder { - public: - ProcessContextBuilderImpl(std::string_view name, const minifi::utils::Identifier &uuid); - - explicit ProcessContextBuilderImpl(std::string_view name); - - ~ProcessContextBuilderImpl() override = default; - - MINIFIAPI static constexpr auto Properties = std::array<core::PropertyReference, 0>{}; - MINIFIAPI static constexpr bool SupportsDynamicProperties = false; - MINIFIAPI static constexpr bool SupportsDynamicRelationships = false; - - std::shared_ptr<ProcessContextBuilder> withProvider(core::controller::ControllerServiceProvider* controller_service_provider) override; - - std::shared_ptr<ProcessContextBuilder> withProvenanceRepository(const std::shared_ptr<core::Repository> &repo) override; - - std::shared_ptr<ProcessContextBuilder> withFlowFileRepository(const std::shared_ptr<core::Repository> &repo) override; - - std::shared_ptr<ProcessContextBuilder> withContentRepository(const std::shared_ptr<core::ContentRepository> &repo) override; - - std::shared_ptr<ProcessContextBuilder> withConfiguration(const std::shared_ptr<minifi::Configure> &configuration) override; - - std::shared_ptr<core::ProcessContext> build(Processor& processor) override; - - protected: - std::shared_ptr<minifi::Configure> configuration_; - core::controller::ControllerServiceProvider* controller_service_provider_ = nullptr; - std::shared_ptr<core::Repository> prov_repo_; - std::shared_ptr<core::Repository> flow_repo_; - std::shared_ptr<core::ContentRepository> content_repo_; -}; - -} // namespace org::apache::nifi::minifi::core diff --git a/core-framework/src/core/ProcessContextBuilder.cpp b/core-framework/src/core/ProcessContextBuilder.cpp deleted file mode 100644 index 79b2c57d4..000000000 --- a/core-framework/src/core/ProcessContextBuilder.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "core/ProcessContextBuilder.h" -#include <memory> -#include <string> -#include "core/logging/LoggerFactory.h" -#include "core/Resource.h" -#include "minifi-cpp/core/repository/FileSystemRepository.h" - -namespace org::apache::nifi::minifi::core { - -ProcessContextBuilderImpl::ProcessContextBuilderImpl(std::string_view name, const minifi::utils::Identifier &uuid) - : core::CoreComponentImpl(name, uuid), - configuration_{minifi::Configure::create()}, - content_repo_{repository::createFileSystemRepository()} -{} - -ProcessContextBuilderImpl::ProcessContextBuilderImpl(std::string_view name) - : core::CoreComponentImpl(name), - configuration_{minifi::Configure::create()}, - content_repo_{repository::createFileSystemRepository()} -{} - -std::shared_ptr<ProcessContextBuilder> ProcessContextBuilderImpl::withProvider(core::controller::ControllerServiceProvider* controller_service_provider) { - controller_service_provider_ = controller_service_provider; - return sharedFromThis<ProcessContextBuilder>(); -} - -std::shared_ptr<ProcessContextBuilder> ProcessContextBuilderImpl::withProvenanceRepository(const std::shared_ptr<core::Repository> &repo) { - prov_repo_ = repo; - return sharedFromThis<ProcessContextBuilder>(); -} - -std::shared_ptr<ProcessContextBuilder> ProcessContextBuilderImpl::withFlowFileRepository(const std::shared_ptr<core::Repository> &repo) { - flow_repo_ = repo; - return sharedFromThis<ProcessContextBuilder>(); -} - -std::shared_ptr<ProcessContextBuilder> ProcessContextBuilderImpl::withContentRepository(const std::shared_ptr<core::ContentRepository> &repo) { - content_repo_ = repo; - return sharedFromThis<ProcessContextBuilder>(); -} - -std::shared_ptr<ProcessContextBuilder> ProcessContextBuilderImpl::withConfiguration(const std::shared_ptr<minifi::Configure> &configuration) { - configuration_ = configuration; - return sharedFromThis<ProcessContextBuilder>(); -} - -std::shared_ptr<core::ProcessContext> ProcessContextBuilderImpl::build(Processor& processor) { - return std::make_shared<core::ProcessContextImpl>(processor, controller_service_provider_, prov_repo_, flow_repo_, configuration_, content_repo_); -} - -} // namespace org::apache::nifi::minifi::core diff --git a/extensions/aws/tests/CMakeLists.txt b/extensions/aws/tests/CMakeLists.txt index 542211aee..3cdbec8c8 100644 --- a/extensions/aws/tests/CMakeLists.txt +++ b/extensions/aws/tests/CMakeLists.txt @@ -27,12 +27,10 @@ FOREACH(testfile ${AWS_INTEGRATION_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/aws") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/aws/s3") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/aws/processors") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/expression-language") createTests("${testfilename}") target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-aws) target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries(${testfilename} minifi-expression-language-extensions) MATH(EXPR AWS_TEST_COUNT "${AWS_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) set_tests_properties("${testfilename}" PROPERTIES LABELS "aws") diff --git a/extensions/azure/tests/CMakeLists.txt b/extensions/azure/tests/CMakeLists.txt index 55ec534a2..ce006f5f7 100644 --- a/extensions/azure/tests/CMakeLists.txt +++ b/extensions/azure/tests/CMakeLists.txt @@ -26,13 +26,11 @@ FOREACH(testfile ${AZURE_INTEGRATION_TESTS}) target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/azure") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/azure/storage") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/expression-language") target_compile_features(${testfilename} PRIVATE cxx_std_14) createTests("${testfilename}") target_link_libraries(${testfilename} Catch2WithMain) target_link_libraries(${testfilename} minifi-azure) target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries(${testfilename} minifi-expression-language-extensions) MATH(EXPR AZURE_TEST_COUNT "${AZURE_TEST_COUNT}+1") add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) set_tests_properties("${testfilename}" PROPERTIES LABELS "azure;memchecked") diff --git a/extensions/expression-language/ExpressionContextBuilder.cpp b/extensions/expression-language/ExpressionContextBuilder.cpp deleted file mode 100644 index e033e6cdd..000000000 --- a/extensions/expression-language/ExpressionContextBuilder.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ExpressionContextBuilder.h" - -#include <memory> -#include <string> - -#include "ProcessContextExpr.h" -#include "core/Resource.h" - -namespace org::apache::nifi::minifi::core::expressions { - -ExpressionContextBuilder::ExpressionContextBuilder(std::string_view name, const minifi::utils::Identifier &uuid) - : core::ProcessContextBuilderImpl(name, uuid) { -} - -ExpressionContextBuilder::ExpressionContextBuilder(std::string_view name) - : core::ProcessContextBuilderImpl(name) { -} - -ExpressionContextBuilder::~ExpressionContextBuilder() = default; - -std::shared_ptr<core::ProcessContext> ExpressionContextBuilder::build(Processor& processor) { - return std::make_shared<core::ProcessContextExpr>(processor, controller_service_provider_, prov_repo_, flow_repo_, configuration_, content_repo_); -} - -REGISTER_RESOURCE_AS(ExpressionContextBuilder, InternalResource, ("ProcessContextBuilder")); - -} // namespace org::apache::nifi::minifi::core::expressions diff --git a/extensions/expression-language/ExpressionContextBuilder.h b/extensions/expression-language/ExpressionContextBuilder.h deleted file mode 100644 index 5f85ec0e8..000000000 --- a/extensions/expression-language/ExpressionContextBuilder.h +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <string> -#include <memory> -#include <utility> - -#include "core/ProcessContextBuilder.h" - -namespace org::apache::nifi::minifi::core::expressions { - -/** - * Purpose: Creates a context builder that can be used by the class loader to inject EL functionality - * - * Justification: Linking became problematic across platforms since EL was used as a carrier of what was - * effectively core functionality. To eliminate this awkward linking and help with disabling EL entirely - * on some platforms, this builder was placed into the class loader. - */ -class ExpressionContextBuilder : public core::ProcessContextBuilderImpl { - public: - ExpressionContextBuilder(std::string_view name, const minifi::utils::Identifier &uuid); - - explicit ExpressionContextBuilder(std::string_view name); - - ~ExpressionContextBuilder() override; - - EXTENSIONAPI static constexpr auto Properties = std::array<core::PropertyReference, 0>{}; - EXTENSIONAPI static constexpr bool SupportsDynamicProperties = false; - EXTENSIONAPI static constexpr bool SupportsDynamicRelationships = false; - - std::shared_ptr<core::ProcessContext> build(Processor& processor) override; -}; - -} // namespace org::apache::nifi::minifi::core::expressions diff --git a/extensions/expression-language/ProcessContextExpr.cpp b/extensions/expression-language/ProcessContextExpr.cpp deleted file mode 100644 index 71a748b35..000000000 --- a/extensions/expression-language/ProcessContextExpr.cpp +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ProcessContextExpr.h" - -#include <memory> -#include <string> - -#include "asio/detail/mutex.hpp" -#include "utils/PropertyErrors.h" - -namespace org::apache::nifi::minifi::core { - -nonstd::expected<std::string, std::error_code> ProcessContextExpr::getProperty(const std::string_view name, const FlowFile* flow_file) const { - std::lock_guard<std::mutex> lock(mutex_); - const auto property = getProcessorInfo().getSupportedProperty(name); - if (!property) { - return nonstd::make_unexpected(PropertyErrorCode::NotSupportedProperty); - } - - if (!property->supportsExpressionLanguage()) { - return ProcessContextImpl::getProperty(name, flow_file); - } - if (!cached_expressions_.contains(name)) { - auto expression_str = ProcessContextImpl::getProperty(name, flow_file); - if (!expression_str) { return expression_str; } - cached_expressions_.emplace(std::string{name}, expression::compile(*expression_str)); - } - expression::Parameters p(this, flow_file); - auto result = cached_expressions_[std::string{name}](p).asString(); - if (!property->getValidator().validate(result)) { - return nonstd::make_unexpected(PropertyErrorCode::ValidationFailed); - } - return result; -} - -nonstd::expected<std::string, std::error_code> ProcessContextExpr::getDynamicProperty(const std::string_view name, const FlowFile* flow_file) const { - std::lock_guard<std::mutex> lock(mutex_); - if (!cached_dynamic_expressions_.contains(name)) { - auto expression_str = ProcessContextImpl::getDynamicProperty(name, flow_file); - if (!expression_str) { return expression_str; } - cached_dynamic_expressions_.emplace(std::string{name}, expression::compile(*expression_str)); - } - const expression::Parameters p(this, flow_file); - return cached_dynamic_expressions_[std::string{name}](p).asString(); -} - -nonstd::expected<void, std::error_code> ProcessContextExpr::setProperty(const std::string_view name, std::string value) { - std::lock_guard<std::mutex> lock(mutex_); - cached_expressions_.erase(std::string{name}); - return ProcessContextImpl::setProperty(name, std::move(value)); -} - -nonstd::expected<void, std::error_code> ProcessContextExpr::setDynamicProperty(std::string name, std::string value) { - std::lock_guard<std::mutex> lock(mutex_); - cached_dynamic_expressions_.erase(name); - return ProcessContextImpl::setDynamicProperty(std::move(name), std::move(value)); -} - -std::map<std::string, std::string> ProcessContextExpr::getDynamicProperties(const FlowFile* flow_file) const { - std::lock_guard<std::mutex> lock(mutex_); - auto dynamic_props = ProcessContextImpl::getDynamicProperties(flow_file); - const expression::Parameters params{this, flow_file}; - for (auto& [dynamic_property_name, dynamic_property_value]: dynamic_props) { - auto cached_dyn_expr_it = cached_dynamic_expressions_.find(dynamic_property_name); - if (cached_dyn_expr_it == cached_dynamic_expressions_.end()) { - auto expression = expression::compile(dynamic_property_value); - const auto [it, success] = cached_dynamic_expressions_.emplace(dynamic_property_name, expression); - gsl_Assert(success && "getDynamicProperties: no element with the key existed, yet insertion failed"); - cached_dyn_expr_it = it; - } - auto& expression = cached_dyn_expr_it->second; - dynamic_property_value = expression(params).asString(); - } - return dynamic_props; -} - -} // namespace org::apache::nifi::minifi::core diff --git a/extensions/expression-language/ProcessContextExpr.h b/extensions/expression-language/ProcessContextExpr.h deleted file mode 100644 index 9e69e332e..000000000 --- a/extensions/expression-language/ProcessContextExpr.h +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include <memory> -#include <unordered_map> -#include <string> - -#include "core/ProcessContext.h" -#include "impl/expression/Expression.h" - -namespace org::apache::nifi::minifi::core { - -/** - * Purpose and Justification: Used to inject EL functionality into the classloader and remove EL as - * a core functionality. This created linking issues as it was always required even in a disabled - * state. With this case, we can rely on instantiation of a builder to create the necessary - * ProcessContext. * - */ -class ProcessContextExpr final : public core::ProcessContextImpl { - public: - using ProcessContextImpl::ProcessContextImpl; - ~ProcessContextExpr() override = default; - - nonstd::expected<std::string, std::error_code> getProperty(std::string_view name, const FlowFile*) const override; - nonstd::expected<std::string, std::error_code> getDynamicProperty(std::string_view name, const FlowFile*) const override; - - nonstd::expected<void, std::error_code> setProperty(std::string_view name, std::string value) override; - nonstd::expected<void, std::error_code> setDynamicProperty(std::string name, std::string value) override; - - std::map<std::string, std::string> getDynamicProperties(const FlowFile*) const override; - - private: - mutable std::mutex mutex_; - mutable std::unordered_map<std::string, expression::Expression, utils::string::transparent_string_hash, std::equal_to<>> cached_expressions_; - mutable std::unordered_map<std::string, expression::Expression, utils::string::transparent_string_hash, std::equal_to<>> cached_dynamic_expressions_; -}; - -} // namespace org::apache::nifi::minifi::core diff --git a/extensions/expression-language/tests/CMakeLists.txt b/extensions/expression-language/tests/CMakeLists.txt deleted file mode 100644 index d01c18b65..000000000 --- a/extensions/expression-language/tests/CMakeLists.txt +++ /dev/null @@ -1,64 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - - -file(GLOB EXPRESSION_LANGUAGE_TESTS "*.cpp") - -SET(EXTENSIONS_TEST_COUNT 0) -FOREACH(testfile ${EXPRESSION_LANGUAGE_TESTS}) - get_filename_component(testfilename "${testfile}" NAME_WE) - add_minifi_executable(${testfilename} "${testfile}") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language") - createTests(${testfilename}) - target_link_libraries(${testfilename} Catch2WithMain) - target_link_libraries(${testfilename} minifi-expression-language-extensions) - target_link_libraries(${testfilename} minifi-standard-processors) - target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") - - MATH(EXPR EXTENSIONS_TEST_COUNT "${EXTENSIONS_TEST_COUNT}+1") - add_test(NAME ${testfilename} COMMAND ${testfilename} WORKING_DIRECTORY ${TEST_DIR}) - set_tests_properties("${testfilename}" PROPERTIES LABELS "expression-language;memchecked") -ENDFOREACH() - - -### integration tests - -file(GLOB INT_EXPRESSION_LANGUAGE_TESTS "integration/*.cpp") - -SET(INT_EXTENSIONS_TEST_COUNT 0) - -FOREACH(testfile ${INT_EXPRESSION_LANGUAGE_TESTS}) - get_filename_component(testfilename "${testfile}" NAME_WE) - add_minifi_executable(${testfilename} "${testfile}") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/standard-processors/processors") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language") - createIntegrationTests(${testfilename}) - target_link_libraries(${testfilename} Catch2WithMain) - target_link_libraries(${testfilename} minifi-expression-language-extensions) - target_link_libraries(${testfilename} minifi-standard-processors) - target_compile_definitions(${testfilename} PRIVATE TEST_RESOURCES="${TEST_RESOURCES}") - add_test(NAME "${testfilename}" COMMAND "${testfilename}") - MATH(EXPR EXTENSIONS_TEST_COUNT "${INT_EXTENSIONS_TEST_COUNT}+1") -ENDFOREACH() - -message("-- Finished building ${EXTENSIONS_TEST_COUNT} expression language related test file(s)...") diff --git a/extensions/gcp/tests/CMakeLists.txt b/extensions/gcp/tests/CMakeLists.txt index 398a66eac..0acc5241b 100644 --- a/extensions/gcp/tests/CMakeLists.txt +++ b/extensions/gcp/tests/CMakeLists.txt @@ -28,7 +28,6 @@ FOREACH(testfile ${GCS_TESTS}) endif() target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/standard-processors") - target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/expression-language") target_include_directories(${testfilename} PRIVATE BEFORE "${CMAKE_SOURCE_DIR}/extensions/gcp") target_include_directories(${testfilename} SYSTEM BEFORE PRIVATE ${googletest_INCLUDE_DIRS}) @@ -38,7 +37,6 @@ FOREACH(testfile ${GCS_TESTS}) target_link_libraries(${testfilename} minifi-gcp) target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} gtest_main gmock) gtest_add_tests(TARGET "${testfilename}") diff --git a/extensions/gcp/tests/PutGCSObjectTests.cpp b/extensions/gcp/tests/PutGCSObjectTests.cpp index ee7c26ca3..a8355b50f 100644 --- a/extensions/gcp/tests/PutGCSObjectTests.cpp +++ b/extensions/gcp/tests/PutGCSObjectTests.cpp @@ -19,7 +19,6 @@ #include "GCPAttributes.h" #include "core/Resource.h" #include "unit/SingleProcessorTestController.h" -#include "ProcessContextExpr.h" #include "google/cloud/storage/testing/mock_client.h" #include "google/cloud/storage/internal/object_metadata_parser.h" #include "google/cloud/storage/retry_policy.h" diff --git a/extensions/libarchive/tests/CompressContentTests.cpp b/extensions/libarchive/tests/CompressContentTests.cpp index 693b4915a..206c717fc 100644 --- a/extensions/libarchive/tests/CompressContentTests.cpp +++ b/extensions/libarchive/tests/CompressContentTests.cpp @@ -32,7 +32,7 @@ #include "catch2/generators/catch_generators.hpp" #include "core/Core.h" #include "minifi-cpp/core/FlowFile.h" -#include "core/ProcessContext.h" +#include "core/ProcessContextImpl.h" #include "core/ProcessSession.h" #include "core/ProcessSessionFactory.h" #include "core/Processor.h" diff --git a/extensions/libarchive/tests/MergeFileTests.cpp b/extensions/libarchive/tests/MergeFileTests.cpp index 3da5b84df..7b19e5fc9 100644 --- a/extensions/libarchive/tests/MergeFileTests.cpp +++ b/extensions/libarchive/tests/MergeFileTests.cpp @@ -28,7 +28,7 @@ #include "core/Relationship.h" #include "core/Core.h" #include "core/Processor.h" -#include "core/ProcessContext.h" +#include "core/ProcessContextImpl.h" #include "core/ProcessSession.h" #include "core/ProcessSessionFactory.h" #include "FlowController.h" diff --git a/extensions/rocksdb-repos/tests/RepoTests.cpp b/extensions/rocksdb-repos/tests/RepoTests.cpp index 138cc7954..275e4ba27 100644 --- a/extensions/rocksdb-repos/tests/RepoTests.cpp +++ b/extensions/rocksdb-repos/tests/RepoTests.cpp @@ -23,6 +23,7 @@ #include <optional> #include "core/Core.h" +#include "core/ProcessContextImpl.h" #include "core/repository/AtomicRepoEntries.h" #include "core/RepositoryFactory.h" #include "minifi-cpp/FlowFileRecord.h" @@ -41,7 +42,6 @@ #include "core/repository/FileSystemRepository.h" #include "core/Processor.h" #include "Connection.h" -#include "core/ProcessContext.h" #include "ResourceClaim.h" using namespace std::literals::chrono_literals; diff --git a/extensions/rocksdb-repos/tests/SwapTests.cpp b/extensions/rocksdb-repos/tests/SwapTests.cpp index 3c20b60ba..af7707c58 100644 --- a/extensions/rocksdb-repos/tests/SwapTests.cpp +++ b/extensions/rocksdb-repos/tests/SwapTests.cpp @@ -23,6 +23,7 @@ #include "io/StreamPipe.h" #include "unit/TestUtils.h" #include "core/ProcessorImpl.h" +#include "core/ProcessContextImpl.h" #include "core/ProcessSession.h" #include "core/ProcessSessionFactory.h" #include "minifi-cpp/core/PropertyDefinition.h" @@ -30,7 +31,6 @@ #include "unit/ProvenanceTestHelper.h" #include "core/repository/FileSystemRepository.h" #include "Connection.h" -#include "core/ProcessContext.h" namespace org::apache::nifi::minifi::test { diff --git a/extensions/sftp/processors/PutSFTP.cpp b/extensions/sftp/processors/PutSFTP.cpp index 4554f982e..b04feb457 100644 --- a/extensions/sftp/processors/PutSFTP.cpp +++ b/extensions/sftp/processors/PutSFTP.cpp @@ -28,11 +28,11 @@ #include "core/Resource.h" #include "minifi-cpp/core/logging/Logger.h" #include "io/BufferStream.h" +#include "io/validation.h" #include "utils/ConfigurationUtils.h" #include "utils/StringUtils.h" #include "utils/file/FileUtils.h" #include "utils/ProcessorConfigUtils.h" -#include "io/validation.h" namespace org::apache::nifi::minifi::processors { diff --git a/extensions/sftp/processors/SFTPProcessorBase.cpp b/extensions/sftp/processors/SFTPProcessorBase.cpp index c06cf285d..1bcf5c585 100644 --- a/extensions/sftp/processors/SFTPProcessorBase.cpp +++ b/extensions/sftp/processors/SFTPProcessorBase.cpp @@ -37,10 +37,10 @@ #include "core/Relationship.h" #include "minifi-cpp/core/logging/Logger.h" #include "io/BufferStream.h" +#include "io/validation.h" #include "utils/ByteArrayCallback.h" #include "utils/StringUtils.h" #include "utils/ProcessorConfigUtils.h" -#include "io/validation.h" namespace org::apache::nifi::minifi::processors { diff --git a/extensions/sftp/tests/CMakeLists.txt b/extensions/sftp/tests/CMakeLists.txt index a8767d7bc..a285ae514 100644 --- a/extensions/sftp/tests/CMakeLists.txt +++ b/extensions/sftp/tests/CMakeLists.txt @@ -23,7 +23,7 @@ find_package(Maven) message(STATUS "JAVA_HOME: '$ENV{JAVA_HOME}'") message(STATUS "MAVEN: ${MAVEN_EXECUTABLE}") -if (NOT SKIP_TESTS AND Java_FOUND AND Maven_FOUND AND ENABLE_EXPRESSION_LANGUAGE) +if (NOT SKIP_TESTS AND Java_FOUND AND Maven_FOUND) file(GLOB SFTP_INTEGRATION_TESTS "*.cpp") SET(SFTP-EXTENSIONS_TEST_COUNT 0) FOREACH(testfile ${SFTP_INTEGRATION_TESTS}) @@ -42,7 +42,6 @@ if (NOT SKIP_TESTS AND Java_FOUND AND Maven_FOUND AND ENABLE_EXPRESSION_LANGUAGE createTests("${testfilename}") target_link_libraries(${testfilename} Catch2WithMain core-minifi sftp-test-tools) target_link_libraries(${testfilename} minifi-sftp) - target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) if (ENABLE_ROCKSDB) target_link_libraries(${testfilename} minifi-rocksdb-repos) diff --git a/extensions/sql/tests/CMakeLists.txt b/extensions/sql/tests/CMakeLists.txt index 81184f066..33af393ac 100644 --- a/extensions/sql/tests/CMakeLists.txt +++ b/extensions/sql/tests/CMakeLists.txt @@ -38,7 +38,6 @@ foreach(testfile ${SQL_TESTS}) target_link_libraries(${testfilename} minifi-sql-mocks) target_link_libraries(${testfilename} minifi-sql) target_link_libraries(${testfilename} minifi-standard-processors) - target_link_libraries(${testfilename} minifi-expression-language-extensions) add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR}) math(EXPR SQL_TEST_COUNT "${SQL_TEST_COUNT}+1") diff --git a/extensions/standard-processors/processors/EvaluateJsonPath.cpp b/extensions/standard-processors/processors/EvaluateJsonPath.cpp index 11533abf3..9273cee6f 100644 --- a/extensions/standard-processors/processors/EvaluateJsonPath.cpp +++ b/extensions/standard-processors/processors/EvaluateJsonPath.cpp @@ -19,7 +19,7 @@ #include <unordered_map> #include "core/ProcessSession.h" -#include "core/ProcessContext.h" +#include "minifi-cpp/core/ProcessContext.h" #include "core/Resource.h" #include "utils/ProcessorConfigUtils.h" diff --git a/extensions/standard-processors/processors/SplitJson.cpp b/extensions/standard-processors/processors/SplitJson.cpp index eacbf0001..0e3d90cd4 100644 --- a/extensions/standard-processors/processors/SplitJson.cpp +++ b/extensions/standard-processors/processors/SplitJson.cpp @@ -19,7 +19,7 @@ #include <unordered_map> #include "core/ProcessSession.h" -#include "core/ProcessContext.h" +#include "minifi-cpp/core/ProcessContext.h" #include "core/Resource.h" #include "utils/ProcessorConfigUtils.h" #include "utils/Id.h" diff --git a/extensions/standard-processors/tests/CMakeLists.txt b/extensions/standard-processors/tests/CMakeLists.txt index edef43a46..aecfc84ea 100644 --- a/extensions/standard-processors/tests/CMakeLists.txt +++ b/extensions/standard-processors/tests/CMakeLists.txt @@ -34,10 +34,8 @@ FOREACH(testfile ${PROCESSOR_UNIT_TESTS}) target_include_directories(${testfilename} BEFORE PRIVATE "../processors") target_include_directories(${testfilename} BEFORE PRIVATE ./include) target_include_directories(${testfilename} BEFORE PRIVATE "${CIVETWEB_INCLUDE_DIRS}") - target_include_directories(${testfilename} BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/extensions/expression-language/") createTests("${testfilename}") target_link_libraries(${testfilename} Catch2WithMain) - target_link_libraries(${testfilename} minifi-expression-language-extensions) target_link_libraries(${testfilename} minifi-standard-processors) target_compile_definitions("${testfilename}" PRIVATE JOLT_TESTS_DIR="${jolt_tests_SOURCE_DIR}/jolt-core/src/test/resources/json/shiftr") target_compile_definitions("${testfilename}" PRIVATE TZ_DATA_DIR="${CMAKE_BINARY_DIR}/tzdata") diff --git a/extensions/standard-processors/tests/unit/ExpressionLanguageInDynamicPropertiesTests.cpp b/extensions/standard-processors/tests/unit/ExpressionLanguageInDynamicPropertiesTests.cpp new file mode 100644 index 000000000..ca67eae73 --- /dev/null +++ b/extensions/standard-processors/tests/unit/ExpressionLanguageInDynamicPropertiesTests.cpp @@ -0,0 +1,94 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <ExtractText.h> +#include <GetFile.h> +#include <PutFile.h> +#include <UpdateAttribute.h> +#include <LogAttribute.h> +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "unit/ProvenanceTestHelper.h" + +namespace org::apache::nifi::minifi::test { + +TEST_CASE("GetFile PutFile dynamic attribute", "[expressionLanguageTestGetFilePutFileDynamicAttribute]") { + TestController testController; + + LogTestController::getInstance().setTrace<TestPlan>(); + LogTestController::getInstance().setTrace<minifi::processors::PutFile>(); + LogTestController::getInstance().setTrace<minifi::processors::ExtractText>(); + LogTestController::getInstance().setTrace<minifi::processors::GetFile>(); + LogTestController::getInstance().setTrace<minifi::processors::PutFile>(); + LogTestController::getInstance().setTrace<minifi::processors::LogAttribute>(); + LogTestController::getInstance().setTrace<minifi::processors::UpdateAttribute>(); + + auto conf = std::make_shared<minifi::ConfigureImpl>(); + + conf->set("nifi.my.own.property", "custom_value"); + + auto plan = testController.createPlan(conf); + + auto in_dir = testController.createTempDirectory(); + REQUIRE(!in_dir.empty()); + + auto in_file = in_dir / "file"; + auto out_dir = testController.createTempDirectory(); + REQUIRE(!out_dir.empty()); + + auto out_file = out_dir / "extracted_attr" / "file"; + + // Build MiNiFi processing graph + auto get_file = plan->addProcessor("GetFile", "GetFile"); + plan->setProperty(get_file, minifi::processors::GetFile::Directory, in_dir.string()); + plan->setProperty(get_file, minifi::processors::GetFile::KeepSourceFile, "false"); + auto update = plan->addProcessor("UpdateAttribute", "UpdateAttribute", core::Relationship("success", "description"), true); + REQUIRE(update->setDynamicProperty("prop_attr", "${'nifi.my.own.property'}_added")); + plan->addProcessor("LogAttribute", "LogAttribute", core::Relationship("success", "description"), true); + auto extract_text = plan->addProcessor("ExtractText", "ExtractText", core::Relationship("success", "description"), true); + plan->setProperty(extract_text, minifi::processors::ExtractText::Attribute, "extracted_attr_name"); + plan->addProcessor("LogAttribute", "LogAttribute", core::Relationship("success", "description"), true); + auto put_file = plan->addProcessor("PutFile", "PutFile", core::Relationship("success", "description"), true); + plan->setProperty(put_file, minifi::processors::PutFile::Directory, (out_dir / "${extracted_attr_name}").string()); + plan->setProperty(put_file, minifi::processors::PutFile::ConflictResolution, magic_enum::enum_name(minifi::processors::PutFile::FileExistsResolutionStrategy::replace)); + plan->setProperty(put_file, minifi::processors::PutFile::CreateDirs, "true"); + + // Write test input + { + std::ofstream in_file_stream(in_file); + in_file_stream << "extracted_attr"; + } + + plan->runNextProcessor(); // GetFile + plan->runNextProcessor(); // Update + plan->runNextProcessor(); // Log + plan->runNextProcessor(); // ExtractText + plan->runNextProcessor(); // Log + plan->runNextProcessor(); // PutFile + + // Verify output + { + std::stringstream output_str; + std::ifstream out_file_stream(out_file); + output_str << out_file_stream.rdbuf(); + REQUIRE("extracted_attr" == output_str.str()); + } + + REQUIRE(LogTestController::getInstance().contains("key:prop_attr value:custom_value_added")); +} + +} // namespace org::apache::nifi::minifi::test diff --git a/extensions/standard-processors/tests/unit/GetFileTests.cpp b/extensions/standard-processors/tests/unit/GetFileTests.cpp index 20e368530..5529330c0 100644 --- a/extensions/standard-processors/tests/unit/GetFileTests.cpp +++ b/extensions/standard-processors/tests/unit/GetFileTests.cpp @@ -29,7 +29,7 @@ #include "unit/TestUtils.h" #include "unit/ProvenanceTestHelper.h" #ifdef WIN32 -#include "impl/expression/Expression.h" +#include "expression-language/Expression.h" #endif using namespace std::literals::chrono_literals; diff --git a/extensions/standard-processors/tests/unit/ProcessorTests.cpp b/extensions/standard-processors/tests/unit/ProcessorTests.cpp index 70c8ea4e2..554fd7b41 100644 --- a/extensions/standard-processors/tests/unit/ProcessorTests.cpp +++ b/extensions/standard-processors/tests/unit/ProcessorTests.cpp @@ -41,7 +41,7 @@ #include "core/Core.h" #include "minifi-cpp/core/FlowFile.h" #include "core/Processor.h" -#include "core/ProcessContext.h" +#include "core/ProcessContextImpl.h" #include "core/ProcessSession.h" #include "core/ProcessSessionFactory.h" #include "core/reporting/SiteToSiteProvenanceReportingTask.h" diff --git a/extensions/expression-language/tests/RouteOnAttributeTests.cpp b/extensions/standard-processors/tests/unit/RouteOnAttributeTests.cpp similarity index 100% rename from extensions/expression-language/tests/RouteOnAttributeTests.cpp rename to extensions/standard-processors/tests/unit/RouteOnAttributeTests.cpp diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index e0db77b08..a4b8cbf8d 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -32,13 +32,7 @@ set_cpp_version() include_directories(include) -if(WIN32) - include_directories(opsys/win) - set(SOCKET_SOURCES "src/io/win/*.cpp") -else() - include_directories(opsys/posix) - set(SOCKET_SOURCES "src/io/posix/*.cpp") -endif() +include(ExpressionLanguage) set(TLS_SOURCES "src/utils/tls/*.cpp" "src/io/tls/*.cpp") @@ -50,7 +44,7 @@ file(GLOB SOURCES "src/controllers/*.cpp" "src/core/*.cpp" "src/core/repository/*.cpp" "src/core/flow/*.cpp" "src/core/json/*.cpp" "src/core/yaml/*.cpp" "src/core/reporting/*.cpp" "src/core/extension/*.cpp" "src/serialization/*.cpp" "src/provenance/*.cpp" "src/utils/*.cpp" "src/*.cpp" "src/http/*.cpp" - "src/parameter-providers/*.cpp") + "src/parameter-providers/*.cpp" "src/expression-language/*.cpp") # manually add this as it might not yet be present when this executes list(APPEND SOURCES "${CMAKE_CURRENT_BINARY_DIR}/agent_version.cpp") @@ -68,7 +62,12 @@ endif() file(GLOB PROCESSOR_SOURCES "src/processors/*.cpp" ) -add_minifi_library(core-minifi SHARED ${SOURCES}) +add_minifi_library(core-minifi SHARED ${SOURCES} ${BISON_el-parser_OUTPUTS} ${FLEX_el-scanner_OUTPUTS}) +target_include_directories(core-minifi SYSTEM PRIVATE ${EL_GENERATED_INCLUDE_DIR}) +if (BREW_FLEX_INCLUDE) + target_include_directories(core-minifi SYSTEM PRIVATE ${BREW_FLEX_INCLUDE}) +endif() + target_compile_definitions(core-minifi PRIVATE "LIBMINIFI=1") if(WIN32) set_target_properties(core-minifi PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS TRUE) diff --git a/core-framework/include/core/ProcessContext.h b/libminifi/include/core/ProcessContextImpl.h similarity index 96% rename from core-framework/include/core/ProcessContext.h rename to libminifi/include/core/ProcessContextImpl.h index 5c533721b..31f6ad188 100644 --- a/core-framework/include/core/ProcessContext.h +++ b/libminifi/include/core/ProcessContextImpl.h @@ -42,6 +42,7 @@ #include "minifi-cpp/core/controller/ControllerServiceLookup.h" #include "minifi-cpp/core/controller/ControllerServiceProvider.h" #include "minifi-cpp/core/repository/FileSystemRepository.h" +#include "expression-language/Expression.h" namespace org::apache::nifi::minifi::core { @@ -204,6 +205,10 @@ class ProcessContextImpl : public core::VariableRegistryImpl, public virtual Pro Processor& processor_; gsl::not_null<std::shared_ptr<Configure>> configure_; std::unique_ptr<ProcessorInfo> info_; + + mutable std::mutex mutex_; + mutable std::unordered_map<std::string, expression::Expression, utils::string::transparent_string_hash, std::equal_to<>> cached_expressions_; + mutable std::unordered_map<std::string, expression::Expression, utils::string::transparent_string_hash, std::equal_to<>> cached_dynamic_expressions_; }; } // namespace org::apache::nifi::minifi::core diff --git a/extensions/expression-language/Driver.h b/libminifi/include/expression-language/Driver.h similarity index 97% rename from extensions/expression-language/Driver.h rename to libminifi/include/expression-language/Driver.h index 6d511dd7e..5a0b0daf1 100644 --- a/extensions/expression-language/Driver.h +++ b/libminifi/include/expression-language/Driver.h @@ -19,7 +19,7 @@ #include <string> #include <map> -#include "expression/Expression.h" +#include "expression-language/Expression.h" #undef yyFlexLexer #include <FlexLexer.h> diff --git a/extensions/expression-language/impl/expression/Expression.h b/libminifi/include/expression-language/Expression.h similarity index 98% rename from extensions/expression-language/impl/expression/Expression.h rename to libminifi/include/expression-language/Expression.h index db113b817..039f6fc24 100644 --- a/extensions/expression-language/impl/expression/Expression.h +++ b/libminifi/include/expression-language/Expression.h @@ -23,8 +23,8 @@ #include <utility> #include <vector> -#include "common/Value.h" -#include "FlowFile.h" +#include "expression-language/Value.h" +#include "core/FlowFile.h" #include "minifi-cpp/core/VariableRegistry.h" namespace org::apache::nifi::minifi::expression { diff --git a/extensions/expression-language/Parser.yy b/libminifi/include/expression-language/Parser.yy similarity index 98% rename from extensions/expression-language/Parser.yy rename to libminifi/include/expression-language/Parser.yy index 17b98200f..7a9ea140e 100644 --- a/extensions/expression-language/Parser.yy +++ b/libminifi/include/expression-language/Parser.yy @@ -32,7 +32,7 @@ { #include <string> - #include <expression/Expression.h> + #include "expression-language/Expression.h" #include "location.hh" #if defined(WIN32) && defined(S_FALSE) #undef S_FALSE // winerror.h #defines S_FALSE, which conflicts with the generated token @@ -49,8 +49,8 @@ #include <memory> #include <functional> - #include <expression/Expression.h> - #include "Driver.h" + #include "expression-language/Expression.h" + #include "expression-language/Driver.h" #undef yylex #define yylex driver->lex diff --git a/extensions/expression-language/Scanner.ll b/libminifi/include/expression-language/Scanner.ll similarity index 98% rename from extensions/expression-language/Scanner.ll rename to libminifi/include/expression-language/Scanner.ll index df4c1d794..72081b050 100644 --- a/extensions/expression-language/Scanner.ll +++ b/libminifi/include/expression-language/Scanner.ll @@ -22,7 +22,7 @@ #include <string> #include <string> #include <sstream> - #include "Driver.h" + #include "expression-language/Driver.h" #include "Parser.hpp" %} diff --git a/extensions/expression-language/common/Value.h b/libminifi/include/expression-language/Value.h similarity index 100% rename from extensions/expression-language/common/Value.h rename to libminifi/include/expression-language/Value.h diff --git a/libminifi/src/ThreadedSchedulingAgent.cpp b/libminifi/src/ThreadedSchedulingAgent.cpp index c809a3535..05cbb11e6 100644 --- a/libminifi/src/ThreadedSchedulingAgent.cpp +++ b/libminifi/src/ThreadedSchedulingAgent.cpp @@ -31,9 +31,8 @@ #include "core/ClassLoader.h" #include "core/Processor.h" -#include "minifi-cpp/core/ProcessContext.h" +#include "core/ProcessContextImpl.h" #include "core/ProcessSessionFactory.h" -#include "minifi-cpp/core/ProcessContextBuilder.h" #include "utils/ValueParser.h" using namespace std::literals::chrono_literals; @@ -72,12 +71,7 @@ void ThreadedSchedulingAgent::schedule(core::Processor* processor) { return; } - std::shared_ptr<core::ProcessContextBuilder> contextBuilder = core::ClassLoader::getDefaultClassLoader().instantiate<core::ProcessContextBuilder>("ProcessContextBuilder", "ProcessContextBuilder"); - - contextBuilder = contextBuilder->withContentRepository(content_repo_)->withFlowFileRepository(flow_repo_)->withProvider(controller_service_provider_)->withProvenanceRepository(repo_) - ->withConfiguration(configure_); - - auto process_context = contextBuilder->build(*processor); + auto process_context = std::make_shared<core::ProcessContextImpl>(*processor, controller_service_provider_, repo_, flow_repo_, configure_, content_repo_); auto session_factory = std::make_shared<core::ProcessSessionFactoryImpl>(process_context); diff --git a/libminifi/src/c2/C2Agent.cpp b/libminifi/src/c2/C2Agent.cpp index 363cf9577..5d08757e7 100644 --- a/libminifi/src/c2/C2Agent.cpp +++ b/libminifi/src/c2/C2Agent.cpp @@ -29,7 +29,7 @@ #include <utility> #include <vector> -#include "core/ProcessContext.h" +#include "core/ProcessContextImpl.h" #include "core/state/UpdateController.h" #include "minifi-cpp/core/logging/Logger.h" #include "core/logging/LoggerConfiguration.h" diff --git a/libminifi/src/core/ClassLoader.cpp b/libminifi/src/core/ClassLoader.cpp index 67d241f9d..520f63465 100644 --- a/libminifi/src/core/ClassLoader.cpp +++ b/libminifi/src/core/ClassLoader.cpp @@ -168,7 +168,7 @@ std::optional<std::string> ClassLoaderImpl::getGroupForClass(const std::string & std::unique_ptr<CoreComponent> ClassLoaderImpl::instantiate(const std::string &class_name, const std::string &name, std::function<bool(CoreComponent*)> filter) { std::lock_guard<std::mutex> lock(internal_mutex_); - // allow subsequent classes to override functionality (like ProcessContextBuilder) + // allow subsequent classes to override functionality for (auto& child_loader : class_loaders_) { if (auto result = child_loader.second.instantiate(class_name, name, filter)) { return result; @@ -186,7 +186,7 @@ std::unique_ptr<CoreComponent> ClassLoaderImpl::instantiate(const std::string &c std::unique_ptr<CoreComponent> ClassLoaderImpl::instantiate(const std::string &class_name, const utils::Identifier &uuid, std::function<bool(CoreComponent*)> filter) { std::lock_guard<std::mutex> lock(internal_mutex_); - // allow subsequent classes to override functionality (like ProcessContextBuilder) + // allow subsequent classes to override functionality for (auto& child_loader : class_loaders_) { if (auto result = child_loader.second.instantiate(class_name, uuid, filter)) { return result; @@ -204,7 +204,7 @@ std::unique_ptr<CoreComponent> ClassLoaderImpl::instantiate(const std::string &c std::unique_ptr<CoreComponent> ClassLoaderImpl::instantiate(const std::string &class_name, const std::string &name, const utils::Identifier &uuid, std::function<bool(CoreComponent*)> filter) { std::lock_guard<std::mutex> lock(internal_mutex_); - // allow subsequent classes to override functionality (like ProcessContextBuilder) + // allow subsequent classes to override functionality for (auto& child_loader : class_loaders_) { if (auto result = child_loader.second.instantiate(class_name, name, uuid, filter)) { return result; @@ -222,7 +222,7 @@ std::unique_ptr<CoreComponent> ClassLoaderImpl::instantiate(const std::string &c gsl::owner<CoreComponent*> ClassLoaderImpl::instantiateRaw(const std::string &class_name, const std::string &name, std::function<bool(CoreComponent*)> filter) { std::lock_guard<std::mutex> lock(internal_mutex_); - // allow subsequent classes to override functionality (like ProcessContextBuilder) + // allow subsequent classes to override functionality for (auto& child_loader : class_loaders_) { if (gsl::owner<CoreComponent*> result = child_loader.second.instantiateRaw(class_name, name, filter)) { return result; diff --git a/libminifi/src/core/ProcessContextBuilder.cpp b/libminifi/src/core/ProcessContextBuilder.cpp deleted file mode 100644 index 9e9d3f8f2..000000000 --- a/libminifi/src/core/ProcessContextBuilder.cpp +++ /dev/null @@ -1,24 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "core/ProcessContextBuilder.h" -#include "core/Resource.h" - -namespace org::apache::nifi::minifi::core { - -REGISTER_RESOURCE_IMPLEMENTATION(ProcessContextBuilderImpl, "ProcessContextBuilder", InternalResource); - -} // namespace org::apache::nifi::minifi::core diff --git a/libminifi/src/core/ProcessContext.cpp b/libminifi/src/core/ProcessContextImpl.cpp similarity index 69% rename from libminifi/src/core/ProcessContext.cpp rename to libminifi/src/core/ProcessContextImpl.cpp index 97028ae45..94caeab6c 100644 --- a/libminifi/src/core/ProcessContext.cpp +++ b/libminifi/src/core/ProcessContextImpl.cpp @@ -15,8 +15,10 @@ * limitations under the License. */ -#include "core/ProcessContext.h" +#include "core/ProcessContextImpl.h" #include "core/Processor.h" +#include "utils/PropertyErrors.h" +#include "minifi-cpp/utils/gsl.h" namespace org::apache::nifi::minifi::core { @@ -74,7 +76,23 @@ bool ProcessContextImpl::hasNonEmptyProperty(std::string_view name) const { std::vector<std::string> ProcessContextImpl::getDynamicPropertyKeys() const { return processor_.getDynamicPropertyKeys(); } -std::map<std::string, std::string> ProcessContextImpl::getDynamicProperties(const FlowFile*) const { return processor_.getDynamicProperties(); } +std::map<std::string, std::string> ProcessContextImpl::getDynamicProperties(const FlowFile* flow_file) const { + std::lock_guard<std::mutex> lock(mutex_); + auto dynamic_props = processor_.getDynamicProperties(); + const expression::Parameters params{this, flow_file}; + for (auto& [dynamic_property_name, dynamic_property_value]: dynamic_props) { + auto cached_dyn_expr_it = cached_dynamic_expressions_.find(dynamic_property_name); + if (cached_dyn_expr_it == cached_dynamic_expressions_.end()) { + auto expression = expression::compile(dynamic_property_value); + const auto [it, success] = cached_dynamic_expressions_.emplace(dynamic_property_name, expression); + gsl_Assert(success && "getDynamicProperties: no element with the key existed, yet insertion failed"); + cached_dyn_expr_it = it; + } + auto& expression = cached_dyn_expr_it->second; + dynamic_property_value = expression(params).asString(); + } + return dynamic_props; +} bool ProcessContextImpl::isAutoTerminated(Relationship relationship) const { return processor_.isAutoTerminated(relationship); } @@ -82,20 +100,50 @@ uint8_t ProcessContextImpl::getMaxConcurrentTasks() const { return processor_.ge void ProcessContextImpl::yield() { processor_.yield(); } -nonstd::expected<std::string, std::error_code> ProcessContextImpl::getProperty(const std::string_view name, const FlowFile* const) const { - return getProcessor().getProperty(name); +nonstd::expected<std::string, std::error_code> ProcessContextImpl::getProperty(const std::string_view name, const FlowFile* flow_file) const { + std::lock_guard<std::mutex> lock(mutex_); + const auto property = getProcessorInfo().getSupportedProperty(name); + if (!property) { + return nonstd::make_unexpected(PropertyErrorCode::NotSupportedProperty); + } + + if (!property->supportsExpressionLanguage()) { + return getProcessor().getProperty(name); + } + if (!cached_expressions_.contains(name)) { + auto expression_str = getProcessor().getProperty(name); + if (!expression_str) { return expression_str; } + cached_expressions_.emplace(std::string{name}, expression::compile(*expression_str)); + } + expression::Parameters p(this, flow_file); + auto result = cached_expressions_[std::string{name}](p).asString(); + if (!property->getValidator().validate(result)) { + return nonstd::make_unexpected(PropertyErrorCode::ValidationFailed); + } + return result; } nonstd::expected<void, std::error_code> ProcessContextImpl::setProperty(const std::string_view name, std::string value) { + std::lock_guard<std::mutex> lock(mutex_); + cached_expressions_.erase(std::string{name}); return getProcessor().setProperty(name, std::move(value)); } nonstd::expected<void, std::error_code> ProcessContextImpl::clearProperty(const std::string_view name) { + std::lock_guard<std::mutex> lock(mutex_); + cached_expressions_.erase(std::string{name}); return getProcessor().clearProperty(name); } -nonstd::expected<std::string, std::error_code> ProcessContextImpl::getDynamicProperty(const std::string_view name, const FlowFile* const) const { - return getProcessor().getDynamicProperty(name); +nonstd::expected<std::string, std::error_code> ProcessContextImpl::getDynamicProperty(const std::string_view name, const FlowFile* flow_file) const { + std::lock_guard<std::mutex> lock(mutex_); + if (!cached_dynamic_expressions_.contains(name)) { + auto expression_str = getProcessor().getDynamicProperty(name); + if (!expression_str) { return expression_str; } + cached_dynamic_expressions_.emplace(std::string{name}, expression::compile(*expression_str)); + } + const expression::Parameters p(this, flow_file); + return cached_dynamic_expressions_[std::string{name}](p).asString(); } nonstd::expected<std::string, std::error_code> ProcessContextImpl::getRawProperty(const std::string_view name) const { @@ -107,6 +155,8 @@ nonstd::expected<std::string, std::error_code> ProcessContextImpl::getRawDynamic } nonstd::expected<void, std::error_code> ProcessContextImpl::setDynamicProperty(std::string name, std::string value) { + std::lock_guard<std::mutex> lock(mutex_); + cached_dynamic_expressions_.erase(name); return getProcessor().setDynamicProperty(std::move(name), std::move(value)); } diff --git a/extensions/expression-language/Expression.cpp b/libminifi/src/expression-language/Expression.cpp similarity index 99% rename from extensions/expression-language/Expression.cpp rename to libminifi/src/expression-language/Expression.cpp index 7a6cc8308..ae2d17f60 100644 --- a/extensions/expression-language/Expression.cpp +++ b/libminifi/src/expression-language/Expression.cpp @@ -32,7 +32,7 @@ #include "utils/StringUtils.h" #include "utils/OsUtils.h" -#include "expression/Expression.h" +#include "expression-language/Expression.h" #include "utils/RegexUtils.h" #include "utils/TimeUtil.h" @@ -62,7 +62,7 @@ #include <unistd.h> #endif -#include "Driver.h" +#include "expression-language/Driver.h" #include "core/logging/LoggerFactory.h" diff --git a/libminifi/test/flow-tests/SessionTests.cpp b/libminifi/test/flow-tests/SessionTests.cpp index 99eff2231..f86540d52 100644 --- a/libminifi/test/flow-tests/SessionTests.cpp +++ b/libminifi/test/flow-tests/SessionTests.cpp @@ -26,7 +26,7 @@ #include "repository/FileSystemRepository.h" #include "utils/Id.h" #include "io/BufferStream.h" -#include "core/ProcessContext.h" +#include "core/ProcessContextImpl.h" #include "core/ProcessSession.h" #include "core/ProcessorImpl.h" #include "core/RepositoryFactory.h" diff --git a/extensions/expression-language/tests/integration/UpdateAttributeIntegrationTest.cpp b/libminifi/test/integration/UpdateAttributeIntegrationTest.cpp similarity index 97% rename from extensions/expression-language/tests/integration/UpdateAttributeIntegrationTest.cpp rename to libminifi/test/integration/UpdateAttributeIntegrationTest.cpp index dc4ce3ea7..5a258a518 100644 --- a/extensions/expression-language/tests/integration/UpdateAttributeIntegrationTest.cpp +++ b/libminifi/test/integration/UpdateAttributeIntegrationTest.cpp @@ -19,9 +19,9 @@ #include <chrono> #include <memory> #include <string> +#include "core/ProcessContextImpl.h" #include "processors/LogAttribute.h" #include "integration/IntegrationBase.h" -#include "ProcessContextExpr.h" #include "unit/TestBase.h" #include "unit/TestUtils.h" #include "unit/Catch.h" @@ -35,7 +35,7 @@ class TestHarness : public IntegrationBase { void testSetup() override { LogTestController::getInstance().setTrace<minifi::FlowController>(); LogTestController::getInstance().setTrace<core::ProcessSession>(); - LogTestController::getInstance().setTrace<core::ProcessContextExpr>(); + LogTestController::getInstance().setTrace<core::ProcessContextImpl>(); LogTestController::getInstance().setInfo<minifi::processors::LogAttribute>(); } diff --git a/libminifi/test/libtest/integration/IntegrationBase.cpp b/libminifi/test/libtest/integration/IntegrationBase.cpp index 3f76eb053..cb29d5a02 100644 --- a/libminifi/test/libtest/integration/IntegrationBase.cpp +++ b/libminifi/test/libtest/integration/IntegrationBase.cpp @@ -26,7 +26,7 @@ #include "utils/FifoExecutor.h" #include "utils/file/AssetManager.h" #include "core/ConfigurationFactory.h" -#include "core/ProcessContext.h" +#include "core/ProcessContextImpl.h" namespace org::apache::nifi::minifi::test { diff --git a/libminifi/test/libtest/unit/TestBase.cpp b/libminifi/test/libtest/unit/TestBase.cpp index 193885d62..135805d50 100644 --- a/libminifi/test/libtest/unit/TestBase.cpp +++ b/libminifi/test/libtest/unit/TestBase.cpp @@ -25,7 +25,7 @@ #include <utility> #include "core/Processor.h" -#include "minifi-cpp/core/ProcessContextBuilder.h" +#include "core/ProcessContextImpl.h" #include "minifi-cpp/core/PropertyDefinition.h" #include "core/logging/LoggerConfiguration.h" #include "core/state/nodes/FlowInformation.h" @@ -39,7 +39,7 @@ #include "LogUtils.h" #include "utils/GeneralUtils.h" #include "Connection.h" -#include "core/ProcessContext.h" +#include "minifi-cpp/core/ProcessContext.h" #include "core/ProcessSessionFactory.h" #include "ResourceClaim.h" @@ -286,11 +286,7 @@ minifi::core::Processor* TestPlan::addProcessor(std::unique_ptr<minifi::core::Pr } relationships_.push_back(std::move(connection)); } - std::shared_ptr<minifi::core::ProcessContextBuilder> contextBuilder = - minifi::core::ClassLoader::getDefaultClassLoader().instantiate<minifi::core::ProcessContextBuilder>("ProcessContextBuilder", "ProcessContextBuilder"); - contextBuilder = contextBuilder->withContentRepository(content_repo_)->withFlowFileRepository(flow_repo_)->withProvider(controller_services_provider_.get()) - ->withProvenanceRepository(prov_repo_)->withConfiguration(configuration_); - auto context = contextBuilder->build(*processor); + auto context = std::make_shared<minifi::core::ProcessContextImpl>(*processor, controller_services_provider_.get(), prov_repo_, flow_repo_, configuration_, content_repo_); processor_contexts_.push_back(context); processor_queue_.push_back(processor.get()); auto raw_ptr = processor.get(); diff --git a/libminifi/test/persistence-tests/PersistenceTests.cpp b/libminifi/test/persistence-tests/PersistenceTests.cpp index 4fed83d11..43dd56d31 100644 --- a/libminifi/test/persistence-tests/PersistenceTests.cpp +++ b/libminifi/test/persistence-tests/PersistenceTests.cpp @@ -22,6 +22,7 @@ #include <thread> #include "core/Core.h" +#include "core/ProcessContextImpl.h" #include "core/repository/AtomicRepoEntries.h" #include "core/RepositoryFactory.h" #include "FlowFileRecord.h" @@ -38,7 +39,6 @@ #include "core/repository/FileSystemRepository.h" #include "core/ProcessorImpl.h" #include "Connection.h" -#include "core/ProcessContext.h" #include "core/ProcessSessionFactory.h" using ConnectionImpl = minifi::ConnectionImpl; diff --git a/extensions/expression-language/tests/ExpressionLanguageTests.cpp b/libminifi/test/unit/ExpressionLanguageTests.cpp similarity index 95% rename from extensions/expression-language/tests/ExpressionLanguageTests.cpp rename to libminifi/test/unit/ExpressionLanguageTests.cpp index 3c493c5df..f05bbdd91 100644 --- a/extensions/expression-language/tests/ExpressionLanguageTests.cpp +++ b/libminifi/test/unit/ExpressionLanguageTests.cpp @@ -31,18 +31,12 @@ #pragma comment(lib, "crypt32.lib") #endif #include <curl/curl.h> -#include "impl/expression/Expression.h" -#include <ExtractText.h> -#include <GetFile.h> -#include <PutFile.h> -#include <UpdateAttribute.h> +#include "expression-language/Expression.h" #include "minifi-cpp/core/FlowFile.h" -#include <LogAttribute.h> #include "minifi-cpp/utils/gsl.h" #include "unit/TestBase.h" #include "unit/Catch.h" #include "catch2/catch_approx.hpp" -#include "unit/ProvenanceTestHelper.h" #include "date/tz.h" #include "unit/TestUtils.h" @@ -181,72 +175,6 @@ TEST_CASE("ToLower function", "[expressionLanguageTestToLowerFunction]") { REQUIRE("text_before__flow_a_attr_value_a__text_after" == expr(expression::Parameters{ flow_file_a.get() }).asString()); } -TEST_CASE("GetFile PutFile dynamic attribute", "[expressionLanguageTestGetFilePutFileDynamicAttribute]") { - TestController testController; - - LogTestController::getInstance().setTrace<TestPlan>(); - LogTestController::getInstance().setTrace<minifi::processors::PutFile>(); - LogTestController::getInstance().setTrace<minifi::processors::ExtractText>(); - LogTestController::getInstance().setTrace<minifi::processors::GetFile>(); - LogTestController::getInstance().setTrace<minifi::processors::PutFile>(); - LogTestController::getInstance().setTrace<minifi::processors::LogAttribute>(); - LogTestController::getInstance().setTrace<minifi::processors::UpdateAttribute>(); - - auto conf = std::make_shared<minifi::ConfigureImpl>(); - - conf->set("nifi.my.own.property", "custom_value"); - - auto plan = testController.createPlan(conf); - auto repo = std::make_shared<TestRepository>(); - - auto in_dir = testController.createTempDirectory(); - REQUIRE(!in_dir.empty()); - - auto in_file = in_dir / "file"; - auto out_dir = testController.createTempDirectory(); - REQUIRE(!out_dir.empty()); - - auto out_file = out_dir / "extracted_attr" / "file"; - - // Build MiNiFi processing graph - auto get_file = plan->addProcessor("GetFile", "GetFile"); - plan->setProperty(get_file, minifi::processors::GetFile::Directory, in_dir.string()); - plan->setProperty(get_file, minifi::processors::GetFile::KeepSourceFile, "false"); - auto update = plan->addProcessor("UpdateAttribute", "UpdateAttribute", core::Relationship("success", "description"), true); - REQUIRE(update->setDynamicProperty("prop_attr", "${'nifi.my.own.property'}_added")); - plan->addProcessor("LogAttribute", "LogAttribute", core::Relationship("success", "description"), true); - auto extract_text = plan->addProcessor("ExtractText", "ExtractText", core::Relationship("success", "description"), true); - plan->setProperty(extract_text, minifi::processors::ExtractText::Attribute, "extracted_attr_name"); - plan->addProcessor("LogAttribute", "LogAttribute", core::Relationship("success", "description"), true); - auto put_file = plan->addProcessor("PutFile", "PutFile", core::Relationship("success", "description"), true); - plan->setProperty(put_file, minifi::processors::PutFile::Directory, (out_dir / "${extracted_attr_name}").string()); - plan->setProperty(put_file, minifi::processors::PutFile::ConflictResolution, magic_enum::enum_name(minifi::processors::PutFile::FileExistsResolutionStrategy::replace)); - plan->setProperty(put_file, minifi::processors::PutFile::CreateDirs, "true"); - - // Write test input - { - std::ofstream in_file_stream(in_file); - in_file_stream << "extracted_attr"; - } - - plan->runNextProcessor(); // GetFile - plan->runNextProcessor(); // Update - plan->runNextProcessor(); // Log - plan->runNextProcessor(); // ExtractText - plan->runNextProcessor(); // Log - plan->runNextProcessor(); // PutFile - - // Verify output - { - std::stringstream output_str; - std::ifstream out_file_stream(out_file); - output_str << out_file_stream.rdbuf(); - REQUIRE("extracted_attr" == output_str.str()); - } - - REQUIRE(LogTestController::getInstance().contains("key:prop_attr value:custom_value_added")); -} - TEST_CASE("Substring 2 arg", "[expressionLanguageSubstring2]") { auto expr = expression::compile("text_before${attr:substring(6, 8)}text_after"); diff --git a/extensions/expression-language/tests/ProcessContextExprTests.cpp b/libminifi/test/unit/ProcessContextExprTests.cpp similarity index 96% rename from extensions/expression-language/tests/ProcessContextExprTests.cpp rename to libminifi/test/unit/ProcessContextExprTests.cpp index 9b9635653..08d38479f 100644 --- a/extensions/expression-language/tests/ProcessContextExprTests.cpp +++ b/libminifi/test/unit/ProcessContextExprTests.cpp @@ -18,9 +18,8 @@ #include <memory> #include <string> -#include "minifi-cpp/core/ProcessContext.h" +#include "core/ProcessContextImpl.h" #include "core/Resource.h" -#include "ProcessContextExpr.h" #include "core/ProcessorImpl.h" #include "minifi-cpp/core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" @@ -62,7 +61,7 @@ TEST_CASE("ProcessContextExpr can update existing processor properties", "[setPr std::shared_ptr<TestPlan> test_plan = test_controller.createPlan(); [[maybe_unused]] minifi::core::Processor* dummy_processor = test_plan->addProcessor("DummyProcessContextExprProcessor", "dummy_processor"); std::shared_ptr<minifi::core::ProcessContext> context = [test_plan] { test_plan->runNextProcessor(); return test_plan->getCurrentContext(); }(); - REQUIRE(dynamic_pointer_cast<minifi::core::ProcessContextExpr>(context) != nullptr); + REQUIRE(dynamic_pointer_cast<minifi::core::ProcessContextImpl>(context) != nullptr); SECTION("Set and get simple property") { SECTION("Using a Property reference parameter") { @@ -107,7 +106,7 @@ TEST_CASE("ProcessContextExpr can use expression language in dynamic properties" std::ignore = test_plan->addProcessor("DummyProcessor", "dummy_processor"); test_plan->runNextProcessor(); const auto context = test_plan->getCurrentContext(); - REQUIRE(dynamic_pointer_cast<core::ProcessContextExpr>(context) != nullptr); + REQUIRE(dynamic_pointer_cast<core::ProcessContextImpl>(context) != nullptr); core::FlowFileImpl flow_file; flow_file.setAttribute("attr_a", "myAttributeValue"); @@ -136,7 +135,7 @@ TEST_CASE("ProcessContextExpr is mutex guarded properly") { std::ignore = test_plan->addProcessor("DummyProcessor", "dummy_processor"); test_plan->runNextProcessor(); const auto context = test_plan->getCurrentContext(); - REQUIRE(dynamic_pointer_cast<core::ProcessContextExpr>(context) != nullptr); + REQUIRE(dynamic_pointer_cast<core::ProcessContextImpl>(context) != nullptr); auto play_with_context = [=]() { for (auto i = 0; i < 100; ++i) { diff --git a/libminifi/test/unit/ProcessorConfigUtilsTests.cpp b/libminifi/test/unit/ProcessorConfigUtilsTests.cpp index d7c66a0d6..8070ce7a1 100644 --- a/libminifi/test/unit/ProcessorConfigUtilsTests.cpp +++ b/libminifi/test/unit/ProcessorConfigUtilsTests.cpp @@ -27,7 +27,7 @@ #include "utils/Enum.h" #include "utils/Id.h" #include "unit/TestUtils.h" -#include "core/ProcessContext.h" +#include "core/ProcessContextImpl.h" namespace org::apache::nifi::minifi::core { namespace { @@ -64,7 +64,7 @@ TEST_CASE("Parse enum property") { self.setSupportedProperties(std::to_array<core::PropertyReference>({prop})); }; proc->initialize(); - ProcessContextImpl context(*proc, nullptr, nullptr, nullptr, nullptr, nullptr); + core::ProcessContextImpl context(*proc, nullptr, nullptr, nullptr, nullptr, nullptr); SECTION("Valid") { REQUIRE(proc->setProperty(prop.name, "B")); const auto val = utils::parseEnumProperty<TestEnum>(context, prop); @@ -135,7 +135,7 @@ TEST_CASE("Parse controller service property") { }; processor->initialize(); auto configuration = minifi::Configure::create(); - ProcessContextImpl context(*processor, &test_controller_service_provider, nullptr, nullptr, configuration, nullptr); + core::ProcessContextImpl context(*processor, &test_controller_service_provider, nullptr, nullptr, configuration, nullptr); SECTION("Required controller service property") { SECTION("... is valid") { diff --git a/libminifi/test/unit/SchedulingAgentTests.cpp b/libminifi/test/unit/SchedulingAgentTests.cpp index 61cc5e931..ed79f7f18 100644 --- a/libminifi/test/unit/SchedulingAgentTests.cpp +++ b/libminifi/test/unit/SchedulingAgentTests.cpp @@ -22,7 +22,7 @@ #include "unit/ProvenanceTestHelper.h" #include "unit/TestUtils.h" #include "utils/TimeUtil.h" -#include "core/ProcessContext.h" +#include "core/ProcessContextImpl.h" #include "core/ProcessSessionFactory.h" using namespace std::literals::chrono_literals; diff --git a/minifi-api/include/minifi-cpp/core/ProcessContextBuilder.h b/minifi-api/include/minifi-cpp/core/ProcessContextBuilder.h deleted file mode 100644 index a579d5ba8..000000000 --- a/minifi-api/include/minifi-cpp/core/ProcessContextBuilder.h +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include <memory> -#include "minifi-cpp/core/Core.h" -#include "minifi-cpp/core/ContentRepository.h" -#include "minifi-cpp/properties/Configure.h" -#include "minifi-cpp/core/controller/ControllerServiceProvider.h" -#include "ProcessContext.h" -#include "minifi-cpp/core/Repository.h" - -namespace org::apache::nifi::minifi::core { -/** - * Could use instantiate<T> from core, which uses a simple compile time check to figure out if a destructor is defined - * and thus that will allow us to know if the context instance exists, but I like using the build because it allows us - * to eventually share the builder across different contexts and shares up the construction ever so slightly. - * - * While this incurs a tiny cost to look up, it allows us to have a replaceable builder that erases the type we are - * constructing. - */ -class Processor; - -class ProcessContextBuilder : public virtual core::CoreComponent, public virtual utils::EnableSharedFromThis { - public: - virtual std::shared_ptr<ProcessContextBuilder> withProvider(core::controller::ControllerServiceProvider* controller_service_provider) = 0; - virtual std::shared_ptr<ProcessContextBuilder> withProvenanceRepository(const std::shared_ptr<core::Repository> &repo) = 0; - virtual std::shared_ptr<ProcessContextBuilder> withFlowFileRepository(const std::shared_ptr<core::Repository> &repo) = 0; - virtual std::shared_ptr<ProcessContextBuilder> withContentRepository(const std::shared_ptr<core::ContentRepository> &repo) = 0; - virtual std::shared_ptr<ProcessContextBuilder> withConfiguration(const std::shared_ptr<minifi::Configure> &configuration) = 0; - virtual std::shared_ptr<core::ProcessContext> build(Processor& processor) = 0; -}; - -} // namespace org::apache::nifi::minifi::core
