This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 49e5b937084a5639ea2f84300bb388175610ee1e Author: james94 <[email protected]> AuthorDate: Fri Oct 18 17:07:47 2024 +0200 MINIFICPP-2346-P1: Integrated Conan2 for OpenSSL, CURL, ZLIB & Build MiNiFi Closes #1793 Signed-off-by: Marton Szasz <[email protected]> --- CMakeLists.txt | 33 +-- CONAN.md | 137 ++++++++++++ README.md | 9 + cmake/BuildTests.cmake | 3 +- cmake/BundledOpenSSL.cmake | 1 + cmake/BundledRocksDB.cmake | 6 +- cmake/Bustache.cmake | 4 +- cmake/{Bustache.cmake => GetBZip2.cmake} | 21 +- cmake/{Bustache.cmake => GetCatch2.cmake} | 21 +- cmake/{Bustache.cmake => GetCivetWeb.cmake} | 20 +- cmake/{Bustache.cmake => GetFmt.cmake} | 20 +- cmake/{Bustache.cmake => GetLibCURL.cmake} | 21 +- cmake/{Bustache.cmake => GetLibXml2.cmake} | 21 +- cmake/{Bustache.cmake => GetOpenSSL.cmake} | 21 +- cmake/{Bustache.cmake => GetRocksDB.cmake} | 28 ++- cmake/{Bustache.cmake => GetSpdlog.cmake} | 25 ++- cmake/{Bustache.cmake => GetZLIB.cmake} | 21 +- cmake/{Bustache.cmake => GetZstd.cmake} | 21 +- cmake/MiNiFiOptions.cmake | 41 +++- cmake/Spdlog.cmake | 2 +- conanfile.py | 90 ++++++++ etc/conan/profiles/release-linux | 12 + extensions/civetweb/CMakeLists.txt | 2 +- extensions/opc/CMakeLists.txt | 2 +- extensions/rocksdb-repos/CMakeLists.txt | 9 +- libminifi/CMakeLists.txt | 2 +- libminifi/test/libtest/unit/CMakeLists.txt | 2 +- libminifi/test/unit/LoggerTests.cpp | 4 +- thirdparty/rocksdb/all/conandata.yml | 15 ++ thirdparty/rocksdb/all/conanfile.py | 243 +++++++++++++++++++++ thirdparty/rocksdb/{ => all/patches}/arm7.patch | 0 thirdparty/rocksdb/{ => all/patches}/cstdint.patch | 0 .../patches}/dboptions_equality_operator.patch | 0 thirdparty/rocksdb/config.yml | 3 + 34 files changed, 708 insertions(+), 152 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1699e95f9..9b31d6c0e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -221,9 +221,15 @@ else() message(VERBOSE "No custom malloc implementation") endif() +# OpenSSL +include(GetOpenSSL) +get_openssl("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/ssl") + + if (ENABLE_BZIP2 AND (ENABLE_LIBARCHIVE OR (ENABLE_ROCKSDB AND NOT WIN32))) - include(BundledBZip2) - use_bundled_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) + include(GetBZip2) + get_bzip2(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/bzip2/dummy") endif() @@ -238,11 +244,6 @@ if(NOT WIN32) use_bundled_osspuuid(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) endif() -# OpenSSL -include(BundledOpenSSL) -use_openssl("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") -list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/ssl") - # libsodium include(BundledLibSodium) use_bundled_libsodium("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") @@ -250,17 +251,18 @@ use_bundled_libsodium("${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR} list(APPEND MINIFI_CPP_COMPILE_DEFINITIONS SODIUM_STATIC=1) # zlib -include(BundledZLIB) -use_bundled_zlib(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +include(GetZLIB) +get_zlib(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/zlib/dummy") # cURL -include(BundledLibcURL) -use_bundled_curl(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +include(GetLibCURL) +get_curl(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/curl/dummy") # spdlog -include(Spdlog) +include(GetSpdlog) +get_spdlog() # yaml-cpp include(BundledYamlCpp) @@ -340,13 +342,14 @@ include(Extensions) add_subdirectory(libminifi) if (ENABLE_ALL OR ENABLE_AZURE) - include(BundledLibXml2) - use_bundled_libxml2(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + include(GetLibXml2) + get_libxml2(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/libxml2/dummy") endif() if (ENABLE_ALL OR ENABLE_PROMETHEUS OR ENABLE_CIVET) - include(CivetWeb) + include(GetCivetWeb) + get_civetweb() list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/civetweb/dummy") endif() diff --git a/CONAN.md b/CONAN.md new file mode 100644 index 000000000..f4a5d04f2 --- /dev/null +++ b/CONAN.md @@ -0,0 +1,137 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Apache NiFi - MiNiFi - C++ Conan Build Guide + +We will walk through the steps to build MiNiFi using conan version 2 that comes with CMake integration. We will also go through the process of creating a MiNiFi conan package for easier integration into other C++ projects. We will start with a discussion on building MiNiFi using the standalone CMake approach and the conan approach. After we have built MiNiFi and created a MiNiFi conan package, we will conclude by elaborating on some of the benefits that can come with integrating a MiNiFi [...] + +Conan build support is experimental, and not yet suitable for general use. + +## Table of Contents + +- [Apache NiFi - MiNiFi - C++ Conan Build Guide](#apache-nifi---minifi---c---conan-build-guide) + - [Table of Contents](#table-of-contents) + - [Description](#description) + - [Build MiNiFi C++ with Conan](#build-minifi-c---with-conan) + - [Create MiNiFi C++ Conan Package](#create-minifi-c---conan-package) + - [Conclusion](#conclusion) + - [Appendix](#appendix) + +## Description + +Apache NiFi MiNiFi C++ is normally built using standalone CMake. Similarly, we can also create a MiNiFi C++ package using make package. However, each time we build MiNiFi from +source, it uses CMake to install the third party external libraries that are enabled by downloading the source code and then building each library until it is available to use toward building MiNiFi. If a change is made toward MiNiFi's CMake code in regards to one or more of the external libraries, then potentially the long process of installing one or more of the externl library(ies) happens again before the MiNiFi source code is built. Another way we can build MiNiFi and create MiNiFi [...] + +## Build MiNiFi C++ with Conan + +To build MiNiFi using conan, first we install conan version 2, then we create a **default** conan profile that will later be ignored for our custom conan profile, create a MINIFI_HOME environment variable, then we install prebuilt conan packages representing the MiNiFi external libraries and finally we compile MiNiFi. + +~~~bash +sudo pip install --force-reinstall -v "conan==2.0.17" + +# create a "default" conan profile, so conan has it on record in ~/.conan2/, before using your own custom profile. +conan profile detect + +# conanfile.py is in root dir of MiNiFi C++ project +cd $HOME/nifi-minifi-cpp + +# create MINIFI_HOME env variable for binary executable minifi +export MINIFI_HOME=$(pwd) + +# install conan packages for MiNiFi C++ using conanfile.py invoking Conan +# since we created default profile earlier, we can override it with our own minifi profile +conan install . --build=missing --output-folder=build_conan -pr=etc/conan/profiles/release-linux + +# build MiNiFi C++ using conanfile.py invoking Conan & CMake +conan build . --output-folder=build_conan -pr=etc/conan/profiles/release-linux +~~~ + +- **NOTE**: After building MiNiFi, we must have the MINIFI_HOME environment variable created in order to successfully run the minifi binary executable. + +- **NOTE**: When we install the prebuilt conan package representing the MiNiFi third party libraries, we add the `--build=missing` in case some of the prebuilt missing conan packages are not found from conancenter, jfrog, or one of our conan repositories, then conan will build the conan packages from source. We can upload new prebuilt conan packages by **[conan upload](https://docs.conan.io/2/reference/commands/upload.html)**. + +- **NOTE**: When we tell conan to build MiNiFi, conan first installs prebuilt conan packages and then it compiles MiNiFi from source, so we have the MiNiFi library and its binary executable available. + +### Run MiNiFi CTESTs from Build Folder + +~~~bash +pushd build_conan/ +ctest +popd # build_conan/ +~~~ + +### Run MiNiFi Executable + +~~~bash +# verify we can run minifi binary executable +./build_conan/bin/minifi +~~~ + +## Create MiNiFi C++ Conan Package + +To create a MiNiFi package, we will follow the similar steps we took to build MiNiFi. Then right after we install the prebuilt conan packages representing MiNiFi's external libraries, we create that MiNiFi conan package. + +~~~bash +# make sure to install conan2 for your environment +sudo pip install --force-reinstall -v "conan==2.0.17" + +# create a "default" conan profile, so conan has it on record, before using your own custom profile. Gets created in ~/.conan2/ +conan profile detect + +# conanfile.py is in root dir of MiNiFi C++ project +cd $HOME/nifi-minifi-cpp + +# create MINIFI_HOME env variable for binary executable minifi +export MINIFI_HOME=$(pwd) + +# install conan packages for MiNiFi C++ using conanfile.py invoking Conan +# since we created default profile earlier, we can override it with our own minifi profile +# make sure path is correct +conan install . --build=missing --output-folder=build_conan -pr=etc/conan/profiles/release-linux + +# create MiNiFi C++ conan package using conanfile.py invoking Conan & CMake +conan create . --user=minifi --channel=develop -pr=etc/conan/profiles/release-linux +~~~ + +- **NOTE**: When we tell conan to create the MiNiFi conan package, conan first installs prebuilt conan packages, then it compiles MiNiFi from source inside `~/.conan2/p/b/minif<UUID>/b`, and then it copies over MiNiFi's libraries and its binary executables into the conan package folder `~/.conan2/p/b/minif<UUID>/p`. Once we have the MiNiFi conan package, we can integrate it into other C++ infrastructure using CMake. + +### Run MiNiFi CTESTs from Its Conan Package + +~~~bash +# Example of minif<UUID> conan package folder name: minifbd17f6a02da35 +pushd ~/.conan2/p/b/minif<UUID>/b +ctest +popd # build_conan/ +~~~ + +## Conclusion + +To have a more consistent quick build process for MiNiFi, we can use conan version 2. When we use conan with MiNiFi's build process, we can install prebuilt conan packages that can represent MiNiFi's external libraries and then build MiNiFi quickly. We also have a bit more control over when we want to make updates to MiNiFi's external libraries that impact their build and then re-build them as prebuilt conan packages. So these external library conan packages can be used toward building M [...] + +There are multiple benefits of having MiNiFi prebuilt conan packages. We can upload these MiNiFi conan packages to a conan repository like jfrog for version management. We can easily integrate MiNiFi's edge data pipeline features into other C++ software infrastructure using conan's CMake support. We can still use MiNiFi for edge data collection from the IoT devices embedded on robotic systems. We can integrate MiNiFi into self-driving cars (sensor examples: cameras, lidar, radar, inertia [...] + +By leveraging MiNiFi as a conan package, we can leverage MiNiFi that comes with the best practices of building data pipelines from NiFi and bring them into existing C++ real-time robotics infrastructure. Some teams across companies typically have their own custom edge data pipelines that process data for the different events to eventually perform actions on that data. As an alternative to all these companies and their teams having their own custom edge data pipeline libraries, MiNiFi C++ [...] + +## Appendix + +### Create Custom RocksDB Conan Package + +~~~bash +pushd nifi-minifi-cpp/thirdparty/rocksdb/all + +conan create . --user=minifi --channel=develop --version=8.10.2 -pr=../../../etc/conan/profiles/release-linux + +popd +~~~ diff --git a/README.md b/README.md index 13c83f08f..b80108b8d 100644 --- a/README.md +++ b/README.md @@ -358,6 +358,8 @@ advanced features. ### Building +#### Build MiNiFi using Standalone CMake + - From your source checkout, create a directory to perform the build (e.g. build) and cd into that directory. ``` # ~/Development/code/apache/nifi-minifi-cpp on git:master @@ -403,6 +405,8 @@ advanced features. [100%] Built target run-tests ``` +#### Create MiNiFi Package using Standalone CMake + - Create a binary assembly located in your build directory with suffix .tar.gz ``` ~/Development/code/apache/nifi-minifi-cpp/build @@ -427,6 +431,11 @@ advanced features. CPack: - package: ~/Development/code/apache/nifi-minifi-cpp/build/nifi-minifi-cpp-0.99.0-source.tar.gz generated. ``` +#### Build MiNiFi & Create MiNiFi Package using Conan v2 + +Building MiNiFi and creating MiNiFi package supporting a portion of the extensions has been tested with Conan version 2 using VS code as an alternative to standalone CMake. By building MiNiFi using prebuilt conan packages for the external libraries as an alternative to CMake building the sources of those external libraries, we maybe able to speed up MiNiFi builds. Additionally, by creating a MiNiFi package as a conan package, it should be easier to integrate MiNiFi library and its execut [...] +project infrastructure to build out data pipelines on the edge. For instance, once we create the MiNiFi conan package, we can upload it to jfrog, conancenter or some other supported conan cloud repository and then download the prebuilt MiNiFi conan package to our new C++ project by adding it to our conanfile. For more details on the conan commands to build MiNiFi and create a MiNiFi conan package, see [CONAN.md](CONAN.md). + ### Building a docker image #### Building your own custom image diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake index e55500242..37e19fa10 100644 --- a/cmake/BuildTests.cmake +++ b/cmake/BuildTests.cmake @@ -15,7 +15,8 @@ # specific language governing permissions and limitations # under the License. -include(Catch2) +include(GetCatch2) +get_catch2() ### test functions MACRO(GETSOURCEFILES result curdir) diff --git a/cmake/BundledOpenSSL.cmake b/cmake/BundledOpenSSL.cmake index 275e9ca1d..4ffa51e80 100644 --- a/cmake/BundledOpenSSL.cmake +++ b/cmake/BundledOpenSSL.cmake @@ -148,4 +148,5 @@ function(use_openssl SOURCE_DIR BINARY_DIR) set_property(TARGET OpenSSL::Crypto APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib ) set_property(TARGET OpenSSL::SSL APPEND PROPERTY INTERFACE_LINK_LIBRARIES crypt32.lib) endif() + endfunction(use_openssl) diff --git a/cmake/BundledRocksDB.cmake b/cmake/BundledRocksDB.cmake index 2d4211e94..ac2a6e203 100644 --- a/cmake/BundledRocksDB.cmake +++ b/cmake/BundledRocksDB.cmake @@ -27,9 +27,9 @@ function(use_bundled_rocksdb SOURCE_DIR BINARY_DIR) endif() # Patch to fix build issue on ARM7 architecture: https://github.com/facebook/rocksdb/issues/8609#issuecomment-1009572506 - set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/rocksdb/arm7.patch") - set(PATCH_FILE_2 "${SOURCE_DIR}/thirdparty/rocksdb/dboptions_equality_operator.patch") - set(PATCH_FILE_3 "${SOURCE_DIR}/thirdparty/rocksdb/cstdint.patch") + set(PATCH_FILE_1 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/arm7.patch") + set(PATCH_FILE_2 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch") + set(PATCH_FILE_3 "${SOURCE_DIR}/thirdparty/rocksdb/all/patches/cstdint.patch") set(PC ${Bash_EXECUTABLE} -c "set -x &&\ (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_1}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_1}\") &&\ (\"${Patch_EXECUTABLE}\" -p1 -R -s -f --dry-run -i \"${PATCH_FILE_2}\" || \"${Patch_EXECUTABLE}\" -p1 -N -i \"${PATCH_FILE_2}\") &&\ diff --git a/cmake/Bustache.cmake b/cmake/Bustache.cmake index 116028775..6c037af6c 100644 --- a/cmake/Bustache.cmake +++ b/cmake/Bustache.cmake @@ -17,7 +17,9 @@ # under the License. # include(FetchContent) -include(fmt) +include(GetFmt) +get_fmt() + set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) FetchContent_Declare(Bustache GIT_REPOSITORY https://github.com/jamboree/bustache.git diff --git a/cmake/Bustache.cmake b/cmake/GetBZip2.cmake similarity index 66% copy from cmake/Bustache.cmake copy to cmake/GetBZip2.cmake index 116028775..2a7d47e9e 100644 --- a/cmake/Bustache.cmake +++ b/cmake/GetBZip2.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -include(FetchContent) -include(fmt) -set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) -FetchContent_Declare(Bustache - GIT_REPOSITORY https://github.com/jamboree/bustache.git - GIT_TAG 47096caa8e1f9f7ebe34e3a022dbb822c174011d -) -FetchContent_MakeAvailable(Bustache) + +function(get_bzip2 SOURCE_DIR BINARY_DIR) + if(MINIFI_BZIP2_SOURCE STREQUAL "CONAN") + message("Using Conan to install bzip2") + find_package(BZip2 REQUIRED) + elseif(MINIFI_BZIP2_SOURCE STREQUAL "BUILD") + message("Using CMake to build bzip2 from source") + include(BundledBZip2) + use_bundled_bzip2(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_bzip2) diff --git a/cmake/Bustache.cmake b/cmake/GetCatch2.cmake similarity index 67% copy from cmake/Bustache.cmake copy to cmake/GetCatch2.cmake index 116028775..d6c1d6acc 100644 --- a/cmake/Bustache.cmake +++ b/cmake/GetCatch2.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -include(FetchContent) -include(fmt) -set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) -FetchContent_Declare(Bustache - GIT_REPOSITORY https://github.com/jamboree/bustache.git - GIT_TAG 47096caa8e1f9f7ebe34e3a022dbb822c174011d -) -FetchContent_MakeAvailable(Bustache) + +function(get_catch2) + if(MINIFI_CATCH2_SOURCE STREQUAL "CONAN") + message("Using Conan to install Catch2") + find_package(Catch2 REQUIRED) + add_library(Catch2WithMain ALIAS Catch2::Catch2WithMain) + elseif(MINIFI_CATCH2_SOURCE STREQUAL "BUILD") + message("Using CMake to build Catch2 from source") + include(Catch2) + endif() +endfunction(get_catch2) diff --git a/cmake/Bustache.cmake b/cmake/GetCivetWeb.cmake similarity index 69% copy from cmake/Bustache.cmake copy to cmake/GetCivetWeb.cmake index 116028775..f30d22289 100644 --- a/cmake/Bustache.cmake +++ b/cmake/GetCivetWeb.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -include(FetchContent) -include(fmt) -set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) -FetchContent_Declare(Bustache - GIT_REPOSITORY https://github.com/jamboree/bustache.git - GIT_TAG 47096caa8e1f9f7ebe34e3a022dbb822c174011d -) -FetchContent_MakeAvailable(Bustache) + +function(get_civetweb) + if(MINIFI_CIVETWEB_SOURCE STREQUAL "CONAN") + message("Using Conan to install CivetWeb") + find_package(civetweb REQUIRED) + elseif(MINIFI_CIVETWEB_SOURCE STREQUAL "BUILD") + message("Using CMake to build CivetWeb from source") + include(CivetWeb) + endif() +endfunction(get_civetweb) diff --git a/cmake/Bustache.cmake b/cmake/GetFmt.cmake similarity index 72% copy from cmake/Bustache.cmake copy to cmake/GetFmt.cmake index 116028775..f4f28458d 100644 --- a/cmake/Bustache.cmake +++ b/cmake/GetFmt.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -include(FetchContent) -include(fmt) -set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) -FetchContent_Declare(Bustache - GIT_REPOSITORY https://github.com/jamboree/bustache.git - GIT_TAG 47096caa8e1f9f7ebe34e3a022dbb822c174011d -) -FetchContent_MakeAvailable(Bustache) + +function(get_fmt) + if(MINIFI_FMT_SOURCE STREQUAL "CONAN") + message("Using Conan to install Fmt") + find_package(fmt REQUIRED) + elseif(MINIFI_FMT_SOURCE STREQUAL "BUILD") + message("Using CMake to build Fmt from source") + include(fmt) + endif() +endfunction(get_fmt) diff --git a/cmake/Bustache.cmake b/cmake/GetLibCURL.cmake similarity index 64% copy from cmake/Bustache.cmake copy to cmake/GetLibCURL.cmake index 116028775..afd38f3c9 100644 --- a/cmake/Bustache.cmake +++ b/cmake/GetLibCURL.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -include(FetchContent) -include(fmt) -set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) -FetchContent_Declare(Bustache - GIT_REPOSITORY https://github.com/jamboree/bustache.git - GIT_TAG 47096caa8e1f9f7ebe34e3a022dbb822c174011d -) -FetchContent_MakeAvailable(Bustache) + +function(get_curl SOURCE_DIR BINARY_DIR) + if(MINIFI_LIBCURL_SOURCE STREQUAL "CONAN") + message("Using Conan to install libcurl") + find_package(CURL REQUIRED) + elseif(MINIFI_LIBCURL_SOURCE STREQUAL "BUILD") + message("Using CMake to build libcurl from source") + include(BundledLibcURL) + use_bundled_curl(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_curl SOURCE_DIR BINARY_DIR) diff --git a/cmake/Bustache.cmake b/cmake/GetLibXml2.cmake similarity index 65% copy from cmake/Bustache.cmake copy to cmake/GetLibXml2.cmake index 116028775..5915b117d 100644 --- a/cmake/Bustache.cmake +++ b/cmake/GetLibXml2.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -include(FetchContent) -include(fmt) -set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) -FetchContent_Declare(Bustache - GIT_REPOSITORY https://github.com/jamboree/bustache.git - GIT_TAG 47096caa8e1f9f7ebe34e3a022dbb822c174011d -) -FetchContent_MakeAvailable(Bustache) + +function(get_libxml2 SOURCE_DIR BINARY_DIR) + if(MINIFI_LIBXML2_SOURCE STREQUAL "CONAN") + message("Using Conan to install libxml2") + find_package(libxml2 REQUIRED) + elseif(MINIFI_LIBXML2_SOURCE STREQUAL "BUILD") + message("Using CMake to build libxml2 from source") + include(BundledLibXml2) + use_bundled_libxml2(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_libxml2) diff --git a/cmake/Bustache.cmake b/cmake/GetOpenSSL.cmake similarity index 65% copy from cmake/Bustache.cmake copy to cmake/GetOpenSSL.cmake index 116028775..58132c6bf 100644 --- a/cmake/Bustache.cmake +++ b/cmake/GetOpenSSL.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -include(FetchContent) -include(fmt) -set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) -FetchContent_Declare(Bustache - GIT_REPOSITORY https://github.com/jamboree/bustache.git - GIT_TAG 47096caa8e1f9f7ebe34e3a022dbb822c174011d -) -FetchContent_MakeAvailable(Bustache) + +function(get_openssl SOURCE_DIR BINARY_DIR) + if(MINIFI_OPENSSL_SOURCE STREQUAL "CONAN") + message("Using Conan to install OpenSSL") + find_package(OpenSSL REQUIRED) + elseif(MINIFI_OPENSSL_SOURCE STREQUAL "BUILD") + message("Using CMake to build OpenSSL from source") + include(BundledOpenSSL) + use_openssl(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_openssl) diff --git a/cmake/Bustache.cmake b/cmake/GetRocksDB.cmake similarity index 53% copy from cmake/Bustache.cmake copy to cmake/GetRocksDB.cmake index 116028775..0968afc38 100644 --- a/cmake/Bustache.cmake +++ b/cmake/GetRocksDB.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +14,21 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -include(FetchContent) -include(fmt) -set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) -FetchContent_Declare(Bustache - GIT_REPOSITORY https://github.com/jamboree/bustache.git - GIT_TAG 47096caa8e1f9f7ebe34e3a022dbb822c174011d -) -FetchContent_MakeAvailable(Bustache) + +function(get_rocksdb SOURCE_DIR BINARY_DIR) + if(MINIFI_ROCKSDB_SOURCE STREQUAL "CONAN") + message("Using Conan to install RocksDB") + find_package(RocksDB REQUIRED) + add_library(RocksDB::RocksDB ALIAS RocksDB::rocksdb) + elseif(MINIFI_ROCKSDB_SOURCE STREQUAL "BUILD") + message("Using CMake to build RocksDB from source") + + if (BUILD_ROCKSDB) + include(BundledRocksDB) + use_bundled_rocksdb(${SOURCE_DIR} ${BINARY_DIR}) + else() + list(APPEND CMAKE_MODULE_PATH "${SOURCE_DIR}/cmake/rocksdb/sys") + find_package(RocksDB REQUIRED) + endif() + endif() +endfunction(get_rocksdb SOURCE_DIR BINARY_DIR) diff --git a/cmake/Bustache.cmake b/cmake/GetSpdlog.cmake similarity index 65% copy from cmake/Bustache.cmake copy to cmake/GetSpdlog.cmake index 116028775..747894984 100644 --- a/cmake/Bustache.cmake +++ b/cmake/GetSpdlog.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +14,18 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -include(FetchContent) -include(fmt) -set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) -FetchContent_Declare(Bustache - GIT_REPOSITORY https://github.com/jamboree/bustache.git - GIT_TAG 47096caa8e1f9f7ebe34e3a022dbb822c174011d -) -FetchContent_MakeAvailable(Bustache) + +function(get_spdlog) + include(GetFmt) + get_fmt() + + if(MINIFI_SPDLOG_SOURCE STREQUAL "CONAN") + message("Using Conan to install spdlog") + find_package(spdlog REQUIRED) + + add_library(spdlog ALIAS spdlog::spdlog) + elseif(MINIFI_SPDLOG_SOURCE STREQUAL "BUILD") + message("Using CMake to build spdlog from source") + include(Spdlog) + endif() +endfunction(get_spdlog) diff --git a/cmake/Bustache.cmake b/cmake/GetZLIB.cmake similarity index 66% copy from cmake/Bustache.cmake copy to cmake/GetZLIB.cmake index 116028775..6e7bea1e3 100644 --- a/cmake/Bustache.cmake +++ b/cmake/GetZLIB.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -include(FetchContent) -include(fmt) -set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) -FetchContent_Declare(Bustache - GIT_REPOSITORY https://github.com/jamboree/bustache.git - GIT_TAG 47096caa8e1f9f7ebe34e3a022dbb822c174011d -) -FetchContent_MakeAvailable(Bustache) + +function(get_zlib SOURCE_DIR BINARY_DIR) + if(MINIFI_ZLIB_SOURCE STREQUAL "CONAN") + message("Using Conan to install zlib") + find_package(ZLIB REQUIRED) + elseif(MINIFI_ZLIB_SOURCE STREQUAL "BUILD") + message("Using CMake to build zlib from source") + include(BundledZLIB) + use_bundled_zlib(${SOURCE_DIR} ${BINARY_DIR}) + endif() +endfunction(get_zlib) diff --git a/cmake/Bustache.cmake b/cmake/GetZstd.cmake similarity index 68% copy from cmake/Bustache.cmake copy to cmake/GetZstd.cmake index 116028775..a0334b02d 100644 --- a/cmake/Bustache.cmake +++ b/cmake/GetZstd.cmake @@ -1,4 +1,3 @@ -# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,12 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# -include(FetchContent) -include(fmt) -set(BUSTACHE_USE_FMT ON CACHE STRING "" FORCE) -FetchContent_Declare(Bustache - GIT_REPOSITORY https://github.com/jamboree/bustache.git - GIT_TAG 47096caa8e1f9f7ebe34e3a022dbb822c174011d -) -FetchContent_MakeAvailable(Bustache) + +function(get_zstd) + if(MINIFI_ZSTD_SOURCE STREQUAL "CONAN") + message("Using Conan to install zstd") + find_package(zstd REQUIRED) + add_library(zstd::zstd ALIAS zstd::libzstd_static) + elseif(MINIFI_ZSTD_SOURCE STREQUAL "BUILD") + message("Using CMake to build zstd from source") + include(Zstd) + endif() +endfunction(get_zstd) diff --git a/cmake/MiNiFiOptions.cmake b/cmake/MiNiFiOptions.cmake index 536e93557..8a7ccc58c 100644 --- a/cmake/MiNiFiOptions.cmake +++ b/cmake/MiNiFiOptions.cmake @@ -23,6 +23,13 @@ function(add_minifi_option OPTION_NAME OPTION_DESCRIPTION OPTION_VALUE) set(MINIFI_OPTIONS ${MINIFI_OPTIONS} PARENT_SCOPE) endfunction() +function(add_minifi_multi_option OPTION_NAME OPTION_DESCRIPTION OPTION_VALUES DEFAULT_VALUE) + set(${OPTION_NAME} ${DEFAULT_VALUE} CACHE STRING ${OPTION_DESCRIPTION}) + set_property(CACHE ${OPTION_NAME} PROPERTY STRINGS ${OPTION_VALUES}) + list(APPEND MINIFI_OPTIONS ${OPTION_NAME}) + set(MINIFI_OPTIONS ${MINIFI_OPTIONS} PARENT_SCOPE) +endfunction() + function(add_minifi_dependent_option OPTION_NAME OPTION_DESCRIPTION OPTION_VALUE DEPENDS FORCE) cmake_dependent_option(${OPTION_NAME} ${OPTION_DESCRIPTION} ${OPTION_VALUE} ${DEPENDS} ${FORCE}) list(APPEND MINIFI_OPTIONS ${OPTION_NAME}) @@ -65,15 +72,6 @@ add_minifi_option(AWS_ENABLE_UNITY_BUILD "If enabled, AWS SDK libraries will be add_minifi_dependent_option(MINIFI_ADVANCED_ASAN_BUILD "Uses AddressSanitizer to instrument the code" OFF "NOT WIN32" OFF) add_minifi_dependent_option(MINIFI_ADVANCED_CODE_COVERAGE "Use coverage build options and enable coverage build target" OFF "NOT WIN32" OFF) -# Option: STRICT_GSL_CHECKS -# AUDIT: Enable all checks, including gsl_ExpectsAudit() and gsl_EnsuresAudit() -# ON: Enable all checks, excluding gsl_ExpectsAudit() and gsl_EnsuresAudit() (GSL default) -# DEBUG_ONLY: Like ON in the Debug configuration, OFF in others (MiNiFi C++ default) -# OFF: Throw on contract checking and assertion failures instead of calling std::terminate() -set(STRICT_GSL_CHECKS "DEBUG_ONLY" CACHE STRING "Contract checking and assertion failures call terminate") -list(APPEND STRICT_GSL_CHECKS_Values AUDIT ON DEBUG_ONLY OFF) -set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values}) - if (WIN32) add_minifi_option(MINIFI_INCLUDE_VC_REDIST_MERGE_MODULES "Include merge modules for Visual C++ Redistributable with the MSI generated by CPack. The resulting MSI is not distributable under Apache 2.0." OFF) add_minifi_option(MINIFI_INCLUDE_VC_REDIST_DLLS "Include Visual C++ Redistributable DLLs with the MSI generated by CPack. The resulting MSI is not distributable under Apache 2.0." OFF) @@ -123,6 +121,31 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_minifi_option(ENABLE_PROCFS "Enables the procfs extension." ON) endif() +## Each Option provides a list of possible values + +# Option: STRICT_GSL_CHECKS +# AUDIT: Enable all checks, including gsl_ExpectsAudit() and gsl_EnsuresAudit() +# ON: Enable all checks, excluding gsl_ExpectsAudit() and gsl_EnsuresAudit() (GSL default) +# DEBUG_ONLY: Like ON in the Debug configuration, OFF in others (MiNiFi C++ default) +# OFF: Throw on contract checking and assertion failures instead of calling std::terminate() +set(STRICT_GSL_CHECKS "DEBUG_ONLY" CACHE STRING "Contract checking and assertion failures call terminate") +list(APPEND STRICT_GSL_CHECKS_Values AUDIT ON DEBUG_ONLY OFF) +set_property(CACHE STRICT_GSL_CHECKS PROPERTY STRINGS ${STRICT_GSL_CHECKS_Values}) + +# BUILD: Fetch and build from source using CMake FetchContent or ExternalProject +# SYSTEM: Use find_package to use the system version +# CONAN: Use Conan packages +add_minifi_multi_option(MINIFI_LIBCURL_SOURCE "Retrieves LibCURL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_OPENSSL_SOURCE "Retrieves OpenSSL from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_ZLIB_SOURCE "Retrieves ZLib from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_ROCKSDB_SOURCE "Retrieves RocksDB from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_ZSTD_SOURCE "Retrieves Zstd from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_BZIP2_SOURCE "Retrieves BZip2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_CIVETWEB_SOURCE "Retrieves CivetWeb from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_LIBXML2_SOURCE "Retrieves LibXml2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_FMT_SOURCE "Retrieves Fmt from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_SPDLOG_SOURCE "Retrieves Spdlog from provided source" "BUILD;SYSTEM;CONAN" "BUILD") +add_minifi_multi_option(MINIFI_CATCH2_SOURCE "Retrieves Catch2 from provided source" "BUILD;SYSTEM;CONAN" "BUILD") # Docker options diff --git a/cmake/Spdlog.cmake b/cmake/Spdlog.cmake index 07eb0f103..3b0729829 100644 --- a/cmake/Spdlog.cmake +++ b/cmake/Spdlog.cmake @@ -17,7 +17,7 @@ # under the License. # include(FetchContent) -include(fmt) + set(SPDLOG_FMT_EXTERNAL ON CACHE STRING "" FORCE) FetchContent_Declare(Spdlog URL https://github.com/gabime/spdlog/archive/refs/tags/v1.12.0.tar.gz diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 000000000..5dd4db1fc --- /dev/null +++ b/conanfile.py @@ -0,0 +1,90 @@ +from conan import ConanFile +from conan.tools.cmake import CMake, CMakeToolchain +from conan.tools.files import collect_libs, copy +from conan.errors import ConanException +import os +import shutil + +required_conan_version = ">=2.0" + +shared_requires = ("openssl/3.2.1", "libcurl/8.4.0", "civetweb/1.16", "libxml2/2.12.6", "fmt/10.2.1", "spdlog/1.14.0", "catch2/3.5.4", "zlib/1.2.11", "zstd/1.5.2", "bzip2/1.0.8", "rocksdb/8.10.2@minifi/develop") + +shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "nanofi/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "encrypt-config/*", "etc/*", "examples/*", "msi/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", "README.md", "C2.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "JNI.md", "METRICS.md", "OPS.md", "PROCESSORS.md", "ThirdParties.md", "Windows.md", "aptitude.sh", "arch.sh", "bootstrap.sh" [...] + + +class MiNiFiCppMain(ConanFile): + name = "minifi-cpp" + version = "0.99.1" + license = "Apache-2.0" + requires = shared_requires + settings = "os", "compiler", "build_type", "arch" + generators = "CMakeDeps" + options = {"shared": [True, False], "fPIC": [True, False]} + + default_options = {"shared": False, "fPIC": True} + + exports_sources = shared_sources + + def generate(self): + tc = CMakeToolchain(self) + + tc.variables["MINIFI_LIBCURL_SOURCE"] = "CONAN" + tc.variables["MINIFI_OPENSSL_SOURCE"] = "CONAN" + tc.variables["MINIFI_ZLIB_SOURCE"] = "CONAN" + tc.variables["MINIFI_ROCKSDB_SOURCE"] = "CONAN" + tc.variables["MINIFI_ZSTD_SOURCE"] = "CONAN" + tc.variables["MINIFI_BZIP2_SOURCE"] = "CONAN" + tc.variables["MINIFI_CIVETWEB_SOURCE"] = "CONAN" + tc.variables["MINIFI_LIBXML2_SOURCE"] = "CONAN" + tc.variables["MINIFI_FMT_SOURCE"] = "CONAN" + tc.variables["MINIFI_SPDLOG_SOURCE"] = "CONAN" + tc.variables["MINIFI_CATCH2_SOURCE"] = "CONAN" + + tc.variables["SKIP_TESTS"] = "OFF" + tc.variables["ENABLE_CIVET"] = "ON" + tc.variables["ENABLE_EXPRESSION_LANGUAGE"] = "ON" + tc.variables["ENABLE_LIBARCHIVE"] = "OFF" + tc.variables["ENABLE_AWS"] = "OFF" + tc.variables["ENABLE_SQL"] = "OFF" + tc.variables["ENABLE_GCP"] = "OFF" + tc.variables["ENABLE_LUA_SCRIPTING"] = "OFF" + + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def overwrite_libfile(self, oldfile, newfile): + print("Copying {} to {}".format(oldfile, newfile)) + if os.path.exists(oldfile): + try: + if os.path.exists(newfile): + os.remove(newfile) + + shutil.copy2(oldfile, newfile) + except Exception as e: + raise ConanException(f"Error copying {newfile}: {e}") + else: + raise ConanException(f"Error: {oldfile} does not exist") + + def package(self): + cmake = CMake(self) + cmake.install() + include_dir = os.path.join(self.source_folder) + built_dir = os.path.join(self.source_folder, self.folders.build) + copy(self, pattern="*.h*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) + copy(self, pattern="*.i*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) + copy(self, pattern="*.a", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) + copy(self, pattern="*.so*", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) + + minifi_py_ext_oldfile = os.path.join(self.package_folder, "lib", "libminifi-python-script-extension.so") + minifi_py_ext_copynewfile = os.path.join(self.package_folder, "lib", "libminifi_native.so") + self.overwrite_libfile(minifi_py_ext_oldfile, minifi_py_ext_copynewfile) + + def package_info(self): + self.cpp_info.libs = collect_libs(self, folder=os.path.join(self.package_folder, "lib")) + self.cpp_info.set_property("cmake_file_name", "minifi-cpp") + self.cpp_info.set_property("cmake_target_name", "minifi-cpp::minifi-cpp") + self.cpp_info.set_property("pkg_config_name", "minifi-cpp") diff --git a/etc/conan/profiles/release-linux b/etc/conan/profiles/release-linux new file mode 100644 index 000000000..9233901a5 --- /dev/null +++ b/etc/conan/profiles/release-linux @@ -0,0 +1,12 @@ +[settings] +os=Linux +arch=x86_64 +build_type=Release +compiler=gcc +compiler.cppstd=20 +compiler.libcxx=libstdc++11 +compiler.version=11 +[conf] +tools.system.package_manager:mode=install +tools.system.package_manager:sudo=True +[options] diff --git a/extensions/civetweb/CMakeLists.txt b/extensions/civetweb/CMakeLists.txt index 0eab2b98c..c6f45d280 100644 --- a/extensions/civetweb/CMakeLists.txt +++ b/extensions/civetweb/CMakeLists.txt @@ -32,7 +32,7 @@ target_include_directories(minifi-civet-extensions BEFORE PUBLIC ${CMAKE_SOURCE_DIR}/thirdparty/ ${civetweb_SOURCE_DIR}/include ./include) -target_link_libraries(minifi-civet-extensions ${LIBMINIFI} Threads::Threads civetweb::civetweb-cpp civetweb::c-library) +target_link_libraries(minifi-civet-extensions ${LIBMINIFI} Threads::Threads civetweb::civetweb-cpp) register_extension(minifi-civet-extensions CIVETWEB CIVETWEB "This enables ListenHTTP" "extensions/civetweb/tests") register_extension_linter(minifi-civet-extensions-linter) diff --git a/extensions/opc/CMakeLists.txt b/extensions/opc/CMakeLists.txt index 9f1267cf9..0d120de83 100644 --- a/extensions/opc/CMakeLists.txt +++ b/extensions/opc/CMakeLists.txt @@ -38,7 +38,7 @@ file(GLOB SOURCES "src/*.cpp") add_minifi_library(minifi-opc-extensions SHARED ${SOURCES}) target_link_libraries(minifi-opc-extensions ${LIBMINIFI} Threads::Threads) -target_link_libraries(minifi-opc-extensions ${CMAKE_DL_LIBS} spdlog open62541::open62541) +target_link_libraries(minifi-opc-extensions ${CMAKE_DL_LIBS} spdlog::spdlog open62541::open62541) register_extension(minifi-opc-extensions "OPC EXTENSIONS" OPC-EXTENSIONS "This enables OPC-UA support") register_extension_linter(minifi-opc-extensions-linter) diff --git a/extensions/rocksdb-repos/CMakeLists.txt b/extensions/rocksdb-repos/CMakeLists.txt index e9d0a7af8..2f43753c7 100644 --- a/extensions/rocksdb-repos/CMakeLists.txt +++ b/extensions/rocksdb-repos/CMakeLists.txt @@ -21,13 +21,8 @@ if (NOT ENABLE_ROCKSDB) return() endif() -if (BUILD_ROCKSDB) - include(BundledRocksDB) - use_bundled_rocksdb(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) -else() - list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/rocksdb/sys") - find_package(RocksDB REQUIRED) -endif() +include(GetRocksDB) +get_rocksdb(${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) include(${CMAKE_SOURCE_DIR}/extensions/ExtensionHeader.txt) diff --git a/libminifi/CMakeLists.txt b/libminifi/CMakeLists.txt index 3a7778236..73a3fc262 100644 --- a/libminifi/CMakeLists.txt +++ b/libminifi/CMakeLists.txt @@ -76,7 +76,7 @@ endif() include(RangeV3) include(Asio) include(MagicEnum) -list(APPEND LIBMINIFI_LIBRARIES yaml-cpp ZLIB::ZLIB concurrentqueue RapidJSON spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON) +list(APPEND LIBMINIFI_LIBRARIES yaml-cpp ZLIB::ZLIB concurrentqueue RapidJSON spdlog::spdlog Threads::Threads gsl-lite libsodium range-v3 expected-lite date::date date::tz asio magic_enum OpenSSL::Crypto OpenSSL::SSL CURL::libcurl RapidJSON fmt::fmt) if(NOT WIN32) list(APPEND LIBMINIFI_LIBRARIES OSSP::libuuid++) endif() diff --git a/libminifi/test/libtest/unit/CMakeLists.txt b/libminifi/test/libtest/unit/CMakeLists.txt index 75847685e..3b90524e9 100644 --- a/libminifi/test/libtest/unit/CMakeLists.txt +++ b/libminifi/test/libtest/unit/CMakeLists.txt @@ -18,7 +18,7 @@ # file(GLOB TEST_BASE_SOURCES "*.cpp") add_minifi_library(libminifi-unittest STATIC ${TEST_BASE_SOURCES}) -target_link_libraries(libminifi-unittest core-minifi Catch2) +target_link_libraries(libminifi-unittest core-minifi Catch2WithMain) target_include_directories(libminifi-unittest BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/include/") if(WIN32) target_include_directories(libminifi-unittest BEFORE PRIVATE "${CMAKE_SOURCE_DIR}/libminifi/opsys/win") diff --git a/libminifi/test/unit/LoggerTests.cpp b/libminifi/test/unit/LoggerTests.cpp index 573ad2029..fc87ee500 100644 --- a/libminifi/test/unit/LoggerTests.cpp +++ b/libminifi/test/unit/LoggerTests.cpp @@ -380,7 +380,9 @@ TEST_CASE("Setting max log entry length to unlimited results in unlimited log en TEST_CASE("fmt formatting works with the logger") { LogTestController::getInstance().setTrace<logging::Logger>(); std::shared_ptr<logging::Logger> logger = logging::LoggerFactory<logging::Logger>::getLogger(); - logger->log_critical("{} equals to {}", 1min, 60s); + std::chrono::minutes duration{1}; + std::string formatted_min_duration = fmt::format("{0}m", duration.count()); + logger->log_critical("{} equals to {}", formatted_min_duration, 60s); logger->log_critical("{} in hex is {:#x}", 13, 13); logger->log_critical("Unix epoch: {}", std::chrono::system_clock::time_point()); logger->log_critical("{:%Q %q} equals to {:%Q %q}", 2h, std::chrono::duration_cast<std::chrono::seconds>(2h)); diff --git a/thirdparty/rocksdb/all/conandata.yml b/thirdparty/rocksdb/all/conandata.yml new file mode 100644 index 000000000..f5c3ecd30 --- /dev/null +++ b/thirdparty/rocksdb/all/conandata.yml @@ -0,0 +1,15 @@ +sources: + "8.10.2": + url: "https://github.com/facebook/rocksdb/archive/refs/tags/v8.10.2.tar.gz"; + sha256: "44b6ec2f4723a0d495762da245d4a59d38704e0d9d3d31c45af4014bee853256" +patches: + "8.10.2": + - patch_file: "patches/arm7.patch" + patch_description: "Fix for ARM7" + patch_type: "portability" + patch_source: "https://github.com/apache/nifi-minifi-cpp/commit/36244376bb46c5c85657c6dfe92a6a0c4cea650a"; + - patch_file: "patches/dboptions_equality_operator.patch" + patch_description: "Fix for missing equality operator in DBOptions and make DBOptions configurable" + patch_type: "portability" + patch_source: "https://github.com/apache/nifi-minifi-cpp/commit/545236fd06b613f2d2b38f74ff9ee85df9190d59"; + - patch_file: "patches/cstdint.patch" diff --git a/thirdparty/rocksdb/all/conanfile.py b/thirdparty/rocksdb/all/conanfile.py new file mode 100644 index 000000000..a1ff97ad8 --- /dev/null +++ b/thirdparty/rocksdb/all/conanfile.py @@ -0,0 +1,243 @@ +import os +import glob +import shutil + +from conan import ConanFile +from conan.errors import ConanInvalidConfiguration +from conan.tools.build import check_min_cppstd +from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout +from conan.tools.files import patch, collect_libs, copy, export_conandata_patches, get, rm, rmdir +from conan.tools.microsoft import check_min_vs, is_msvc, is_msvc_static_runtime +from conan.tools.scm import Version + +required_conan_version = ">=1.53.0" + + +class RocksDBConan(ConanFile): + name = "rocksdb" + homepage = "https://github.com/facebook/rocksdb"; + license = ("GPL-2.0-only", "Apache-2.0") + url = "https://github.com/conan-io/conan-center-index"; + description = "A library that provides an embeddable, persistent key-value store for fast storage" + topics = ("database", "leveldb", "facebook", "key-value") + package_type = "library" + settings = "os", "arch", "compiler", "build_type" + options = { + "shared": [True, False], + "fPIC": [True, False], + "lite": [True, False], + "with_gflags": [True, False], + "with_snappy": [True, False], + "with_lz4": [True, False], + "with_zlib": [True, False], + "with_bz2": [True, False], + "with_zstd": [True, False], + "with_tbb": [True, False], + "with_jemalloc": [True, False], + "enable_sse": [False, "sse42", "avx2"], + "use_rtti": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + "lite": False, + "with_snappy": False, + "with_lz4": True, + "with_zlib": True, + "with_bz2": True, + "with_zstd": True, + "with_gflags": False, + "with_tbb": False, + "with_jemalloc": False, + "enable_sse": False, + "use_rtti": True, + } + + @property + def _min_cppstd(self): + return "11" if Version(self.version) < "8.8.1" else "20" + + @property + def _compilers_minimum_version(self): + return {} if self._min_cppstd == "11" else { + "apple-clang": "10", + "clang": "7", + "gcc": "7", + "msvc": "191", + "Visual Studio": "15", + } + + def export_sources(self): + export_conandata_patches(self) + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + if self.settings.arch != "x86_64": + del self.options.with_tbb + if self.settings.build_type == "Debug": + self.options.use_rtti = True # Rtti are used in asserts for debug mode... + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def layout(self): + cmake_layout(self, src_folder="src") + + def requirements(self): + if self.options.with_gflags: + self.requires("gflags/2.2.2") + if self.options.with_snappy: + self.requires("snappy/1.1.10") + if self.options.with_lz4: + self.requires("lz4/1.9.4") + if self.options.with_zlib: + self.requires("zlib/[>=1.2.11 <2]") + if self.options.with_bz2: + self.requires("bzip2/1.0.8") + if self.options.with_zstd: + self.requires("zstd/1.5.2") + if self.options.get_safe("with_tbb"): + self.requires("onetbb/2021.10.0") + if self.options.with_jemalloc: + self.requires("jemalloc/5.3.0") + + def validate(self): + if self.settings.compiler.get_safe("cppstd"): + check_min_cppstd(self, self._min_cppstd) + + minimum_version = self._compilers_minimum_version.get(str(self.settings.compiler), False) + if minimum_version and Version(self.settings.compiler.version) < minimum_version: + raise ConanInvalidConfiguration( + f"{self.ref} requires C++{self._min_cppstd}, which your compiler does not support." + ) + + if self.settings.arch not in ["x86_64", "ppc64le", "ppc64", "mips64", "armv8"]: + raise ConanInvalidConfiguration("Rocksdb requires 64 bits") + + check_min_vs(self, "191") + + if self.version == "6.20.3" and \ + self.settings.os == "Linux" and \ + self.settings.compiler == "gcc" and \ + Version(self.settings.compiler.version) < "5": + raise ConanInvalidConfiguration("Rocksdb 6.20.3 is not compilable with gcc <5.") # See https://github.com/facebook/rocksdb/issues/3522 + + def source(self): + get(self, **self.conan_data["sources"][self.version], strip_root=True) + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["FAIL_ON_WARNINGS"] = False + tc.variables["WITH_TESTS"] = False + tc.variables["WITH_TOOLS"] = True + tc.variables["WITH_CORE_TOOLS"] = False + tc.variables["WITH_BENCHMARK_TOOLS"] = False + tc.variables["WITH_FOLLY_DISTRIBUTED_MUTEX"] = False + if is_msvc(self): + tc.variables["WITH_MD_LIBRARY"] = not is_msvc_static_runtime(self) + tc.variables["ROCKSDB_INSTALL_ON_WINDOWS"] = self.settings.os == "Windows" + tc.variables["ROCKSDB_LITE"] = self.options.lite + tc.variables["WITH_GFLAGS"] = self.options.with_gflags + tc.variables["WITH_SNAPPY"] = self.options.with_snappy + tc.variables["WITH_LZ4"] = self.options.with_lz4 + tc.variables["WITH_ZLIB"] = self.options.with_zlib + tc.variables["WITH_BZ2"] = self.options.with_bz2 + tc.variables["WITH_ZSTD"] = self.options.with_zstd + tc.variables["WITH_TBB"] = self.options.get_safe("with_tbb", False) + tc.variables["WITH_JEMALLOC"] = self.options.with_jemalloc + tc.variables["ROCKSDB_BUILD_SHARED"] = self.options.shared + tc.variables["ROCKSDB_LIBRARY_EXPORTS"] = self.settings.os == "Windows" and self.options.shared + tc.variables["ROCKSDB_DLL" ] = self.settings.os == "Windows" and self.options.shared + tc.variables["USE_RTTI"] = self.options.use_rtti + if not bool(self.options.enable_sse): + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = False + elif self.options.enable_sse == "sse42": + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = True + elif self.options.enable_sse == "avx2": + tc.variables["PORTABLE"] = True + tc.variables["FORCE_SSE42"] = False + # not available yet in CCI + tc.variables["WITH_NUMA"] = False + tc.generate() + + deps = CMakeDeps(self) + if self.options.with_jemalloc: + deps.set_property("jemalloc", "cmake_file_name", "JeMalloc") + deps.set_property("jemalloc", "cmake_target_name", "JeMalloc::JeMalloc") + if self.options.with_zstd: + deps.set_property("zstd", "cmake_target_name", "zstd::zstd") + deps.generate() + + def build(self): + for patch_data in self.conan_data.get("patches", {}).get(self.version, []): + patch(self, patch_file=patch_data["patch_file"], base_path=self.source_folder, strip=1, fuzz=True) + cmake = CMake(self) + cmake.configure() + cmake.build() + + def _remove_static_libraries(self): + rm(self, "rocksdb.lib", os.path.join(self.package_folder, "lib")) + for lib in glob.glob(os.path.join(self.package_folder, "lib", "*.a")): + if not lib.endswith(".dll.a"): + os.remove(lib) + + def _remove_cpp_headers(self): + for path in glob.glob(os.path.join(self.package_folder, "include", "rocksdb", "*")): + if path != os.path.join(self.package_folder, "include", "rocksdb", "c.h"): + if os.path.isfile(path): + os.remove(path) + else: + shutil.rmtree(path) + + def package(self): + copy(self, "COPYING", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + copy(self, "LICENSE*", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses")) + cmake = CMake(self) + cmake.install() + if self.options.shared: + self._remove_static_libraries() + self._remove_cpp_headers() # Force stable ABI for shared libraries + rmdir(self, os.path.join(self.package_folder, "lib", "cmake")) + rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig")) + + def package_info(self): + cmake_target = "rocksdb-shared" if self.options.shared else "rocksdb" + self.cpp_info.set_property("cmake_file_name", "RocksDB") + self.cpp_info.set_property("cmake_target_name", f"RocksDB::{cmake_target}") + # TODO: back to global scope in conan v2 once cmake_find_package* generators removed + self.cpp_info.components["librocksdb"].libs = collect_libs(self) + if self.settings.os == "Windows": + self.cpp_info.components["librocksdb"].system_libs = ["shlwapi", "rpcrt4"] + if self.options.shared: + self.cpp_info.components["librocksdb"].defines = ["ROCKSDB_DLL"] + elif self.settings.os in ["Linux", "FreeBSD"]: + self.cpp_info.components["librocksdb"].system_libs = ["pthread", "m"] + if self.options.lite: + self.cpp_info.components["librocksdb"].defines.append("ROCKSDB_LITE") + + # TODO: to remove in conan v2 once cmake_find_package* generators removed + self.cpp_info.names["cmake_find_package"] = "RocksDB" + self.cpp_info.names["cmake_find_package_multi"] = "RocksDB" + self.cpp_info.components["librocksdb"].names["cmake_find_package"] = cmake_target + self.cpp_info.components["librocksdb"].names["cmake_find_package_multi"] = cmake_target + self.cpp_info.components["librocksdb"].set_property("cmake_target_name", f"RocksDB::{cmake_target}") + if self.options.with_gflags: + self.cpp_info.components["librocksdb"].requires.append("gflags::gflags") + if self.options.with_snappy: + self.cpp_info.components["librocksdb"].requires.append("snappy::snappy") + if self.options.with_lz4: + self.cpp_info.components["librocksdb"].requires.append("lz4::lz4") + if self.options.with_zlib: + self.cpp_info.components["librocksdb"].requires.append("zlib::zlib") + if self.options.with_bz2: + self.cpp_info.components["librocksdb"].requires.append("bzip2::bzip2") + if self.options.with_zstd: + self.cpp_info.components["librocksdb"].requires.append("zstd::zstd") + if self.options.get_safe("with_tbb"): + self.cpp_info.components["librocksdb"].requires.append("onetbb::onetbb") + if self.options.with_jemalloc: + self.cpp_info.components["librocksdb"].requires.append("jemalloc::jemalloc") diff --git a/thirdparty/rocksdb/arm7.patch b/thirdparty/rocksdb/all/patches/arm7.patch similarity index 100% rename from thirdparty/rocksdb/arm7.patch rename to thirdparty/rocksdb/all/patches/arm7.patch diff --git a/thirdparty/rocksdb/cstdint.patch b/thirdparty/rocksdb/all/patches/cstdint.patch similarity index 100% rename from thirdparty/rocksdb/cstdint.patch rename to thirdparty/rocksdb/all/patches/cstdint.patch diff --git a/thirdparty/rocksdb/dboptions_equality_operator.patch b/thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch similarity index 100% rename from thirdparty/rocksdb/dboptions_equality_operator.patch rename to thirdparty/rocksdb/all/patches/dboptions_equality_operator.patch diff --git a/thirdparty/rocksdb/config.yml b/thirdparty/rocksdb/config.yml new file mode 100644 index 000000000..21a3c25ea --- /dev/null +++ b/thirdparty/rocksdb/config.yml @@ -0,0 +1,3 @@ +versions: + "8.10.2": + folder: all
