This is an automated email from the ASF dual-hosted git repository. martinzink pushed a commit to branch MINIFICPP-2514 in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit ca28a9859e8df75207bf5d2233e86e5ef5de3f96 Author: Martin Zink <[email protected]> AuthorDate: Fri Jan 24 17:41:19 2025 +0100 MINIFICPP-2514 Fix Python libloader Fix and extend verify package to include aarch64 --- .github/workflows/verify-package.yml | 124 ++++++++++++++------- cmake/VerifyPackageWithDocker.cmake | 14 +-- docker/requirements.txt | 6 +- docker/test/integration/cluster/ImageStore.py | 6 +- .../integration/features/fetch_modbus_tcp.feature | 1 + .../features/google_cloud_storage.feature | 1 + docker/test/integration/features/steps/steps.py | 1 - .../python/pythonlibloader/PythonLibLoader.cpp | 2 +- 8 files changed, 97 insertions(+), 58 deletions(-) diff --git a/.github/workflows/verify-package.yml b/.github/workflows/verify-package.yml index 7e6c0c45c..b5a788324 100644 --- a/.github/workflows/verify-package.yml +++ b/.github/workflows/verify-package.yml @@ -6,9 +6,9 @@ env: -DENABLE_ELASTICSEARCH=OFF -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DDOCKER_BUILD_ONLY=ON CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache jobs: - rocky-build: - name: "Build package (Rocky 8)" - runs-on: ubuntu-22.04 + rocky-x86-64-build: + name: "Build package (Rocky 8 x86_64)" + runs-on: ubuntu-24.04 timeout-minutes: 180 steps: - id: checkout @@ -38,40 +38,76 @@ jobs: key: verify-package-ccache-${{github.ref}}-${{github.sha}} - uses: actions/upload-artifact@v4 with: - name: minifi-tar + name: minifi-x86-64-tar path: build/nifi-minifi-cpp-*-bin-rockylinux.tar.gz if-no-files-found: error - docker-tests: + rocky-aarch64-build: + name: "Build package (Rocky 8 aarch64)" + runs-on: ubuntu-24.04-arm + timeout-minutes: 180 + steps: + - id: checkout + uses: actions/checkout@v4 + - name: cache restore + uses: actions/cache/restore@v4 + with: + path: ~/.ccache + key: verify-package-arm-ccache-${{github.ref}}-${{github.sha}} + restore-keys: | + verify-package-arm-ccache-${{github.ref}}- + verify-package-arm-ccache-refs/heads/main- + - id: install_deps + run: | + sudo apt update + sudo apt install -y ccache cmake + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - id: build + run: | + if [ -d ~/.ccache ]; then mv ~/.ccache .; fi + mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make rocky + - name: cache save + uses: actions/cache/save@v4 + if: always() + with: + path: ${{ env.CCACHE_DIR }} + key: verify-package-arm-ccache-${{github.ref}}-${{github.sha}} + - uses: actions/upload-artifact@v4 + with: + name: minifi-aarch64-tar + path: build/nifi-minifi-cpp-*-bin-rockylinux.tar.gz + if-no-files-found: error + + docker-x86-64-tests: name: "${{ matrix.platform.name }}" - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 180 - needs: rocky-build + needs: rocky-x86-64-build strategy: fail-fast: false matrix: platform: [ - { name: "Rocky Linux 8", + { name: "Rocky Linux 8 x86-64", id: "rocky8", build_cmd: "make rocky8_from_rocky_package", verify_cmd: "make docker-verify-rocky8" }, - { name: "Rocky Linux 9", + { name: "Rocky Linux 9 x86-64", id: "rocky9", build_cmd: "make rocky9_from_rocky_package", verify_cmd: "make docker-verify-rocky9" }, - { name: "Ubuntu 22.04", + { name: "Ubuntu 22.04 x86-64", id: "ubuntu_jammy", build_cmd: "make jammy_from_rocky_package", verify_cmd: "make docker-verify-jammy" }, - { name: "Ubuntu 24.04", + { name: "Ubuntu 24.04 x86-64", id: "ubuntu_noble", build_cmd: "make noble_from_rocky_package", verify_cmd: "make docker-verify-noble" }, - { name: "Debian 11", + { name: "Debian 11 x86-64", id: "debian_bullseye", build_cmd: "make bullseye_from_rocky_package", verify_cmd: "make docker-verify-bullseye" }, - { name: "Debian 12", + { name: "Debian 12 x86-64", id: "debian_bookworm", build_cmd: "make bookworm_from_rocky_package", verify_cmd: "make docker-verify-bookworm" }, @@ -82,7 +118,7 @@ jobs: uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: - name: minifi-tar + name: minifi-x86-64-tar path: build - id: install_deps run: | @@ -104,41 +140,43 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: ${{ matrix.platform.id }}_behavex_output + name: ${{ matrix.platform.id }}_x86_64_behavex_output path: build/behavex_output - docker-python-tests: + + + docker-aarch64-tests: name: "${{ matrix.platform.name }}" - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04-arm timeout-minutes: 180 - needs: rocky-build + needs: rocky-aarch64-build strategy: fail-fast: false matrix: platform: [ - { name: "Rocky Linux 8 Python", + { name: "Rocky Linux 8 aarch64", id: "rocky8", - build_cmd: "make rocky8_py_from_rocky_package", - verify_cmd: "make docker-verify-rocky8_py" }, - { name: "Rocky Linux 9 Python", + build_cmd: "make rocky8_from_rocky_package", + verify_cmd: "make docker-verify-rocky8" }, + { name: "Rocky Linux 9 aarch64", id: "rocky9", - build_cmd: "make rocky9_py_from_rocky_package", - verify_cmd: "make docker-verify-rocky9_py" }, - { name: "Patchelf-ed Ubuntu 22.04 Python", - id: "patchelfed_ubuntu22.04", - build_cmd: "make patched_jammy_py_from_rocky_package", - verify_cmd: "make docker-verify-patched_jammy_py" }, - { name: "Patchelf-ed Debian 11 Python", - id: "patchelfed_debian11", - build_cmd: "make patched_bullseye_py_from_rocky_package", - verify_cmd: "make docker-verify-patched_bullseye_py" }, - { name: "Ubuntu 22.04 with Anaconda Python", - id: "ubuntu22.04_anaconda", - build_cmd: "make jammy_py_from_rocky_package && make conda_jammy_py_from_rocky_package", - verify_cmd: "make docker-verify-conda_jammy_py" }, - { name: "Rocky Linux 9 in virtualenv Python", - id: "rocky9_venv", - build_cmd: "make rocky9_py_from_rocky_package && make venv_rocky9_py_from_rocky_package", - verify_cmd: "make docker-verify-venv_rocky9_py" } + build_cmd: "make rocky9_from_rocky_package", + verify_cmd: "make docker-verify-rocky9" }, + { name: "Ubuntu 22.04 aarch64", + id: "ubuntu_jammy", + build_cmd: "make jammy_from_rocky_package", + verify_cmd: "make docker-verify-jammy" }, + { name: "Ubuntu 24.04 aarch64", + id: "ubuntu_noble", + build_cmd: "make noble_from_rocky_package", + verify_cmd: "make docker-verify-noble" }, + { name: "Debian 11 aarch64", + id: "debian_bullseye", + build_cmd: "make bullseye_from_rocky_package", + verify_cmd: "make docker-verify-bullseye" }, + { name: "Debian 12 aarch64", + id: "debian_bookworm", + build_cmd: "make bookworm_from_rocky_package", + verify_cmd: "make docker-verify-bookworm" }, ] steps: @@ -146,13 +184,13 @@ jobs: uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: - name: minifi-tar + name: minifi-aarch64-tar path: build - id: install_deps run: | sudo apt update sudo apt install -y python3-virtualenv - - run: cd build && cmake -DENABLE_PYTHON_SCRIPTING=ON -DDOCKER_BUILD_ONLY=ON .. \ + - run: cd build && cmake ${DOCKER_CMAKE_FLAGS} .. \ && ${{ matrix.platform.build_cmd }} && ${{ matrix.platform.verify_cmd }} - name: Test Reporter if: always() @@ -168,5 +206,5 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: ${{ matrix.platform.id }}_behavex_output + name: ${{ matrix.platform.id }}_aarch64_behavex_output path: build/behavex_output diff --git a/cmake/VerifyPackageWithDocker.cmake b/cmake/VerifyPackageWithDocker.cmake index 8c926c0e7..ab4b34a63 100644 --- a/cmake/VerifyPackageWithDocker.cmake +++ b/cmake/VerifyPackageWithDocker.cmake @@ -28,7 +28,7 @@ function(ADD_PACKAGE_VERIFY TAG_PREFIX) endif() endforeach() - set(DISABLED_TAGS "SKIP_CI,ENABLE_PYTHON_SCRIPTING") + set(DISABLED_TAGS "SKIP_CI,NEEDS_NUMPY,ENABLE_KAFKA") add_custom_target( docker-verify-${TAG_PREFIX} @@ -36,12 +36,12 @@ function(ADD_PACKAGE_VERIFY TAG_PREFIX) endfunction() -CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:8 rocky8 "yum install -y wget") -CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:9 rocky9 "yum install -y wget") -CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:jammy jammy "apt update \\&\\& apt install -y wget") -CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:noble noble "apt update \\&\\& apt install -y wget") -CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bookworm bookworm "apt update \\&\\& apt install -y wget") -CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bullseye bullseye "apt update \\&\\& apt install -y wget") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:8 rocky8 "dnf install -y wget python3.12-devel python3.12-pip gcc gcc-c++") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:9 rocky9 "dnf install -y wget python3-devel python3-pip gcc gcc-c++") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:jammy jammy "apt update \\&\\& apt install -y wget python3-dev python3-venv python3-pip") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:noble noble "apt update \\&\\& apt install -y wget python3-dev python3-venv python3-pip") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bookworm bookworm "apt update \\&\\& apt install -y wget python3-dev python3-venv python3-pip") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bullseye bullseye "apt update \\&\\& apt install -y wget python3-dev python3-venv python3-pip") if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features) ADD_PACKAGE_VERIFY(rocky8) diff --git a/docker/requirements.txt b/docker/requirements.txt index 09e799589..5c01145e5 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -2,10 +2,10 @@ pyjks==20.0.0 shortuuid==1.0.11 behavex==2.0.1 docker==5.0.0 -kafka-python==2.0.2 -confluent-kafka==1.7.0 +kafka-python-ng==2.2.3 +confluent-kafka==2.8.0 PyYAML==6.0.1 -m2crypto==0.41.0 +m2crypto==0.43.0 watchdog==2.1.2 pyopenssl==23.0.0 azure-storage-blob==12.13.0 diff --git a/docker/test/integration/cluster/ImageStore.py b/docker/test/integration/cluster/ImageStore.py index 781ce420d..b75cd3051 100644 --- a/docker/test/integration/cluster/ImageStore.py +++ b/docker/test/integration/cluster/ImageStore.py @@ -80,10 +80,10 @@ class ImageStore: so_location = "psqlodbca.so" elif "bullseye" in MinifiContainer.MINIFI_TAG_PREFIX or "bookworm" in MinifiContainer.MINIFI_TAG_PREFIX: install_sql_cmd = "apt -y install odbc-postgresql" - so_location = "/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so" + so_location = "/usr/lib/$(gcc -dumpmachine)/odbc/psqlodbca.so" elif "jammy" in MinifiContainer.MINIFI_TAG_PREFIX or "noble" in MinifiContainer.MINIFI_TAG_PREFIX: install_sql_cmd = "apt -y install odbc-postgresql" - so_location = "/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so" + so_location = "/usr/lib/$(gcc -dumpmachine)/odbc/psqlodbca.so" else: install_sql_cmd = "apk --update --no-cache add psqlodbc" so_location = "psqlodbca.so" @@ -143,7 +143,7 @@ class ImageStore: parse_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}" /opt/minifi/minifi-current/minifi-python/nifi_python_processors/ParseDocument.py && \\' chunk_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}" /opt/minifi/minifi-current/minifi-python/nifi_python_processors/ChunkDocument.py && \\' if python_option == PythonOptions.SYSTEM_INSTALLED_PACKAGES: - if not MinifiContainer.MINIFI_TAG_PREFIX: + if not MinifiContainer.MINIFI_TAG_PREFIX or "bookworm" in MinifiContainer.MINIFI_TAG_PREFIX or "noble" in MinifiContainer.MINIFI_TAG_PREFIX: additional_cmd = "RUN pip3 install --break-system-packages 'langchain<=0.17.0'" else: additional_cmd = "RUN pip3 install 'langchain<=0.17.0'" diff --git a/docker/test/integration/features/fetch_modbus_tcp.feature b/docker/test/integration/features/fetch_modbus_tcp.feature index 99ce18f46..d160a01f2 100644 --- a/docker/test/integration/features/fetch_modbus_tcp.feature +++ b/docker/test/integration/features/fetch_modbus_tcp.feature @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +@x86_x64_only @CORE Feature: Minifi C++ can act as a modbus tcp master diff --git a/docker/test/integration/features/google_cloud_storage.feature b/docker/test/integration/features/google_cloud_storage.feature index de64caff8..2d0b49c86 100644 --- a/docker/test/integration/features/google_cloud_storage.feature +++ b/docker/test/integration/features/google_cloud_storage.feature @@ -28,6 +28,7 @@ Feature: Sending data to Google Cloud Storage using PutGCSObject And a PutFile processor with the "Directory" property set to "/tmp/output" And the "success" relationship of the GetFile processor is connected to the PutGCSObject And the "success" relationship of the PutGCSObject processor is connected to the PutFile + And the "failure" relationship of the PutGCSObject processor is connected to the PutGCSObject When all instances start up diff --git a/docker/test/integration/features/steps/steps.py b/docker/test/integration/features/steps/steps.py index 70a193888..fa9daae1d 100644 --- a/docker/test/integration/features/steps/steps.py +++ b/docker/test/integration/features/steps/steps.py @@ -735,7 +735,6 @@ def step_impl(context, content, topic_name): "ssl.ca.location": ca_cert_file.name, "ssl.certificate.location": client_cert_file.name, "ssl.key.location": client_key_file.name, - "ssl.key.password": "", "client.id": socket.gethostname()}) producer.produce(topic_name, content.encode("utf-8"), callback=delivery_report) producer.flush(10) diff --git a/extensions/python/pythonlibloader/PythonLibLoader.cpp b/extensions/python/pythonlibloader/PythonLibLoader.cpp index 8c2d7ec55..a21eddb3a 100644 --- a/extensions/python/pythonlibloader/PythonLibLoader.cpp +++ b/extensions/python/pythonlibloader/PythonLibLoader.cpp @@ -38,7 +38,7 @@ class PythonLibLoader { python_command = python_binary.value(); } std::string command = python_command + - " -c \"import sysconfig, os, glob; print(min(glob.glob(os.path.join(sysconfig.get_config_var('LIBDIR'), f\\\"libpython{sysconfig.get_config_var('VERSION')}.so*\\\")), key=len, default=''))\""; + R"###( -c "import sysconfig, os, glob; v = sysconfig.get_config_vars(); lib_dir = v['LIBDIR']; ld_lib = v['LDLIBRARY']; so_paths = glob.glob(f'{lib_dir}/*{ld_lib}*'); so_paths.extend(glob.glob(f'{lib_dir}/*/*{ld_lib}*')); print(list(filter(os.path.exists, so_paths))[0])")###"; auto lib_python_path = execCommand(command); if (lib_python_path.empty()) { logger_->log_error("Failed to find libpython path from specified python binary: {}", python_command);
