This is an automated email from the ASF dual-hosted git repository. lordgamez pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 0af586c78e0d387568c6f9a5812cfabff3591a43 Author: Martin Zink <[email protected]> AuthorDate: Thu Apr 25 17:35:48 2024 +0200 MINIFICPP-2345 Upgrade CI matrix Signed-off-by: Gabor Gyimesi <[email protected]> This closes #1776 --- .github/workflows/ci.yml | 44 +++++++------- .github/workflows/verify-python-compatibility.yml | 67 ++++++++++++--------- README.md | 14 ++--- cmake/DockerConfig.cmake | 69 +++++----------------- cmake/VerifyPythonCompatibility.cmake | 47 ++++++--------- docker/bionic/Dockerfile | 56 ------------------ docker/fedora/Dockerfile | 58 ------------------ docker/focal/Dockerfile | 56 ------------------ docker/rockylinux/Dockerfile | 10 +++- libminifi/src/utils/Cron.cpp | 4 +- .../libstdc++/avoid_bogus_Wrestrict_PR105651.patch | 32 ++++++++++ 11 files changed, 138 insertions(+), 319 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab4412322..951643f27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ env: jobs: macos_xcode: name: "macos-xcode" - runs-on: macos-13 + runs-on: macos-14 timeout-minutes: 180 env: MACOS_MINIFI_OPTIONS: >- @@ -72,20 +72,18 @@ jobs: uses: actions/cache/restore@v4 with: path: ${{ env.CCACHE_DIR }} - key: macos-xcode-ccache-${{github.ref}}-${{github.sha}} + key: macos-arm-xcode-ccache-${{github.ref}}-${{github.sha}} restore-keys: | - macos-xcode-ccache-${{github.ref}}- - macos-xcode-ccache-refs/heads/main- + macos-arm-xcode-ccache-${{github.ref}}- + macos-arm-xcode-ccache-refs/heads/main- - id: install_dependencies run: | - # Skip brew update until https://github.com/actions/setup-python/issues/577 is fixed - # brew update - HOMEBREW_NO_AUTO_UPDATE=1 brew install ossp-uuid bison flex ccache sqliteodbc automake autoconf ninja + brew update + brew install ossp-uuid bison flex ccache sqliteodbc automake autoconf ninja - id: setup_env name: setup enviroment run: | - echo "PATH=/usr/lib/ccache:/usr/local/opt/ccache/bin:/usr/local/opt/ccache/libexec:$PATH" >> $GITHUB_ENV - # macOS no longer searches /usr/local/lib for dynamic libraries, e.g. for libsqlite3odbc.so + echo "PATH=/opt/homebrew/opt/ccache:/opt/homebrew/opt/ccache/bin:/opt/homebrew/opt/ccache/libexec:$PATH" >> $GITHUB_ENV echo "DYLD_LIBRARY_PATH=$(brew --prefix)/lib" >> $GITHUB_ENV echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null - name: build @@ -97,7 +95,7 @@ jobs: if: always() with: path: ${{ env.CCACHE_DIR }} - key: macos-xcode-ccache-${{github.ref}}-${{github.sha}} + key: macos-arm-xcode-ccache-${{github.ref}}-${{github.sha}} - name: test id: test run: | @@ -502,8 +500,8 @@ jobs: with: name: ubuntu-clang-binaries path: build/bin - centos: - name: "centos" + rocky: + name: "rocky" runs-on: ubuntu-20.04 timeout-minutes: 180 steps: @@ -513,10 +511,10 @@ jobs: uses: actions/cache/restore@v4 with: path: ${{ env.CCACHE_DIR }} - key: centos-ccache-${{github.ref}}-${{github.sha}} + key: rocky-ccache-${{github.ref}}-${{github.sha}} restore-keys: | - centos-ccache-${{github.ref}}- - centos-ccache-refs/heads/main- + rocky-ccache-${{github.ref}}- + rocky-ccache-refs/heads/main- - id: install_deps run: | sudo apt update @@ -534,18 +532,18 @@ jobs: sudo rm -rf "$AGENT_TOOLSDIRECTORY" - id: build run: | - # centos build can run out of the github runners' disk space if built with RelWithDebInfo so we keep the Release build here - mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DSTRICT_GSL_CHECKS=AUDIT -DMINIFI_FAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON \ + # rocky build can run out of the github runners' disk space if built with RelWithDebInfo so we keep the Release build here + mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DSTRICT_GSL_CHECKS=AUDIT -DMINIFI_FAIL_ON_WARNINGS=OFF -DENABLE_AWS=ON -DENABLE_AZURE=ON \ -DENABLE_COAP=ON -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON \ -DENABLE_OPENCV=ON -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \ -DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON \ - -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make centos-test + -DENABLE_ELASTICSEARCH=ON -DENABLE_GRAFANA_LOKI=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make rocky-test - name: cache save uses: actions/cache/save@v4 if: always() with: path: ${{ env.CCACHE_DIR }} - key: centos-ccache-${{github.ref}}-${{github.sha}} + key: rocky-ccache-${{github.ref}}-${{github.sha}} - id: test run: | # Set core file size limit to unlimited @@ -553,24 +551,24 @@ jobs: echo '/cores/core.%e.%p' | sudo tee /proc/sys/kernel/core_pattern sudo mkdir /cores sudo chmod 777 /cores - docker run --name centos-test --init --ulimit core=-1 --mount type=bind,source=/cores,target=/cores apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}') bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"' + docker run --name rocky-test --init --ulimit core=-1 --mount type=bind,source=/cores,target=/cores apacheminificpp:$(docker images | grep apacheminificpp | grep rocky | awk '{print $2}') bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"' - name: check-cores if: ${{ failure() && steps.test.conclusion == 'failure' }} run: | if [ "$(ls -A /cores)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; - docker cp centos-test:/opt/minifi/build/bin /tmp + docker cp rocky-test:/opt/minifi/build/bin /tmp sudo chmod -R 777 /cores fi - uses: actions/upload-artifact@v4 if: ${{ failure() && env.CORES_EXIST == 'true' }} with: - name: centos-crashes + name: rocky-crashes path: /cores - uses: actions/upload-artifact@v4 if: ${{ failure() && env.CORES_EXIST == 'true' }} with: - name: centos-binaries + name: rocky-binaries path: /tmp/bin docker_build: name: "Docker build for integration tests" diff --git a/.github/workflows/verify-python-compatibility.yml b/.github/workflows/verify-python-compatibility.yml index d757fb4db..db36235e5 100644 --- a/.github/workflows/verify-python-compatibility.yml +++ b/.github/workflows/verify-python-compatibility.yml @@ -1,8 +1,13 @@ name: "MiNiFi-CPP Verify Python Compatibility" on: [workflow_dispatch] +env: + DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ + -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \ + -DENABLE_ELASTICSEARCH=OFF -DENABLE_GRAFANA_LOKI=ON -DDOCKER_BUILD_ONLY=ON + CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache jobs: - centos-build: - name: "Build centos" + rocky-build: + name: "Build package (Rocky 8)" runs-on: ubuntu-22.04 timeout-minutes: 180 steps: @@ -12,10 +17,10 @@ jobs: uses: actions/cache@v3 with: path: ~/.ccache - key: centos-build-with-python-ccache-${{github.ref}}-${{github.sha}} + key: rocky-build-py-ccache-${{github.ref}}-${{github.sha}} restore-keys: | - centos-build-with-python-ccache-${{github.ref}}- - centos-build-with-python-ccache-refs/heads/main- + rocky-build-py-ccache-${{github.ref}}- + rocky-build-py-ccache-refs/heads/main- - id: install_deps run: | sudo apt update @@ -24,48 +29,52 @@ jobs: - id: build run: | if [ -d ~/.ccache ]; then mv ~/.ccache .; fi - mkdir build && cd build && cmake -DENABLE_AWS=OFF -DENABLE_LIBRDKAFKA=OFF -DENABLE_KUBERNETES=OFF -DENABLE_SYSTEMD=OFF -DENABLE_SQL=OFF -DENABLE_MQTT=OFF -DENABLE_GCP=OFF \ - -DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_ELASTICSEARCH=OFF -DENABLE_PROMETHEUS=OFF -DDOCKER_BUILD_ONLY=ON -DENABLE_PROCFS=OFF -DENABLE_OPC=OFF \ - -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos + mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make rocky + - name: cache save + uses: actions/cache/save@v4 + if: always() + with: + path: ${{ env.CCACHE_DIR }} + key: rocky-build-py-ccache-${{github.ref}}-${{github.sha}} - uses: actions/upload-artifact@v3 with: name: minifi-tar - path: build/nifi-minifi-cpp-*-bin-centos.tar.gz + path: build/nifi-minifi-cpp-*-bin-rockylinux.tar.gz if-no-files-found: error - docker-tests: + docker-python-tests: name: "${{ matrix.platform.name }}" runs-on: ubuntu-20.04 timeout-minutes: 180 - needs: centos-build + needs: rocky-build strategy: fail-fast: false matrix: platform: [ - { name: "Rocky Linux 8", + { name: "Rocky Linux 8 Python", id: "rocky8", - build_cmd: "make rocky8_docker_from_centos_build", - verify_cmd: "make docker-verify-python-rocky8" }, - { name: "Rocky Linux 9", + build_cmd: "make rocky8_py_from_rocky_package", + verify_cmd: "make docker-verify-rocky8_py" }, + { name: "Rocky Linux 9 Python", id: "rocky9", - build_cmd: "make rocky9_docker_from_centos_build", - verify_cmd: "make docker-verify-python-rocky9" }, - { name: "Patchelf-ed Ubuntu 22.04", + build_cmd: "make rocky9_py_from_rocky_package", + verify_cmd: "make docker-verify-rocky9_py" }, + { name: "Patchelf-ed Ubuntu 22.04 Python", id: "patchelfed_ubuntu22.04", - build_cmd: "make patched_jammy_docker_from_centos_build", - verify_cmd: "make docker-verify-python-patched_jammy" }, - { name: "Patchelf-ed Debian 11", + build_cmd: "make patched_jammy_py_from_rocky_package", + verify_cmd: "make docker-verify-patched_jammy_py" }, + { name: "Patchelf-ed Debian 11 Python", id: "patchelfed_debian11", - build_cmd: "make patched_bullseye_docker_from_centos_build", - verify_cmd: "make docker-verify-python-patched_bullseye" }, - { name: "Ubuntu 22.04 with Anaconda", + build_cmd: "make patched_bullseye_py_from_rocky_package", + verify_cmd: "make docker-verify-patched_bullseye_py" }, + { name: "Ubuntu 22.04 with Anaconda Python", id: "ubuntu22.04_anaconda", - build_cmd: "make jammy_docker_from_centos_build && make conda_jammy_docker_from_centos_build", - verify_cmd: "make docker-verify-python-conda_jammy" }, - { name: "Rocky Linux 9 in virtualenv", + build_cmd: "make jammy_py_from_rocky_package && make conda_jammy_py_docker_from_centos_build", + verify_cmd: "make docker-verify-conda_jammy_py" }, + { name: "Rocky Linux 9 in virtualenv Python", id: "rocky9_venv", - build_cmd: "make rocky9_docker_from_centos_build && make venv_rocky9_docker_from_centos_build", - verify_cmd: "make docker-verify-python-venv_rocky9" } + build_cmd: "make rocky9_py_from_rocky_package && make venv_rocky9_py_docker_from_centos_build", + verify_cmd: "make docker-verify-venv_rocky9_py" } ] steps: diff --git a/README.md b/README.md index ee85355b5..c3c703fa8 100644 --- a/README.md +++ b/README.md @@ -497,16 +497,12 @@ $ make docker-verify ``` ### Building For Other Distros -If you have docker installed on your machine you can build for CentOS 7, Fedora 34, Ubuntu 18.04, and Ubuntu 20.04 via our make docker commands. The following table -provides the command to build your distro and the output file in your build directory. Since the versions are limited ( except for Ubuntu ) we output the archive based on the distro's name. +Since only glibc and libstdc++ is dynamically linked to MiNiFi, the binary built with RHEL devtoolset should be distro-agnostic, and work on any relatively modern distro. - -| Distro | command | Output File | -|-----------------------|:------------|:---------------------------------------| -| CentOS 7 | make centos | nifi-minifi-cpp-centos-$VERSION.tar.gz | -| Fedora 34 | make fedora | nifi-minifi-cpp-fedora-$VERSION.tar.gz | -| Ubuntu 18.04 (bionic) | make u18 | nifi-minifi-cpp-bionic-$VERSION.tar.gz | -| Ubuntu 20.04 (focal) | make u20 | nifi-minifi-cpp-focal-$VERSION.tar.gz | +| Distro | command | Output File | +|---------------|:------------|:-------------------------------------------| +| centos 7 | make centos | nifi-minifi-cpp-centos-$VERSION.tar.gz | +| Rockylinux 8 | make rocky | nifi-minifi-cpp-rockylinux-$VERSION.tar.gz | You can avoid the requirement of an up-to-date compiler when generating the build system by adding `-DDOCKER_BUILD_ONLY=ON` to the cmake command line. This disables all cmake targets except the docker build and test ones. diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake index 383472787..67282a61c 100644 --- a/cmake/DockerConfig.cmake +++ b/cmake/DockerConfig.cmake @@ -87,13 +87,13 @@ add_custom_target( WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) add_custom_target( - centos-test + rocky-test COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh -u 1000 -g 1000 -v ${PROJECT_VERSION_STR} -o ${MINIFI_DOCKER_OPTIONS_STR} - -d centos + -d rockylinux -c BUILD_NUMBER=${BUILD_NUMBER} -c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION} -c DOCKER_SKIP_TESTS=OFF @@ -104,59 +104,9 @@ add_custom_target( -c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) -add_custom_target( - fedora - COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh - -u 1000 - -g 1000 - -v ${PROJECT_VERSION_STR} - -o ${MINIFI_DOCKER_OPTIONS_STR} - -l ${CMAKE_BINARY_DIR} - -d fedora - -c BUILD_NUMBER=${BUILD_NUMBER} - -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS} - -c DOCKER_PUSH=${DOCKER_PUSH} - -c DOCKER_TAGS=${DOCKER_TAGS} - -c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE} - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) - -add_custom_target( - u18 - COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh - -u 1000 - -g 1000 - -v ${PROJECT_VERSION_STR} - -o ${MINIFI_DOCKER_OPTIONS_STR} - -l ${CMAKE_BINARY_DIR} - -d bionic - -c BUILD_NUMBER=${BUILD_NUMBER} - -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS} - -c DOCKER_PUSH=${DOCKER_PUSH} - -c DOCKER_TAGS=${DOCKER_TAGS} - -c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE} - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) - -add_custom_target( - u20 - COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh - -u 1000 - -g 1000 - -v ${PROJECT_VERSION_STR} - -o ${MINIFI_DOCKER_OPTIONS_STR} - -l ${CMAKE_BINARY_DIR} - -d focal - -c BUILD_NUMBER=${BUILD_NUMBER} - -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -c DOCKER_PLATFORMS=${DOCKER_PLATFORMS} - -c DOCKER_PUSH=${DOCKER_PUSH} - -c DOCKER_TAGS=${DOCKER_TAGS} - -c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE} - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) add_custom_target( - rockylinux + rocky COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh -u 1000 -g 1000 @@ -192,4 +142,17 @@ if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features) COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh ${PROJECT_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS}) endif() +function(CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE BASE_IMAGE TAG_PREFIX INSTALL_PACKAGE_CMD) + add_custom_target( + ${TAG_PREFIX}_from_rocky_package + COMMAND DOCKER_BUILDKIT=1 docker build + --build-arg MINIFI_VERSION=${MINIFI_VERSION_STR} + --build-arg BASE_IMAGE=${BASE_IMAGE} + --build-arg ARCHIVE_LOCATION=nifi-minifi-cpp-${MINIFI_VERSION_STR}-bin-rockylinux.tar.gz + --build-arg INSTALL_PACKAGE_CMD=${INSTALL_PACKAGE_CMD} + -t apacheminificpp:${TAG_PREFIX}-${MINIFI_VERSION_STR} + -f ${CMAKE_SOURCE_DIR}/docker/python-verify/installed.Dockerfile + ${CMAKE_BINARY_DIR}) +endfunction() + include(VerifyPythonCompatibility) diff --git a/cmake/VerifyPythonCompatibility.cmake b/cmake/VerifyPythonCompatibility.cmake index e537e2dd1..4eca2be9b 100644 --- a/cmake/VerifyPythonCompatibility.cmake +++ b/cmake/VerifyPythonCompatibility.cmake @@ -21,34 +21,21 @@ endif() set(MINIFI_VERSION_STR ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}) -function(ADD_DOCKER_TARGET_FROM_CENTOS BASE_IMAGE TAG_PREFIX INSTALL_PACKAGE_CMD) - add_custom_target( - ${TAG_PREFIX}_docker_from_centos_build - COMMAND DOCKER_BUILDKIT=1 docker build - --build-arg MINIFI_VERSION=${MINIFI_VERSION_STR} - --build-arg BASE_IMAGE=${BASE_IMAGE} - --build-arg ARCHIVE_LOCATION=nifi-minifi-cpp-${MINIFI_VERSION_STR}-bin-centos.tar.gz - --build-arg INSTALL_PACKAGE_CMD=${INSTALL_PACKAGE_CMD} - -t apacheminificpp:${TAG_PREFIX}-${MINIFI_VERSION_STR} - -f ${CMAKE_SOURCE_DIR}/docker/python-verify/installed.Dockerfile - ${CMAKE_BINARY_DIR}) -endfunction() - function(ADD_DOCKER_VERIFY_PYTHON TAG_PREFIX HAS_MODULES) if (HAS_MODULES) add_custom_target( - docker-verify-python-${TAG_PREFIX} + docker-verify-${TAG_PREFIX} COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ENABLE_PYTHON_SCRIPTING) else() add_custom_target( - docker-verify-python-${TAG_PREFIX} + docker-verify-${TAG_PREFIX} COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ENABLE_PYTHON_SCRIPTING --tags_to_exclude NEEDS_NUMPY) endif() endfunction() function(ADD_CONDA_TO_DOCKER TAG_PREFIX) add_custom_target( - conda_${TAG_PREFIX}_docker_from_centos_build + conda_${TAG_PREFIX}_docker_from_rocky_build COMMAND DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE=apacheminificpp:${TAG_PREFIX}-${MINIFI_VERSION_STR} -t apacheminificpp:conda_${TAG_PREFIX}-${MINIFI_VERSION_STR} @@ -58,7 +45,7 @@ endfunction() function(ADD_VENV_TO_DOCKER TAG_PREFIX) add_custom_target( - venv_${TAG_PREFIX}_docker_from_centos_build + venv_${TAG_PREFIX}_docker_from_rocky_build COMMAND DOCKER_BUILDKIT=1 docker build --build-arg BASE_IMAGE=apacheminificpp:${TAG_PREFIX}-${MINIFI_VERSION_STR} -t apacheminificpp:venv_${TAG_PREFIX}-${MINIFI_VERSION_STR} @@ -67,19 +54,19 @@ function(ADD_VENV_TO_DOCKER TAG_PREFIX) endfunction() -ADD_DOCKER_TARGET_FROM_CENTOS(debian:bullseye patched_bullseye "apt update \\&\\& apt install -y patchelf libpython3-dev python3-venv python3-pip wget \\&\\& patchelf /opt/minifi/minifi-current/extensions/libminifi-python-script-extension.so --replace-needed libpython3.so libpython3.9.so") -ADD_DOCKER_TARGET_FROM_CENTOS(ubuntu:jammy patched_jammy "apt update \\&\\& apt install -y patchelf libpython3.10-dev python3.10-venv python3-pip wget \\&\\& patchelf /opt/minifi/minifi-current/extensions/libminifi-python-script-extension.so --replace-needed libpython3.so libpython3.10.so.1.0") -ADD_DOCKER_TARGET_FROM_CENTOS(rockylinux:8 rocky8 "yum install -y python3-libs python3-pip python3-devel gcc-c++ wget") -ADD_DOCKER_TARGET_FROM_CENTOS(rockylinux:9 rocky9 "yum install -y python3-libs python3-pip wget") -ADD_DOCKER_TARGET_FROM_CENTOS(ubuntu:jammy jammy "apt update \\&\\& apt install -y wget") -ADD_CONDA_TO_DOCKER(jammy) -ADD_VENV_TO_DOCKER(rocky9) +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bullseye patched_bullseye_py "apt update \\&\\& apt install -y patchelf libpython3-dev python3-venv python3-pip wget \\&\\& patchelf /opt/minifi/minifi-current/extensions/libminifi-python-script-extension.so --replace-needed libpython3.so libpython3.9.so") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:jammy patched_jammy_py "apt update \\&\\& apt install -y patchelf libpython3.10-dev python3.10-venv python3-pip wget \\&\\& patchelf /opt/minifi/minifi-current/extensions/libminifi-python-script-extension.so --replace-needed libpython3.so libpython3.10.so.1.0") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:8 rocky8_py "yum install -y python3-libs python3-pip python3-devel gcc-c++ wget") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:9 rocky9_py "yum install -y python3-libs python3-pip wget") +CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:jammy jammy_py "apt update \\&\\& apt install -y wget") +ADD_CONDA_TO_DOCKER(jammy_py) +ADD_VENV_TO_DOCKER(rocky9_py) if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features) - ADD_DOCKER_VERIFY_PYTHON(rocky8 FALSE) - ADD_DOCKER_VERIFY_PYTHON(rocky9 FALSE) - ADD_DOCKER_VERIFY_PYTHON(patched_jammy FALSE) - ADD_DOCKER_VERIFY_PYTHON(patched_bullseye FALSE) - ADD_DOCKER_VERIFY_PYTHON(conda_jammy TRUE) - ADD_DOCKER_VERIFY_PYTHON(venv_rocky9 TRUE) + ADD_DOCKER_VERIFY_PYTHON(rocky8_py FALSE) + ADD_DOCKER_VERIFY_PYTHON(rocky9_py FALSE) + ADD_DOCKER_VERIFY_PYTHON(patched_jammy_py FALSE) + ADD_DOCKER_VERIFY_PYTHON(patched_bullseye_py FALSE) + ADD_DOCKER_VERIFY_PYTHON(conda_jammy_py TRUE) + ADD_DOCKER_VERIFY_PYTHON(venv_rocky9_py TRUE) endif() diff --git a/docker/bionic/Dockerfile b/docker/bionic/Dockerfile deleted file mode 100644 index ca66810e5..000000000 --- a/docker/bionic/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# First stage: the build environment -ARG BASE_IMAGE="ubuntu:bionic" - -FROM ${BASE_IMAGE} AS build -LABEL maintainer="Apache NiFi <[email protected]>" - -ARG MINIFI_VERSION -ARG DOCKER_MAKE_TARGET="all" - -# Install the system dependencies needed for a build - -ENV MINIFI_BASE_DIR /opt/minifi -ENV MINIFI_HOME $MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ - -RUN apt-get update \ - && DEBIAN_FRONTEND="noninteractive" apt-get install -y openjdk-8-jdk openjdk-8-source sudo git maven autogen autoconf automake pkgconf libtool libfl-dev libusb-1.0-0-dev \ - libpng-dev libgps-dev libsqliteodbc \ - && mkdir -p $MINIFI_BASE_DIR -COPY . ${MINIFI_BASE_DIR} - - -FROM build_deps AS release - -# MINIFI_OPTIONS will be passed directly to cmake -# use it to define cmake options (e.g. -DENABLE_AWS=ON -DENABLE_AZURE=ON) -ARG MINIFI_OPTIONS="" -ARG CMAKE_BUILD_TYPE=Release - -ENV CC gcc-11 -ENV CXX g++-11 - -# Run bootstrap and build -RUN cd $MINIFI_BASE_DIR \ - && ./bootstrap.sh -t \ - && cd $MINIFI_BASE_DIR/build \ - && cmake -DSTATIC_BUILD= -DSKIP_TESTS=true ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. \ - && make -j "$(nproc)" ${DOCKER_MAKE_TARGET} diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile deleted file mode 100644 index cd62a492e..000000000 --- a/docker/fedora/Dockerfile +++ /dev/null @@ -1,58 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# First stage: the build environment -ARG BASE_IMAGE="fedora:34" - -FROM ${BASE_IMAGE} AS build -LABEL maintainer="Apache NiFi <[email protected]>" - -ARG MINIFI_VERSION -ARG DOCKER_MAKE_TARGET="all" - -# Install the system dependencies needed for a build - -ENV MINIFI_BASE_DIR /opt/minifi -ENV MINIFI_HOME $MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION - -RUN echo "fastestmirror=True" | tee -a /etc/dnf/dnf.conf -RUN for iter in {1..10}; do yum update -y && \ - yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel flex bison make patch sudo git which maven libtool autoconf automake java-1.8.0-openjdk java-1.8.0-openjdk-devel sudo \ - git which maven make libarchive libusb-devel libpng-devel gpsd-devel libpcap-devel perl && \ - yum clean all && exit_code=0 && break || exit_code=$? && echo "yum error: retry $iter in 10s" && sleep 10; done; \ - (exit $exit_code) - -RUN mkdir -p $MINIFI_BASE_DIR -COPY . ${MINIFI_BASE_DIR} - - -FROM build_deps AS release - -# MINIFI_OPTIONS will be passed directly to cmake -# use it to define cmake options (e.g. -DENABLE_AWS=ON -DENABLE_AZURE=ON) -ARG MINIFI_OPTIONS="" -ARG CMAKE_BUILD_TYPE=Release -# Perform the build -RUN cd $MINIFI_BASE_DIR \ - && ./bootstrap.sh -e \ - && rm -rf build \ - && mkdir build \ - && cd build \ - && cmake3 -DSTATIC_BUILD= -DSKIP_TESTS=true ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. \ - && make -j "$(nproc)" ${DOCKER_MAKE_TARGET} - diff --git a/docker/focal/Dockerfile b/docker/focal/Dockerfile deleted file mode 100644 index 1db4cc21f..000000000 --- a/docker/focal/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# First stage: the build environment -ARG BASE_IMAGE="ubuntu:focal" - -FROM ${BASE_IMAGE} AS build -LABEL maintainer="Apache NiFi <[email protected]>" - -ARG MINIFI_VERSION -ARG DOCKER_MAKE_TARGET="all" - -# Install the system dependencies needed for a build - -ENV MINIFI_BASE_DIR /opt/minifi -ENV MINIFI_HOME $MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ - -RUN apt update \ - && DEBIAN_FRONTEND="noninteractive" apt install -y openjdk-8-jdk openjdk-8-source python3.9-dev sudo git maven autogen autoconf automake cmake software-properties-common pkgconf libtool \ - libfl-dev libusb-1.0-0-dev libpng-dev libgps-dev libsqliteodbc \ - && mkdir -p "${MINIFI_BASE_DIR}" - -COPY . ${MINIFI_BASE_DIR} - - -FROM build_deps AS release - -# MINIFI_OPTIONS will be passed directly to cmake -# use it to define cmake options (e.g. -DENABLE_AWS=ON -DENABLE_AZURE=ON) -ARG MINIFI_OPTIONS="" -ARG CMAKE_BUILD_TYPE=Release -ENV CC gcc-11 -ENV CXX g++-11 - -# Run bootstrap and build -RUN cd $MINIFI_BASE_DIR \ - && ./bootstrap.sh -t \ - && cd $MINIFI_BASE_DIR/build \ - && cmake -DSTATIC_BUILD= -DSKIP_TESTS=true ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. \ - && make -j "$(nproc)" ${DOCKER_MAKE_TARGET} diff --git a/docker/rockylinux/Dockerfile b/docker/rockylinux/Dockerfile index a6504d841..345f259f3 100644 --- a/docker/rockylinux/Dockerfile +++ b/docker/rockylinux/Dockerfile @@ -40,26 +40,30 @@ COPY . ${MINIFI_BASE_DIR} # Install the system dependencies needed for a build # gpsd-devel and ccache are in EPEL -RUN dnf -y install epel-release && dnf -y install sudo git which make libarchive ccache ca-certificates perl && \ +RUN dnf -y install epel-release && dnf -y install gcc-toolset-12 sudo git which make libarchive ccache ca-certificates perl patch bison flex libtool cmake && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_ALL=ON"; then dnf -y --enablerepo=devel install gpsd-devel libpng-devel libusbx-devel python3-devel java-1.8.0-openjdk maven libpcap-devel; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_GPS=ON"; then dnf -y install gpsd-devel; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_JNI=ON"; then dnf -y install java-1.8.0-openjdk maven; fi && \ - if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PCAP=ON"; then dnf -y install libpcap-devel; fi && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PCAP=ON"; then dnf -y --enablerepo=devel install libpcap-devel; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_USB_CAMERA=ON"; then dnf -y install libpng-devel libusbx-devel; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PYTHON_SCRIPTING=ON"; then dnf -y install python3-devel; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SFTP=ON" && [ "${DOCKER_SKIP_TESTS}" == "OFF" ]; then dnf -y install java-1.8.0-openjdk maven; fi RUN cd $MINIFI_BASE_DIR && \ - ./bootstrap.sh -t && \ ln -s /usr/bin/ccache /usr/lib64/ccache/c++ # Setup minificpp user RUN groupadd -g ${GID} ${USER} && useradd -g ${GID} ${USER} && \ chown -R ${USER}:${USER} ${MINIFI_BASE_DIR} +# Patching standard header to avoid https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105651 +RUN patch -p1 /opt/rh/gcc-toolset-12/root/usr/include/c++/12/bits/basic_string.tcc ${MINIFI_BASE_DIR}/thirdparty/libstdc++/avoid_bogus_Wrestrict_PR105651.patch + USER ${USER} # Perform the build RUN cd $MINIFI_BASE_DIR && \ + mkdir build && \ cd build && \ source /opt/rh/gcc-toolset-12/enable && \ export PATH=/usr/lib64/ccache${PATH:+:${PATH}} && \ diff --git a/libminifi/src/utils/Cron.cpp b/libminifi/src/utils/Cron.cpp index ec0373ffa..ff06abbee 100644 --- a/libminifi/src/utils/Cron.cpp +++ b/libminifi/src/utils/Cron.cpp @@ -48,9 +48,9 @@ namespace { // https://github.com/HowardHinnant/date/issues/550 // Due to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78714 // the month parsing with '%b' and the weekday parsing with '%a' is case-sensitive in gcc11 -// This has been fixed in gcc12.2 +// This has been fixed in gcc13 std::stringstream getCaseInsensitiveCStream(const std::string& str) { -#if defined(__GNUC__) && (__GNUC__ < 12 || (__GNUC__ == 12 && __GNUC_MINOR__ < 2)) +#if defined(__GNUC__) && (__GNUC__ < 13) auto patched_str = string::toLower(str); if (!patched_str.empty()) patched_str[0] = static_cast<char>(std::toupper(static_cast<unsigned char>(patched_str[0]))); diff --git a/thirdparty/libstdc++/avoid_bogus_Wrestrict_PR105651.patch b/thirdparty/libstdc++/avoid_bogus_Wrestrict_PR105651.patch new file mode 100644 index 000000000..8ac0a762b --- /dev/null +++ b/thirdparty/libstdc++/avoid_bogus_Wrestrict_PR105651.patch @@ -0,0 +1,32 @@ +From 50dd920299f8f905f47475bf77c299d9a84512b8 Mon Sep 17 00:00:00 2001 +From: Jason Merrill <[email protected]> +Date: Thu, 18 Aug 2022 23:53:16 -0400 +Subject: [PATCH] libstdc++: avoid bogus -Wrestrict [PR105651] +To: [email protected] + + PR tree-optimization/105651 + +libstdc++-v3/ChangeLog: + + * include/bits/basic_string.tcc (_M_replace): Add an assert + to avoid -Wrestrict false positive. +--- + libstdc++-v3/include/bits/basic_string.tcc | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/libstdc++-v3/include/bits/basic_string.tcc b/libstdc++-v3/include/bits/basic_string.tcc +index 4563c61429a..0d5c4d6191e 100644 +--- a/libstdc++-v3/include/bits/basic_string.tcc ++++ b/libstdc++-v3/include/bits/basic_string.tcc +@@ -529,6 +529,9 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION + { + const size_type __nleft = (__p + __len1) - __s; + this->_S_move(__p, __s, __nleft); ++ // Tell the middle-end that the copy can't overlap ++ // (PR105651). ++ if (__len2 < __nleft) __builtin_unreachable (); + this->_S_copy(__p + __nleft, __p + __len2, + __len2 - __nleft); + } +-- +2.31.1
