This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 7942e0e7f344d449bb8b4635b1a8737caf57bce5 Author: Gabor Gyimesi <[email protected]> AuthorDate: Fri Mar 3 14:16:42 2023 +0100 MINIFICPP-2053 Reduce CentOS image size in CI build It started to occur that the CentOS build exceeded the 14GB disk space Github Actions VMs provide (for example https://github.com/apache/nifi-minifi-cpp/actions/runs/4263514879/jobs/7420351598). JDK and the packaged tar.gz are removed from the CI build and JNI and SFTP builds are moved to the ubuntu build. This reduces the image size by about 2GBs. Closes #1513 Signed-off-by: Marton Szasz <[email protected]> --- .github/workflows/ci.yml | 22 ++++++++++++---------- cmake/DockerConfig.cmake | 13 +++++++++++++ docker/DockerBuild.sh | 7 +++++-- docker/bionic/Dockerfile | 3 ++- docker/centos/Dockerfile | 18 +++++++++++++----- docker/fedora/Dockerfile | 3 ++- docker/focal/Dockerfile | 3 ++- 7 files changed, 49 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9aa080554..9047e87d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: export CC=gcc-11 export CXX=g++-11 cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_SQL=OFF -DENABLE_LIBRDKAFKA=OFF -DENABLE_AWS=OFF \ - -DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_GCP=OFF -DENABLE_PROCFS=OFF -DENABLE_BUSTACHE=ON -DENABLE_PCAP=ON .. + -DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_GCP=OFF -DENABLE_PROCFS=OFF -DENABLE_BUSTACHE=ON -DENABLE_PCAP=ON -DENABLE_JNI=ON -DENABLE_SFTP=ON .. make -j$(nproc) VERBOSE=1 - name: test run: cd build && make test ARGS="--timeout 300 -j2 --output-on-failure" @@ -143,8 +143,8 @@ jobs: export CXXFLAGS="${CXXFLAGS} -stdlib=libc++" export LDFLAGS="${LDFLAGS} -stdlib=libc++" cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_BUSTACHE=ON -DENABLE_COAP=ON \ - -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_JNI=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \ - -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PCAP=ON -DENABLE_PYTHON=ON -DENABLE_SENSORS=ON -DENABLE_SFTP=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON -DENABLE_TENSORFLOW=OFF \ + -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \ + -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PCAP=ON -DENABLE_PYTHON=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON -DENABLE_TENSORFLOW=OFF \ -DENABLE_USB_CAMERA=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. cmake --build . --parallel $(nproc) @@ -196,13 +196,13 @@ jobs: - id: build run: | if [ -d ~/.ccache ]; then mv ~/.ccache .; fi - mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_COAP=ON \ - -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_JNI=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \ - -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PYTHON=ON -DENABLE_SENSORS=ON -DENABLE_SFTP=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON -DENABLE_TENSORFLOW=OFF \ - -DENABLE_USB_CAMERA=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON \ - -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos + mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON \ + -DENABLE_COAP=ON -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON \ + -DENABLE_OPENCV=ON -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PYTHON=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \ + -DENABLE_USB_CAMERA=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON \ + -DENABLE_ELASTICSEARCH=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos-test - id: test - run: docker run --rm apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}') bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"' + run: docker run --rm apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}' | head -1) bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"' docker_integration_tests: name: "Docker integration tests" runs-on: ubuntu-20.04 @@ -223,7 +223,9 @@ jobs: if [ -d ~/.ccache ]; then mv ~/.ccache .; fi mkdir build cd build - cmake -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DENABLE_JNI=OFF -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. + cmake -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ + -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \ + -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. make docker - id: install_deps run: | diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake index 04b9bb92d..dc555fead 100644 --- a/cmake/DockerConfig.cmake +++ b/cmake/DockerConfig.cmake @@ -64,6 +64,19 @@ add_custom_target( -c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) +add_custom_target( + centos-test + COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh + -u 1000 + -g 1000 + -v ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} + -o ${MINIFI_DOCKER_OPTIONS_STR} + -d centos + -c BUILD_NUMBER=${BUILD_NUMBER} + -c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION} + -c DOCKER_SKIP_TESTS=OFF + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) + add_custom_target( fedora COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerBuild.sh diff --git a/docker/DockerBuild.sh b/docker/DockerBuild.sh index 0f53f7f77..14c5efd11 100755 --- a/docker/DockerBuild.sh +++ b/docker/DockerBuild.sh @@ -160,10 +160,13 @@ BUILD_ARGS+=("--build-arg" "UID=${UID_ARG}" "--build-arg" "DUMP_LOCATION=${DUMP_LOCATION}" "--build-arg" "DISTRO_NAME=${DISTRO_NAME}" "--build-arg" "DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}") +if [ -n "${DUMP_LOCATION}" ]; then + BUILD_ARGS+=("--build-arg" "DOCKER_MAKE_TARGET=package") +fi if [ -n "${DISTRO_NAME}" ]; then - echo DOCKER_BUILDKIT=0 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" .. - DOCKER_BUILDKIT=0 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" .. + echo DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" .. + DOCKER_BUILDKIT=1 docker build "${BUILD_ARGS[@]}" -f "${DOCKERFILE}" -t apacheminificpp:"${TAG}" .. if [ -n "${DOCKER_CCACHE_DUMP_LOCATION}" ]; then dump_ccache "apacheminificpp:${TAG}" "${DOCKER_CCACHE_DUMP_LOCATION}" diff --git a/docker/bionic/Dockerfile b/docker/bionic/Dockerfile index a715d2afa..41ac52461 100644 --- a/docker/bionic/Dockerfile +++ b/docker/bionic/Dockerfile @@ -21,6 +21,7 @@ FROM ubuntu:bionic AS build_deps LABEL maintainer="Apache NiFi <[email protected]>" ARG MINIFI_VERSION +ARG DOCKER_MAKE_TARGET="all" # Install the system dependencies needed for a build @@ -50,4 +51,4 @@ RUN cd $MINIFI_BASE_DIR \ && ./bootstrap.sh -t \ && cd $MINIFI_BASE_DIR/build \ && cmake -DSTATIC_BUILD= -DSKIP_TESTS=true ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. \ - && make -j "$(nproc)" package + && make -j "$(nproc)" ${DOCKER_MAKE_TARGET} diff --git a/docker/centos/Dockerfile b/docker/centos/Dockerfile index be82d8571..f0e55107b 100644 --- a/docker/centos/Dockerfile +++ b/docker/centos/Dockerfile @@ -29,6 +29,7 @@ ARG GID=1000 ARG MINIFI_OPTIONS="" ARG CMAKE_BUILD_TYPE=Release ARG DOCKER_SKIP_TESTS=ON +ARG DOCKER_MAKE_TARGET="all" # Install the system dependencies needed for a build @@ -36,13 +37,20 @@ ENV MINIFI_BASE_DIR /opt/minifi ENV MINIFI_HOME $MINIFI_BASE_DIR/nifi-minifi-cpp-$MINIFI_VERSION ENV USER minificpp -# gpsd-devel is in EPEL -RUN yum -y install epel-release && yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel python36-devel sudo git which maven make libarchive boost-devel lua-devel libusbx-devel libpng-devel \ - gpsd-devel libpcap-devel ccache ca-certificates - RUN mkdir -p $MINIFI_BASE_DIR COPY . ${MINIFI_BASE_DIR} +# gpsd-devel and ccache are in EPEL +RUN yum -y install epel-release && yum -y install sudo git which make libarchive ccache ca-certificates && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_GPS=ON"; then yum -y install gpsd-devel; fi && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_JNI=ON"; then yum -y install java-1.8.0-openjdk maven; fi && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_PCAP=ON"; then yum -y install libpcap-devel; fi && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_USB_CAMERA=ON"; then yum -y install libpng-devel libusbx-devel; fi && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_OPENCV=ON" || echo "$MINIFI_OPTIONS" | grep -q "ENABLE_BUSTACHE=ON"; then yum -y install boost-devel; fi && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SCRIPTING=ON" && echo "$MINIFI_OPTIONS" | grep -q "ENABLE_LUA_SCRIPTING=ON"; then yum -y install lua-devel; fi && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SCRIPTING=ON" && echo "$MINIFI_OPTIONS" | grep -q "DISABLE_PYTHON_SCRIPTING=OFF"; then yum -y install python36-devel; fi && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_SFTP=ON" && [ "${DOCKER_SKIP_TESTS}" == "OFF" ]; then yum -y install java-1.8.0-openjdk maven; fi + RUN cd $MINIFI_BASE_DIR && \ ./bootstrap.sh -t && \ ln -s /usr/bin/ccache /usr/lib64/ccache/c++ @@ -60,4 +68,4 @@ RUN cd $MINIFI_BASE_DIR && \ export PATH=/usr/lib64/ccache${PATH:+:${PATH}} && \ export CCACHE_DIR=${MINIFI_BASE_DIR}/.ccache && \ cmake3 -DSTATIC_BUILD= -DSKIP_TESTS=${DOCKER_SKIP_TESTS} ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \ - make -j "$(nproc)" package + make -j "$(nproc)" ${DOCKER_MAKE_TARGET} diff --git a/docker/fedora/Dockerfile b/docker/fedora/Dockerfile index b6155d034..1f34ba776 100644 --- a/docker/fedora/Dockerfile +++ b/docker/fedora/Dockerfile @@ -21,6 +21,7 @@ FROM fedora:34 AS build_deps LABEL maintainer="Apache NiFi <[email protected]>" ARG MINIFI_VERSION +ARG DOCKER_MAKE_TARGET="all" # Install the system dependencies needed for a build @@ -51,5 +52,5 @@ RUN cd $MINIFI_BASE_DIR \ && mkdir build \ && cd build \ && cmake3 -DSTATIC_BUILD= -DSKIP_TESTS=true ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. \ - && make -j "$(nproc)" package + && make -j "$(nproc)" ${DOCKER_MAKE_TARGET} diff --git a/docker/focal/Dockerfile b/docker/focal/Dockerfile index 77b3ef74d..564c74154 100644 --- a/docker/focal/Dockerfile +++ b/docker/focal/Dockerfile @@ -21,6 +21,7 @@ FROM ubuntu:focal AS build_deps LABEL maintainer="Apache NiFi <[email protected]>" ARG MINIFI_VERSION +ARG DOCKER_MAKE_TARGET="all" # Install the system dependencies needed for a build @@ -50,4 +51,4 @@ RUN cd $MINIFI_BASE_DIR \ && ./bootstrap.sh -t \ && cd $MINIFI_BASE_DIR/build \ && cmake -DSTATIC_BUILD= -DSKIP_TESTS=true ${MINIFI_OPTIONS} -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. \ - && make -j "$(nproc)" package + && make -j "$(nproc)" ${DOCKER_MAKE_TARGET}
