This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit dcf5e7194a1dcf11dbfee725b4405a86d8507112 Author: Martin Zink <[email protected]> AuthorDate: Fri Apr 14 18:08:24 2023 +0200 MINIFICPP-2073 Separate docker build from docker tests in CI Closes #1538 Signed-off-by: Marton Szasz <[email protected]> --- .github/workflows/ci.yml | 135 ++++++++++++++++++++++-- cmake/DockerConfig.cmake | 16 +++ docker/test/integration/features/CMakeLists.txt | 26 ++++- 3 files changed, 167 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 185639af3..7a8b50ebc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,9 @@ name: "MiNiFi-CPP CI" on: [push, pull_request, workflow_dispatch] +env: + DOCKER_CMAKE_FLAGS: -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ + -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \ + -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache jobs: macos_xcode: name: "macos-xcode" @@ -282,8 +286,8 @@ jobs: with: name: centos-binaries path: /tmp/bin - docker_integration_tests: - name: "Docker integration tests" + docker_build: + name: "Docker build for integration tests" runs-on: ubuntu-20.04 timeout-minutes: 180 steps: @@ -302,13 +306,132 @@ jobs: if [ -d ~/.ccache ]; then mv ~/.ccache .; fi mkdir build cd build - cmake -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \ - -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \ - -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. + cmake ${DOCKER_CMAKE_FLAGS} .. make docker + - name: Save docker image + run: cd build && docker save -o minifi_docker.tar apacheminificpp:$(grep CMAKE_PROJECT_VERSION:STATIC CMakeCache.txt | cut -d "=" -f2) + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: minifi_docker + path: build/minifi_docker.tar + docker_tests_q1: + name: "Docker integration tests 1/4" + needs: docker_build + runs-on: ubuntu-20.04 + timeout-minutes: 180 + steps: + - id: checkout + uses: actions/checkout@v3 + - id: run_cmake + name: Run CMake + run: | + mkdir build + cd build + cmake ${DOCKER_CMAKE_FLAGS} .. + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: minifi_docker + path: build + - name: Load Docker image + run: | + docker load --input ./build/minifi_docker.tar + - id: install_deps + name: Install dependencies for Docker Verify + run: | + sudo apt update + sudo apt install -y python3-virtualenv + - id: test + name: Docker Verify + run: cd build && make docker-verify-q1 + docker_tests_q2: + name: "Docker integration tests 2/4" + needs: docker_build + runs-on: ubuntu-20.04 + timeout-minutes: 180 + steps: + - id: checkout + uses: actions/checkout@v3 + - id: run_cmake + name: Run CMake + run: | + mkdir build + cd build + cmake ${DOCKER_CMAKE_FLAGS} .. + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: minifi_docker + path: build + - name: Load Docker image + run: | + docker load --input ./build/minifi_docker.tar + - id: install_deps + name: Install dependencies for Docker Verify + run: | + sudo apt update + sudo apt install -y python3-virtualenv + - id: test + name: Docker Verify + run: cd build && make docker-verify-q2 + docker_tests_q3: + name: "Docker integration tests 3/4" + needs: docker_build + runs-on: ubuntu-20.04 + timeout-minutes: 180 + steps: + - id: checkout + uses: actions/checkout@v3 + - id: run_cmake + name: Run CMake + run: | + mkdir build + cd build + cmake ${DOCKER_CMAKE_FLAGS} .. + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: minifi_docker + path: build + - name: Load Docker image + run: | + docker load --input ./build/minifi_docker.tar + - id: install_deps + name: Install dependencies for Docker Verify + run: | + sudo apt update + sudo apt install -y python3-virtualenv + - id: test + name: Docker Verify + run: cd build && make docker-verify-q3 + docker_tests_q4: + name: "Docker integration tests 4/4" + needs: docker_build + runs-on: ubuntu-20.04 + timeout-minutes: 180 + steps: + - id: checkout + uses: actions/checkout@v3 + - id: run_cmake + name: Run CMake + run: | + mkdir build + cd build + cmake ${DOCKER_CMAKE_FLAGS} .. + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: minifi_docker + path: build + - name: Load Docker image + run: | + docker load --input ./build/minifi_docker.tar - id: install_deps + name: Install dependencies for Docker Verify run: | sudo apt update sudo apt install -y python3-virtualenv - id: test - run: cd build && make docker-verify + name: Docker Verify + run: cd build && make docker-verify-q4 diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake index 1b294b824..bc533acbb 100644 --- a/cmake/DockerConfig.cmake +++ b/cmake/DockerConfig.cmake @@ -128,6 +128,22 @@ if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features) add_custom_target( docker-verify COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ${ENABLED_BEHAVE_TESTS}) + + add_custom_target( + docker-verify-q1 + COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ${ENABLED_BEHAVE_TESTS_FIRST_QUADRANT}) + + add_custom_target( + docker-verify-q2 + COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ${ENABLED_BEHAVE_TESTS_SECOND_QUADRANT}) + + add_custom_target( + docker-verify-q3 + COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ${ENABLED_BEHAVE_TESTS_THIRD_QUADRANT}) + + add_custom_target( + docker-verify-q4 + COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH} ${ENABLED_BEHAVE_TESTS_LAST_QUADRANT}) endif() include(VerifyPythonCompatibility) diff --git a/docker/test/integration/features/CMakeLists.txt b/docker/test/integration/features/CMakeLists.txt index 251e4d6ef..f0ca97a77 100644 --- a/docker/test/integration/features/CMakeLists.txt +++ b/docker/test/integration/features/CMakeLists.txt @@ -15,6 +15,10 @@ # specific language governing permissions and limitations # under the License. +if (ENABLE_LIBRDKAFKA) + set(ENABLED_BEHAVE_TESTS "${ENABLED_BEHAVE_TESTS};${CMAKE_CURRENT_SOURCE_DIR}/kafka.feature") +endif() + set(ENABLED_BEHAVE_TESTS "${ENABLED_BEHAVE_TESTS};${CMAKE_CURRENT_SOURCE_DIR}/attributes_to_json.feature") set(ENABLED_BEHAVE_TESTS "${ENABLED_BEHAVE_TESTS};${CMAKE_CURRENT_SOURCE_DIR}/core_functionality.feature") set(ENABLED_BEHAVE_TESTS "${ENABLED_BEHAVE_TESTS};${CMAKE_CURRENT_SOURCE_DIR}/defragtextflowfiles.feature") @@ -40,10 +44,6 @@ if (NOT DISABLE_CIVET) set(ENABLED_BEHAVE_TESTS "${ENABLED_BEHAVE_TESTS};${CMAKE_CURRENT_SOURCE_DIR}/https.feature") endif() -if (ENABLE_LIBRDKAFKA) - set(ENABLED_BEHAVE_TESTS "${ENABLED_BEHAVE_TESTS};${CMAKE_CURRENT_SOURCE_DIR}/kafka.feature") -endif() - if (ENABLE_KUBERNETES) set(ENABLED_BEHAVE_TESTS "${ENABLED_BEHAVE_TESTS};${CMAKE_CURRENT_SOURCE_DIR}/kubernetes_logging.feature") set(ENABLED_BEHAVE_TESTS "${ENABLED_BEHAVE_TESTS};${CMAKE_CURRENT_SOURCE_DIR}/kubernetes_metrics.feature") @@ -89,5 +89,23 @@ endif() file(GLOB FEATURE_FILES "*.feature") +list(LENGTH ENABLED_BEHAVE_TESTS ENABLED_BEHAVE_TESTS_LENGTH) + +math(EXPR ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH ${ENABLED_BEHAVE_TESTS_LENGTH}/4) + +math(EXPR ENABLED_BEHAVE_TESTS_SECOND_QUADRANT_START ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH}) +math(EXPR ENABLED_BEHAVE_TESTS_THIRD_QUADRANT_START ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH}*2) +math(EXPR ENABLED_BEHAVE_TESTS_LAST_QUADRANT_START ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH}*3) + + +list(SUBLIST ENABLED_BEHAVE_TESTS 0 ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH} ENABLED_BEHAVE_TESTS_FIRST_QUADRANT) +list(SUBLIST ENABLED_BEHAVE_TESTS ${ENABLED_BEHAVE_TESTS_SECOND_QUADRANT_START} ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH} ENABLED_BEHAVE_TESTS_SECOND_QUADRANT) +list(SUBLIST ENABLED_BEHAVE_TESTS ${ENABLED_BEHAVE_TESTS_THIRD_QUADRANT_START} ${ENABLED_BEHAVE_TESTS_QUADRANT_LENGTH} ENABLED_BEHAVE_TESTS_THIRD_QUADRANT) +list(SUBLIST ENABLED_BEHAVE_TESTS ${ENABLED_BEHAVE_TESTS_LAST_QUADRANT_START} -1 ENABLED_BEHAVE_TESTS_LAST_QUADRANT) + set(ENABLED_BEHAVE_TESTS ${ENABLED_BEHAVE_TESTS} PARENT_SCOPE) +set(ENABLED_BEHAVE_TESTS_FIRST_QUADRANT ${ENABLED_BEHAVE_TESTS_FIRST_QUADRANT} PARENT_SCOPE) +set(ENABLED_BEHAVE_TESTS_SECOND_QUADRANT ${ENABLED_BEHAVE_TESTS_SECOND_QUADRANT} PARENT_SCOPE) +set(ENABLED_BEHAVE_TESTS_THIRD_QUADRANT ${ENABLED_BEHAVE_TESTS_THIRD_QUADRANT} PARENT_SCOPE) +set(ENABLED_BEHAVE_TESTS_LAST_QUADRANT ${ENABLED_BEHAVE_TESTS_LAST_QUADRANT} PARENT_SCOPE) set(ALL_BEHAVE_TESTS ${FEATURE_FILES} PARENT_SCOPE)
