This is an automated email from the ASF dual-hosted git repository. szaszm pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/nifi-minifi-cpp.git
commit 31e6e6b6a9fc53ca011776aea30892a5290e1d3d Author: Gabor Gyimesi <[email protected]> AuthorDate: Mon Apr 3 17:43:02 2023 +0200 MINIFICPP-1716 Recover core dumps from CI up to 1G - Keep RelWithDebInfo in clang build only Closes #1511 Signed-off-by: Marton Szasz <[email protected]> --- .github/workflows/ci.yml | 91 ++++++++++++++++++++++++++++++++++++++++++++---- cmake/DockerConfig.cmake | 6 ++++ docker/DockerBuild.sh | 6 +++- 3 files changed, 96 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 402871f95..185639af3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,27 @@ jobs: # CPPFLAGS are not recognized by cmake, so we have to force them to CFLAGS and CXXFLAGS to have flex 2.6 working ./bootstrap.sh -e -t && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DCMAKE_C_FLAGS="${CPPFLAGS} ${CFLAGS}" -DCMAKE_CXX_FLAGS="${CPPFLAGS} ${CXXFLAGS}" -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_SQL=ON -DUSE_REAL_ODBC_TEST_DRIVER=ON -DENABLE_AZURE=ON -DENABLE_GCP=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES=OFF -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON .. && cmake --build . --parallel 4 - name: test - run: cd build && make test ARGS="--timeout 300 -j4 --output-on-failure" + id: test + run: | + # Set core file size limit to 1GiB + ulimit -c 1048576 + cd build && make test ARGS="--timeout 300 -j4 --output-on-failure" - name: linter run: cd build && make -j4 linter + - name: check-cores + if: ${{ failure() && steps.test.conclusion == 'failure' }} + run: | + if [ "$(ls -A /cores)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi + - uses: actions/[email protected] + if: ${{ failure() && env.CORES_EXIST == 'true' }} + with: + name: macos-coredumps + path: /cores + - uses: actions/[email protected] + if: ${{ failure() && env.CORES_EXIST == 'true' }} + with: + name: macos-binaries + path: build/bin windows_VS2019: name: "windows-2019" runs-on: windows-2019 @@ -107,7 +125,25 @@ jobs: -DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_GCP=OFF -DENABLE_PROCFS=OFF -DENABLE_BUSTACHE=ON -DENABLE_PCAP=ON -DENABLE_JNI=ON -DENABLE_SFTP=ON .. make -j$(nproc) VERBOSE=1 - name: test - run: cd build && make test ARGS="--timeout 300 -j2 --output-on-failure" + id: test + run: | + # Set core file size limit to 1GiB + ulimit -c 1048576 + cd build && make test ARGS="--timeout 300 -j8 --output-on-failure" + - name: check-cores + if: ${{ failure() && steps.test.conclusion == 'failure' }} + run: | + if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi + - uses: actions/[email protected] + if: ${{ failure() && env.CORES_EXIST == 'true' }} + with: + name: ubuntu-coredumps + path: /var/lib/apport/coredump/ + - uses: actions/[email protected] + if: ${{ failure() && env.CORES_EXIST == 'true' }} + with: + name: ubuntu-binaries + path: build/bin ubuntu_20_04_clang: name: "ubuntu-20.04-clang" runs-on: ubuntu-20.04 @@ -142,14 +178,18 @@ jobs: export CXX=clang++-14 export CXXFLAGS="${CXXFLAGS} -stdlib=libc++" export LDFLAGS="${LDFLAGS} -stdlib=libc++" - cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_BUSTACHE=ON -DENABLE_COAP=ON \ + cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_BUSTACHE=ON -DENABLE_COAP=ON \ -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \ -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PCAP=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON -DENABLE_TENSORFLOW=OFF \ -DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON .. cmake --build . --parallel $(nproc) - name: test - run: cd build && make test ARGS="--timeout 300 -j8 --output-on-failure" + id: test + run: | + # Set core file size limit to 1GiB + ulimit -c 1048576 + cd build && make test ARGS="--timeout 300 -j8 --output-on-failure" - name: linter run: cd build && make -j$(nproc) linter - name: shellcheck @@ -173,6 +213,20 @@ jobs: sed -i -e 's/\/usr\/lib\/ccache\/clang-14/\/lib\/llvm-14\/bin\/clang/g' build/compile_commands.json ./run_clang_tidy.sh "${FILES}" + - name: check-cores + if: ${{ failure() && steps.test.conclusion == 'failure' }} + run: | + if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi + - uses: actions/[email protected] + if: ${{ failure() && env.CORES_EXIST == 'true' }} + with: + name: ubuntu-clang-coredumps + path: /var/lib/apport/coredump/ + - uses: actions/[email protected] + if: ${{ failure() && env.CORES_EXIST == 'true' }} + with: + name: ubuntu-clang-binaries + path: build/bin centos: name: "centos" runs-on: ubuntu-20.04 @@ -196,13 +250,38 @@ jobs: - id: build run: | if [ -d ~/.ccache ]; then mv ~/.ccache .; fi - mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON \ + # centos build can run out of the github runners' disk space if built with RelWithDebInfo so we keep the Release build here + mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON \ -DENABLE_COAP=ON -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON \ -DENABLE_OPENCV=ON -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \ -DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON \ -DENABLE_ELASTICSEARCH=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos-test - id: test - run: docker run --rm apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}' | head -1) bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"' + run: | + # Set core file size limit to 1GiB + ulimit -c 1048576 + echo '/cores/core.%e.%p' | sudo tee /proc/sys/kernel/core_pattern + sudo mkdir /cores + sudo chmod 777 /cores + docker run --name centos-test --init --ulimit core=1048576 --mount type=bind,source=/cores,target=/cores apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}') bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"' + - name: check-cores + if: ${{ failure() && steps.test.conclusion == 'failure' }} + run: | + if [ "$(ls -A /cores)" ]; then + echo "CORES_EXIST=true" >> $GITHUB_ENV; + docker cp centos-test:/opt/minifi/build/bin /tmp + sudo chmod -R 777 /cores + fi + - uses: actions/[email protected] + if: ${{ failure() && env.CORES_EXIST == 'true' }} + with: + name: centos-crashes + path: /cores + - uses: actions/[email protected] + if: ${{ failure() && env.CORES_EXIST == 'true' }} + with: + name: centos-binaries + path: /tmp/bin docker_integration_tests: name: "Docker integration tests" runs-on: ubuntu-20.04 diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake index cb2a1df18..1b294b824 100644 --- a/cmake/DockerConfig.cmake +++ b/cmake/DockerConfig.cmake @@ -26,6 +26,7 @@ add_custom_target( -c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE} -c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION} -c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS} + -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -c BUILD_NUMBER=${BUILD_NUMBER} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) @@ -47,6 +48,7 @@ add_custom_target( -c DOCKER_BASE_IMAGE=${DOCKER_BASE_IMAGE} -c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS} -c BUILD_NUMBER=${BUILD_NUMBER} + -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) add_custom_target( @@ -61,6 +63,7 @@ add_custom_target( -c BUILD_NUMBER=${BUILD_NUMBER} -c DOCKER_CCACHE_DUMP_LOCATION=${DOCKER_CCACHE_DUMP_LOCATION} -c DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS} + -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) add_custom_target( @@ -86,6 +89,7 @@ add_custom_target( -l ${CMAKE_BINARY_DIR} -d fedora -c BUILD_NUMBER=${BUILD_NUMBER} + -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) add_custom_target( @@ -98,6 +102,7 @@ add_custom_target( -l ${CMAKE_BINARY_DIR} -d bionic -c BUILD_NUMBER=${BUILD_NUMBER} + -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) add_custom_target( @@ -110,6 +115,7 @@ add_custom_target( -l ${CMAKE_BINARY_DIR} -d focal -c BUILD_NUMBER=${BUILD_NUMBER} + -c CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/docker/) if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features) diff --git a/docker/DockerBuild.sh b/docker/DockerBuild.sh index 14c5efd11..4ac0c4050 100755 --- a/docker/DockerBuild.sh +++ b/docker/DockerBuild.sh @@ -30,6 +30,7 @@ DISTRO_NAME= BUILD_NUMBER= DOCKER_CCACHE_DUMP_LOCATION= DOCKER_SKIP_TESTS=ON +CMAKE_BUILD_TYPE=Release function usage { echo "Usage: ./DockerBuild.sh -v <MINIFI_VERSION> [additional options]" @@ -100,6 +101,8 @@ while [[ $# -gt 0 ]]; do DOCKER_CCACHE_DUMP_LOCATION="${ARR[1]}" elif [ "${ARR[0]}" == "DOCKER_SKIP_TESTS" ]; then DOCKER_SKIP_TESTS="${ARR[1]}" + elif [ "${ARR[0]}" == "CMAKE_BUILD_TYPE" ]; then + CMAKE_BUILD_TYPE="${ARR[1]}" else BUILD_ARGS+=("--build-arg" "${ARR[0]}=${ARR[1]}") fi @@ -159,7 +162,8 @@ BUILD_ARGS+=("--build-arg" "UID=${UID_ARG}" "--build-arg" "MINIFI_VERSION=${MINIFI_VERSION}" "--build-arg" "DUMP_LOCATION=${DUMP_LOCATION}" "--build-arg" "DISTRO_NAME=${DISTRO_NAME}" - "--build-arg" "DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}") + "--build-arg" "DOCKER_SKIP_TESTS=${DOCKER_SKIP_TESTS}" + "--build-arg" "CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}") if [ -n "${DUMP_LOCATION}" ]; then BUILD_ARGS+=("--build-arg" "DOCKER_MAKE_TARGET=package") fi
