build: add option to ignore test failures Sometimes when running build-and-test.sh, we may not care about the failure of any given test, so long as the tests ran, e.g. if running only for the sake of reporting the results of the tests. As such, this patch adds an option to ignore test failures during runs of build-and-test.sh.
I tested this by injecting errors into a test and doing a build-and-test.sh run with the new ERROR_ON_TEST_FAILURE=1, verifying that it still returns an error; and with ERROR_ON_TEST_FAILURE=0, verifying that it returns no error. I also broke the build and verified that, despite ERROR_ON_TEST_FAILURE=0, it returns an error. Change-Id: I27ecacd499423755dbf742bb70db6a7f1e5c9db7 Reviewed-on: http://gerrit.cloudera.org:8080/11399 Reviewed-by: Adar Dembo <[email protected]> Tested-by: Kudu Jenkins Reviewed-by: Grant Henke <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d91a2525 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d91a2525 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d91a2525 Branch: refs/heads/master Commit: d91a2525060d52bc27661b2b5978a288a71bcec2 Parents: 7856ec9 Author: Andrew Wong <[email protected]> Authored: Thu Sep 6 18:10:45 2018 -0700 Committer: Andrew Wong <[email protected]> Committed: Tue Sep 11 21:24:22 2018 +0000 ---------------------------------------------------------------------- build-support/jenkins/build-and-test.sh | 33 ++++++++++++++++++---------- 1 file changed, 21 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/d91a2525/build-support/jenkins/build-and-test.sh ---------------------------------------------------------------------- diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build-and-test.sh index d1a3e42..312ff65 100755 --- a/build-support/jenkins/build-and-test.sh +++ b/build-support/jenkins/build-and-test.sh @@ -82,6 +82,9 @@ # # GRADLE_FLAGS Default: "" # Extra flags which are passed to 'gradle' when running Gradle commands. +# +# ERROR_ON_TEST_FAILURE Default: 1 +# Whether test failures will cause this script to return an error. # If a commit messages contains a line that says 'DONT_BUILD', exit # immediately. @@ -121,6 +124,7 @@ BUILD_MAVEN=${BUILD_MAVEN:-0} BUILD_GRADLE=${BUILD_GRADLE:-1} BUILD_PYTHON=${BUILD_PYTHON:-1} BUILD_PYTHON3=${BUILD_PYTHON3:-1} +ERROR_ON_TEST_FAILURE=${ERROR_ON_TEST_FAILURE:-1} # Ensure that the test data directory is usable. mkdir -p "$TEST_TMPDIR" @@ -329,6 +333,7 @@ if [ "$RUN_FLAKY_ONLY" == "1" ] ; then BUILD_JAVA=0 fi +TESTS_FAILED=0 EXIT_STATUS=0 FAILURES="" @@ -350,7 +355,7 @@ if [ "$ENABLE_DIST_TEST" == "1" ]; then fi if ! $THIRDPARTY_BIN/ctest -j$NUM_PROCS $EXTRA_TEST_FLAGS ; then - EXIT_STATUS=1 + TESTS_FAILED=1 FAILURES="$FAILURES"$'C++ tests failed\n' fi @@ -388,7 +393,7 @@ if [ "$BUILD_JAVA" == "1" ]; then EXTRA_MVN_FLAGS="$EXTRA_MVN_FLAGS -Dmaven.javadoc.skip" EXTRA_MVN_FLAGS="$EXTRA_MVN_FLAGS $MVN_FLAGS" if ! mvn $EXTRA_MVN_FLAGS clean verify ; then - EXIT_STATUS=1 + TESTS_FAILED=1 FAILURES="$FAILURES"$'Java Maven build/test failed\n' fi fi @@ -418,7 +423,7 @@ if [ "$BUILD_JAVA" == "1" ]; then else # TODO: Run `gradle check` in BUILD_TYPE DEBUG when static code analysis is fixed if ! ./gradlew $EXTRA_GRADLE_FLAGS clean test ; then - EXIT_STATUS=1 + TESTS_FAILED=1 FAILURES="$FAILURES"$'Java Gradle build/test failed\n' fi fi @@ -501,7 +506,7 @@ if [ "$BUILD_PYTHON" == "1" ]; then if ! python setup.py test \ --addopts="kudu --junit-xml=$TEST_LOGDIR/python_client.xml" \ 2> $TEST_LOGDIR/python_client.log ; then - EXIT_STATUS=1 + TESTS_FAILED=1 FAILURES="$FAILURES"$'Python tests failed\n' fi @@ -559,7 +564,7 @@ if [ "$BUILD_PYTHON3" == "1" ]; then if ! python setup.py test \ --addopts="kudu --junit-xml=$TEST_LOGDIR/python3_client.xml" \ 2> $TEST_LOGDIR/python3_client.log ; then - EXIT_STATUS=1 + TESTS_FAILED=1 FAILURES="$FAILURES"$'Python 3 tests failed\n' fi @@ -574,7 +579,7 @@ if [ "$ENABLE_DIST_TEST" == "1" ]; then echo ------------------------------------------------------------ C_DIST_TEST_ID=`cat $BUILD_ROOT/c-dist-test-job-id` if ! $DIST_TEST_HOME/bin/client watch $C_DIST_TEST_ID ; then - EXIT_STATUS=1 + TESTS_FAILED=1 FAILURES="$FAILURES"$'Distributed C++ tests failed\n' fi DT_DIR=$TEST_LOGDIR/dist-test-out @@ -594,7 +599,7 @@ if [ "$ENABLE_DIST_TEST" == "1" ]; then echo ------------------------------------------------------------ JAVA_DIST_TEST_ID=`cat $BUILD_ROOT/java-dist-test-job-id` if ! $DIST_TEST_HOME/bin/client watch $JAVA_DIST_TEST_ID ; then - EXIT_STATUS=1 + TESTS_FAILED=1 FAILURES="$FAILURES"$'Distributed Java tests failed\n' fi DT_DIR=$TEST_LOGDIR/java-dist-test-out @@ -610,7 +615,7 @@ if [ "$ENABLE_DIST_TEST" == "1" ]; then fi fi -if [ $EXIT_STATUS != 0 ]; then +if [ "$TESTS_FAILED" != "0" -o "$EXIT_STATUS" != "0" ]; then echo echo Tests failed, making sure we have XML files for all tests. echo ------------------------------------------------------------ @@ -626,10 +631,8 @@ if [ $EXIT_STATUS != 0 ]; then zcat $GTEST_OUTFILE | $SOURCE_ROOT/build-support/parse_test_failure.py -x > $GTEST_XMLFILE fi done -fi - -# If all tests passed, ensure that they cleaned up their test output. -if [ $EXIT_STATUS == 0 ]; then +else + # If all tests passed, ensure that they cleaned up their test output. TEST_TMPDIR_CONTENTS=$(ls $TEST_TMPDIR) if [ -n "$TEST_TMPDIR_CONTENTS" ]; then echo "All tests passed, yet some left behind their test output." @@ -652,4 +655,10 @@ if [ -n "$FAILURES" ]; then echo fi +# If any of the tests failed and we are honoring test failures, set the exit +# status to 1. Note that it may have already been set to 1 above. +if [ "$ERROR_ON_TEST_FAILURE" == "1" -a "$TESTS_FAILED" == "1" ]; then + EXIT_STATUS=1 +fi + exit $EXIT_STATUS
