[
https://issues.apache.org/jira/browse/FLINK-9257?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16486885#comment-16486885
]
ASF GitHub Bot commented on FLINK-9257:
---------------------------------------
Github user zentol commented on a diff in the pull request:
https://github.com/apache/flink/pull/6053#discussion_r190156721
--- Diff: flink-end-to-end-tests/run-nightly-tests.sh ---
@@ -30,168 +30,50 @@ if [ -z "$FLINK_DIR" ] ; then
exit 1
fi
-source "$(dirname "$0")"/test-scripts/common.sh
+source "$(dirname "$0")"/test-scripts/test-runner-common.sh
FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd )`" # absolutized and normalized
echo "flink-end-to-end-test directory: $END_TO_END_DIR"
echo "Flink distribution directory: $FLINK_DIR"
-EXIT_CODE=0
-
# Template for adding a test:
-# if [ $EXIT_CODE == 0 ]; then
-# run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>"
-# EXIT_CODE=$?
-# fi
-
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Running HA (file, async) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh file true false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Running HA (file, sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh file false false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Running HA (rocks, non-incremental) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh rocks true false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Running HA (rocks, incremental) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh rocks true true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, async, no parallelism change)
end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
file true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, sync, no parallelism change)
end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
file false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, async, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, sync, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, async, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, sync, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (rocks, no parallelism change) end-to-end
test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 rocks"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (rocks, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 rocks"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (rocks, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 rocks"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint (file, async) end-to-end
test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh
file true false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint (file, sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file
false false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint (rocks) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh rocks
false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint after terminal failure (file,
async) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file true
true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint after terminal failure (file,
sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file
false true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint after terminal failure
(rocks) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh rocks
true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "DataSet allround end-to-end test"
"$END_TO_END_DIR/test-scripts/test_batch_allround.sh"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Streaming SQL end-to-end test"
"$END_TO_END_DIR/test-scripts/test_streaming_sql.sh"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Streaming bucketing end-to-end test"
"$END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Stateful stream job upgrade end-to-end test"
"$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test \
- "Elasticsearch (v1.7.1) sink end-to-end test" \
- "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 1
https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz";
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test \
- "Elasticsearch (v2.3.5) sink end-to-end test" \
- "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 2
https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.5/elasticsearch-2.3.5.tar.gz";
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test \
- "Elasticsearch (v5.1.2) sink end-to-end test" \
- "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 5
https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.2.tar.gz";
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Local recovery and sticky scheduling nightly end-to-end test"
"$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Quickstarts nightly end-to-end test"
"$END_TO_END_DIR/test-scripts/test_quickstarts.sh"
- EXIT_CODE=$?
-fi
-
-# Exit code for Travis build success/failure
-exit $EXIT_CODE
+# run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>"
+
+run_test "Running HA (file, async) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh file true false"
+run_test "Running HA (file, sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh file false false"
+run_test "Running HA (rocks, non-incremental) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh rocks true false"
+run_test "Running HA (rocks, incremental) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh rocks true true"
+
+run_test "Resuming Savepoint (file, async, no parallelism change)
end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
file true"
+run_test "Resuming Savepoint (file, sync, no parallelism change)
end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
file false"
+run_test "Resuming Savepoint (file, async, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file true"
+run_test "Resuming Savepoint (file, sync, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file false"
+run_test "Resuming Savepoint (file, async, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file true"
+run_test "Resuming Savepoint (file, sync, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file false"
+run_test "Resuming Savepoint (rocks, no parallelism change) end-to-end
test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 rocks"
+run_test "Resuming Savepoint (rocks, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 rocks"
+run_test "Resuming Savepoint (rocks, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 rocks"
+
+run_test "Resuming Externalized Checkpoint (file, async) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file true"
+run_test "Resuming Externalized Checkpoint (file, sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file
false"
+run_test "Resuming Externalized Checkpoint (rocks) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh rocks"
+
+run_test "Resuming Externalized Checkpoint after terminal failure (file,
async) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file true
true"
+run_test "Resuming Externalized Checkpoint after terminal failure (file,
sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file
false true"
+run_test "Resuming Externalized Checkpoint after terminal failure (rocks)
end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh rocks
true"
+
+run_test "DataSet allround end-to-end test"
"$END_TO_END_DIR/test-scripts/test_batch_allround.sh"
+run_test "Streaming SQL end-to-end test"
"$END_TO_END_DIR/test-scripts/test_streaming_sql.sh"
+run_test "Streaming bucketing end-to-end test"
"$END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh"
+run_test "Stateful stream job upgrade end-to-end test"
"$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4"
+run_test "Local recovery and sticky scheduling end-to-end test"
"$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh"
--- End diff --
+1 to having each execution as a separate test
> End-to-end tests prints "All tests PASS" even if individual test-script
> returns non-zero exit code
> --------------------------------------------------------------------------------------------------
>
> Key: FLINK-9257
> URL: https://issues.apache.org/jira/browse/FLINK-9257
> Project: Flink
> Issue Type: Bug
> Components: Tests
> Affects Versions: 1.5.0
> Reporter: Florian Schmidt
> Assignee: Florian Schmidt
> Priority: Critical
> Fix For: 1.5.0
>
>
> In some cases the test-suite exits with non-zero exit code but still prints
> "All tests PASS" to stdout. This happens because how the test runner works,
> which is roughly as follows
> # Either run-nightly-tests.sh or run-precommit-tests.sh executes a suite of
> tests consisting of one multiple bash scripts.
> # As soon as one of those bash scripts exists with non-zero exit code, the
> tests won't continue to run and the test-suite will also exit with non-zero
> exit code.
> # *During the cleanup hook (trap cleanup EXIT in common.sh) it will be
> checked whether there are non-empty out files or log files with certain
> exceptions. If a tests fails with non-zero exit code, but does not have any
> exceptions or .out files, this will still print "All tests PASS" to stdout,
> even though they don't*
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
