[
https://issues.apache.org/jira/browse/FLINK-9257?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16486733#comment-16486733
]
ASF GitHub Bot commented on FLINK-9257:
---------------------------------------
Github user tzulitai commented on a diff in the pull request:
https://github.com/apache/flink/pull/6053#discussion_r190126277
--- Diff: flink-end-to-end-tests/run-nightly-tests.sh ---
@@ -30,168 +30,50 @@ if [ -z "$FLINK_DIR" ] ; then
exit 1
fi
-source "$(dirname "$0")"/test-scripts/common.sh
+source "$(dirname "$0")"/test-scripts/test-runner-common.sh
FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd )`" # absolutized and normalized
echo "flink-end-to-end-test directory: $END_TO_END_DIR"
echo "Flink distribution directory: $FLINK_DIR"
-EXIT_CODE=0
-
# Template for adding a test:
-# if [ $EXIT_CODE == 0 ]; then
-# run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>"
-# EXIT_CODE=$?
-# fi
-
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Running HA (file, async) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh file true false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Running HA (file, sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh file false false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Running HA (rocks, non-incremental) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh rocks true false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Running HA (rocks, incremental) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh rocks true true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, async, no parallelism change)
end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
file true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, sync, no parallelism change)
end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
file false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, async, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, sync, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, async, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (file, sync, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (rocks, no parallelism change) end-to-end
test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 rocks"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (rocks, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 rocks"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Savepoint (rocks, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 rocks"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint (file, async) end-to-end
test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh
file true false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint (file, sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file
false false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint (rocks) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh rocks
false"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint after terminal failure (file,
async) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file true
true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint after terminal failure (file,
sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file
false true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Resuming Externalized Checkpoint after terminal failure
(rocks) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh rocks
true"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "DataSet allround end-to-end test"
"$END_TO_END_DIR/test-scripts/test_batch_allround.sh"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Streaming SQL end-to-end test"
"$END_TO_END_DIR/test-scripts/test_streaming_sql.sh"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Streaming bucketing end-to-end test"
"$END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Stateful stream job upgrade end-to-end test"
"$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test \
- "Elasticsearch (v1.7.1) sink end-to-end test" \
- "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 1
https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.1.tar.gz";
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test \
- "Elasticsearch (v2.3.5) sink end-to-end test" \
- "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 2
https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.5/elasticsearch-2.3.5.tar.gz";
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test \
- "Elasticsearch (v5.1.2) sink end-to-end test" \
- "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 5
https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.2.tar.gz";
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Local recovery and sticky scheduling nightly end-to-end test"
"$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh"
- EXIT_CODE=$?
-fi
-
-if [ $EXIT_CODE == 0 ]; then
- run_test "Quickstarts nightly end-to-end test"
"$END_TO_END_DIR/test-scripts/test_quickstarts.sh"
- EXIT_CODE=$?
-fi
-
-# Exit code for Travis build success/failure
-exit $EXIT_CODE
+# run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>"
+
+run_test "Running HA (file, async) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh file true false"
+run_test "Running HA (file, sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh file false false"
+run_test "Running HA (rocks, non-incremental) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh rocks true false"
+run_test "Running HA (rocks, incremental) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_ha.sh rocks true true"
+
+run_test "Resuming Savepoint (file, async, no parallelism change)
end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
file true"
+run_test "Resuming Savepoint (file, sync, no parallelism change)
end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
file false"
+run_test "Resuming Savepoint (file, async, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file true"
+run_test "Resuming Savepoint (file, sync, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file false"
+run_test "Resuming Savepoint (file, async, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file true"
+run_test "Resuming Savepoint (file, sync, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file false"
+run_test "Resuming Savepoint (rocks, no parallelism change) end-to-end
test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 rocks"
+run_test "Resuming Savepoint (rocks, scale up) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 rocks"
+run_test "Resuming Savepoint (rocks, scale down) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 rocks"
+
+run_test "Resuming Externalized Checkpoint (file, async) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file true"
+run_test "Resuming Externalized Checkpoint (file, sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file
false"
+run_test "Resuming Externalized Checkpoint (rocks) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh rocks"
+
+run_test "Resuming Externalized Checkpoint after terminal failure (file,
async) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file true
true"
+run_test "Resuming Externalized Checkpoint after terminal failure (file,
sync) end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file
false true"
+run_test "Resuming Externalized Checkpoint after terminal failure (rocks)
end-to-end test"
"$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh rocks
true"
+
+run_test "DataSet allround end-to-end test"
"$END_TO_END_DIR/test-scripts/test_batch_allround.sh"
+run_test "Streaming SQL end-to-end test"
"$END_TO_END_DIR/test-scripts/test_streaming_sql.sh"
+run_test "Streaming bucketing end-to-end test"
"$END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh"
+run_test "Stateful stream job upgrade end-to-end test"
"$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4"
+run_test "Local recovery and sticky scheduling end-to-end test"
"$END_TO_END_DIR/test-scripts/test_local_recovery_and_scheduling.sh"
--- End diff --
This test currently performs log verifications and cleanups within a single
execution of the test script, since it specifies multiple executions with
different state backend configurations.
Should we break this up, so that each configuration variant is explicitly
executed by the `run_test` method (like what we currently do with the savepoint
/ externalized checkpoint tests)
> End-to-end tests prints "All tests PASS" even if individual test-script
> returns non-zero exit code
> --------------------------------------------------------------------------------------------------
>
> Key: FLINK-9257
> URL: https://issues.apache.org/jira/browse/FLINK-9257
> Project: Flink
> Issue Type: Bug
> Components: Tests
> Affects Versions: 1.5.0
> Reporter: Florian Schmidt
> Assignee: Florian Schmidt
> Priority: Critical
> Fix For: 1.5.0
>
>
> In some cases the test-suite exits with non-zero exit code but still prints
> "All tests PASS" to stdout. This happens because how the test runner works,
> which is roughly as follows
> # Either run-nightly-tests.sh or run-precommit-tests.sh executes a suite of
> tests consisting of one multiple bash scripts.
> # As soon as one of those bash scripts exists with non-zero exit code, the
> tests won't continue to run and the test-suite will also exit with non-zero
> exit code.
> # *During the cleanup hook (trap cleanup EXIT in common.sh) it will be
> checked whether there are non-empty out files or log files with certain
> exceptions. If a tests fails with non-zero exit code, but does not have any
> exceptions or .out files, this will still print "All tests PASS" to stdout,
> even though they don't*
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
