[flink] branch master updated: [FLINK-25225][e2e] Add e2e TPCDS tests to run against the AdaptiveBatchScheduler

[zhuzh]

This is an automated email from the ASF dual-hosted git repository.

zhuzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new 5c7a432  [FLINK-25225][e2e] Add e2e TPCDS tests to run against the 
AdaptiveBatchScheduler
5c7a432 is described below

commit 5c7a432778f2f8494e104fa89ea08a71e83978b1
Author: Lijie Wang <wangdachui9...@gmail.com>
AuthorDate: Thu Feb 10 16:38:32 2022 +0800

    [FLINK-25225][e2e] Add e2e TPCDS tests to run against the 
AdaptiveBatchScheduler
    
    This closes #18720.
---
 .../apache/flink/table/tpcds/TpcdsTestProgram.java    |  3 ---
 flink-end-to-end-tests/run-nightly-tests.sh           |  1 +
 flink-end-to-end-tests/test-scripts/test_tpcds.sh     | 19 +++++++++++++++++--
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git 
a/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java
 
b/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java
index 9ae3d80..c6251aa 100644
--- 
a/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java
+++ 
b/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java
@@ -138,9 +138,6 @@ public class TpcdsTestProgram {
         TableEnvironment tEnv = TableEnvironment.create(environmentSettings);
 
         // config Optimizer parameters
-        tEnv.getConfig()
-                .getConfiguration()
-                
.setInteger(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 4);
         // TODO use the default shuffle mode of batch runtime mode once 
FLINK-23470 is implemented
         tEnv.getConfig()
                 .getConfiguration()
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh 
b/flink-end-to-end-tests/run-nightly-tests.sh
index 566284f..20c4aa1 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -217,6 +217,7 @@ function run_group_2 {
 
     run_test "TPC-H end-to-end test" 
"$END_TO_END_DIR/test-scripts/test_tpch.sh"
     run_test "TPC-DS end-to-end test" 
"$END_TO_END_DIR/test-scripts/test_tpcds.sh"
+    run_test "TPC-DS end-to-end test with adaptive batch scheduler" 
"$END_TO_END_DIR/test-scripts/test_tpcds.sh AdaptiveBatch"
 
     run_test "Heavy deployment end-to-end test" 
"$END_TO_END_DIR/test-scripts/test_heavy_deployment.sh" "skip_check_exceptions"
 
diff --git a/flink-end-to-end-tests/test-scripts/test_tpcds.sh 
b/flink-end-to-end-tests/test-scripts/test_tpcds.sh
index 4f88c1f..ce87326 100755
--- a/flink-end-to-end-tests/test-scripts/test_tpcds.sh
+++ b/flink-end-to-end-tests/test-scripts/test_tpcds.sh
@@ -54,10 +54,25 @@ cd "$END_TO_END_DIR"
 
 echo "[INFO]Preparing Flink cluster..."
 
+SCHEDULER="${1:-Ng}"
+
+set_config_key "jobmanager.scheduler" "${SCHEDULER}"
 set_config_key "taskmanager.memory.process.size" "4096m"
-set_config_key "taskmanager.numberOfTaskSlots" "4"
-set_config_key "parallelism.default" "4"
 set_config_key "taskmanager.memory.network.fraction" "0.2"
+
+if [ "${SCHEDULER}" == "Ng" ]; then
+    set_config_key "taskmanager.numberOfTaskSlots" "4"
+    set_config_key "parallelism.default" "4"
+elif [ "${SCHEDULER}" == "AdaptiveBatch" ]; then
+    set_config_key "taskmanager.numberOfTaskSlots" "8"
+    set_config_key "parallelism.default" "-1"
+    set_config_key "jobmanager.adaptive-batch-scheduler.max-parallelism" "8"
+    set_config_key "jobmanager.adaptive-batch-scheduler.data-volume-per-task" 
"6m"
+else
+    echo "ERROR: Scheduler ${SCHEDULER} is unsupported for tpcds test. 
Aborting..."
+    exit 1
+fi
+
 start_cluster