[FLINK-9301][tests] Enable high-parallellism-iterations test This closes #6024.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/885640f7 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/885640f7 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/885640f7 Branch: refs/heads/master Commit: 885640f781aa66359d929eb387f27a6024d75025 Parents: 81d1355 Author: Andrey Zagrebin <azagre...@gmail.com> Authored: Wed Jun 27 11:16:51 2018 +0200 Committer: zentol <ches...@apache.org> Committed: Tue Jul 3 21:08:45 2018 +0200 ---------------------------------------------------------------------- flink-end-to-end-tests/run-nightly-tests.sh | 2 ++ flink-end-to-end-tests/test-scripts/common.sh | 29 ++++++++++++++++++++ .../test_high_parallelism_iterations.sh | 24 ++++------------ 3 files changed, 37 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/885640f7/flink-end-to-end-tests/run-nightly-tests.sh ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh index a493f49..15c73d5 100755 --- a/flink-end-to-end-tests/run-nightly-tests.sh +++ b/flink-end-to-end-tests/run-nightly-tests.sh @@ -43,6 +43,8 @@ echo "Flink distribution directory: $FLINK_DIR" # run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>" +run_test "ConnectedComponents iterations with high parallelism end-to-end test" "$END_TO_END_DIR/test-scripts/test_high_parallelism_iterations.sh 25" + run_test "Queryable state (rocksdb) end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state.sh rocksdb" run_test "Queryable state (rocksdb) with TM restart end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state_restart_tm.sh" http://git-wip-us.apache.org/repos/asf/flink/blob/885640f7/flink-end-to-end-tests/test-scripts/common.sh ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh index 4ed83b0..e5ad458 100644 --- a/flink-end-to-end-tests/test-scripts/common.sh +++ b/flink-end-to-end-tests/test-scripts/common.sh @@ -24,6 +24,14 @@ if [[ -z $FLINK_DIR ]]; then exit 1 fi +case "$(uname -s)" in + Linux*) OS_TYPE=linux;; + Darwin*) OS_TYPE=mac;; + CYGWIN*) OS_TYPE=cygwin;; + MINGW*) OS_TYPE=mingw;; + *) OS_TYPE="UNKNOWN:${unameOut}" +esac + export EXIT_CODE=0 echo "Flink dist directory: $FLINK_DIR" @@ -39,6 +47,27 @@ cd $TEST_ROOT export TEST_DATA_DIR=$TEST_INFRA_DIR/temp-test-directory-$(date +%S%N) echo "TEST_DATA_DIR: $TEST_DATA_DIR" +function print_mem_use_osx { + declare -a mem_types=("active" "inactive" "wired down") + used="" + for mem_type in "${mem_types[@]}" + do + used_type=$(vm_stat | grep "Pages ${mem_type}:" | awk '{print $NF}' | rev | cut -c 2- | rev) + let used_type="(${used_type}*4096)/1024/1024" + used="$used $mem_type=${used_type}MB" + done + let mem=$(sysctl -n hw.memsize)/1024/1024 + echo "Memory Usage: ${used} total=${mem}MB" +} + +function print_mem_use { + if [[ "$OS_TYPE" == "mac" ]]; then + print_mem_use_osx + else + free -m | awk 'NR==2{printf "Memory Usage: used=%sMB total=%sMB %.2f%%\n", $3,$2,$3*100/$2 }' + fi +} + function backup_config() { # back up the masters and flink-conf.yaml cp $FLINK_DIR/conf/masters $FLINK_DIR/conf/masters.bak http://git-wip-us.apache.org/repos/asf/flink/blob/885640f7/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh ---------------------------------------------------------------------- diff --git a/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh b/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh index dbdacef..aa937ae 100755 --- a/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh +++ b/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh @@ -19,15 +19,14 @@ source "$(dirname "$0")"/common.sh -PARALLELISM="${PARALLELISM:-100}" +PARALLELISM="${1:-25}" TEST=flink-high-parallelism-iterations-test TEST_PROGRAM_NAME=HighParallelismIterationsTestProgram TEST_PROGRAM_JAR=${END_TO_END_DIR}/$TEST/target/$TEST_PROGRAM_NAME.jar -echo "Run Not So MiniCluster Iterations Graph Connected Components Program" +backup_config -cp $FLINK_DIR/conf/flink-conf.yaml $FLINK_DIR/conf/flink-conf.yaml.bak echo "taskmanager.heap.mb: 52" >> $FLINK_DIR/conf/flink-conf.yaml # 52Mb x 100 TMs = 5Gb total heap @@ -41,27 +40,16 @@ echo "taskmanager.network.netty.client.numThreads: 1" >> $FLINK_DIR/conf/flink-c echo "taskmanager.numberOfTaskSlots: 1" >> $FLINK_DIR/conf/flink-conf.yaml +print_mem_use start_cluster +print_mem_use let TMNUM=$PARALLELISM-1 echo "Start $TMNUM more task managers" for i in `seq 1 $TMNUM`; do $FLINK_DIR/bin/taskmanager.sh start + print_mem_use done -function test_cleanup { - # don't call ourselves again for another signal interruption - trap "exit -1" INT - # don't call ourselves again for normal exit - trap "" EXIT - - stop_cluster - $FLINK_DIR/bin/taskmanager.sh stop-all - - # revert our modifications to the Flink distribution - mv -f $FLINK_DIR/conf/flink-conf.yaml.bak $FLINK_DIR/conf/flink-conf.yaml -} -trap test_cleanup INT -trap test_cleanup EXIT - $FLINK_DIR/bin/flink run -p $PARALLELISM $TEST_PROGRAM_JAR +print_mem_use