This is an automated email from the ASF dual-hosted git repository. azagrebin pushed a commit to branch flip116 in repository https://gitbox.apache.org/repos/asf/flink.git
commit e6240429aab65cad048b650b4e9f76e3c4ea7f08 Author: Xintong Song <[email protected]> AuthorDate: Thu Mar 26 15:37:22 2020 +0800 [FLINK-16742][dist] config.sh does not assume BashJavaUtils always return two lines of results. --- flink-dist/src/main/flink-bin/bin/config.sh | 25 ++++++++++++++----- flink-dist/src/main/flink-bin/bin/taskmanager.sh | 18 +++++--------- flink-dist/src/test/bin/runBashJavaUtilsCmd.sh | 6 ++--- ...JavaUtilsCmd.sh => runExtractLoggingOutputs.sh} | 11 ++++----- .../org/apache/flink/dist/BashJavaUtilsITCase.java | 28 ++++++++++++++++++++-- .../apache/flink/runtime/util/BashJavaUtils.java | 4 +++- 6 files changed, 61 insertions(+), 31 deletions(-) diff --git a/flink-dist/src/main/flink-bin/bin/config.sh b/flink-dist/src/main/flink-bin/bin/config.sh index 3c5e99d..a3ea0c0 100755 --- a/flink-dist/src/main/flink-bin/bin/config.sh +++ b/flink-dist/src/main/flink-bin/bin/config.sh @@ -640,16 +640,29 @@ runBashJavaUtilsCmd() { echo "$output" } -extractExecutionParams() { - local execution_config=$1 +extractExecutionResults() { + local output="$1" + local expected_lines="$2" local EXECUTION_PREFIX="BASH_JAVA_UTILS_EXEC_RESULT:" + local execution_results - if ! [[ $execution_config =~ ^${EXECUTION_PREFIX}.* ]]; then - echo "[ERROR] Unexpected result: $execution_config" 1>&2 - echo "[ERROR] The last line of the BashJavaUtils outputs is expected to be the execution result, following the prefix '${EXECUTION_PREFIX}'" 1>&2 + IFS=$'\n' execution_results=($(echo "${output}" | grep ${EXECUTION_PREFIX})) + if [[ ${#execution_results[@]} != ${expected_lines} ]]; then + echo "[ERROR] The execution results has unexpected number of lines, expected: ${expected_lines}, actual: ${#execution_results[@]}." 1>&2 + echo "[ERROR] An execution result line is expected following the prefix '${EXECUTION_PREFIX}'" 1>&2 echo "$output" 1>&2 exit 1 fi - echo ${execution_config} | sed "s/$EXECUTION_PREFIX//" + for result in ${execution_results[@]} + do + echo ${result} | sed "s/$EXECUTION_PREFIX//" + done +} + +extractLoggingOutputs() { + local output="$1" + local EXECUTION_PREFIX="BASH_JAVA_UTILS_EXEC_RESULT:" + + echo "${output}" | grep -v ${EXECUTION_PREFIX} } diff --git a/flink-dist/src/main/flink-bin/bin/taskmanager.sh b/flink-dist/src/main/flink-bin/bin/taskmanager.sh index fd1e1d4..3f71208 100755 --- a/flink-dist/src/main/flink-bin/bin/taskmanager.sh +++ b/flink-dist/src/main/flink-bin/bin/taskmanager.sh @@ -50,24 +50,18 @@ if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then java_utils_output=$(runBashJavaUtilsCmd GET_TM_RESOURCE_PARAMS ${FLINK_CONF_DIR}) - num_lines=$(echo "${java_utils_output}" | wc -l) - logging_output=$(echo "${java_utils_output}" | head -n $((${num_lines} - 2))) - params_output=$(echo "${java_utils_output}" | tail -n 2) + logging_output=$(extractLoggingOutputs "${java_utils_output}") + params_output=$(extractExecutionResults "${java_utils_output}" 2) - jvm_params=$(extractExecutionParams "$(echo "$params_output" | head -n 1)") if [[ $? -ne 0 ]]; then - echo "[ERROR] Could not get JVM parameters properly." + echo "[ERROR] Could not get JVM parameters and dynamic configurations properly." exit 1 fi - export JVM_ARGS="${JVM_ARGS} ${jvm_params}" - IFS=$" " + jvm_params=$(echo "${params_output}" | head -n 1) + export JVM_ARGS="${JVM_ARGS} ${jvm_params}" - dynamic_configs=$(extractExecutionParams "$(echo "$params_output" | tail -n 1)") - if [[ $? -ne 0 ]]; then - echo "[ERROR] Could not get dynamic configurations properly." - exit 1 - fi + IFS=$" " dynamic_configs=$(echo "${params_output}" | tail -n 1) ARGS+=("--configDir" "${FLINK_CONF_DIR}" ${dynamic_configs[@]}) export FLINK_INHERITED_LOGS=" diff --git a/flink-dist/src/test/bin/runBashJavaUtilsCmd.sh b/flink-dist/src/test/bin/runBashJavaUtilsCmd.sh index a69610d..1c4dd1d 100755 --- a/flink-dist/src/test/bin/runBashJavaUtilsCmd.sh +++ b/flink-dist/src/test/bin/runBashJavaUtilsCmd.sh @@ -21,6 +21,7 @@ USAGE="Usage: runBashJavaUtilsCmd.sh <command>" COMMAND=$1 +EXPECTED_LINES=$2 if [[ -z "${COMMAND}" ]]; then echo "$USAGE" @@ -36,6 +37,5 @@ FLINK_DIST_JAR=`find $FLINK_TARGET_DIR -name 'flink-dist*.jar'` . ${bin}/../../main/flink-bin/bin/config.sh > /dev/null -output=$(runBashJavaUtilsCmd ${COMMAND} ${FLINK_CONF_DIR} "$FLINK_TARGET_DIR/bash-java-utils.jar:$FLINK_DIST_JAR}" | tail -n 2) -extractExecutionParams "$(echo "$output" | head -n 1)" -extractExecutionParams "$(echo "$output" | tail -n 1)" +output=$(runBashJavaUtilsCmd ${COMMAND} ${FLINK_CONF_DIR} "$FLINK_TARGET_DIR/bash-java-utils.jar:$FLINK_DIST_JAR}") +extractExecutionResults "${output}" ${EXPECTED_LINES} diff --git a/flink-dist/src/test/bin/runBashJavaUtilsCmd.sh b/flink-dist/src/test/bin/runExtractLoggingOutputs.sh similarity index 75% copy from flink-dist/src/test/bin/runBashJavaUtilsCmd.sh copy to flink-dist/src/test/bin/runExtractLoggingOutputs.sh index a69610d..d892901 100755 --- a/flink-dist/src/test/bin/runBashJavaUtilsCmd.sh +++ b/flink-dist/src/test/bin/runExtractLoggingOutputs.sh @@ -17,12 +17,11 @@ # limitations under the License. ################################################################################ -# Wrapper script to compare the TM heap size calculation of config.sh with Java -USAGE="Usage: runBashJavaUtilsCmd.sh <command>" +USAGE="Usage: runExtractLoggingOutputs.sh <input>" -COMMAND=$1 +INPUT="$1" -if [[ -z "${COMMAND}" ]]; then +if [[ -z "${INPUT}" ]]; then echo "$USAGE" exit 1 fi @@ -36,6 +35,4 @@ FLINK_DIST_JAR=`find $FLINK_TARGET_DIR -name 'flink-dist*.jar'` . ${bin}/../../main/flink-bin/bin/config.sh > /dev/null -output=$(runBashJavaUtilsCmd ${COMMAND} ${FLINK_CONF_DIR} "$FLINK_TARGET_DIR/bash-java-utils.jar:$FLINK_DIST_JAR}" | tail -n 2) -extractExecutionParams "$(echo "$output" | head -n 1)" -extractExecutionParams "$(echo "$output" | tail -n 1)" +extractLoggingOutputs "${INPUT}" diff --git a/flink-dist/src/test/java/org/apache/flink/dist/BashJavaUtilsITCase.java b/flink-dist/src/test/java/org/apache/flink/dist/BashJavaUtilsITCase.java index 875ef21..56fb84e 100644 --- a/flink-dist/src/test/java/org/apache/flink/dist/BashJavaUtilsITCase.java +++ b/flink-dist/src/test/java/org/apache/flink/dist/BashJavaUtilsITCase.java @@ -23,6 +23,7 @@ import org.apache.flink.runtime.util.BashJavaUtils; import org.junit.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -36,14 +37,37 @@ import static org.junit.Assert.assertEquals; public class BashJavaUtilsITCase extends JavaBashTestBase { private static final String RUN_BASH_JAVA_UTILS_CMD_SCRIPT = "src/test/bin/runBashJavaUtilsCmd.sh"; + private static final String RUN_EXTRACT_LOGGING_OUTPUTS_SCRIPT = "src/test/bin/runExtractLoggingOutputs.sh"; @Test public void testGetTmResourceParamsConfigs() throws Exception { - String[] commands = {RUN_BASH_JAVA_UTILS_CMD_SCRIPT, BashJavaUtils.Command.GET_TM_RESOURCE_PARAMS.toString()}; + int expectedResultLines = 2; + String[] commands = {RUN_BASH_JAVA_UTILS_CMD_SCRIPT, BashJavaUtils.Command.GET_TM_RESOURCE_PARAMS.toString(), String.valueOf(expectedResultLines)}; List<String> lines = Arrays.asList(executeScript(commands).split(System.lineSeparator())); - assertEquals(2, lines.size()); + assertEquals(expectedResultLines, lines.size()); ConfigurationUtils.parseJvmArgString(lines.get(0)); ConfigurationUtils.parseTmResourceDynamicConfigs(lines.get(1)); } + + @Test + public void testExtractLoggingOutputs() throws Exception { + StringBuilder input = new StringBuilder(); + List<String> expectedOutput = new ArrayList<>(); + + for (int i = 0; i < 5; ++i) { + String line = "BashJavaUtils output line " + i + " `~!@#$%^&*()-_=+;:,.'\"\\\t/?"; + if (i % 2 == 0) { + expectedOutput.add(line); + } else { + line = BashJavaUtils.EXECUTION_PREFIX + line; + } + input.append(line + "\n"); + } + + String[] commands = {RUN_EXTRACT_LOGGING_OUTPUTS_SCRIPT, input.toString()}; + List<String> actualOutput = Arrays.asList(executeScript(commands).split(System.lineSeparator())); + + assertEquals(expectedOutput, actualOutput); + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/BashJavaUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/BashJavaUtils.java index 14d24f5..e00d667 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/BashJavaUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/BashJavaUtils.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.util; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.clusterframework.TaskExecutorProcessSpec; @@ -34,7 +35,8 @@ import static org.apache.flink.util.Preconditions.checkArgument; */ public class BashJavaUtils { - private static final String EXECUTION_PREFIX = "BASH_JAVA_UTILS_EXEC_RESULT:"; + @VisibleForTesting + public static final String EXECUTION_PREFIX = "BASH_JAVA_UTILS_EXEC_RESULT:"; public static void main(String[] args) throws Exception { checkArgument(args.length > 0, "Command not specified.");
