zentol commented on a change in pull request #15853:
URL: https://github.com/apache/flink/pull/15853#discussion_r628945894
##########
File path: flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
##########
@@ -138,35 +138,65 @@ END
function debug_copy_and_show_logs {
echo "Debugging failed YARN Docker test:"
- echo "\nCurrently running containers"
+ echo -e "\nCurrently running containers"
docker ps
- echo "\n\nCurrently running JVMs"
+ echo -e "\n\nCurrently running JVMs"
jps -v
- echo "\n\nHadoop logs:"
- mkdir -p $TEST_DATA_DIR/logs
- docker cp master:/var/log/hadoop/ $TEST_DATA_DIR/logs/
- ls -lisah $TEST_DATA_DIR/logs/hadoop
- for f in $TEST_DATA_DIR/logs/hadoop/*; do
- echo "$f:"
- cat $f
- done
+ local log_directory="$TEST_DATA_DIR/logs"
+ local yarn_docker_containers="master $(docker ps --format '{{.Names}}' |
grep worker)"
- echo "\n\nDocker logs:"
- docker logs master
+ extract_hadoop_logs ${log_directory} ${yarn_docker_containers}
+ print_logs ${log_directory} ${yarn_docker_containers}
- echo "\n\nFlink logs:"
+ echo -e "\n\n ==== Flink logs ===="
docker exec master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab
hadoop-user"
docker exec master bash -c "yarn application -list -appStates ALL"
application_id=`docker exec master bash -c "yarn application -list
-appStates ALL" | grep -i "Flink" | grep -i "cluster" | awk '{print \$1}'`
- echo "Application ID: '$application_id'"
+ echo -e "\n\nApplication ID: '$application_id'"
docker exec master bash -c "yarn logs -applicationId $application_id"
docker exec master bash -c "kdestroy"
}
+function extract_hadoop_logs() {
+ local log_folder="$1"
Review comment:
same as below
##########
File path: flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
##########
@@ -138,35 +138,65 @@ END
function debug_copy_and_show_logs {
echo "Debugging failed YARN Docker test:"
- echo "\nCurrently running containers"
+ echo -e "\nCurrently running containers"
docker ps
- echo "\n\nCurrently running JVMs"
+ echo -e "\n\nCurrently running JVMs"
jps -v
- echo "\n\nHadoop logs:"
- mkdir -p $TEST_DATA_DIR/logs
- docker cp master:/var/log/hadoop/ $TEST_DATA_DIR/logs/
- ls -lisah $TEST_DATA_DIR/logs/hadoop
- for f in $TEST_DATA_DIR/logs/hadoop/*; do
- echo "$f:"
- cat $f
- done
+ local log_directory="$TEST_DATA_DIR/logs"
+ local yarn_docker_containers="master $(docker ps --format '{{.Names}}' |
grep worker)"
- echo "\n\nDocker logs:"
- docker logs master
+ extract_hadoop_logs ${log_directory} ${yarn_docker_containers}
+ print_logs ${log_directory} ${yarn_docker_containers}
- echo "\n\nFlink logs:"
+ echo -e "\n\n ==== Flink logs ===="
docker exec master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab
hadoop-user"
docker exec master bash -c "yarn application -list -appStates ALL"
application_id=`docker exec master bash -c "yarn application -list
-appStates ALL" | grep -i "Flink" | grep -i "cluster" | awk '{print \$1}'`
- echo "Application ID: '$application_id'"
+ echo -e "\n\nApplication ID: '$application_id'"
docker exec master bash -c "yarn logs -applicationId $application_id"
docker exec master bash -c "kdestroy"
}
+function extract_hadoop_logs() {
+ local log_folder="$1"
+ shift
+ docker_container_aliases="$@"
+
+ for docker_container_alias in $(echo ${docker_container_aliases}); do
+ local container_logs="${log_folder}/${docker_container_alias}"
+ echo "Extracting ${docker_container_alias} Hadoop logs into
${container_logs}"
+ mkdir -p "${container_logs}"
+ docker cp "${docker_container_alias}:/var/log/hadoop/"
"${container_logs}"
+ done
+}
+
+function print_logs() {
+ local log_folder="$1"
+ shift
+ local docker_container_aliases="$@"
+
+ for docker_container_alias in $(echo ${docker_container_aliases}); do
+ echo -e "\n\n ==== ${docker_container_alias} Hadoop logs ===="
+ local container_logs="${log_folder}/${docker_container_alias}/hadoop"
+ ls -lisah "${container_logs}"
+ # find will list all files (even hidden ones) but no subfolders
+ for f in $(find "${container_logs}" -maxdepth 1 -mindepth 1 -type f); do
+ echo "$f:"
+ cat $f
+ done
+ done
+
+ echo -e "\n\n ==== Docker logs ===="
+ for docker_container_alias in $(echo ${docker_container_aliases}); do
+ echo "\n\nDocker logs for ${docker_container_alias}"
+ docker logs "${docker_container_alias}"
Review comment:
how come we aren't copying these as well?
##########
File path: flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
##########
@@ -138,35 +138,65 @@ END
function debug_copy_and_show_logs {
echo "Debugging failed YARN Docker test:"
- echo "\nCurrently running containers"
+ echo -e "\nCurrently running containers"
docker ps
- echo "\n\nCurrently running JVMs"
+ echo -e "\n\nCurrently running JVMs"
jps -v
- echo "\n\nHadoop logs:"
- mkdir -p $TEST_DATA_DIR/logs
- docker cp master:/var/log/hadoop/ $TEST_DATA_DIR/logs/
- ls -lisah $TEST_DATA_DIR/logs/hadoop
- for f in $TEST_DATA_DIR/logs/hadoop/*; do
- echo "$f:"
- cat $f
- done
+ local log_directory="$TEST_DATA_DIR/logs"
+ local yarn_docker_containers="master $(docker ps --format '{{.Names}}' |
grep worker)"
- echo "\n\nDocker logs:"
- docker logs master
+ extract_hadoop_logs ${log_directory} ${yarn_docker_containers}
+ print_logs ${log_directory} ${yarn_docker_containers}
- echo "\n\nFlink logs:"
+ echo -e "\n\n ==== Flink logs ===="
docker exec master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab
hadoop-user"
docker exec master bash -c "yarn application -list -appStates ALL"
application_id=`docker exec master bash -c "yarn application -list
-appStates ALL" | grep -i "Flink" | grep -i "cluster" | awk '{print \$1}'`
- echo "Application ID: '$application_id'"
+ echo -e "\n\nApplication ID: '$application_id'"
docker exec master bash -c "yarn logs -applicationId $application_id"
docker exec master bash -c "kdestroy"
}
+function extract_hadoop_logs() {
+ local log_folder="$1"
+ shift
+ docker_container_aliases="$@"
+
+ for docker_container_alias in $(echo ${docker_container_aliases}); do
+ local container_logs="${log_folder}/${docker_container_alias}"
+ echo "Extracting ${docker_container_alias} Hadoop logs into
${container_logs}"
+ mkdir -p "${container_logs}"
+ docker cp "${docker_container_alias}:/var/log/hadoop/"
"${container_logs}"
+ done
+}
+
+function print_logs() {
+ local log_folder="$1"
+ shift
+ local docker_container_aliases="$@"
+
+ for docker_container_alias in $(echo ${docker_container_aliases}); do
+ echo -e "\n\n ==== ${docker_container_alias} Hadoop logs ===="
+ local container_logs="${log_folder}/${docker_container_alias}/hadoop"
+ ls -lisah "${container_logs}"
+ # find will list all files (even hidden ones) but no subfolders
+ for f in $(find "${container_logs}" -maxdepth 1 -mindepth 1 -type f); do
Review comment:
what is the key difference to the previous code?
##########
File path: flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
##########
@@ -138,35 +138,65 @@ END
function debug_copy_and_show_logs {
echo "Debugging failed YARN Docker test:"
- echo "\nCurrently running containers"
+ echo -e "\nCurrently running containers"
docker ps
- echo "\n\nCurrently running JVMs"
+ echo -e "\n\nCurrently running JVMs"
jps -v
- echo "\n\nHadoop logs:"
- mkdir -p $TEST_DATA_DIR/logs
- docker cp master:/var/log/hadoop/ $TEST_DATA_DIR/logs/
- ls -lisah $TEST_DATA_DIR/logs/hadoop
- for f in $TEST_DATA_DIR/logs/hadoop/*; do
- echo "$f:"
- cat $f
- done
+ local log_directory="$TEST_DATA_DIR/logs"
+ local yarn_docker_containers="master $(docker ps --format '{{.Names}}' |
grep worker)"
- echo "\n\nDocker logs:"
- docker logs master
+ extract_hadoop_logs ${log_directory} ${yarn_docker_containers}
+ print_logs ${log_directory} ${yarn_docker_containers}
- echo "\n\nFlink logs:"
+ echo -e "\n\n ==== Flink logs ===="
docker exec master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab
hadoop-user"
docker exec master bash -c "yarn application -list -appStates ALL"
application_id=`docker exec master bash -c "yarn application -list
-appStates ALL" | grep -i "Flink" | grep -i "cluster" | awk '{print \$1}'`
- echo "Application ID: '$application_id'"
+ echo -e "\n\nApplication ID: '$application_id'"
docker exec master bash -c "yarn logs -applicationId $application_id"
docker exec master bash -c "kdestroy"
}
+function extract_hadoop_logs() {
+ local log_folder="$1"
+ shift
+ docker_container_aliases="$@"
+
+ for docker_container_alias in $(echo ${docker_container_aliases}); do
+ local container_logs="${log_folder}/${docker_container_alias}"
Review comment:
name is unclear as to whether it describes the source or target directory
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
