XComp commented on a change in pull request #15853:
URL: https://github.com/apache/flink/pull/15853#discussion_r629062034
##########
File path: flink-end-to-end-tests/test-scripts/common_yarn_docker.sh
##########
@@ -138,35 +138,65 @@ END
function debug_copy_and_show_logs {
echo "Debugging failed YARN Docker test:"
- echo "\nCurrently running containers"
+ echo -e "\nCurrently running containers"
docker ps
- echo "\n\nCurrently running JVMs"
+ echo -e "\n\nCurrently running JVMs"
jps -v
- echo "\n\nHadoop logs:"
- mkdir -p $TEST_DATA_DIR/logs
- docker cp master:/var/log/hadoop/ $TEST_DATA_DIR/logs/
- ls -lisah $TEST_DATA_DIR/logs/hadoop
- for f in $TEST_DATA_DIR/logs/hadoop/*; do
- echo "$f:"
- cat $f
- done
+ local log_directory="$TEST_DATA_DIR/logs"
+ local yarn_docker_containers="master $(docker ps --format '{{.Names}}' |
grep worker)"
- echo "\n\nDocker logs:"
- docker logs master
+ extract_hadoop_logs ${log_directory} ${yarn_docker_containers}
+ print_logs ${log_directory} ${yarn_docker_containers}
- echo "\n\nFlink logs:"
+ echo -e "\n\n ==== Flink logs ===="
docker exec master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab
hadoop-user"
docker exec master bash -c "yarn application -list -appStates ALL"
application_id=`docker exec master bash -c "yarn application -list
-appStates ALL" | grep -i "Flink" | grep -i "cluster" | awk '{print \$1}'`
- echo "Application ID: '$application_id'"
+ echo -e "\n\nApplication ID: '$application_id'"
docker exec master bash -c "yarn logs -applicationId $application_id"
docker exec master bash -c "kdestroy"
}
+function extract_hadoop_logs() {
+ local log_folder="$1"
+ shift
+ docker_container_aliases="$@"
+
+ for docker_container_alias in $(echo ${docker_container_aliases}); do
+ local container_logs="${log_folder}/${docker_container_alias}"
+ echo "Extracting ${docker_container_alias} Hadoop logs into
${container_logs}"
+ mkdir -p "${container_logs}"
+ docker cp "${docker_container_alias}:/var/log/hadoop/"
"${container_logs}"
+ done
+}
+
+function print_logs() {
+ local log_folder="$1"
+ shift
+ local docker_container_aliases="$@"
+
+ for docker_container_alias in $(echo ${docker_container_aliases}); do
+ echo -e "\n\n ==== ${docker_container_alias} Hadoop logs ===="
+ local container_logs="${log_folder}/${docker_container_alias}/hadoop"
+ ls -lisah "${container_logs}"
+ # find will list all files (even hidden ones) but no subfolders
+ for f in $(find "${container_logs}" -maxdepth 1 -mindepth 1 -type f); do
Review comment:
Iterating over the entries of the directory as we did it before didn't
make dotfiles available:
```
➜ test ls -lisahR
total 0
53223531 0 drwxr-xr-x 5 mapohl staff 160B May 10 07:36 .
52512433 0 drwxr-xr-x 14 mapohl staff 448B May 10 07:36 ..
53223571 0 -rw-r--r-- 1 mapohl staff 0B May 10 07:36 .hidden
53223593 0 -rw-r--r-- 1 mapohl staff 0B May 10 07:36 not-hidden
53223619 0 drwxr-xr-x 4 mapohl staff 128B May 10 07:39 subdir
./subdir:
total 0
53223619 0 drwxr-xr-x 4 mapohl staff 128B May 10 07:39 .
53223531 0 drwxr-xr-x 5 mapohl staff 160B May 10 07:36 ..
53223667 0 -rw-r--r-- 1 mapohl staff 0B May 10 07:36 .hidden-in-subdir
53223710 0 -rw-r--r-- 1 mapohl staff 0B May 10 07:36
not-hidden-in-subdir
➜ test for f in ./*; do echo $f; done
./not-hidden
./subdir
➜ test for f in $(find . -maxdepth 1 -mindepth 1 -type f); do echo $f; done
./.hidden
./not-hidden
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
