aljoscha closed pull request #6469: [FLINK-10016] Make YARN/Kerberos end-to-end
test stricter
URL: https://github.com/apache/flink/pull/6469
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml
b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml
index 62bea95301c..9b17acc1656 100644
---
a/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml
+++
b/flink-end-to-end-tests/test-scripts/docker-hadoop-secure-cluster/config/yarn-site.xml
@@ -21,11 +21,26 @@ under the License.
<value>mapreduce_shuffle</value>
</property>
+ <!-- this is ignored by the default scheduler but we have to set it
because Flink would
+ complain if we didn't have it -->
<property>
<name>yarn.nodemanager.resource.cpu-vcores</name>
<value>1</value>
</property>
+ <!-- the combination of this and the next setting ensures that the
YARN/Kerberos test starts
+ containers on all available NMs. If the memory is to big it could happen
that all containers
+ are scheduled on one NM, which wouldn't provoke a previously fixed
Kerberos keytab bug. -->
+ <property>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>4100</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.minimum-allocation-mb</name>
+ <value>2000</value>
+ </property>
+
<property>
<name>yarn.application.classpath</name>
<value>/usr/local/hadoop/etc/hadoop,
/usr/local/hadoop/share/hadoop/common/*,
/usr/local/hadoop/share/hadoop/common/lib/*,
/usr/local/hadoop/share/hadoop/hdfs/*,
/usr/local/hadoop/share/hadoop/hdfs/lib/*,
/usr/local/hadoop/share/hadoop/mapreduce/*,
/usr/local/hadoop/share/hadoop/mapreduce/lib/*,
/usr/local/hadoop/share/hadoop/yarn/*,
/usr/local/hadoop/share/hadoop/yarn/lib/*</value>
diff --git a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
index e7726c92ece..02b794fea7d 100755
--- a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
+++ b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
@@ -72,6 +72,7 @@ docker exec -it master bash -c "tar xzf
/home/hadoop-user/$FLINK_TARBALL --direc
# minimal Flink config, bebe
docker exec -it master bash -c "echo \"security.kerberos.login.keytab:
/home/hadoop-user/hadoop-user.keytab\" >
/home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
docker exec -it master bash -c "echo \"security.kerberos.login.principal:
hadoop-user\" >> /home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
+docker exec -it master bash -c "echo \"slot.request.timeout: 60000\" >>
/home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
echo "Flink config:"
docker exec -it master bash -c "cat
/home/hadoop-user/$FLINK_DIRNAME/conf/flink-conf.yaml"
@@ -83,7 +84,7 @@ OUTPUT_PATH=hdfs:///user/hadoop-user/wc-out-$RANDOM
start_time=$(date +%s)
# it's important to run this with higher parallelism, otherwise we might risk
that
# JM and TM are on the same YARN node and that we therefore don't test the
keytab shipping
-until docker exec -it master bash -c "export HADOOP_CLASSPATH=\`hadoop
classpath\` && /home/hadoop-user/$FLINK_DIRNAME/bin/flink run -m yarn-cluster
-yn 3 -ys 1 -ytm 1200 -yjm 800 -p 3
/home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar --output
$OUTPUT_PATH"; do
+until docker exec -it master bash -c "export HADOOP_CLASSPATH=\`hadoop
classpath\` && /home/hadoop-user/$FLINK_DIRNAME/bin/flink run -m yarn-cluster
-yn 3 -ys 1 -ytm 1000 -yjm 1000 -p 3
/home/hadoop-user/$FLINK_DIRNAME/examples/streaming/WordCount.jar --output
$OUTPUT_PATH"; do
current_time=$(date +%s)
time_diff=$((current_time - start_time))
@@ -106,12 +107,22 @@ until docker exec -it master bash -c "export
HADOOP_CLASSPATH=\`hadoop classpath
done
docker exec -it master bash -c "kinit -kt /home/hadoop-user/hadoop-user.keytab
hadoop-user"
+docker exec -it master bash -c "hdfs dfs -ls $OUTPUT_PATH"
OUTPUT=$(docker exec -it master bash -c "hdfs dfs -cat $OUTPUT_PATH/*")
docker exec -it master bash -c "kdestroy"
echo "$OUTPUT"
if [[ ! "$OUTPUT" =~ "consummation,1" ]]; then
echo "Output does not contain (consummation, 1) as required"
+ mkdir -p $TEST_DATA_DIR/logs
+ echo "Hadoop logs:"
+ docker cp master:/var/log/hadoop/* $TEST_DATA_DIR/logs/
+ for f in $TEST_DATA_DIR/logs/*; do
+ echo "$f:"
+ cat $f
+ done
+ echo "Docker logs:"
+ docker logs master
exit 1
fi
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
