This is an automated email from the ASF dual-hosted git repository.
mchades pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 1b6a1556b [#3862] improvement(docker-image): Start multiple HMSs in
Kerberos Hive docker image (#3863)
1b6a1556b is described below
commit 1b6a1556bdf9d61d299ed23003aa44e050e13b67
Author: Qi Yu <[email protected]>
AuthorDate: Sun Jul 28 21:01:44 2024 +0800
[#3862] improvement(docker-image): Start multiple HMSs in Kerberos Hive
docker image (#3863)
### What changes were proposed in this pull request?
Add another Hive metastore server instance in the Hive docker iamge.
### Why are the changes needed?
We are going to check if Gravitino supports Kerberos authentication with
multiple HMS, so starting another HMS in the existing Kerberos Hive
docker image is in great need.
Fix: #3862
Fix: #4240
### Does this PR introduce _any_ user-facing change?
N/A.
### How was this patch tested?
N/A.
---
catalogs/catalog-hadoop/build.gradle.kts | 2 +-
catalogs/catalog-hive/build.gradle.kts | 2 +-
catalogs/catalog-lakehouse-iceberg/build.gradle.kts | 2 +-
dev/docker/kerberos-hive/Dockerfile | 4 +++-
dev/docker/kerberos-hive/hive-site.xml | 2 +-
.../kerberos-hive/{hive-site.xml => hive-site1.xml} | 8 +++++++-
dev/docker/kerberos-hive/start.sh | 15 +++++++++++----
docs/docker-image-details.md | 9 +++++++++
8 files changed, 34 insertions(+), 10 deletions(-)
diff --git a/catalogs/catalog-hadoop/build.gradle.kts
b/catalogs/catalog-hadoop/build.gradle.kts
index e7659ef7b..ba85c3b6e 100644
--- a/catalogs/catalog-hadoop/build.gradle.kts
+++ b/catalogs/catalog-hadoop/build.gradle.kts
@@ -126,7 +126,7 @@ tasks.test {
doFirst {
environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-hive:0.1.13")
- environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-kerberos-hive:0.1.4")
+ environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-kerberos-hive:0.1.5")
}
val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
diff --git a/catalogs/catalog-hive/build.gradle.kts
b/catalogs/catalog-hive/build.gradle.kts
index da7284292..c9c7422c6 100644
--- a/catalogs/catalog-hive/build.gradle.kts
+++ b/catalogs/catalog-hive/build.gradle.kts
@@ -180,7 +180,7 @@ tasks.test {
doFirst {
environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-hive:0.1.13")
- environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-kerberos-hive:0.1.4")
+ environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-kerberos-hive:0.1.5")
}
val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
index 154cbc65a..f7a251480 100644
--- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
+++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
@@ -151,7 +151,7 @@ tasks.test {
doFirst {
environment("GRAVITINO_CI_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-hive:0.1.13")
- environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-kerberos-hive:0.1.4")
+ environment("GRAVITINO_CI_KERBEROS_HIVE_DOCKER_IMAGE",
"datastrato/gravitino-ci-kerberos-hive:0.1.5")
}
val init = project.extra.get("initIntegrationTest") as (Test) -> Unit
diff --git a/dev/docker/kerberos-hive/Dockerfile
b/dev/docker/kerberos-hive/Dockerfile
index ae44ad6b7..2dbb3af58 100644
--- a/dev/docker/kerberos-hive/Dockerfile
+++ b/dev/docker/kerberos-hive/Dockerfile
@@ -141,6 +141,8 @@ ADD check-status.sh /tmp/check-status.sh
RUN mkdir ${HIVE_HOME}
RUN tar -xz -C ${HIVE_HOME} --strip-components 1 -f
/tmp/packages/${HIVE_PACKAGE_NAME}
ADD hive-site.xml ${HIVE_HOME}/conf/hive-site.xml
+RUN mkdir ${HIVE_HOME}/conf1 && cp ${HIVE_HOME}/conf/* ${HIVE_HOME}/conf1/
+ADD hive-site1.xml ${HIVE_HOME}/conf1/hive-site.xml
################################################################################
# install MySQL
@@ -178,7 +180,7 @@ RUN rm -rf /tmp/packages
################################################################################
# expose port
-EXPOSE 3306 9000 9083 10000 10002 50070 50075 50010 88
+EXPOSE 3306 9000 9083 10000 10002 50070 50075 50010 88 19083
################################################################################
# create startup script and set ENTRYPOINT
diff --git a/dev/docker/kerberos-hive/hive-site.xml
b/dev/docker/kerberos-hive/hive-site.xml
index 2256e5ae7..2acc7eecc 100644
--- a/dev/docker/kerberos-hive/hive-site.xml
+++ b/dev/docker/kerberos-hive/hive-site.xml
@@ -2,7 +2,7 @@
<property>
<name>hive.server2.enable.doAs</name>
<value>true</value>
- <description>Disable user impersonation for HiveServer2</description>
+ <description>Enable user impersonation for HiveServer2</description>
</property>
<property>
diff --git a/dev/docker/kerberos-hive/hive-site.xml
b/dev/docker/kerberos-hive/hive-site1.xml
similarity index 91%
copy from dev/docker/kerberos-hive/hive-site.xml
copy to dev/docker/kerberos-hive/hive-site1.xml
index 2256e5ae7..eaebfaf99 100644
--- a/dev/docker/kerberos-hive/hive-site.xml
+++ b/dev/docker/kerberos-hive/hive-site1.xml
@@ -2,7 +2,7 @@
<property>
<name>hive.server2.enable.doAs</name>
<value>true</value>
- <description>Disable user impersonation for HiveServer2</description>
+ <description>Enable user impersonation for HiveServer2</description>
</property>
<property>
@@ -68,4 +68,10 @@
<name>hive.metastore.kerberos.principal</name>
<value>hive/mockhost@HADOOPKRB</value>
</property>
+
+ <property>
+ <name>hive.metastore.port</name>
+ <value>19083</value>
+ </property>
+
</configuration>
diff --git a/dev/docker/kerberos-hive/start.sh
b/dev/docker/kerberos-hive/start.sh
index b457a3c53..a38c7edb0 100644
--- a/dev/docker/kerberos-hive/start.sh
+++ b/dev/docker/kerberos-hive/start.sh
@@ -21,10 +21,12 @@
# start ssh
HOSTNAME=`hostname`
service ssh start
-ssh-keyscan ${HOSTNAME} >> /root/.ssh/known_hosts
-ssh-keyscan localhost >> /root/.ssh/known_hosts
-ssh-keyscan 0.0.0.0 >> /root/.ssh/known_hosts
-ssh-keyscan 127.0.0.1 >> /root/.ssh/known_hosts
+
+for host in ${HOSTNAME} localhost 127.0.0.1 0.0.0.0; do
+ ssh-keygen -R ${host}
+ ssh-keyscan ${host} >> /root/.ssh/known_hosts
+ ssh -o BatchMode=yes -o ConnectTimeout=5 ${host} "exit" &>/dev/null || {
echo "failed to connect to ${host}"; exit 1; }
+done
# init the Kerberos database
echo -e "${PASS}\n${PASS}" | kdb5_util create -s
@@ -71,6 +73,7 @@ echo -e "${PASS}\n" | kinit hive/${HOSTNAME}
sed -i "s/mockhost/${HOSTNAME}/g" ${HADOOP_CONF_DIR}/hdfs-site.xml
sed -i "s/mockhost/${HOSTNAME}/g" ${HADOOP_CONF_DIR}/core-site.xml
sed -i "s/mockhost/${HOSTNAME}/g" ${HIVE_HOME}/conf/hive-site.xml
+sed -i "s/mockhost/${HOSTNAME}/g" ${HIVE_HOME}/conf1/hive-site.xml
# format HDFS
${HADOOP_HOME}/bin/hdfs namenode -format -nonInteractive
@@ -151,5 +154,9 @@ ${HIVE_HOME}/bin/schematool -initSchema -dbType mysql
${HIVE_HOME}/bin/hive --service hiveserver2 > /dev/null 2>&1 &
${HIVE_HOME}/bin/hive --service metastore > /dev/null 2>&1 &
+# Start another metastore
+export HIVE_CONF_DIR=${HIVE_HOME}/conf1
+${HIVE_HOME}/bin/hive --service metastore > /dev/null 2>&1 &
+
# persist the container
tail -f /dev/null
diff --git a/docs/docker-image-details.md b/docs/docker-image-details.md
index db526c3d1..d9f7638e8 100644
--- a/docs/docker-image-details.md
+++ b/docs/docker-image-details.md
@@ -96,6 +96,15 @@ You can use these kinds of Docker images to facilitate
integration testing of al
You can use this kind of image to test the catalog of Apache Hive with
kerberos enable
Changelog
+- gravitino-ci-kerberos-hive:0.1.5
+ - Start another HMS for the Hive cluster in the container with port 19083.
This is to test whether Kerberos authentication works for a Kerberos-enabled
Hive cluster with multiple HMS.
+ - Refresh ssh keys in the startup script.
+ - Add test logic to log in localhost via ssh without password.
+
+- gravitino-ci-kerberos-hive:0.1.4
+ - Increase the total check time for the status of DataNode to 150s.
+ - Output the log of the DataNode fails to start
+
- gravitino-ci-kerberos-hive:0.1.3
- Add more proxy users in the core-site.xml file.
- fix bugs in the `start.sh` script.
