This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 4ecc3edc217 [test](hive)revert hive container to host mode (#34322)
4ecc3edc217 is described below
commit 4ecc3edc2170e5744c0137db0b58e8070c196b72
Author: 苏小刚 <[email protected]>
AuthorDate: Mon May 6 10:58:13 2024 +0800
[test](hive)revert hive container to host mode (#34322)
Revert hive container to host mode to fix pipeline problem
---
.../docker-compose/hive/hadoop-hive.env | 52 ----------------------
...-hive-metastore.env.tpl => hadoop-hive.env.tpl} | 8 ++--
.../docker-compose/hive/hive-2x.yaml.tpl | 44 +++++++++---------
.../docker-compose/hive/hive-2x_settings.env | 8 ++--
.../docker-compose/hive/hive-3x.yaml.tpl | 45 +++++++++----------
.../docker-compose/hive/hive-3x_settings.env | 8 ++--
docker/thirdparties/run-thirdparties-docker.sh | 4 +-
7 files changed, 55 insertions(+), 114 deletions(-)
diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env
b/docker/thirdparties/docker-compose/hive/hadoop-hive.env
deleted file mode 100644
index 90412d74e66..00000000000
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive.env
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore-postgresql/metastore
-HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver
-HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive
-HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive
-HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
-HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083
-HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
-HIVE_SITE_CONF_hive_server2_thrift_port=10000
-HIVE_SITE_CONF_hive_compactor_initiator_on=true
-HIVE_SITE_CONF_hive_compactor_worker_threads=2
-HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader
-
-CORE_CONF_fs_defaultFS=hdfs://namenode:8020
-CORE_CONF_hadoop_http_staticuser_user=root
-CORE_CONF_hadoop_proxyuser_hue_hosts=*
-CORE_CONF_hadoop_proxyuser_hue_groups=*
-
-HDFS_CONF_dfs_webhdfs_enabled=true
-HDFS_CONF_dfs_permissions_enabled=false
-HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false
-
-YARN_CONF_yarn_log___aggregation___enable=true
-YARN_CONF_yarn_resourcemanager_recovery_enabled=true
-YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
-YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate
-YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
-YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/
-YARN_CONF_yarn_timeline___service_enabled=true
-YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true
-YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true
-YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
-YARN_CONF_yarn_timeline___service_hostname=historyserver
-YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032
-YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030
-YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031
diff --git
a/docker/thirdparties/docker-compose/hive/hadoop-hive-metastore.env.tpl
b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
similarity index 91%
rename from
docker/thirdparties/docker-compose/hive/hadoop-hive-metastore.env.tpl
rename to docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
index 4497b37d450..0e074228410 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive-metastore.env.tpl
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl
@@ -15,14 +15,16 @@
# limitations under the License.
#
-HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore-postgresql/metastore
+HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://${IP_HOST}:${PG_PORT}/metastore
HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver
HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive
HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive
HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
-HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083
+HIVE_SITE_CONF_hive_metastore_port=${HMS_PORT}
+HIVE_SITE_CONF_hive_metastore_uris=thrift://${IP_HOST}:${HMS_PORT}
HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0
-HIVE_SITE_CONF_hive_server2_thrift_port=10000
+HIVE_SITE_CONF_hive_server2_thrift_port=${HS_PORT}
+HIVE_SITE_CONF_hive_server2_webui_port=0
HIVE_SITE_CONF_hive_compactor_initiator_on=true
HIVE_SITE_CONF_hive_compactor_worker_threads=2
HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
index bf4cf010963..388ea0fc04f 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl
@@ -26,62 +26,68 @@ services:
env_file:
- ./hadoop-hive.env
container_name: ${CONTAINER_UID}hadoop2-namenode
- ports:
- - "${FS_PORT}:8020"
+ expose:
+ - "50070"
+ - "${FS_PORT}"
healthcheck:
test: [ "CMD", "curl", "http://localhost:50070/"; ]
interval: 5s
timeout: 120s
retries: 120
+ network_mode: "host"
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
env_file:
- ./hadoop-hive.env
environment:
- SERVICE_PRECONDITION: "namenode:50070"
+ SERVICE_PRECONDITION: "${IP_HOST}:50070"
container_name: ${CONTAINER_UID}hadoop2-datanode
+ expose:
+ - "50075"
healthcheck:
test: [ "CMD", "curl", "http://localhost:50075"; ]
interval: 5s
timeout: 60s
retries: 120
+ network_mode: "host"
hive-server:
image: bde2020/hive:2.3.2-postgresql-metastore
env_file:
- - ./hadoop-hive-metastore.env
+ - ./hadoop-hive.env
environment:
- HIVE_CORE_CONF_javax_jdo_option_ConnectionURL:
"jdbc:postgresql://hive-metastore/metastore"
- SERVICE_PRECONDITION: "hive-metastore:9083"
+ HIVE_CORE_CONF_javax_jdo_option_ConnectionURL:
"jdbc:postgresql://${IP_HOST}:${PG_PORT}/metastore"
+ SERVICE_PRECONDITION: "${IP_HOST}:${HMS_PORT}"
container_name: ${CONTAINER_UID}hive2-server
- ports:
- - "${HS_PORT}:10000"
+ expose:
+ - "${HS_PORT}"
depends_on:
- datanode
- namenode
healthcheck:
- test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check
-e "show databases;"
+ test: beeline -u "jdbc:hive2://127.0.0.1:${HS_PORT}/default" -n
health_check -e "show databases;"
interval: 10s
timeout: 120s
retries: 120
+ network_mode: "host"
hive-metastore:
image: bde2020/hive:2.3.2-postgresql-metastore
env_file:
- - ./hadoop-hive-metastore.env
+ - ./hadoop-hive.env
command: /bin/bash /mnt/scripts/hive-metastore.sh
- # command: /opt/hive/bin/hive --service metastore
environment:
- SERVICE_PRECONDITION: "namenode:50070 datanode:50075
hive-metastore-postgresql:5432"
+ SERVICE_PRECONDITION: "${IP_HOST}:50070 ${IP_HOST}:50075
${IP_HOST}:${PG_PORT}"
container_name: ${CONTAINER_UID}hive2-metastore
- ports:
- - "${HMS_PORT}:9083"
+ expose:
+ - "${HMS_PORT}"
volumes:
- ./scripts:/mnt/scripts
depends_on:
- hive-metastore-postgresql
+ network_mode: "host"
hive-metastore-postgresql:
image: bde2020/hive-metastore-postgresql:2.3.0
@@ -93,13 +99,3 @@ services:
interval: 5s
timeout: 60s
retries: 120
-
-# solve HiveServer2 connect error:
-# java.net.URISyntaxException Illegal character in hostname
:thrift://${CONTAINER_UID}hive2_default:9083
-networks:
- default:
- name: ${CONTAINER_UID}hive2-default
- ipam:
- driver: default
- config:
- - subnet: 168.58.0.0/24
diff --git a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
index 6e1109dcb47..ddd4625aaab 100644
--- a/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env
@@ -20,7 +20,7 @@
# Do not use "_" or other sepcial characters, only number and alphabeta.
# NOTICE: change this uid will modify hive-*.yaml
-export FS_PORT=8020 #should be same as hive2HdfsPort in regression-conf.groovy
-export HMS_PORT=9083 #should be same as hive2HmsPort in regression-conf.groovy
-export HS_PORT=10000 #should be same as hive2ServerPort in
regression-conf.groovy
-export PG_PORT=5432 #should be same as hive2PgPort in regression-conf.groovy
\ No newline at end of file
+export FS_PORT=8020 # should be same as hive3HmsPort in regression-conf.groovy
+export HMS_PORT=9083 # should be same as hive2HmsPort in regression-conf.groovy
+export HS_PORT=10000 # should be same as hive2ServerPort in
regression-conf.groovy
+export PG_PORT=5432 # should be same as hive2PgPort in regression-conf.groovy
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
index 72c0a7cfd21..c6d637b77e2 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
+++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl
@@ -26,62 +26,68 @@ services:
env_file:
- ./hadoop-hive.env
container_name: ${CONTAINER_UID}hadoop3-namenode
- ports:
- - "${FS_PORT}:8020"
+ expose:
+ - "9870"
+ - "${FS_PORT}"
healthcheck:
test: [ "CMD", "curl", "http://localhost:9870/"; ]
interval: 5s
timeout: 120s
retries: 120
+ network_mode: "host"
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
env_file:
- ./hadoop-hive.env
environment:
- SERVICE_PRECONDITION: "namenode:9870"
+ SERVICE_PRECONDITION: "${IP_HOST}:9870"
container_name: ${CONTAINER_UID}hadoop3-datanode
+ expose:
+ - "9864"
healthcheck:
test: [ "CMD", "curl", "http://localhost:9864"; ]
interval: 5s
timeout: 60s
retries: 120
+ network_mode: "host"
hive-server:
image: lishizhen/hive:3.1.2-postgresql-metastore
env_file:
- - ./hadoop-hive-metastore.env
+ - ./hadoop-hive.env
environment:
- HIVE_CORE_CONF_javax_jdo_option_ConnectionURL:
"jdbc:postgresql://hive-metastore/metastore"
- SERVICE_PRECONDITION: "hive-metastore:9083"
+ HIVE_CORE_CONF_javax_jdo_option_ConnectionURL:
"jdbc:postgresql://${IP_HOST}:${PG_PORT}/metastore"
+ SERVICE_PRECONDITION: "${IP_HOST}:${HMS_PORT}"
container_name: ${CONTAINER_UID}hive3-server
- ports:
- - "${HS_PORT}:10000"
+ expose:
+ - "${HS_PORT}"
depends_on:
- datanode
- namenode
healthcheck:
- test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check
-e "show databases;"
+ test: beeline -u "jdbc:hive2://127.0.0.1:${HS_PORT}/default" -n
health_check -e "show databases;"
interval: 10s
timeout: 120s
retries: 120
+ network_mode: "host"
hive-metastore:
image: lishizhen/hive:3.1.2-postgresql-metastore
env_file:
- - ./hadoop-hive-metastore.env
+ - ./hadoop-hive.env
command: /bin/bash /mnt/scripts/hive-metastore.sh
- # command: /opt/hive/bin/hive --service metastore
environment:
- SERVICE_PRECONDITION: "namenode:9870 datanode:9864
hive-metastore-postgresql:5432"
+ SERVICE_PRECONDITION: "${IP_HOST}:9870 ${IP_HOST}:9864
${IP_HOST}:${PG_PORT}"
container_name: ${CONTAINER_UID}hive3-metastore
- ports:
- - "${HMS_PORT}:9083"
+ expose:
+ - "${HMS_PORT}"
volumes:
- ./scripts:/mnt/scripts
depends_on:
- hive-metastore-postgresql
+ network_mode: "host"
hive-metastore-postgresql:
image: bde2020/hive-metastore-postgresql:3.1.0
@@ -93,14 +99,3 @@ services:
interval: 5s
timeout: 60s
retries: 120
-
-# solve HiveServer2 connect error:
-# java.net.URISyntaxException Illegal character in hostname
:thrift://${CONTAINER_UID}hive3_default:9083
-
-networks:
- default:
- name: ${CONTAINER_UID}hive3-default
- ipam:
- driver: default
- config:
- - subnet: 168.59.0.0/24
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
index 9f89b1f07b8..7322f71f307 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
@@ -20,7 +20,7 @@
# Do not use "_" or other sepcial characters, only number and alphabeta.
# NOTICE: change this uid will modify hive-*.yaml
-export FS_PORT=8320 #should be same as hive3HdfsPort in regression-conf.groovy
-export HMS_PORT=9383 #should be same as hive3HmsPort in regression-conf.groovy
-export HS_PORT=13000 #should be same as hive3ServerPort in
regression-conf.groovy
-export PG_PORT=5732 #should be same as hive3PgPort in regression-conf.groovy
\ No newline at end of file
+export FS_PORT=8320 # should be same as hive3HmsPort in regression-conf.groovy
+export HMS_PORT=9383 # should be same as hive3HmsPort in regression-conf.groovy
+export HS_PORT=13000 # should be same as hive3ServerPort in
regression-conf.groovy
+export PG_PORT=5732 # should be same as hive3PgPort in regression-conf.groovy
diff --git a/docker/thirdparties/run-thirdparties-docker.sh
b/docker/thirdparties/run-thirdparties-docker.sh
index a6c0b6c0ef3..edf2f96c6a6 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -319,7 +319,7 @@ if [[ "${RUN_HIVE2}" -eq 1 ]]; then
export CONTAINER_UID=${CONTAINER_UID}
. "${ROOT}"/docker-compose/hive/hive-2x_settings.env
envsubst < "${ROOT}"/docker-compose/hive/hive-2x.yaml.tpl >
"${ROOT}"/docker-compose/hive/hive-2x.yaml
- envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env.tpl >
"${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env
+ envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl >
"${ROOT}"/docker-compose/hive/hadoop-hive.env
sudo docker compose -p ${CONTAINER_UID}hive2 -f
"${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file
"${ROOT}"/docker-compose/hive/hadoop-hive.env down
if [[ "${STOP}" -ne 1 ]]; then
sudo docker compose -p ${CONTAINER_UID}hive2 -f
"${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file
"${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d
@@ -345,7 +345,7 @@ if [[ "${RUN_HIVE3}" -eq 1 ]]; then
export CONTAINER_UID=${CONTAINER_UID}
. "${ROOT}"/docker-compose/hive/hive-3x_settings.env
envsubst < "${ROOT}"/docker-compose/hive/hive-3x.yaml.tpl >
"${ROOT}"/docker-compose/hive/hive-3x.yaml
- envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env.tpl >
"${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env
+ envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl >
"${ROOT}"/docker-compose/hive/hadoop-hive.env
sudo docker compose -p ${CONTAINER_UID}hive3 -f
"${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file
"${ROOT}"/docker-compose/hive/hadoop-hive.env down
if [[ "${STOP}" -ne 1 ]]; then
sudo docker compose -p ${CONTAINER_UID}hive3 -f
"${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file
"${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
