This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 775ffdfc5bfe266dbb63da7f8b1d9e65ab204334 Author: 苏小刚 <[email protected]> AuthorDate: Tue Apr 16 16:43:49 2024 +0800 [docker](hive) add hive3 docker compose and modify scripts (#33115) add hive3 docker compose from: big-data-europe/docker-hive#56 --- ...e.env.tpl.tpl => hadoop-hive-metastore.env.tpl} | 11 ++- .../{hadoop-hive.env.tpl.tpl => hadoop-hive.env} | 11 ++- .../docker-compose/hive/hive-2x.yaml.tpl | 70 +++++++++--------- .../hive/{gen_env.sh.tpl => hive-2x_settings.env} | 29 ++------ .../hive/{hive-2x.yaml.tpl => hive-3x.yaml.tpl} | 85 +++++++++++----------- .../hive/{gen_env.sh.tpl => hive-3x_settings.env} | 29 ++------ .../hive/scripts/create_preinstalled_table.hql | 4 + docker/thirdparties/run-thirdparties-docker.sh | 57 +++++++++++---- regression-test/conf/regression-conf.groovy | 4 +- .../export/hive_read/orc/test_hive_read_orc.out | 3 - .../orc/test_hive_read_orc_complex_type.out | 2 +- .../hive_read/parquet/test_hive_read_parquet.out | 3 - .../export/hive_read/orc/test_hive_read_orc.groovy | 14 +--- .../orc/test_hive_read_orc_complex_type.groovy | 8 +- .../parquet/test_hive_read_parquet.groovy | 14 +--- .../test_hive_read_parquet_comlex_type.groovy | 8 +- .../hive/ddl/test_hive_ddl.groovy | 2 +- .../mtmv_p0/test_hive_multi_partition_mtmv.groovy | 6 ++ .../suites/mtmv_p0/test_hive_refresh_mtmv.groovy | 8 ++ .../mv_contain_external_table.groovy | 4 + 20 files changed, 179 insertions(+), 193 deletions(-) diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl.tpl b/docker/thirdparties/docker-compose/hive/hadoop-hive-metastore.env.tpl similarity index 90% copy from docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl.tpl copy to docker/thirdparties/docker-compose/hive/hadoop-hive-metastore.env.tpl index 7fb85d72fa8..828a194e2b3 100644 --- a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl.tpl +++ b/docker/thirdparties/docker-compose/hive/hadoop-hive-metastore.env.tpl @@ -15,25 +15,24 @@ # limitations under the License. # -HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://externalEnvIp:5432/metastore +HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore-postgresql/metastore HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive HIVE_SITE_CONF_datanucleus_autoCreateSchema=false -HIVE_SITE_CONF_hive_metastore_uris=thrift://externalEnvIp:9083 -HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false +HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083 HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0 HIVE_SITE_CONF_hive_server2_thrift_port=10000 -HIVE_SITE_CONF_hive_compactor_initiator_on=true -HIVE_SITE_CONF_hive_compactor_worker_threads=2 +HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader +CORE_CONF_fs_defaultFS=hdfs://${IP_HOST}:${FS_PORT} CORE_CONF_hadoop_http_staticuser_user=root CORE_CONF_hadoop_proxyuser_hue_hosts=* CORE_CONF_hadoop_proxyuser_hue_groups=* -CORE_CONF_hadoop_proxyuser_hive_hosts=* HDFS_CONF_dfs_webhdfs_enabled=true HDFS_CONF_dfs_permissions_enabled=false +HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false YARN_CONF_yarn_log___aggregation___enable=true YARN_CONF_yarn_resourcemanager_recovery_enabled=true diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl.tpl b/docker/thirdparties/docker-compose/hive/hadoop-hive.env similarity index 90% rename from docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl.tpl rename to docker/thirdparties/docker-compose/hive/hadoop-hive.env index 7fb85d72fa8..0e092766dc3 100644 --- a/docker/thirdparties/docker-compose/hive/hadoop-hive.env.tpl.tpl +++ b/docker/thirdparties/docker-compose/hive/hadoop-hive.env @@ -15,25 +15,24 @@ # limitations under the License. # -HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://externalEnvIp:5432/metastore +HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore-postgresql/metastore HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive HIVE_SITE_CONF_datanucleus_autoCreateSchema=false -HIVE_SITE_CONF_hive_metastore_uris=thrift://externalEnvIp:9083 -HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false +HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083 HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0 HIVE_SITE_CONF_hive_server2_thrift_port=10000 -HIVE_SITE_CONF_hive_compactor_initiator_on=true -HIVE_SITE_CONF_hive_compactor_worker_threads=2 +HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader +CORE_CONF_fs_defaultFS=hdfs://namenode:8020 CORE_CONF_hadoop_http_staticuser_user=root CORE_CONF_hadoop_proxyuser_hue_hosts=* CORE_CONF_hadoop_proxyuser_hue_groups=* -CORE_CONF_hadoop_proxyuser_hive_hosts=* HDFS_CONF_dfs_webhdfs_enabled=true HDFS_CONF_dfs_permissions_enabled=false +HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false YARN_CONF_yarn_log___aggregation___enable=true YARN_CONF_yarn_resourcemanager_recovery_enabled=true diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl index 32b99a0b0cc..9b201a5416e 100644 --- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl +++ b/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl @@ -19,87 +19,83 @@ version: "3.8" services: - doris--namenode: + namenode: image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 environment: - CLUSTER_NAME=test env_file: - ./hadoop-hive.env - container_name: doris--namenode - expose: - - "50070" - - "8020" - - "9000" + container_name: ${CONTAINER_UID}hadoop2-namenode + ports: + - "${FS_PORT}:8020" healthcheck: test: [ "CMD", "curl", "http://localhost:50070/"; ] interval: 5s timeout: 120s retries: 120 - network_mode: "host" - doris--datanode: + datanode: image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 env_file: - ./hadoop-hive.env environment: - SERVICE_PRECONDITION: "externalEnvIp:50070" - container_name: doris--datanode - expose: - - "50075" + SERVICE_PRECONDITION: "namenode:50070" + container_name: ${CONTAINER_UID}hadoop2-datanode healthcheck: test: [ "CMD", "curl", "http://localhost:50075"; ] interval: 5s timeout: 60s retries: 120 - network_mode: "host" - doris--hive-server: + hive-server: image: bde2020/hive:2.3.2-postgresql-metastore env_file: - - ./hadoop-hive.env + - ./hadoop-hive-metastore.env environment: - HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://externalEnvIp:5432/metastore" - SERVICE_PRECONDITION: "externalEnvIp:9083" - container_name: doris--hive-server - expose: - - "10000" + HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore" + SERVICE_PRECONDITION: "hive-metastore:9083" + container_name: ${CONTAINER_UID}hive2-server + ports: + - "${HS_PORT}:10000" depends_on: - - doris--datanode - - doris--namenode + - datanode + - namenode healthcheck: test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;" interval: 10s timeout: 120s retries: 120 - network_mode: "host" - doris--hive-metastore: + hive-metastore: image: bde2020/hive:2.3.2-postgresql-metastore env_file: - - ./hadoop-hive.env + - ./hadoop-hive-metastore.env command: /bin/bash /mnt/scripts/hive-metastore.sh # command: /opt/hive/bin/hive --service metastore environment: - SERVICE_PRECONDITION: "externalEnvIp:50070 externalEnvIp:50075 externalEnvIp:5432" - container_name: doris--hive-metastore - expose: - - "9083" + SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432" + container_name: ${CONTAINER_UID}hive2-metastore + ports: + - "${HMS_PORT}:9083" volumes: - ./scripts:/mnt/scripts depends_on: - - doris--hive-metastore-postgresql - network_mode: "host" + - hive-metastore-postgresql - doris--hive-metastore-postgresql: + hive-metastore-postgresql: image: bde2020/hive-metastore-postgresql:2.3.0 - restart: always - container_name: doris--hive-metastore-postgresql - expose: - - "5432" + container_name: ${CONTAINER_UID}hive2-metastore-postgresql + ports: + - "${PG_PORT}:5432" healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 5s timeout: 60s retries: 120 - network_mode: "host" + +# solve HiveServer2 connect error: +# java.net.URISyntaxException Illegal character in hostname :thrift://${CONTAINER_UID}hive2_default:9083 +networks: + default: + name: ${CONTAINER_UID}hive2-default diff --git a/docker/thirdparties/docker-compose/hive/gen_env.sh.tpl b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env old mode 100755 new mode 100644 similarity index 50% copy from docker/thirdparties/docker-compose/hive/gen_env.sh.tpl copy to docker/thirdparties/docker-compose/hive/hive-2x_settings.env index a2c708841da..1d021ba20c9 --- a/docker/thirdparties/docker-compose/hive/gen_env.sh.tpl +++ b/docker/thirdparties/docker-compose/hive/hive-2x_settings.env @@ -16,26 +16,11 @@ # specific language governing permissions and limitations # under the License. -#################################################################### -# This script will generate hadoop-hive.env from hadoop-hive.env.tpl -#################################################################### +# Change this to a specific string. +# Do not use "_" or other sepcial characters, only number and alphabeta. +# NOTICE: change this uid will modify hive-*.yaml -set -eo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -FS_PORT=8020 -HMS_PORT=9083 - -cp "${ROOT}"/hadoop-hive.env.tpl "${ROOT}"/hadoop-hive.env -# Need to set hostname of container to same as host machine's. -# Otherwise, the doris process can not connect to namenode directly. -HOST_NAME="doris--" - -{ - echo "FS_PORT=${FS_PORT}" - echo "HMS_PORT=${HMS_PORT}" - echo "CORE_CONF_fs_defaultFS=hdfs://${externalEnvIp}:${FS_PORT}" - echo "HOST_NAME=${HOST_NAME}" - echo "externalEnvIp=${externalEnvIp}" - -} >>"${ROOT}"/hadoop-hive.env +export FS_PORT=8220 #should be same in regression-conf.groovy +export HMS_PORT=9283 #should be same in regression-conf.groovy +export HS_PORT=12000 #should be same in regression-conf.groovy +export PG_PORT=5632 #should be same in regression-conf.groovy \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl similarity index 54% copy from docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl copy to docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl index 32b99a0b0cc..5ce81774c69 100644 --- a/docker/thirdparties/docker-compose/hive/hive-2x.yaml.tpl +++ b/docker/thirdparties/docker-compose/hive/hive-3x.yaml.tpl @@ -19,87 +19,84 @@ version: "3.8" services: - doris--namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 + namenode: + image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8 environment: - CLUSTER_NAME=test env_file: - ./hadoop-hive.env - container_name: doris--namenode - expose: - - "50070" - - "8020" - - "9000" + container_name: ${CONTAINER_UID}hadoop3-namenode + ports: + - "${FS_PORT}:8020" healthcheck: - test: [ "CMD", "curl", "http://localhost:50070/"; ] + test: [ "CMD", "curl", "http://localhost:9870/"; ] interval: 5s timeout: 120s retries: 120 - network_mode: "host" - doris--datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 + datanode: + image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 env_file: - ./hadoop-hive.env environment: - SERVICE_PRECONDITION: "externalEnvIp:50070" - container_name: doris--datanode - expose: - - "50075" + SERVICE_PRECONDITION: "namenode:9870" + container_name: ${CONTAINER_UID}hadoop3-datanode healthcheck: - test: [ "CMD", "curl", "http://localhost:50075"; ] + test: [ "CMD", "curl", "http://localhost:9864"; ] interval: 5s timeout: 60s retries: 120 - network_mode: "host" - doris--hive-server: - image: bde2020/hive:2.3.2-postgresql-metastore + hive-server: + image: lishizhen/hive:3.1.2-postgresql-metastore env_file: - - ./hadoop-hive.env + - ./hadoop-hive-metastore.env environment: - HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://externalEnvIp:5432/metastore" - SERVICE_PRECONDITION: "externalEnvIp:9083" - container_name: doris--hive-server - expose: - - "10000" + HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore" + SERVICE_PRECONDITION: "hive-metastore:9083" + container_name: ${CONTAINER_UID}hive3-server + ports: + - "${HS_PORT}:10000" depends_on: - - doris--datanode - - doris--namenode + - datanode + - namenode healthcheck: test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;" interval: 10s timeout: 120s retries: 120 - network_mode: "host" - doris--hive-metastore: - image: bde2020/hive:2.3.2-postgresql-metastore + hive-metastore: + image: lishizhen/hive:3.1.2-postgresql-metastore env_file: - - ./hadoop-hive.env + - ./hadoop-hive-metastore.env command: /bin/bash /mnt/scripts/hive-metastore.sh # command: /opt/hive/bin/hive --service metastore environment: - SERVICE_PRECONDITION: "externalEnvIp:50070 externalEnvIp:50075 externalEnvIp:5432" - container_name: doris--hive-metastore - expose: - - "9083" + SERVICE_PRECONDITION: "namenode:9870 datanode:9864 hive-metastore-postgresql:5432" + container_name: ${CONTAINER_UID}hive3-metastore + ports: + - "${HMS_PORT}:9083" volumes: - ./scripts:/mnt/scripts depends_on: - - doris--hive-metastore-postgresql - network_mode: "host" + - hive-metastore-postgresql - doris--hive-metastore-postgresql: - image: bde2020/hive-metastore-postgresql:2.3.0 - restart: always - container_name: doris--hive-metastore-postgresql - expose: - - "5432" + hive-metastore-postgresql: + image: bde2020/hive-metastore-postgresql:3.1.0 + container_name: ${CONTAINER_UID}hive3-metastore-postgresql + ports: + - "${PG_PORT}:5432" healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 5s timeout: 60s retries: 120 - network_mode: "host" + +# solve HiveServer2 connect error: +# java.net.URISyntaxException Illegal character in hostname :thrift://${CONTAINER_UID}hive3_default:9083 + +networks: + default: + name: ${CONTAINER_UID}hive3-default diff --git a/docker/thirdparties/docker-compose/hive/gen_env.sh.tpl b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env old mode 100755 new mode 100644 similarity index 50% rename from docker/thirdparties/docker-compose/hive/gen_env.sh.tpl rename to docker/thirdparties/docker-compose/hive/hive-3x_settings.env index a2c708841da..82486a17f0a --- a/docker/thirdparties/docker-compose/hive/gen_env.sh.tpl +++ b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env @@ -16,26 +16,11 @@ # specific language governing permissions and limitations # under the License. -#################################################################### -# This script will generate hadoop-hive.env from hadoop-hive.env.tpl -#################################################################### +# Change this to a specific string. +# Do not use "_" or other sepcial characters, only number and alphabeta. +# NOTICE: change this uid will modify hive-*.yaml -set -eo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -FS_PORT=8020 -HMS_PORT=9083 - -cp "${ROOT}"/hadoop-hive.env.tpl "${ROOT}"/hadoop-hive.env -# Need to set hostname of container to same as host machine's. -# Otherwise, the doris process can not connect to namenode directly. -HOST_NAME="doris--" - -{ - echo "FS_PORT=${FS_PORT}" - echo "HMS_PORT=${HMS_PORT}" - echo "CORE_CONF_fs_defaultFS=hdfs://${externalEnvIp}:${FS_PORT}" - echo "HOST_NAME=${HOST_NAME}" - echo "externalEnvIp=${externalEnvIp}" - -} >>"${ROOT}"/hadoop-hive.env +export FS_PORT=8020 #should be same in regression-conf.groovy +export HMS_PORT=9083 #should be same in regression-conf.groovy +export HS_PORT=10000 #should be same in regression-conf.groovy +export PG_PORT=5432 #should be same in regression-conf.groovy \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql index 730c8bda04d..dbeeab972f6 100644 --- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_table.hql @@ -601,6 +601,8 @@ CREATE TABLE `unsupported_type_table`( k6 int ); +set hive.stats.column.autogather=false; + CREATE TABLE `schema_evo_test_text`( id int, name string @@ -628,6 +630,8 @@ insert into `schema_evo_test_orc` select 1, "kaka"; alter table `schema_evo_test_orc` ADD COLUMNS (`ts` timestamp); insert into `schema_evo_test_orc` select 2, "messi", from_unixtime(to_unix_timestamp('20230101 13:01:03','yyyyMMdd HH:mm:ss')); +set hive.stats.column.autogather=true; + -- Currently docker is hive 2.x version. Hive 2.x versioned full-acid tables need to run major compaction. SET hive.support.concurrency=true; SET hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 5a17a761d1c..a1247dc5bb6 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -37,7 +37,7 @@ Usage: $0 <options> --stop stop the specified components All valid components: - mysql,pg,oracle,sqlserver,clickhouse,es,hive,iceberg,hudi,trino,kafka,mariadb,db2 + mysql,pg,oracle,sqlserver,clickhouse,es,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2 " exit 1 } @@ -60,7 +60,7 @@ STOP=0 if [[ "$#" == 1 ]]; then # default - COMPONENTS="mysql,es,hive,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2" + COMPONENTS="mysql,es,hive,hive3,pg,oracle,sqlserver,clickhouse,mariadb,iceberg,db2" else while true; do case "$1" in @@ -92,7 +92,7 @@ else done if [[ "${COMPONENTS}"x == ""x ]]; then if [[ "${STOP}" -eq 1 ]]; then - COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,iceberg,hudi,trino,kafka,mariadb,db2" + COMPONENTS="mysql,es,pg,oracle,sqlserver,clickhouse,hive,hive3,iceberg,hudi,trino,kafka,mariadb,db2" fi fi fi @@ -126,6 +126,7 @@ RUN_ORACLE=0 RUN_SQLSERVER=0 RUN_CLICKHOUSE=0 RUN_HIVE=0 +RUN_HIVE3=0; RUN_ES=0 RUN_ICEBERG=0 RUN_HUDI=0 @@ -150,6 +151,8 @@ for element in "${COMPONENTS_ARR[@]}"; do RUN_ES=1 elif [[ "${element}"x == "hive"x ]]; then RUN_HIVE=1 + elif [[ "${element}"x == "hive3"x ]]; then + RUN_HIVE3=1 elif [[ "${element}"x == "kafka"x ]]; then RUN_KAFKA=1 elif [[ "${element}"x == "iceberg"x ]]; then @@ -299,7 +302,7 @@ if [[ "${RUN_KAFKA}" -eq 1 ]]; then fi if [[ "${RUN_HIVE}" -eq 1 ]]; then - # hive + # hive2 # If the doris cluster you need to test is single-node, you can use the default values; If the doris cluster you need to test is composed of multiple nodes, then you need to set the IP_HOST according to the actual situation of your machine #default value IP_HOST="127.0.0.1" @@ -309,21 +312,43 @@ if [[ "${RUN_HIVE}" -eq 1 ]]; then echo "please set IP_HOST according to your actual situation" fi # before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/" - cp "${ROOT}"/docker-compose/hive/gen_env.sh.tpl "${ROOT}"/docker-compose/hive/gen_env.sh - sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/gen_env.sh - cp "${ROOT}"/docker-compose/hive/hive-2x.yaml.tpl "${ROOT}"/docker-compose/hive/hive-2x.yaml - cp "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl.tpl "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl - sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hive-2x.yaml - sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl - sed -i "s/externalEnvIp/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/hive-2x.yaml - sed -i "s/externalEnvIp/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/hadoop-hive.env.tpl - sed -i "s/\${externalEnvIp}/${IP_HOST}/g" "${ROOT}"/docker-compose/hive/gen_env.sh sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh - sudo bash "${ROOT}"/docker-compose/hive/gen_env.sh - sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down + # generate hive-2x.yaml + export IP_HOST=${IP_HOST} + export CONTAINER_UID=${CONTAINER_UID} + . "${ROOT}"/docker-compose/hive/hive-2x_settings.env + envsubst < "${ROOT}"/docker-compose/hive/hive-2x.yaml.tpl > "${ROOT}"/docker-compose/hive/hive-2x.yaml + envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env.tpl > "${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env + sudo docker compose -p ${CONTAINER_UID}hive2 -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down if [[ "${STOP}" -ne 1 ]]; then - sudo docker compose -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d + sudo docker compose -p ${CONTAINER_UID}hive2 -f "${ROOT}"/docker-compose/hive/hive-2x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d + fi +fi + +if [[ "${RUN_HIVE3}" -eq 1 ]]; then + # hive3 + # If the doris cluster you need to test is single-node, you can use the default values; If the doris cluster you need to test is composed of multiple nodes, then you need to set the IP_HOST according to the actual situation of your machine + #default value + IP_HOST="127.0.0.1" + eth0_num=$(ifconfig -a|grep flags=|grep -n ^eth0|awk -F ':' '{print $1}') + IP_HOST=$(ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d "addr:"|tail -n +${eth0_num}|head -n 1) + if [ "_${IP_HOST}" == "_" ];then + echo "please set IP_HOST according to your actual situation" + exit -1 + fi + # before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/" + sed -i "s/s3Endpoint/${s3Endpoint}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh + sed -i "s/s3BucketName/${s3BucketName}/g" "${ROOT}"/docker-compose/hive/scripts/hive-metastore.sh + # generate hive-3x.yaml + export IP_HOST=${IP_HOST} + export CONTAINER_UID=${CONTAINER_UID} + . "${ROOT}"/docker-compose/hive/hive-3x_settings.env + envsubst < "${ROOT}"/docker-compose/hive/hive-3x.yaml.tpl > "${ROOT}"/docker-compose/hive/hive-3x.yaml + envsubst < "${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env.tpl > "${ROOT}"/docker-compose/hive/hadoop-hive-metastore.env + sudo docker compose -p ${CONTAINER_UID}hive3 -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env down + if [[ "${STOP}" -ne 1 ]]; then + sudo docker compose -p ${CONTAINER_UID}hive3 -f "${ROOT}"/docker-compose/hive/hive-3x.yaml --env-file "${ROOT}"/docker-compose/hive/hadoop-hive.env up --build --remove-orphans -d fi fi diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 041744c9b02..ade2484ab58 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -123,8 +123,8 @@ db2_11_port=50000 // See `docker/thirdparties/start-thirdparties-docker.sh` enableHiveTest=false enablePaimonTest=false -hms_port=9183 -hdfs_port=8120 +hms_port=9083 +hdfs_port=8020 hiveServerPort=10000 hive_pg_port=5432 diff --git a/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc.out b/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc.out index 26799d9a532..690889873a6 100644 --- a/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc.out +++ b/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc.out @@ -33,17 +33,14 @@ 1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 -4 0000-01-01 0000-01-01 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 4 4 true 4 4 4 4.4 4.4 char4 4 4 4 0.4 4.00000000 4.0000000000 4 4.0000000000000000000000000000000000000 0.40000000000000000000000000000000000000 -- !select_tvf2 -- 1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 -4 0000-01-01 0000-01-01 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 4 4 true 4 4 4 4.4 4.4 char4 4 4 4 0.4 4.00000000 4.0000000000 4 4.0000000000000000000000000000000000000 0.40000000000000000000000000000000000000 -- !hive_docker_02 -- 1 2023-04-20 2023-04-20 2023-04-20 00:00:00.0 2023-04-20 00:00:00.0 2023-04-20 00:00:00.0 2023-04-20 00:00:00.0 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 2 9999-12-31 9999-12-31 9999-12-31 23:59:59.0 9999-12-31 23:59:59.0 2023-04-20 00:00:00.12 2023-04-20 00:00:00.3344 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 3 2023-04-21 2023-04-21 2023-04-20 12:34:56.0 2023-04-20 00:00:00.0 2023-04-20 00:00:00.123 2023-04-20 00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235E38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 -4 0001-01-04 0001-01-04 2023-04-20 00:00:00.0 2023-04-20 00:00:00.0 2023-04-20 00:00:00.0 2023-04-20 00:00:00.0 Beijing Haidian 4 4 true 4 4 4 4.4 4.4 char4 4 4 4 0.4 4.00000000 4.0000000000 4 4.0000000000000000000000000000000000000 0.40000000000000000000000000000000000000 diff --git a/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.out b/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.out index d13a47d516f..3f4935a2d29 100644 --- a/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.out +++ b/regression-test/data/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.out @@ -51,7 +51,7 @@ 1 doris1 {"a":"100","b":"111"} 2 doris2 {"a":"200","b":"222"} 3 doris3 {"a":null,"b":"333","c":"399","d":"399999999999999"} -4 doris4 {} +4 doris4 {null:null} 5 doris5 {"b":null} 6 \N \N 7 doris7 \N diff --git a/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out b/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out index 90a5cf00ee3..9002c1c4490 100644 --- a/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out +++ b/regression-test/data/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.out @@ -33,17 +33,14 @@ 1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 -4 0000-01-01 0000-01-01 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 4 4 true 4 4 4 4.4 4.4 char4 4 4 4 0.4 4.00000000 4.0000000000 4 4.0000000000000000000000000000000000000 0.40000000000000000000000000000000000000 -- !select_tvf2 -- 1 2023-04-20 2023-04-20 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 2 9999-12-31 9999-12-31 9999-12-31T23:59:59 9999-12-31T23:59:59 2023-04-20T00:00:00.120 2023-04-20T00:00:00.334400 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 3 2023-04-21 2023-04-21 2023-04-20T12:34:56 2023-04-20T00:00 2023-04-20T00:00:00.123 2023-04-20T00:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235e+38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 -4 0000-01-01 0000-01-01 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 2023-04-20T00:00 Beijing Haidian 4 4 true 4 4 4 4.4 4.4 char4 4 4 4 0.4 4.00000000 4.0000000000 4 4.0000000000000000000000000000000000000 0.40000000000000000000000000000000000000 -- !hive_docker_02 -- 1 2023-04-20 2023-04-20 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 Beijing Haidian 1 1 true 1 1 1 1.1 1.1 char1 1 1 1 0.1 1.00000000 1.0000000000 1 1.0000000000000000000000000000000000000 0.10000000000000000000000000000000000000 2 9999-12-31 9999-12-31 9999-12-31 15:59:59.0 9999-12-31 15:59:59.0 2023-04-19 16:00:00.12 2023-04-19 16:00:00.3344 Haidian -32768 -128 true -2147483648 -9223372036854775808 -170141183460469231731687303715884105728 1.4E-45 4.9E-324 char2 100000000 100000000 4 0.1 0.99999999 9999999999.9999999999 99999999999999999999999999999999999999 9.9999999999999999999999999999999999999 0.99999999999999999999999999999999999999 3 2023-04-21 2023-04-21 2023-04-20 04:34:56.0 2023-04-19 16:00:00.0 2023-04-19 16:00:00.123 2023-04-19 16:00:00.123456 Beijing 32767 127 true 2147483647 9223372036854775807 170141183460469231731687303715884105727 3.4028235E38 1.7976931348623157E308 char3 999999999 999999999 9 0.9 9.99999999 1234567890.0123456789 12345678901234567890123456789012345678 1.2345678901234567890123456789012345678 0.12345678901234567890123456789012345678 -4 0001-01-04 0001-01-04 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 2023-04-19 16:00:00.0 Beijing Haidian 4 4 true 4 4 4 4.4 4.4 char4 4 4 4 0.4 4.00000000 4.0000000000 4 4.0000000000000000000000000000000000000 0.40000000000000000000000000000000000000 diff --git a/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy b/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy index b897263ea86..0fead618e91 100644 --- a/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy +++ b/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc.groovy @@ -144,7 +144,7 @@ suite("test_hive_read_orc", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table};""" } finally { } @@ -235,16 +235,8 @@ suite("test_hive_read_orc", "external,hive,external_docker") { (${++i}, '2023-04-21', '2023-04-21', '2023-04-20 12:34:56', '2023-04-20 00:00:00', '2023-04-20 00:00:00.123', '2023-04-20 00:00:00.123456', 'Beijing', '', ${Short.MAX_VALUE}, ${Byte.MAX_VALUE}, true, ${Integer.MAX_VALUE}, ${Long.MAX_VALUE}, 170141183460469231731687303715884105727, ${Float.MAX_VALUE}, ${Double.MAX_VALUE}, 'char${i}', - 999999999, 999999999, 9, 0.9, 9.99999999, 1234567890.0123456789, 12345678901234567890123456789012345678, 1.2345678901234567890123456789012345678, 0.12345678901234567890123456789012345678), + 999999999, 999999999, 9, 0.9, 9.99999999, 1234567890.0123456789, 12345678901234567890123456789012345678, 1.2345678901234567890123456789012345678, 0.12345678901234567890123456789012345678) """) - - sb.append(""" - (${++i}, '0000-01-01', '0000-01-01', '2023-04-20 00:00:00', '2023-04-20 00:00:00', '2023-04-20 00:00:00', '2023-04-20 00:00:00', - 'Beijing', 'Haidian', - ${i}, ${i % 128}, true, ${i}, ${i}, ${i}, ${i}.${i}, ${i}.${i}, 'char${i}', - ${i}, ${i}, ${i}, 0.${i}, ${i}, ${i}, ${i}, ${i}, 0.${i}) - """) - sql """ INSERT INTO ${export_table_name} VALUES ${sb.toString()} @@ -265,7 +257,7 @@ suite("test_hive_read_orc", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_02""" SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_02""" SELECT * FROM ${hive_database}.${hive_table};""" } finally { } diff --git a/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.groovy b/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.groovy index 9edac8229a1..c5577baa60e 100644 --- a/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.groovy +++ b/regression-test/suites/external_table_p0/export/hive_read/orc/test_hive_read_orc_complex_type.groovy @@ -143,7 +143,7 @@ suite("test_hive_read_orc_complex_type", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table};""" } finally { } @@ -181,7 +181,7 @@ suite("test_hive_read_orc_complex_type", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table};""" } finally { } @@ -221,7 +221,7 @@ suite("test_hive_read_orc_complex_type", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_03 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_03 """ SELECT * FROM ${hive_database}.${hive_table};""" } finally { } @@ -272,7 +272,7 @@ suite("test_hive_read_orc_complex_type", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_04 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_04 """ SELECT * FROM ${hive_database}.${hive_table};""" } finally { } diff --git a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy index 1b69c917835..479183daa93 100644 --- a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy +++ b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet.groovy @@ -144,7 +144,7 @@ suite("test_hive_read_parquet", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_01 """ SELECT * FROM ${hive_database}.${hive_table};""" } finally { } @@ -235,16 +235,8 @@ suite("test_hive_read_parquet", "external,hive,external_docker") { (${++i}, '2023-04-21', '2023-04-21', '2023-04-20 12:34:56', '2023-04-20 00:00:00', '2023-04-20 00:00:00.123', '2023-04-20 00:00:00.123456', 'Beijing', '', ${Short.MAX_VALUE}, ${Byte.MAX_VALUE}, true, ${Integer.MAX_VALUE}, ${Long.MAX_VALUE}, 170141183460469231731687303715884105727, ${Float.MAX_VALUE}, ${Double.MAX_VALUE}, 'char${i}', - 999999999, 999999999, 9, 0.9, 9.99999999, 1234567890.0123456789, 12345678901234567890123456789012345678, 1.2345678901234567890123456789012345678, 0.12345678901234567890123456789012345678), + 999999999, 999999999, 9, 0.9, 9.99999999, 1234567890.0123456789, 12345678901234567890123456789012345678, 1.2345678901234567890123456789012345678, 0.12345678901234567890123456789012345678) """) - - sb.append(""" - (${++i}, '0000-01-01', '0000-01-01', '2023-04-20 00:00:00', '2023-04-20 00:00:00', '2023-04-20 00:00:00', '2023-04-20 00:00:00', - 'Beijing', 'Haidian', - ${i}, ${i % 128}, true, ${i}, ${i}, ${i}, ${i}.${i}, ${i}.${i}, 'char${i}', - ${i}, ${i}, ${i}, 0.${i}, ${i}, ${i}, ${i}, ${i}, 0.${i}) - """) - sql """ INSERT INTO ${export_table_name} VALUES ${sb.toString()} @@ -265,7 +257,7 @@ suite("test_hive_read_parquet", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table};""" } finally { } diff --git a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.groovy b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.groovy index e4e80d74b57..eb741e1e5db 100644 --- a/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.groovy +++ b/regression-test/suites/external_table_p0/export/hive_read/parquet/test_hive_read_parquet_comlex_type.groovy @@ -152,7 +152,7 @@ suite("test_hive_read_parquet_complex_type", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table};""" } finally { } @@ -191,7 +191,7 @@ suite("test_hive_read_parquet_complex_type", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_02 """ SELECT * FROM ${hive_database}.${hive_table};""" } finally { } @@ -231,7 +231,7 @@ suite("test_hive_read_parquet_complex_type", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_03 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_03 """ SELECT * FROM ${hive_database}.${hive_table};""" } finally { } @@ -276,7 +276,7 @@ suite("test_hive_read_parquet_complex_type", "external,hive,external_docker") { "format" = "${format}"); """ - qt_hive_docker_04 """ SELECT * FROM ${hive_database}.${hive_table} ORDER BY user_id;""" + qt_hive_docker_04 """ SELECT * FROM ${hive_database}.${hive_table};""" } finally { } diff --git a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy index 41e7658c17c..b494ffcc7e5 100644 --- a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy +++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy @@ -73,7 +73,7 @@ suite("test_hive_ddl", "p0,external,hive,external_docker,external_docker_hive") properties('location'='tt://${externalEnvIp}:${hdfs_port}/exist_check') """ } catch (Exception e) { - assertTrue(e.getMessage().contains("No FileSystem for scheme: tt")) + assertTrue(e.getMessage().contains("No FileSystem for scheme")) } try { sql """ create database if not exists `test_hive_loc_no_exist` diff --git a/regression-test/suites/mtmv_p0/test_hive_multi_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_multi_partition_mtmv.groovy index c8e5830dcac..269bf95a34d 100644 --- a/regression-test/suites/mtmv_p0/test_hive_multi_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_multi_partition_mtmv.groovy @@ -25,6 +25,8 @@ suite("test_hive_multi_partition_mtmv", "p0,external,hive,external_docker,extern def hive_database = "test_hive_multi_partition_mtmv_db" def hive_table = "partition2" + def autogather_off_str = """ set hive.stats.column.autogather = false; """ + def autogather_on_str = """ set hive.stats.column.autogather = true; """ def drop_table_str = """ drop table if exists ${hive_database}.${hive_table} """ def drop_database_str = """ drop database if exists ${hive_database}""" def create_database_str = """ create database ${hive_database}""" @@ -51,6 +53,8 @@ suite("test_hive_multi_partition_mtmv", "p0,external,hive,external_docker,extern def insert_str5 = """insert into ${hive_database}.${hive_table} PARTITION(year=2022,region="bj") values(5)""" def insert_str6 = """insert into ${hive_database}.${hive_table} PARTITION(year=2022,region="sh") values(6)""" + logger.info("hive sql: " + autogather_off_str) + hive_docker """ ${autogather_off_str} """ logger.info("hive sql: " + drop_table_str) hive_docker """ ${drop_table_str} """ logger.info("hive sql: " + drop_database_str) @@ -247,6 +251,8 @@ suite("test_hive_multi_partition_mtmv", "p0,external,hive,external_docker,extern assertTrue(showPartitionsResult.toString().contains("p_sh")) assertFalse(showPartitionsResult.toString().contains("p_tj")) + logger.info("hive sql: " + autogather_on_str) + hive_docker """ ${autogather_on_str} """ sql """drop materialized view if exists ${mvName};""" sql """drop catalog if exists ${catalog_name}""" } diff --git a/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy index 616ff77ca9a..23135ea62a0 100644 --- a/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy @@ -25,6 +25,8 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke def hive_database = "mtmv_test_db" def hive_table = "test_hive_refresh_mtmv_t1" + def autogather_off_str = """ set hive.stats.column.autogather = false; """ + def autogather_on_str = """ set hive.stats.column.autogather = true; """ def drop_table_str = """ drop table if exists ${hive_database}.${hive_table} """ def drop_database_str = """ drop database if exists ${hive_database}""" def create_database_str = """ create database ${hive_database}""" @@ -40,6 +42,9 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke partition(year=2020); """ def insert_str = """ insert into ${hive_database}.${hive_table} PARTITION(year=2020) values(1,1)""" + + logger.info("hive sql: " + autogather_off_str) + hive_docker """ ${autogather_off_str} """ logger.info("hive sql: " + drop_table_str) hive_docker """ ${drop_table_str} """ logger.info("hive sql: " + drop_database_str) @@ -173,6 +178,9 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke waitingMTMVTaskFinishedNotNeedSuccess(jobName) order_qt_task_recover "select Status from tasks('type'='mv') where JobName = '${jobName}' order by CreateTime DESC limit 1" + logger.info("hive sql: " + autogather_on_str) + hive_docker """ ${autogather_on_str} """ + sql """drop materialized view if exists ${mvName};""" sql """drop catalog if exists ${catalog_name}""" diff --git a/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy b/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy index 392997a82b9..fc630a0e4eb 100644 --- a/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/external_table/mv_contain_external_table.groovy @@ -25,6 +25,8 @@ suite("mv_contain_external_table", "p0,external,hive,external_docker,external_do def hive_database = "test_mv_contain_external_table_rewrite_db" def hive_table = "orders" + def autogather_off_str = """ set hive.stats.column.autogather = false; """ + def autogather_on_str = """ set hive.stats.column.autogather = true; """ def drop_table_str = """ drop table if exists ${hive_database}.${hive_table} """ def drop_database_str = """ drop database if exists ${hive_database}""" def create_database_str = """ create database ${hive_database}""" @@ -60,6 +62,7 @@ suite("mv_contain_external_table", "p0,external,hive,external_docker,external_do def insert_str3 = """ insert into ${hive_database}.${hive_table} PARTITION(o_orderdate='2023-10-19') values(3, 3, 'ok', 99.5, 'a', 'b', 1, 'yy')""" + hive_docker """ ${autogather_off_str} """ hive_docker """ ${drop_table_str} """ hive_docker """ ${drop_database_str} """ hive_docker """ ${create_database_str}""" @@ -211,6 +214,7 @@ suite("mv_contain_external_table", "p0,external,hive,external_docker,external_do } order_qt_query_rewritten_with_new_data """ ${query_sql}""" + hive_docker """ ${autogather_on_str} """ sql """drop materialized view if exists ${mv_name};""" sql """drop catalog if exists ${catalog_name}""" } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
