This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit c082d31490646dde2382def9a98c338e34d3d2ee Author: wuwenchi <[email protected]> AuthorDate: Sun Aug 11 20:32:29 2024 +0800 [improvement](iceberg)]support doris's char/varchar to iceberg's string (#38807) ## Proposed changes Support doris's `char`/`varchar` to iceberg's `string`. --- .../datasource/iceberg/DorisTypeToIcebergType.java | 2 +- .../write/test_iceberg_support_char_varchar.out | 19 +++++ .../write/test_iceberg_support_char_varchar.groovy | 90 ++++++++++++++++++++++ 3 files changed, 110 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java index 52e4b6cf17a..d4d62cd4c5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/DorisTypeToIcebergType.java @@ -110,7 +110,7 @@ public class DorisTypeToIcebergType extends DorisTypeVisitor<Type> { return Types.FloatType.get(); } else if (primitiveType.equals(PrimitiveType.DOUBLE)) { return Types.DoubleType.get(); - } else if (primitiveType.equals(PrimitiveType.STRING)) { + } else if (primitiveType.isCharFamily()) { return Types.StringType.get(); } else if (primitiveType.equals(PrimitiveType.DATE) || primitiveType.equals(PrimitiveType.DATEV2)) { diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_support_char_varchar.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_support_char_varchar.out new file mode 100644 index 00000000000..6edcc50c0ac --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_support_char_varchar.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !qt01 -- +a b 1 + +-- !qt02 -- +a b 1 + +-- !qt03 -- +a b 1 + +-- !qt01 -- +a b 1 + +-- !qt02 -- +a b 1 + +-- !qt03 -- +a b 1 + diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_support_char_varchar.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_support_char_varchar.groovy new file mode 100644 index 00000000000..2a756a360d6 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_support_char_varchar.groovy @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_support_char_varchar", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable hive test.") + return; + } + + for (String hivePrefix : ["hive2", "hive3"]) { + setHivePrefix(hivePrefix) + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String iceberg_catalog_name = "test_iceberg_support_char_varchar_iceberg_${hivePrefix}" + String hive_catalog_name = "test_iceberg_support_char_varchar_hive_${hivePrefix}" + + String db = "write_test" + String tb_iceberg = "tb_iceberg_support_char_varchar_iceberg" + String tb_hive = "tb_iceberg_support_char_varchar_hive" + String tb = "tb_iceberg_support_char_varchar_doris" + + try { + + sql """drop catalog if exists ${iceberg_catalog_name}""" + sql """create catalog if not exists ${iceberg_catalog_name} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}', + 'use_meta_cache' = 'true' + );""" + sql """drop catalog if exists ${hive_catalog_name}""" + sql """create catalog if not exists ${hive_catalog_name} properties ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}', + 'use_meta_cache' = 'true' + );""" + + sql """set enable_fallback_to_original_planner=false;""" + + sql """ create database if not exists internal.${db} """ + sql """ drop table if exists internal.${db}.${tb} """ + sql """ drop table if exists ${hive_catalog_name}.${db}.${tb_hive} """ + sql """ drop table if exists ${iceberg_catalog_name}.${db}.${tb_iceberg} """ + sql """ create table internal.${db}.${tb} (v1 varchar(20), v2 char(10), v3 int) DISTRIBUTED BY HASH(`v1`) BUCKETS 1 PROPERTIES ("replication_allocation" = "tag.location.default: 1") """ + sql """ create table ${hive_catalog_name}.${db}.${tb_hive} (v1 varchar(20), v2 char(10), v3 int); """ + sql """ create table ${iceberg_catalog_name}.${db}.${tb_iceberg} (v1 varchar(20), v2 char(10), v3 int); """ + + sql """ insert into internal.${db}.${tb} values ('a', 'b', 1)""" + sql """ insert into ${hive_catalog_name}.${db}.${tb_hive} values ('a', 'b', 1) """ + sql """ insert into ${iceberg_catalog_name}.${db}.${tb_iceberg} values ('a', 'b', 1) """ + + qt_qt01 """ select * from ${iceberg_catalog_name}.${db}.${tb_iceberg} """ + + // ctas from doris + sql """ drop table ${iceberg_catalog_name}.${db}.${tb_iceberg} """ + sql """ create table ${iceberg_catalog_name}.${db}.${tb_iceberg} as select * from internal.${db}.${tb} """ + qt_qt02 """ select * from ${iceberg_catalog_name}.${db}.${tb_iceberg} """ + + // ctas from hive + sql """ drop table ${iceberg_catalog_name}.${db}.${tb_iceberg} """ + sql """ create table ${iceberg_catalog_name}.${db}.${tb_iceberg} as select * from ${hive_catalog_name}.${db}.${tb_hive} """ + qt_qt03 """ select * from ${iceberg_catalog_name}.${db}.${tb_iceberg} """ + + } finally { + sql """drop table if exists internal.${db}.${tb}""" + sql """drop table if exists ${hive_catalog_name}.${db}.${tb_hive}""" + sql """drop table if exists ${iceberg_catalog_name}.${db}.${tb_iceberg}""" + sql """drop catalog if exists ${iceberg_catalog_name}""" + sql """drop catalog if exists ${hive_catalog_name}""" + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
