This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit d9d63c692b4f1fde94ea101c1cc71e6e94843cd7 Author: wuwenchi <[email protected]> AuthorDate: Mon Aug 26 09:51:18 2024 +0800 [bugfix](iceberg)Restrictions on creating a database (#39641) ## Proposed changes 1. Restrictions on creating a database. Currently, only attributes of the `hms` type of database can be added. Like: ``` create database db properties ('a'='b'); ``` 2. Unify the catalog name of iceberg and doris to be the same. --- .../iceberg/IcebergGlueExternalCatalog.java | 2 +- .../iceberg/IcebergHMSExternalCatalog.java | 2 +- .../iceberg/IcebergHadoopExternalCatalog.java | 2 +- .../datasource/iceberg/IcebergMetadataOps.java | 5 ++ .../iceberg/IcebergRestExternalCatalog.java | 2 +- .../iceberg/test_iceberg_show_create.groovy | 56 ++++++++++++++++------ 6 files changed, 50 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java index 9f3e05757c9..ffe48e68a49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java @@ -58,7 +58,7 @@ public class IcebergGlueExternalCatalog extends IcebergExternalCatalog { catalogProperties.get(S3Properties.Env.ENDPOINT)); catalogProperties.putIfAbsent(S3FileIOProperties.ENDPOINT, endpoint); - glueCatalog.initialize(icebergCatalogType, catalogProperties); + glueCatalog.initialize(getName(), catalogProperties); catalog = glueCatalog; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java index 34e6f0c187e..c1475064934 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java @@ -44,7 +44,7 @@ public class IcebergHMSExternalCatalog extends IcebergExternalCatalog { Map<String, String> catalogProperties = catalogProperty.getProperties(); String metastoreUris = catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, ""); catalogProperties.put(CatalogProperties.URI, metastoreUris); - hiveCatalog.initialize(icebergCatalogType, catalogProperties); + hiveCatalog.initialize(getName(), catalogProperties); catalog = hiveCatalog; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java index 01eee31d3b0..bf9e8c2b3f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java @@ -60,7 +60,7 @@ public class IcebergHadoopExternalCatalog extends IcebergExternalCatalog { String warehouse = catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION); hadoopCatalog.setConf(conf); catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse); - hadoopCatalog.initialize(icebergCatalogType, catalogProperties); + hadoopCatalog.initialize(getName(), catalogProperties); catalog = hadoopCatalog; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index edb83ac5bc8..66c57e37307 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -107,6 +107,11 @@ public class IcebergMetadataOps implements ExternalMetadataOps { ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS, dbName); } } + String icebergCatalogType = dorisCatalog.getIcebergCatalogType(); + if (!properties.isEmpty() && !IcebergExternalCatalog.ICEBERG_HMS.equals(icebergCatalogType)) { + throw new DdlException( + "Not supported: create database with properties for iceberg catalog type: " + icebergCatalogType); + } nsCatalog.createNamespace(Namespace.of(dbName), properties); dorisCatalog.onRefreshCache(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java index 77a6a7404ef..908a4fa9e3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java @@ -47,7 +47,7 @@ public class IcebergRestExternalCatalog extends IcebergExternalCatalog { Configuration conf = replaceS3Properties(getConfiguration()); - catalog = CatalogUtil.buildIcebergCatalog(icebergCatalogType, + catalog = CatalogUtil.buildIcebergCatalog(getName(), convertToRestCatalogProperties(), conf); } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy index 8065998fa71..23705c5494d 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy @@ -22,21 +22,23 @@ suite("test_iceberg_show_create", "p0,external,doris,external_docker,external_do return } - String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") - String minio_port = context.config.otherConfigs.get("iceberg_minio_port") - String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") String catalog_name = "test_iceberg_show_create" + String hivePrefix = "hive2"; + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + String iceberg_catalog_name = "test_iceberg_write_partitions_iceberg_${hivePrefix}" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String default_fs = "hdfs://${externalEnvIp}:${hdfs_port}" + String warehouse = "${default_fs}/warehouse" sql """drop catalog if exists ${catalog_name}""" - sql """ - CREATE CATALOG ${catalog_name} PROPERTIES ( + sql """create catalog if not exists ${catalog_name} properties ( 'type'='iceberg', - 'iceberg.catalog.type'='rest', - 'uri' = 'http://${externalEnvIp}:${rest_port}', - "s3.access_key" = "admin", - "s3.secret_key" = "password", - "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";, - "s3.region" = "us-east-1" + 'iceberg.catalog.type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', + 'fs.defaultFS' = '${default_fs}', + 'warehouse' = '${warehouse}', + 'use_meta_cache' = 'true' );""" sql """ switch ${catalog_name} """ @@ -49,25 +51,49 @@ suite("test_iceberg_show_create", "p0,external,doris,external_docker,external_do sql """ drop database if exists ${db1} """ sql """ drop database if exists ${db2} """ - sql """ create database ${db1} properties ('location'='s3a://warehouse/wh/${db1}') """ + sql """ create database ${db1} properties ('location'='${warehouse}/other_location') """ sql """ create database ${db2} """ String result = "" result = sql "show create database ${db1}" logger.info("${result}") - assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}")) + assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location")) result = sql "show create database ${db2}" logger.info("${result}") - assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db2}")) + assertTrue(result.toString().containsIgnoreCase("${warehouse}/${db2}")) sql """ create table ${db1}.${tb1} (id int) """ result = sql "show create table ${db1}.${tb1}" logger.info("${result}") - assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}/${tb1}")) + assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location/${tb1}")) sql """ drop table ${db1}.${tb1} """ sql """ drop database ${db1} """ sql """ drop database ${db2} """ + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + sql """drop catalog if exists ${catalog_name}""" + sql """ + CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";, + "s3.region" = "us-east-1" + );""" + + sql """ switch ${catalog_name} """ + sql """ drop database if exists ${db1} """ + + test { + sql """ create database ${db1} properties ('location'='${warehouse}/other_location') """ + exception "Not supported: create database with properties for iceberg catalog type" + } + + sql """ drop database if exists ${db1} """ + sql """drop catalog if exists ${catalog_name}""" } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
