This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new a32a2d8bb40 [bugfix](iceberg)Restrictions on creating a database
(#39641) (#40051)
a32a2d8bb40 is described below
commit a32a2d8bb4030b620ddc0af87a906e39e627aea7
Author: Mingyu Chen <[email protected]>
AuthorDate: Wed Aug 28 21:45:03 2024 +0800
[bugfix](iceberg)Restrictions on creating a database (#39641) (#40051)
bp #39641
Co-authored-by: wuwenchi <[email protected]>
---
.../iceberg/IcebergGlueExternalCatalog.java | 2 +-
.../iceberg/IcebergHMSExternalCatalog.java | 2 +-
.../iceberg/IcebergHadoopExternalCatalog.java | 2 +-
.../datasource/iceberg/IcebergMetadataOps.java | 5 ++
.../iceberg/IcebergRestExternalCatalog.java | 2 +-
.../iceberg/test_iceberg_show_create.groovy | 56 ++++++++++++++++------
6 files changed, 50 insertions(+), 19 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java
index 9f3e05757c9..ffe48e68a49 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergGlueExternalCatalog.java
@@ -58,7 +58,7 @@ public class IcebergGlueExternalCatalog extends
IcebergExternalCatalog {
catalogProperties.get(S3Properties.Env.ENDPOINT));
catalogProperties.putIfAbsent(S3FileIOProperties.ENDPOINT, endpoint);
- glueCatalog.initialize(icebergCatalogType, catalogProperties);
+ glueCatalog.initialize(getName(), catalogProperties);
catalog = glueCatalog;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java
index 34e6f0c187e..c1475064934 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHMSExternalCatalog.java
@@ -44,7 +44,7 @@ public class IcebergHMSExternalCatalog extends
IcebergExternalCatalog {
Map<String, String> catalogProperties =
catalogProperty.getProperties();
String metastoreUris =
catalogProperty.getOrDefault(HMSProperties.HIVE_METASTORE_URIS, "");
catalogProperties.put(CatalogProperties.URI, metastoreUris);
- hiveCatalog.initialize(icebergCatalogType, catalogProperties);
+ hiveCatalog.initialize(getName(), catalogProperties);
catalog = hiveCatalog;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java
index dffdc454e18..5053e129ef6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergHadoopExternalCatalog.java
@@ -61,7 +61,7 @@ public class IcebergHadoopExternalCatalog extends
IcebergExternalCatalog {
String warehouse =
catalogProperty.getHadoopProperties().get(CatalogProperties.WAREHOUSE_LOCATION);
hadoopCatalog.setConf(conf);
catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
- hadoopCatalog.initialize(icebergCatalogType, catalogProperties);
+ hadoopCatalog.initialize(getName(), catalogProperties);
catalog = hadoopCatalog;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index dd4792715e8..36654a8e673 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -106,6 +106,11 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
ErrorReport.reportDdlException(ErrorCode.ERR_DB_CREATE_EXISTS,
dbName);
}
}
+ String icebergCatalogType = dorisCatalog.getIcebergCatalogType();
+ if (!properties.isEmpty() &&
!IcebergExternalCatalog.ICEBERG_HMS.equals(icebergCatalogType)) {
+ throw new DdlException(
+ "Not supported: create database with properties for iceberg
catalog type: " + icebergCatalogType);
+ }
nsCatalog.createNamespace(Namespace.of(dbName), properties);
dorisCatalog.onRefreshCache(true);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java
index 77a6a7404ef..908a4fa9e3f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergRestExternalCatalog.java
@@ -47,7 +47,7 @@ public class IcebergRestExternalCatalog extends
IcebergExternalCatalog {
Configuration conf = replaceS3Properties(getConfiguration());
- catalog = CatalogUtil.buildIcebergCatalog(icebergCatalogType,
+ catalog = CatalogUtil.buildIcebergCatalog(getName(),
convertToRestCatalogProperties(),
conf);
}
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy
index 8065998fa71..23705c5494d 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_show_create.groovy
@@ -22,21 +22,23 @@ suite("test_iceberg_show_create",
"p0,external,doris,external_docker,external_do
return
}
- String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
- String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
- String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
String catalog_name = "test_iceberg_show_create"
+ String hivePrefix = "hive2";
+ String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")
+ String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort")
+ String iceberg_catalog_name =
"test_iceberg_write_partitions_iceberg_${hivePrefix}"
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String default_fs = "hdfs://${externalEnvIp}:${hdfs_port}"
+ String warehouse = "${default_fs}/warehouse"
sql """drop catalog if exists ${catalog_name}"""
- sql """
- CREATE CATALOG ${catalog_name} PROPERTIES (
+ sql """create catalog if not exists ${catalog_name} properties (
'type'='iceberg',
- 'iceberg.catalog.type'='rest',
- 'uri' = 'http://${externalEnvIp}:${rest_port}',
- "s3.access_key" = "admin",
- "s3.secret_key" = "password",
- "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
- "s3.region" = "us-east-1"
+ 'iceberg.catalog.type'='hms',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}',
+ 'fs.defaultFS' = '${default_fs}',
+ 'warehouse' = '${warehouse}',
+ 'use_meta_cache' = 'true'
);"""
sql """ switch ${catalog_name} """
@@ -49,25 +51,49 @@ suite("test_iceberg_show_create",
"p0,external,doris,external_docker,external_do
sql """ drop database if exists ${db1} """
sql """ drop database if exists ${db2} """
- sql """ create database ${db1} properties
('location'='s3a://warehouse/wh/${db1}') """
+ sql """ create database ${db1} properties
('location'='${warehouse}/other_location') """
sql """ create database ${db2} """
String result = ""
result = sql "show create database ${db1}"
logger.info("${result}")
-
assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}"))
+
assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location"))
result = sql "show create database ${db2}"
logger.info("${result}")
-
assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db2}"))
+ assertTrue(result.toString().containsIgnoreCase("${warehouse}/${db2}"))
sql """ create table ${db1}.${tb1} (id int) """
result = sql "show create table ${db1}.${tb1}"
logger.info("${result}")
-
assertTrue(result.toString().containsIgnoreCase("s3a://warehouse/wh/${db1}/${tb1}"))
+
assertTrue(result.toString().containsIgnoreCase("${warehouse}/other_location/${tb1}"))
sql """ drop table ${db1}.${tb1} """
sql """ drop database ${db1} """
sql """ drop database ${db2} """
+ String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
+ "s3.region" = "us-east-1"
+ );"""
+
+ sql """ switch ${catalog_name} """
+ sql """ drop database if exists ${db1} """
+
+ test {
+ sql """ create database ${db1} properties
('location'='${warehouse}/other_location') """
+ exception "Not supported: create database with properties for iceberg
catalog type"
+ }
+
+ sql """ drop database if exists ${db1} """
+ sql """drop catalog if exists ${catalog_name}"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
