This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 6fbc8269fdf branch-3.1: [test](catalog)add obs and gcs test #55925
(#56078)
6fbc8269fdf is described below
commit 6fbc8269fdf3d15f080521a834b8a69226dd84d2
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Sep 25 17:03:15 2025 +0800
branch-3.1: [test](catalog)add obs and gcs test #55925 (#56078)
Cherry-picked from #55925
Co-authored-by: Calvin Kirs <[email protected]>
---
.../metastore/AWSGlueMetaStoreBaseProperties.java | 6 +-
.../metastore/AWSGlueMetaStoreProperties.java | 76 ---------
.../property/metastore/DataProcProperties.java | 37 -----
.../metastore/FileMetastoreProperties.java | 46 ------
.../metastore/IcebergGlueMetaStoreProperties.java | 5 -
.../metastore/IcebergHMSMetaStoreProperties.java | 17 +-
.../IcebergS3TablesMetaStoreProperties.java | 4 +-
.../metastore/PaimonHMSMetaStoreProperties.java | 6 +-
.../datasource/property/storage/COSProperties.java | 2 -
.../datasource/property/storage/GCSProperties.java | 2 -
.../datasource/property/storage/S3Properties.java | 1 +
.../AWSGlueMetaStoreBasePropertiesTest.java | 111 +++++++++++++
.../metastore/AWSGlueMetaStorePropertiesTest.java | 174 ---------------------
.../metastore/AbstractIcebergPropertiesTest.java | 125 +++++++++++++++
.../property/metastore/GlueCatalogTest.java | 7 +-
.../HMSAliyunDLFMetaStorePropertiesTest.java | 51 ++++++
.../metastore/HMSGlueMetaStorePropertiesTest.java | 2 +-
.../IcebergAliyunDLFMetaStorePropertiesTest.java | 106 +++++++++++++
.../IcebergFileSystemMetaStorePropertiesTest.java | 69 ++++++++
.../IcebergGlueMetaStorePropertiesTest.java | 66 ++++++++
.../IcebergHMSMetaStorePropertiesTest.java | 69 ++++++++
.../IcebergS3TablesMetaStorePropertiesTest.java | 59 +++++++
.../PaimonAliyunDLFMetaStorePropertiesTest.java | 139 ++++++++++++++++
.../property/metastore/PaimonCatalogTest.java | 5 +-
.../PaimonFileSystemMetaStorePropertiesTest.java | 63 ++++++++
.../PaimonHMSMetaStorePropertiesTest.java | 65 ++++++++
.../property/storage/COSPropertiesTest.java | 15 ++
.../property/storage/GCSPropertiesTest.java | 14 ++
.../property/storage/OBSPropertyTest.java | 15 ++
.../property/storage/OSSPropertiesTest.java | 15 ++
.../property/storage/S3PropertiesTest.java | 10 ++
.../hive/test_hive_basic_type.out | 4 +-
.../hive_on_hms_and_dlf.groovy | 51 +++++-
.../iceberg_on_hms_and_filesystem_and_dlf.groovy | 87 ++++++++++-
34 files changed, 1153 insertions(+), 371 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
index 024c1445e18..af0a91fe682 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
@@ -97,10 +97,10 @@ public class AWSGlueMetaStoreBaseProperties {
return new ParamRules()
.require(glueAccessKey,
- "glue.access_key or aws.glue.access-key or
client.credentials-provider.glue.access_key")
+ "glue.access_key is required")
.require(glueSecretKey,
- "glue.secret_key or aws.glue.secret-key or
client.credentials-provider.glue.secret_key")
- .require(glueEndpoint, "glue.endpoint or aws.endpoint or
aws.glue.endpoint is required");
+ "glue.secret_key is required")
+ .require(glueEndpoint, "glue.endpoint is required");
}
private void checkAndInit() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java
deleted file mode 100644
index 8dcee8eb3ea..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java
+++ /dev/null
@@ -1,76 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.property.metastore;
-
-import org.apache.doris.datasource.property.ConnectorProperty;
-
-import com.google.common.collect.Maps;
-import lombok.Getter;
-
-import java.util.Map;
-
-public class AWSGlueMetaStoreProperties extends MetastoreProperties {
-
- private AWSGlueMetaStoreBaseProperties baseProperties;
-
- @ConnectorProperty(names = {"glue.catalog_id"},
- description = "The catalog id of the AWS Glue.",
- supported = false)
- private String glueCatalogId = "";
-
- public AWSGlueMetaStoreProperties(Map<String, String> origProps) {
- super(Type.HMS, origProps);
- }
-
-
- @Override
- public void initNormalizeAndCheckProps() {
- super.initNormalizeAndCheckProps();
- baseProperties = AWSGlueMetaStoreBaseProperties.of(origProps);
- }
-
-
- public AWSCatalogMetastoreClientCredentials
getAWSCatalogMetastoreClientCredentials() {
- return new
AWSCatalogMetastoreClientCredentials(baseProperties.glueEndpoint,
baseProperties.glueAccessKey,
- baseProperties.glueSecretKey);
- }
-
- public void toIcebergGlueCatalogProperties(Map<String, String>
catalogProps) {
- // See AwsClientProperties.java for property keys
- catalogProps.put("client.credentials-provider",
-
"com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProvider2x");
- catalogProps.put("client.credentials-provider.glue.access_key",
baseProperties.glueAccessKey);
- catalogProps.put("client.credentials-provider.glue.secret_key",
baseProperties.glueSecretKey);
- catalogProps.put("client.region", baseProperties.glueRegion);
- }
-
- @Getter
- public static class AWSCatalogMetastoreClientCredentials {
- private Map<String, String> credentials = Maps.newHashMap();
-
- // Used for AWSCatalogMetastoreClient
- // See AWSGlueClientFactory in AWSCatalogMetastoreClient.java
- public AWSCatalogMetastoreClientCredentials(String endpoint, String
ak, String sk) {
- credentials.put("aws.catalog.credentials.provider.factory.class",
-
"com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProviderFactory");
- credentials.put("aws.glue.access-key", ak);
- credentials.put("aws.glue.secret-key", sk);
- credentials.put("aws.glue.endpoint", endpoint);
- }
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/DataProcProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/DataProcProperties.java
deleted file mode 100644
index 2007fd59801..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/DataProcProperties.java
+++ /dev/null
@@ -1,37 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.property.metastore;
-
-import org.apache.doris.datasource.property.ConnectorProperty;
-
-import java.util.Map;
-
-public class DataProcProperties extends MetastoreProperties {
- @ConnectorProperty(names = {"hive.metastore.uri"},
- description = "The uri of the hive metastore in DataProc")
- private String hiveMetastoreUri = "";
-
- public DataProcProperties(Map<String, String> origProps) {
- super(Type.DATAPROC, origProps);
- }
-
- @Override
- protected void checkRequiredProperties() {
-
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/FileMetastoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/FileMetastoreProperties.java
deleted file mode 100644
index e4c4800b2a6..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/FileMetastoreProperties.java
+++ /dev/null
@@ -1,46 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.property.metastore;
-
-import org.apache.doris.common.UserException;
-import org.apache.doris.datasource.property.storage.StorageProperties;
-
-import java.util.List;
-import java.util.Map;
-
-public class FileMetastoreProperties extends MetastoreProperties {
-
- private List<StorageProperties> storageProperties;
-
- public FileMetastoreProperties(Map<String, String> origProps) {
- super(Type.FILE_SYSTEM, origProps);
- try {
- storageProperties = StorageProperties.createAll(origProps);
- } catch (UserException e) {
- throw new RuntimeException(e);
- }
- }
-
- public List<StorageProperties> getStorageProperties() {
- return storageProperties;
- }
-
- @Override
- protected void checkRequiredProperties() {
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
index 33a0c774c13..7952f8d0bae 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
@@ -28,7 +28,6 @@ import org.apache.iceberg.aws.glue.GlueCatalog;
import org.apache.iceberg.aws.s3.S3FileIOProperties;
import org.apache.iceberg.catalog.Catalog;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -74,10 +73,6 @@ public class IcebergGlueMetaStoreProperties extends
AbstractIcebergProperties {
return catalog;
}
- private Map<String, String> prepareBaseCatalogProps() {
- return new HashMap<>(origProps);
- }
-
private void appendS3Props(Map<String, String> props) {
props.put(S3FileIOProperties.ACCESS_KEY_ID,
s3Properties.getAccessKey());
props.put(S3FileIOProperties.SECRET_ACCESS_KEY,
s3Properties.getSecretKey());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
index 871b3b3aec6..fe04cd18026 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
@@ -22,9 +22,8 @@ import
org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
import org.apache.doris.datasource.property.ConnectorProperty;
import org.apache.doris.datasource.property.storage.StorageProperties;
-import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.hive.HiveCatalog;
@@ -84,12 +83,13 @@ public class IcebergHMSMetaStoreProperties extends
AbstractIcebergProperties {
}
}*/
});
+ buildCatalogProperties(catalogProps);
try {
this.executionAuthenticator.execute(() ->
hiveCatalog.initialize(catalogName, catalogProps));
return hiveCatalog;
} catch (Exception e) {
throw new RuntimeException("Failed to initialize HiveCatalog for
Iceberg. "
- + "CatalogName=" + catalogName + ", warehouse=" +
warehouse, e);
+ + "CatalogName=" + catalogName + ", msg :" +
ExceptionUtils.getRootCauseMessage(e), e);
}
}
@@ -110,17 +110,10 @@ public class IcebergHMSMetaStoreProperties extends
AbstractIcebergProperties {
/**
* Constructs HiveCatalog's property map.
*/
- private Map<String, String> buildCatalogProperties() {
+ private void buildCatalogProperties(Map<String, String> catalogProps) {
Map<String, String> props = new HashMap<>();
- props.put(HiveCatalog.LIST_ALL_TABLES, String.valueOf(listAllTables));
-
- if (StringUtils.isNotBlank(warehouse)) {
- props.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
- }
-
+ catalogProps.put(HiveCatalog.LIST_ALL_TABLES,
String.valueOf(listAllTables));
props.put("uri", hmsBaseProperties.getHiveMetastoreUri());
- props.putAll(origProps); // Keep at end to allow override, but risky
if overlaps exist
- return props;
}
private void checkInitialized() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
index 3694ab76911..0b101623c2b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
@@ -22,6 +22,7 @@ import
org.apache.doris.datasource.iceberg.s3tables.CustomAwsCredentialsProvider
import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.iceberg.catalog.Catalog;
import software.amazon.s3tables.iceberg.S3TablesCatalog;
@@ -60,7 +61,8 @@ public class IcebergS3TablesMetaStoreProperties extends
AbstractIcebergPropertie
return catalog;
} catch (Exception e) {
throw new RuntimeException("Failed to initialize S3TablesCatalog
for Iceberg. "
- + "CatalogName=" + catalogName + ", region=" +
s3Properties.getRegion(), e);
+ + "CatalogName=" + catalogName + ", region=" +
s3Properties.getRegion()
+ + ", msg: " + ExceptionUtils.getRootCauseMessage(e), e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
index a7afa94a9e0..2f67d109dac 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
@@ -22,6 +22,7 @@ import
org.apache.doris.datasource.paimon.PaimonExternalCatalog;
import org.apache.doris.datasource.property.ConnectorProperty;
import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
@@ -55,7 +56,7 @@ public class PaimonHMSMetaStoreProperties extends
AbstractPaimonProperties {
@Override
public String getPaimonCatalogType() {
- return PaimonExternalCatalog.PAIMON_DLF;
+ return PaimonExternalCatalog.PAIMON_HMS;
}
protected PaimonHMSMetaStoreProperties(Map<String, String> props) {
@@ -95,7 +96,8 @@ public class PaimonHMSMetaStoreProperties extends
AbstractPaimonProperties {
try {
return executionAuthenticator.execute(() ->
CatalogFactory.createCatalog(catalogContext));
} catch (Exception e) {
- throw new RuntimeException("Failed to create Paimon catalog with
HMS metastore", e);
+ throw new RuntimeException("Failed to create Paimon catalog with
HMS metastore, msg: "
+ + ExceptionUtils.getRootCause(e), e);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
index b284ba5666e..57576aef44e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
@@ -99,7 +99,6 @@ public class COSProperties extends
AbstractS3CompatibleProperties {
* Flag indicating whether to use path-style URLs for the object storage
system.
* This value is optional and can be configured by the user.
*/
- @Setter
@Getter
@ConnectorProperty(names = {"cos.use_path_style", "use_path_style",
"s3.path-style-access"}, required = false,
description = "Whether to use path style URL for the storage.")
@@ -107,7 +106,6 @@ public class COSProperties extends
AbstractS3CompatibleProperties {
@ConnectorProperty(names = {"cos.force_parsing_by_standard_uri",
"force_parsing_by_standard_uri"}, required = false,
description = "Whether to use path style URL for the storage.")
- @Setter
@Getter
protected String forceParsingByStandardUrl = "false";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java
index d99aec3a461..f222097d1d2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java
@@ -123,7 +123,6 @@ public class GCSProperties extends
AbstractS3CompatibleProperties {
* Flag indicating whether to use path-style URLs for the object storage
system.
* This value is optional and can be configured by the user.
*/
- @Setter
@Getter
@ConnectorProperty(names = {"gs.use_path_style", "use_path_style",
"s3.path-style-access"}, required = false,
description = "Whether to use path style URL for the storage.")
@@ -131,7 +130,6 @@ public class GCSProperties extends
AbstractS3CompatibleProperties {
@ConnectorProperty(names = {"gs.force_parsing_by_standard_uri",
"force_parsing_by_standard_uri"}, required = false,
description = "Whether to use path style URL for the storage.")
- @Setter
@Getter
protected String forceParsingByStandardUrl = "false";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index 3092c254dce..b145c49b26f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -289,6 +289,7 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
}).build();
}
// For anonymous access (no credentials required)
+ //fixme: should return AwsCredentialsProviderChain
if (StringUtils.isBlank(accessKey) && StringUtils.isBlank(secretKey)) {
return AnonymousCredentialsProvider.create();
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBasePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBasePropertiesTest.java
new file mode 100644
index 00000000000..945c862f4b2
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBasePropertiesTest.java
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AWSGlueMetaStoreBasePropertiesTest {
+ private Map<String, String> baseValidProps() {
+ Map<String, String> props = new HashMap<>();
+ props.put("glue.access_key", "ak");
+ props.put("glue.secret_key", "sk");
+ props.put("glue.endpoint", "glue.us-east-1.amazonaws.com");
+ return props;
+ }
+
+ @Test
+ void testValidPropertiesWithRegionFromEndpoint() {
+ Map<String, String> props = baseValidProps();
+ // no region set -> should be extracted from endpoint
+ AWSGlueMetaStoreBaseProperties glueProps =
AWSGlueMetaStoreBaseProperties.of(props);
+ Assertions.assertEquals("ak", glueProps.glueAccessKey);
+ Assertions.assertEquals("sk", glueProps.glueSecretKey);
+ Assertions.assertEquals("us-east-1", glueProps.glueRegion);
+ }
+
+ @Test
+ void testValidPropertiesWithExplicitRegion() {
+ Map<String, String> props = baseValidProps();
+ props.put("glue.region", "ap-southeast-1");
+ AWSGlueMetaStoreBaseProperties glueProps =
AWSGlueMetaStoreBaseProperties.of(props);
+ Assertions.assertEquals("ap-southeast-1", glueProps.glueRegion);
+ }
+
+ @Test
+ void testMissingAccessKeyThrows() {
+ Map<String, String> props = baseValidProps();
+ props.remove("glue.access_key");
+ IllegalArgumentException ex = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> AWSGlueMetaStoreBaseProperties.of(props)
+ );
+ Assertions.assertTrue(ex.getMessage().contains("glue.access_key"));
+ }
+
+ @Test
+ void testMissingSecretKeyThrows() {
+ Map<String, String> props = baseValidProps();
+ props.remove("glue.secret_key");
+
+ IllegalArgumentException ex = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> AWSGlueMetaStoreBaseProperties.of(props)
+ );
+ Assertions.assertTrue(ex.getMessage().contains("glue.secret_key is
required"));
+ }
+
+ @Test
+ void testMissingEndpointThrows() {
+ Map<String, String> props = baseValidProps();
+ props.remove("glue.endpoint");
+
+ IllegalArgumentException ex = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> AWSGlueMetaStoreBaseProperties.of(props)
+ );
+ Assertions.assertTrue(ex.getMessage().contains("glue.endpoint is
required"));
+ }
+
+ @Test
+ void testInvalidEndpointThrows() {
+ Map<String, String> props = baseValidProps();
+ props.put("glue.endpoint", "http://invalid-endpoint.com";);
+
+ IllegalArgumentException ex = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> AWSGlueMetaStoreBaseProperties.of(props)
+ );
+ Assertions.assertTrue(ex.getMessage().contains("Invalid AWS Glue
endpoint"));
+ }
+
+ @Test
+ void testExtractRegionFailsWhenPatternMatchesButNoRegion() {
+ Map<String, String> props = baseValidProps();
+ props.put("glue.endpoint", "glue..amazonaws.com"); // malformed
+
+ IllegalArgumentException ex = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> AWSGlueMetaStoreBaseProperties.of(props)
+ );
+ Assertions.assertTrue(ex.getMessage().contains("Invalid AWS Glue
endpoint"));
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStorePropertiesTest.java
deleted file mode 100644
index 28db8552757..00000000000
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStorePropertiesTest.java
+++ /dev/null
@@ -1,174 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.property.metastore;
-
-import org.apache.doris.common.UserException;
-
-import com.google.common.collect.Maps;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-@Disabled("wait metastore integration")
-public class AWSGlueMetaStorePropertiesTest {
- private static Map<String, String> baseProps = new HashMap<>();
-
- @BeforeAll
- public static void setUp() {
- baseProps.put("iceberg.catalog.type", "glue");
- baseProps.put("type", "iceberg");
- }
-
- @Test
- public void testBasicProperties() throws UserException {
- Map<String, String> props = baseProps;
- props.put("glue.access_key", "test_access_key");
- props.put("glue.secret_key", "test_secret_key");
-
- props.put("glue.endpoint",
"https://glue.ap-northeast-1.amazonaws.com";);
-
- AWSGlueMetaStoreProperties glueProperties =
(AWSGlueMetaStoreProperties) MetastoreProperties.create(props);
-
- Map<String, String> catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
-
Assertions.assertEquals("com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProvider2x",
catalogProps
- .get("client.credentials-provider"));
- Assertions.assertEquals(props.get("glue.access_key"), catalogProps
- .get("client.credentials-provider.glue.access_key"));
- Assertions.assertEquals(props.get("glue.secret_key"), catalogProps
- .get("client.credentials-provider.glue.secret_key"));
- Assertions.assertEquals("ap-northeast-1", catalogProps
- .get("client.region"));
- AWSGlueMetaStoreProperties.AWSCatalogMetastoreClientCredentials
awsCatalogMetastoreClientCredentials =
glueProperties.getAWSCatalogMetastoreClientCredentials();
- Map<String, String> credentials =
awsCatalogMetastoreClientCredentials.getCredentials();
- Assertions.assertEquals("test_access_key",
credentials.get("aws.glue.access-key"));
- Assertions.assertEquals("test_secret_key",
credentials.get("aws.glue.secret-key"));
- Assertions.assertEquals("https://glue.ap-northeast-1.amazonaws.com";,
credentials.get("aws.glue.endpoint"));
- //Test glue.endpoint
- props = new HashMap<>();
- props.put("type", "hms");
- props.put("hive.metastore.type", "glue");
- props.put("glue.endpoint",
"https://glue.ap-northeast-1.amazonaws.com";);
- props.put("aws.glue.secret-key", "test_secret_key");
- props.put("aws.glue.access-key", "test_access_key");
- glueProperties = (AWSGlueMetaStoreProperties)
MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("ap-northeast-1",
catalogProps.get("client.region"));
- Assertions.assertEquals("test_access_key",
catalogProps.get("client.credentials-provider.glue.access_key"));
- Assertions.assertEquals("test_secret_key",
catalogProps.get("client.credentials-provider.glue.secret_key"));
- props = new HashMap<>();
- props.put("type", "hms");
- props.put("hive.metastore.type", "glue");
- props.put("glue.endpoint",
"https://glue.ap-northeast-1.amazonaws.com";);
- props.put("aws.glue.secret-key", "test_secret_key");
- props.put("glue.access_key", "test_glue_access_key");
- glueProperties = (AWSGlueMetaStoreProperties)
MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("ap-northeast-1",
catalogProps.get("client.region"));
- Assertions.assertEquals("test_secret_key",
catalogProps.get("client.credentials-provider.glue.secret_key"));
- Assertions.assertEquals("test_glue_access_key",
catalogProps.get("client.credentials-provider.glue.access_key"));
- }
-
- @Test
- public void testMissingRequiredProperties() {
- Map<String, String> props = Maps.newHashMap();
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- });
- props.put("glue.access_key", "test_access_key");
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- });
- props.put("glue.secret_key", "test_secret_key");
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- });
- }
-
- @Test
- public void testEmptyRequiredProperty() {
- Map<String, String> props = Maps.newHashMap();
- props.put("glue.access_key", " ");
- props.put("glue.secret_key", "test_secret_key");
- props.put("glue.endpoint",
"https://glue.ap-northeast-1.amazonaws.com";);
-
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- }, "AWS Glue properties(glue.access_key, glue.secret_key,
glue.endpoint) are not set correctly.");
- props.put("glue.access_key", "");
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- }, "AWS Glue properties(glue.access_key, glue.secret_key,
glue.endpoint) are not set correctly.");
- props.put("glue.access_key", "test_access_key");
- props.put("glue.secret_key", " ");
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- }, "AWS Glue properties(glue.access_key, glue.secret_key,
glue.endpoint) are not set correctly.");
- }
-
- @Test
- public void testEndpointParams() throws UserException {
- Map<String, String> props = Maps.newHashMap();
- props.put("glue.access_key", "a");
- props.put("glue.secret_key", "test_secret_key");
- props.put("glue.endpoint", "https://glue.us-west-2.amazonaws.com";);
- AWSGlueMetaStoreProperties glueProperties =
(AWSGlueMetaStoreProperties) MetastoreProperties.create(props);
- Map<String, String> catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2",
catalogProps.get("client.region"));
- props.put("glue.endpoint", "https://glue-fips.us-west-2.api.aws";);
- glueProperties = (AWSGlueMetaStoreProperties)
MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2",
catalogProps.get("client.region"));
- props.put("glue.endpoint",
"https://glue-fips.us-west-2.amazonaws.com";);
- glueProperties = (AWSGlueMetaStoreProperties)
MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2",
catalogProps.get("client.region"));
- props.put("glue.endpoint", "https://glue.us-west-2.api.aws";);
- glueProperties = (AWSGlueMetaStoreProperties)
MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2",
catalogProps.get("client.region"));
- props.put("glue.endpoint", "https://glue.us-west-2.amazonaws.com";);
- glueProperties = (AWSGlueMetaStoreProperties)
MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2",
catalogProps.get("client.region"));
-
- props.put("glue.endpoint", "glue.us-west-2.amazonaws.com");
- glueProperties = (AWSGlueMetaStoreProperties)
MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2",
catalogProps.get("client.region"));
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2",
catalogProps.get("client.region"));
- props.put("glue.endpoint", "https://glue.us-west-2.amaaws.com";);
- Assertions.assertThrows(IllegalArgumentException.class, () ->
MetastoreProperties.create(props), "AWS Glue properties (glue.endpoint) are not
set correctly: https://glue.us-west-2.amaaws.com";);
-
-
- }
-}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractIcebergPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractIcebergPropertiesTest.java
new file mode 100644
index 00000000000..275180c387f
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractIcebergPropertiesTest.java
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.catalog.Catalog;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class AbstractIcebergPropertiesTest {
+
+ private static class TestIcebergProperties extends
AbstractIcebergProperties {
+ private final Catalog catalogToReturn;
+ private Map<String, String> capturedCatalogProps;
+
+ TestIcebergProperties(Map<String, String> props, Catalog
catalogToReturn) {
+ super(props);
+ this.catalogToReturn = catalogToReturn;
+ }
+
+ @Override
+ public String getIcebergCatalogType() {
+ return "test";
+ }
+
+ @Override
+ protected Catalog initCatalog(String catalogName,
+ Map<String, String> catalogProps,
+ List<StorageProperties>
storagePropertiesList) {
+ // Capture the catalogProps for verification
+ this.capturedCatalogProps = new HashMap<>(catalogProps);
+ return catalogToReturn;
+ }
+
+ Map<String, String> getCapturedCatalogProps() {
+ return capturedCatalogProps;
+ }
+ }
+
+ @Test
+ void testInitializeCatalogWithWarehouse() {
+ Catalog mockCatalog = Mockito.mock(Catalog.class);
+ Mockito.when(mockCatalog.name()).thenReturn("mocked-catalog");
+ Map<String, String> props = new HashMap<>();
+ props.put("k1", "v1");
+ TestIcebergProperties properties = new TestIcebergProperties(props,
mockCatalog);
+ properties.warehouse = "s3://bucket/warehouse";
+ Catalog result = properties.initializeCatalog("testCatalog",
Collections.emptyList());
+ Assertions.assertNotNull(result);
+ Assertions.assertEquals("mocked-catalog", result.name());
+ // Verify that warehouse is included in catalogProps
+ Assertions.assertTrue(properties.getCapturedCatalogProps()
+ .containsKey(CatalogProperties.WAREHOUSE_LOCATION));
+ Assertions.assertEquals("s3://bucket/warehouse",
+
properties.getCapturedCatalogProps().get(CatalogProperties.WAREHOUSE_LOCATION));
+ Assertions.assertNotNull(properties.getExecutionAuthenticator());
+ }
+
+ @Test
+ void testInitializeCatalogWithoutWarehouse() {
+ Catalog mockCatalog = Mockito.mock(Catalog.class);
+ Mockito.when(mockCatalog.name()).thenReturn("no-warehouse");
+ TestIcebergProperties properties = new TestIcebergProperties(new
HashMap<>(), mockCatalog);
+ properties.warehouse = null;
+ Catalog result = properties.initializeCatalog("testCatalog",
Collections.emptyList());
+ Assertions.assertNotNull(result);
+ Assertions.assertEquals("no-warehouse", result.name());
+ // Verify that warehouse key is not present
+ Assertions.assertFalse(properties.getCapturedCatalogProps()
+ .containsKey(CatalogProperties.WAREHOUSE_LOCATION));
+ }
+
+ @Test
+ void testInitializeCatalogThrowsWhenNull() {
+ AbstractIcebergProperties properties = new
AbstractIcebergProperties(new HashMap<>()) {
+ @Override
+ public String getIcebergCatalogType() {
+ return "test";
+ }
+
+ @Override
+ protected Catalog initCatalog(String catalogName,
+ Map<String, String> catalogProps,
+ List<StorageProperties>
storagePropertiesList) {
+ return null; // Simulate a failure case
+ }
+ };
+
+ IllegalStateException ex = Assertions.assertThrows(
+ IllegalStateException.class,
+ () -> properties.initializeCatalog("testCatalog",
Collections.emptyList())
+ );
+ Assertions.assertEquals("Catalog must not be null after
initialization.", ex.getMessage());
+ }
+
+ @Test
+ void testExecutionAuthenticatorNotNull() {
+ Catalog mockCatalog = Mockito.mock(Catalog.class);
+ TestIcebergProperties properties = new TestIcebergProperties(new
HashMap<>(), mockCatalog);
+ Assertions.assertNotNull(properties.executionAuthenticator);
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/GlueCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/GlueCatalogTest.java
index 66f54d7b1d0..5352907d1d2 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/GlueCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/GlueCatalogTest.java
@@ -35,7 +35,7 @@ import java.util.Map;
public class GlueCatalogTest {
private GlueCatalog glueCatalog;
- private AWSGlueMetaStoreProperties glueProperties;
+ private AWSGlueMetaStoreBaseProperties glueProperties;
private static final Namespace queryNameSpace = Namespace.of("test"); //
Replace with your namespace
private static final String AWS_ACCESS_KEY_ID = "YOUR_ACCESS_KEY_ID"; //
Replace with actual access key
private static final String AWS_SECRET_ACCESS_KEY =
"YOUR_SECRET_ACCESS_KEY"; // Replace with actual secret key
@@ -57,11 +57,12 @@ public class GlueCatalogTest {
// Initialize AWSGlueProperties
- glueProperties = (AWSGlueMetaStoreProperties)
AWSGlueMetaStoreProperties.create(props);
+ glueProperties = (AWSGlueMetaStoreBaseProperties)
AWSGlueMetaStoreBaseProperties.of(props);
// Convert to catalog properties
Map<String, String> catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
+ catalogProps.put("catalog-name", "ck");
+
// Initialize Glue Catalog
glueCatalog.initialize("ck", catalogProps);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSAliyunDLFMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSAliyunDLFMetaStorePropertiesTest.java
new file mode 100644
index 00000000000..b93886e0119
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSAliyunDLFMetaStorePropertiesTest.java
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.common.UserException;
+
+import com.aliyun.datalake.metastore.common.DataLakeConfig;
+import com.google.common.collect.Maps;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+public class HMSAliyunDLFMetaStorePropertiesTest {
+
+ @Test
+ public void testCreate() throws UserException {
+ Map<String, String> props = Maps.newHashMap();
+ props.put("type", "hms");
+ props.put("hive.metastore.type", "dlf");
+ props.put("dlf.endpoint", "dlf.cn-shanghai.aliyuncs.com");
+ props.put("dlf.region", "cn-shanghai");
+ props.put("dlf.access_key", "xxx");
+ props.put("dlf.secret_key", "xxx");
+ props.put("dlf.catalog_id", "5789");
+ HMSAliyunDLFMetaStoreProperties hmsAliyunDLFMetaStoreProperties =
+ (HMSAliyunDLFMetaStoreProperties)
MetastoreProperties.create(props);
+ Assertions.assertEquals("xxx",
hmsAliyunDLFMetaStoreProperties.getHiveConf().get(DataLakeConfig.CATALOG_ACCESS_KEY_ID));
+ Assertions.assertEquals("xxx",
hmsAliyunDLFMetaStoreProperties.getHiveConf().get(DataLakeConfig.CATALOG_ACCESS_KEY_SECRET));
+ Assertions.assertEquals("5789",
hmsAliyunDLFMetaStoreProperties.getHiveConf().get(DataLakeConfig.CATALOG_ID));
+ Assertions.assertEquals("cn-shanghai",
hmsAliyunDLFMetaStoreProperties.getHiveConf().get(DataLakeConfig.CATALOG_REGION_ID));
+ Assertions.assertEquals("dlf.cn-shanghai.aliyuncs.com",
hmsAliyunDLFMetaStoreProperties.getHiveConf().get(DataLakeConfig.CATALOG_ENDPOINT));
+ Assertions.assertEquals("dlf",
hmsAliyunDLFMetaStoreProperties.getHiveConf().get("hive.metastore.type"));
+ Assertions.assertEquals("hms",
hmsAliyunDLFMetaStoreProperties.getHiveConf().get("type"));
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStorePropertiesTest.java
index a1c0f7a1bc2..6ac93e448b6 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStorePropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStorePropertiesTest.java
@@ -74,7 +74,7 @@ public class HMSGlueMetaStorePropertiesTest {
incompleteConfig.remove("aws.glue.secret-key");
HMSGlueMetaStoreProperties props = new
HMSGlueMetaStoreProperties(incompleteConfig);
Exception exception =
Assertions.assertThrows(IllegalArgumentException.class,
props::initNormalizeAndCheckProps);
- Assertions.assertTrue(exception.getMessage().contains("glue.secret_key
or aws.glue.secret-key"));
+ Assertions.assertTrue(exception.getMessage().contains("glue.secret_key
is required"));
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergAliyunDLFMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergAliyunDLFMetaStorePropertiesTest.java
new file mode 100644
index 00000000000..5c773967220
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergAliyunDLFMetaStorePropertiesTest.java
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.datasource.iceberg.dlf.DLFCatalog;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+import
org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
+
+import org.apache.iceberg.catalog.Catalog;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class IcebergAliyunDLFMetaStorePropertiesTest {
+ @Test
+ void testGetIcebergCatalogType() {
+ Map<String, String> props = new HashMap<>();
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ props.put("dlf.endpoint", "custom-endpoint");
+ props.put("dlf.catalog.uid", "123");
+
+ IcebergAliyunDLFMetaStoreProperties properties =
+ new IcebergAliyunDLFMetaStoreProperties(props);
+
+ Assertions.assertEquals("dlf", properties.getIcebergCatalogType());
+ }
+
+ @Test
+ void testInitCatalog() {
+ Map<String, String> props = new HashMap<>();
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ props.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
+ props.put("dlf.region", "cn-hz");
+ props.put("dlf.catalog.uid", "uid-123");
+ props.put("dlf.catalog.id", "id-456");
+ props.put("dlf.proxy.mode", "DLF_ONLY");
+
+ IcebergAliyunDLFMetaStoreProperties properties =
+ new IcebergAliyunDLFMetaStoreProperties(props);
+ // Replace DLFCatalog with a mock
+ Catalog catalog = properties.initCatalog("test_catalog", props,
+
Collections.singletonList(StorageProperties.createPrimary(props)));
+ Assertions.assertEquals(DLFCatalog.class, catalog.getClass());
+ }
+
+ @Test
+ void testAliyunDLFBasePropertiesSuccessWithPublicEndpoint() {
+ Map<String, String> props = new HashMap<>();
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ props.put("dlf.region", "cn-shanghai");
+ props.put("dlf.access.public", "true");
+ props.put("dlf.uid", "uid-001");
+
+ AliyunDLFBaseProperties base = AliyunDLFBaseProperties.of(props);
+
+ Assertions.assertEquals("ak", base.dlfAccessKey);
+ Assertions.assertEquals("sk", base.dlfSecretKey);
+ Assertions.assertEquals("dlf.cn-shanghai.aliyuncs.com",
base.dlfEndpoint);
+ Assertions.assertEquals("uid-001", base.dlfCatalogId); // defaulted to
uid
+ }
+
+ @Test
+ void testAliyunDLFBasePropertiesSuccessWithVpcEndpoint() {
+ Map<String, String> props = new HashMap<>();
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ props.put("dlf.region", "cn-hangzhou");
+ props.put("dlf.access.public", "false");
+ props.put("dlf.uid", "uid-002");
+ AliyunDLFBaseProperties base = AliyunDLFBaseProperties.of(props);
+ Assertions.assertEquals("dlf-vpc.cn-hangzhou.aliyuncs.com",
base.dlfEndpoint);
+ Assertions.assertEquals("uid-002", base.dlfCatalogId);
+ }
+
+ @Test
+ void testAliyunDLFBasePropertiesThrowsWhenEndpointMissing() {
+ Map<String, String> props = new HashMap<>();
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ // No endpoint and no region
+
+ Assertions.assertThrows(StoragePropertiesException.class,
+ () -> AliyunDLFBaseProperties.of(props));
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStorePropertiesTest.java
new file mode 100644
index 00000000000..6aa33c2c47d
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStorePropertiesTest.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.datasource.property.storage.HdfsProperties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class IcebergFileSystemMetaStorePropertiesTest {
+
+ @Test
+ public void testKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "hdfs://mycluster_test");
+ props.put("hadoop.security.authentication", "kerberos");
+ props.put("hadoop.kerberos.principal", "myprincipal");
+ props.put("hadoop.kerberos.keytab", "mykeytab");
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "hadoop");
+ props.put("warehouse", "hdfs://mycluster_test/ice");
+ IcebergFileSystemMetaStoreProperties icebergProps =
(IcebergFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ List<StorageProperties> storagePropertiesList =
Collections.singletonList(StorageProperties.createPrimary(props));
+ //We expect a Kerberos-related exception, but because the messages
vary by environment, we’re only doing a simple check.
+ Assertions.assertThrows(RuntimeException.class, () ->
icebergProps.initializeCatalog("iceberg", storagePropertiesList));
+ }
+
+ @Test
+ public void testNonKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "file:///tmp");
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "hadoop");
+ props.put("warehouse", "file:///tmp");
+ IcebergFileSystemMetaStoreProperties icebergProps =
(IcebergFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ Assertions.assertEquals("hadoop",
icebergProps.getIcebergCatalogType());
+ List<StorageProperties> storagePropertiesList =
Collections.singletonList(StorageProperties.createPrimary(props));
+ Assertions.assertDoesNotThrow(() ->
icebergProps.initializeCatalog("iceberg", storagePropertiesList));
+ props.put("fs.defaultFS", "hdfs://mycluster" +
System.currentTimeMillis());
+ props.put("warehouse", "hdfs://mycluster" +
System.currentTimeMillis());
+ IcebergFileSystemMetaStoreProperties icebergPropsFailed =
(IcebergFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ RuntimeException e = Assertions.assertThrows(RuntimeException.class,
() -> icebergPropsFailed.initializeCatalog("iceberg", storagePropertiesList));
+
Assertions.assertTrue(e.getMessage().contains("UnknownHostException:"));
+ }
+
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStorePropertiesTest.java
new file mode 100644
index 00000000000..2eb5ca36486
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStorePropertiesTest.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.aws.glue.GlueCatalog;
+import org.apache.iceberg.catalog.Catalog;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+public class IcebergGlueMetaStorePropertiesTest {
+
+ @Test
+ public void glueTest() throws UserException {
+ Map<String, String> baseProps = ImmutableMap.of(
+ "type", "iceberg",
+ "iceberg.catalog.type", "glue",
+ "glue.region", "us-west-2",
+ "glue.access_key", "AK",
+ "glue.secret_key", "SK",
+ "glue.endpoint", "https://glue.us-west-2.amazonaws.com";,
+ "warehouse", "s3://my-bucket/warehouse");
+ IcebergGlueMetaStoreProperties properties =
(IcebergGlueMetaStoreProperties) MetastoreProperties.create(baseProps);
+ Assertions.assertEquals("glue", properties.getIcebergCatalogType());
+ Catalog catalog = properties.initializeCatalog("iceberg_catalog",
StorageProperties.createAll(baseProps));
+ Assertions.assertEquals(GlueCatalog.class, catalog.getClass());
+ }
+
+ @Test
+ public void glueAndS3Test() throws UserException {
+ Map<String, String> baseProps = ImmutableMap.of(
+ "type", "iceberg",
+ "iceberg.catalog.type", "glue",
+ "glue.region", "us-west-2",
+ "glue.access_key", "AK",
+ "glue.secret_key", "SK",
+ "glue.endpoint", "https://glue.us-west-2.amazonaws.com";,
+ "warehouse", "s3://my-bucket/warehouse",
+ "s3.region", "us-west-2",
+ "s3.endpoint", "https://s3.us-west-2.amazonaws.com";
+ );
+ IcebergGlueMetaStoreProperties properties =
(IcebergGlueMetaStoreProperties) MetastoreProperties.create(baseProps);
+ Catalog catalog = properties.initializeCatalog("iceberg_catalog",
StorageProperties.createAll(baseProps));
+ Assertions.assertEquals(GlueCatalog.class, catalog.getClass());
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStorePropertiesTest.java
new file mode 100644
index 00000000000..63d3a0b6bf5
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStorePropertiesTest.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import
org.apache.doris.common.security.authentication.HadoopExecutionAuthenticator;
+import org.apache.doris.datasource.property.storage.HdfsProperties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class IcebergHMSMetaStorePropertiesTest {
+ @Test
+ public void testKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "hdfs://mycluster_test");
+ props.put("hadoop.security.authentication", "kerberos");
+ props.put("hadoop.kerberos.principal", "myprincipal");
+ props.put("hadoop.kerberos.keytab", "mykeytab");
+ props.put("type", "iceberg");
+ props.put("hive.metastore.uris", "thrift://localhost:12345");
+ props.put("iceberg.catalog.type", "hms");
+ props.put("warehouse", "hdfs://mycluster_test/ice");
+ IcebergHMSMetaStoreProperties icebergProps =
(IcebergHMSMetaStoreProperties) MetastoreProperties.create(props);
+ List<StorageProperties> storagePropertiesList =
Collections.singletonList(StorageProperties.createPrimary(props));
+ //We expect a Kerberos-related exception, but because the messages
vary by environment, we’re only doing a simple check.
+ Assertions.assertThrows(RuntimeException.class,
+ () -> icebergProps.initializeCatalog("iceberg",
storagePropertiesList));
+ }
+
+ @Test
+ public void testNonKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "file:///tmp");
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "hms");
+ props.put("hive.metastore.uris", "thrift://localhost:9083");
+ props.put("warehouse", "file:///tmp");
+ IcebergHMSMetaStoreProperties paimonProps =
(IcebergHMSMetaStoreProperties) MetastoreProperties.create(props);
+ Assertions.assertEquals("hms", paimonProps.getIcebergCatalogType());
+
+ List<StorageProperties> storagePropertiesList =
Collections.singletonList(StorageProperties.createPrimary(props));
+ Assertions.assertDoesNotThrow(() ->
paimonProps.initializeCatalog("iceberg", storagePropertiesList));
+ Assertions.assertEquals(HadoopExecutionAuthenticator.class,
paimonProps.getExecutionAuthenticator().getClass());
+ }
+
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStorePropertiesTest.java
new file mode 100644
index 00000000000..81d1f004040
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStorePropertiesTest.java
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.catalog.Catalog;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import software.amazon.s3tables.iceberg.S3TablesCatalog;
+
+import java.util.Map;
+
+public class IcebergS3TablesMetaStorePropertiesTest {
+
+ @Test
+ public void s3TablesTest() throws UserException {
+ Map<String, String> baseProps = ImmutableMap.of(
+ "type", "iceberg",
+ "iceberg.catalog.type", "s3tables",
+ "warehouse", "s3://my-bucket/warehouse");
+ Map<String, String> s3Props = ImmutableMap.of(
+ "s3.region", "us-west-2",
+ "s3.access_key", "AK",
+ "s3.secret_key", "SK",
+ "s3.endpoint", "https://s3.us-west-2.amazonaws.com";);
+ IcebergS3TablesMetaStoreProperties failedProperties =
(IcebergS3TablesMetaStoreProperties) MetastoreProperties.create(baseProps);
+ Assertions.assertEquals("s3tables",
failedProperties.getIcebergCatalogType());
+ RuntimeException exception =
Assertions.assertThrows(RuntimeException.class, () -> {
+ failedProperties.initializeCatalog("iceberg_catalog",
StorageProperties.createAll(baseProps));
+ });
+ Assertions.assertTrue(exception.getMessage().contains("region must not
be blank or empty."));
+ Map<String, String> allProps = ImmutableMap.<String, String>builder()
+ .putAll(baseProps)
+ .putAll(s3Props)
+ .build();
+ IcebergS3TablesMetaStoreProperties properties =
(IcebergS3TablesMetaStoreProperties) MetastoreProperties.create(allProps);
+ Catalog catalog = properties.initializeCatalog("iceberg_catalog",
StorageProperties.createAll(allProps));
+ Assertions.assertEquals(S3TablesCatalog.class, catalog.getClass());
+ }
+
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStorePropertiesTest.java
new file mode 100644
index 00000000000..f2eade7239c
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStorePropertiesTest.java
@@ -0,0 +1,139 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.CatalogFactory;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class PaimonAliyunDLFMetaStorePropertiesTest {
+ private Map<String, String> createValidProps() {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "paimon");
+ props.put("paimon.catalog.type", "dlf");
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ props.put("dlf.endpoint", "dlf.cn-hangzhou.aliyuncs.com");
+ props.put("dlf.region", "cn-hangzhou");
+ props.put("dlf.catalog.id", "catalogId");
+ props.put("dlf.uid", "uid");
+ props.put("warehouse", "oss://bucket/warehouse");
+ return props;
+ }
+
+ @Test
+ void testInitNormalizeAndCheckProps() {
+ Map<String, String> props = createValidProps();
+ PaimonAliyunDLFMetaStoreProperties dlfProps =
+ new PaimonAliyunDLFMetaStoreProperties(props);
+
+ dlfProps.initNormalizeAndCheckProps();
+
+ Assertions.assertEquals(
+ "dlf",
+ dlfProps.getPaimonCatalogType(),
+ "Catalog type should be PAIMON_DLF"
+ );
+ Assertions.assertEquals(
+ "hive",
+ dlfProps.getMetastoreType(),
+ "Metastore type should be hive"
+ );
+ }
+
+ @Test
+ void testInitializeCatalogWithValidOssProperties() throws UserException {
+ Map<String, String> props = createValidProps();
+ PaimonAliyunDLFMetaStoreProperties dlfProps =
+ new PaimonAliyunDLFMetaStoreProperties(props);
+ dlfProps.initNormalizeAndCheckProps();
+
+ // Prepare OSSProperties mock
+ Map<String, String> ossProps = new HashMap<>();
+ ossProps.put("oss.access_key", "ak");
+ ossProps.put("oss.secret_key", "sk");
+ ossProps.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com");
+
+
+ List<StorageProperties> storageProperties =
StorageProperties.createAll(ossProps);
+
+ Catalog mockCatalog = Mockito.mock(Catalog.class);
+
+ try (MockedStatic<CatalogFactory> mocked =
Mockito.mockStatic(CatalogFactory.class)) {
+ mocked.when(() ->
CatalogFactory.createCatalog(Mockito.any(CatalogContext.class)))
+ .thenReturn(mockCatalog);
+
+ Catalog catalog = dlfProps.initializeCatalog("testCatalog",
storageProperties);
+
+ Assertions.assertNotNull(catalog, "Catalog should not be null");
+ Assertions.assertEquals(mockCatalog, catalog, "Catalog should be
the mocked one");
+
+ mocked.verify(() ->
CatalogFactory.createCatalog(Mockito.any(CatalogContext.class)));
+ }
+ }
+
+ @Test
+ void testInitializeCatalogWithoutOssPropertiesThrows() {
+ Map<String, String> props = createValidProps();
+ PaimonAliyunDLFMetaStoreProperties dlfProps =
+ new PaimonAliyunDLFMetaStoreProperties(props);
+ dlfProps.initNormalizeAndCheckProps();
+
+ List<StorageProperties> storageProperties = new ArrayList<>(); // No
OSS properties
+
+ IllegalStateException ex = Assertions.assertThrows(
+ IllegalStateException.class,
+ () -> dlfProps.initializeCatalog("testCatalog",
storageProperties)
+ );
+
+ Assertions.assertTrue(ex.getMessage().contains("OSS storage
properties"));
+ }
+
+ @Test
+ void testInitializeCatalogWithNonOssTypeThrows() {
+ Map<String, String> props = createValidProps();
+ PaimonAliyunDLFMetaStoreProperties dlfProps =
+ new PaimonAliyunDLFMetaStoreProperties(props);
+ dlfProps.initNormalizeAndCheckProps();
+
+ StorageProperties nonOssProps = Mockito.mock(StorageProperties.class);
+
Mockito.when(nonOssProps.getType()).thenReturn(StorageProperties.Type.HDFS);
+
+ List<StorageProperties> storageProperties =
Collections.singletonList(nonOssProps);
+
+ IllegalStateException ex = Assertions.assertThrows(
+ IllegalStateException.class,
+ () -> dlfProps.initializeCatalog("testCatalog",
storageProperties)
+ );
+
+ Assertions.assertTrue(ex.getMessage().contains("Paimon DLF metastore
requires OSS storage properties."));
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
index 59766ef9902..85633008a71 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource.property.metastore;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.paimon.catalog.Catalog;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@@ -70,12 +71,10 @@ public class PaimonCatalogTest {
System.out.println("Catalog initialization skipped: Missing valid
S3 Access Key/Secret Key.");
return null;
}
-
AbstractPaimonProperties metaStoreProps =
(AbstractPaimonProperties) MetastoreProperties.create(params);
-
metaStoreProps.initNormalizeAndCheckProps();
-
+ Assertions.assertNotNull(metaStoreProps.getExecutionAuthenticator());
List<StorageProperties> storageProps =
StorageProperties.createAll(params);
return metaStoreProps.initializeCatalog("paimon_catalog",
storageProps);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStorePropertiesTest.java
new file mode 100644
index 00000000000..fa52316357d
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStorePropertiesTest.java
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import
org.apache.doris.common.security.authentication.HadoopExecutionAuthenticator;
+import org.apache.doris.datasource.property.storage.HdfsProperties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.apache.paimon.catalog.FileSystemCatalogFactory;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class PaimonFileSystemMetaStorePropertiesTest {
+
+ @Test
+ public void testKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "hdfs://mycluster_test");
+ props.put("hadoop.security.authentication", "kerberos");
+ props.put("hadoop.kerberos.principal", "myprincipal");
+ props.put("hadoop.kerberos.keytab", "mykeytab");
+ props.put("type", "paimon");
+ props.put("paimon.catalog.type", "filesystem");
+ props.put("warehouse", "hdfs://mycluster_test/paimon");
+ PaimonFileSystemMetaStoreProperties paimonProps =
(PaimonFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ //We expect a Kerberos-related exception, but because the messages
vary by environment, we’re only doing a simple check.
+ Assertions.assertThrows(RuntimeException.class, () ->
paimonProps.initializeCatalog("paimon", StorageProperties.createAll(props))
+ );
+ }
+
+ @Test
+ public void testNonKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("fs.defaultFS", "file:///tmp");
+ props.put("type", "paimon");
+ props.put("paimon.catalog.type", "filesystem");
+ props.put("warehouse", "file:///tmp");
+ PaimonFileSystemMetaStoreProperties paimonProps =
(PaimonFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ Assertions.assertEquals(FileSystemCatalogFactory.IDENTIFIER,
paimonProps.getMetastoreType());
+ Assertions.assertEquals("filesystem",
paimonProps.getPaimonCatalogType());
+ Assertions.assertDoesNotThrow(() ->
paimonProps.initializeCatalog("paimon", StorageProperties.createAll(props)));
+ Assertions.assertEquals(HadoopExecutionAuthenticator.class,
paimonProps.getExecutionAuthenticator().getClass());
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStorePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStorePropertiesTest.java
new file mode 100644
index 00000000000..ef382c2c517
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStorePropertiesTest.java
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.datasource.property.storage.HdfsProperties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class PaimonHMSMetaStorePropertiesTest {
+ @Test
+ public void testKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "hdfs://mycluster_test");
+ props.put("hadoop.security.authentication", "kerberos");
+ props.put("hadoop.kerberos.principal", "myprincipal");
+ props.put("hadoop.kerberos.keytab", "mykeytab");
+ props.put("type", "paimon");
+ props.put("hive.metastore.uris", "thrift://localhost:12345");
+ props.put("paimon.catalog.type", "hms");
+ props.put("warehouse", "hdfs://mycluster/paimon");
+ PaimonHMSMetaStoreProperties paimonProps =
(PaimonHMSMetaStoreProperties) MetastoreProperties.create(props);
+ List<StorageProperties> storagePropertiesList =
Collections.singletonList(StorageProperties.createPrimary(props));
+ //We expect a Kerberos-related exception, but because the messages
vary by environment, we’re only doing a simple check.
+ Assertions.assertThrows(RuntimeException.class,
+ () -> paimonProps.initializeCatalog("paimon",
storagePropertiesList));
+ }
+
+ @Test
+ public void testNonKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "file:///tmp");
+ props.put("type", "paimon");
+ props.put("paimon.catalog.type", "hms");
+ props.put("hive.metastore.uris", "thrift://localhost:9083");
+ props.put("warehouse", "file:///tmp");
+ PaimonHMSMetaStoreProperties paimonProps =
(PaimonHMSMetaStoreProperties) MetastoreProperties.create(props);
+ Assertions.assertEquals("hms", paimonProps.getPaimonCatalogType());
+ //should mock connection to hms
+ }
+
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
index 0cf30e49c04..bd3c091ef37 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
@@ -23,6 +23,8 @@ import
org.apache.doris.datasource.property.storage.exception.StoragePropertiesE
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import java.util.HashMap;
import java.util.Map;
@@ -161,4 +163,17 @@ public class COSPropertiesTest {
origProps.remove("cos.access_key");
Assertions.assertDoesNotThrow(() ->
StorageProperties.createPrimary(origProps));
}
+
+ @Test
+ public void testAwsCredentialsProvider() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("fs.cos.support", "true");
+ props.put("cos.endpoint", "cos.ap-beijing.myqcloud.com");
+ COSProperties obsStorageProperties = (COSProperties)
StorageProperties.createPrimary(props);
+ Assertions.assertEquals(AnonymousCredentialsProvider.class,
obsStorageProperties.getAwsCredentialsProvider().getClass());
+ props.put("cos.access_key", "myAccessKey");
+ props.put("cos.secret_key", "mySecretKey");
+ obsStorageProperties = (COSProperties)
StorageProperties.createPrimary(props);
+ Assertions.assertEquals(StaticCredentialsProvider.class,
obsStorageProperties.getAwsCredentialsProvider().getClass());
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java
index dd986134a1c..c7167a8eba7 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java
@@ -20,6 +20,8 @@ package org.apache.doris.datasource.property.storage;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import java.util.HashMap;
import java.util.Map;
@@ -96,4 +98,16 @@ public class GCSPropertiesTest {
Assertions.assertEquals("888",
s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
Assertions.assertEquals("true", s3Props.get("use_path_style"));
}
+
+ @Test
+ public void testGCSAwsCredentialsProvider() throws Exception {
+ Map<String, String> gcsProps = new HashMap<>();
+ gcsProps.put("fs.gcs.support", "true");
+ GCSProperties gcsStorageProperties = (GCSProperties)
StorageProperties.createPrimary(gcsProps);
+ Assertions.assertEquals(AnonymousCredentialsProvider.class,
gcsStorageProperties.getAwsCredentialsProvider().getClass());
+ gcsProps.put("gs.access_key", "myAccessKey");
+ gcsProps.put("gs.secret_key", "mySecretKey");
+ gcsStorageProperties = (GCSProperties)
StorageProperties.createPrimary(gcsProps);
+ Assertions.assertEquals(StaticCredentialsProvider.class,
gcsStorageProperties.getAwsCredentialsProvider().getClass());
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
index 2c6f5ffab21..acaa21ab9bc 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
@@ -22,6 +22,8 @@ import org.apache.doris.common.UserException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import java.util.HashMap;
import java.util.Map;
@@ -137,6 +139,19 @@ public class OBSPropertyTest {
Assertions.assertDoesNotThrow(() ->
StorageProperties.createPrimary(origProps));
}
+ @Test
+ public void testAwsCredentialsProvider() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("fs.obs.support", "true");
+ props.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com");
+ OBSProperties obsStorageProperties = (OBSProperties)
StorageProperties.createPrimary(props);
+ Assertions.assertEquals(AnonymousCredentialsProvider.class,
obsStorageProperties.getAwsCredentialsProvider().getClass());
+ props.put("obs.access_key", "myAccessKey");
+ props.put("obs.secret_key", "mySecretKey");
+ obsStorageProperties = (OBSProperties)
StorageProperties.createPrimary(props);
+ Assertions.assertEquals(StaticCredentialsProvider.class,
obsStorageProperties.getAwsCredentialsProvider().getClass());
+ }
+
@Test
public void testMissingSecretKey() {
origProps.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index 31b56a2bd9b..f00fb0215e7 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -22,6 +22,8 @@ import org.apache.doris.common.UserException;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import java.util.HashMap;
import java.util.Map;
@@ -203,4 +205,17 @@ public class OSSPropertiesTest {
Assertions.assertEquals(1,
StorageProperties.createAll(origProps).size());
}
+ @Test
+ public void testAwsCredentialsProvider() throws Exception {
+ Map<String, String> ossProps = new HashMap<>();
+ ossProps.put("fs.oss.support", "true");
+ ossProps.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com");
+ OSSProperties ossStorageProperties = (OSSProperties)
StorageProperties.createPrimary(ossProps);
+ Assertions.assertEquals(AnonymousCredentialsProvider.class,
ossStorageProperties.getAwsCredentialsProvider().getClass());
+ ossProps.put("oss.access_key", "myAccessKey");
+ ossProps.put("oss.secret_key", "mySecretKey");
+ ossStorageProperties = (OSSProperties)
StorageProperties.createPrimary(ossProps);
+ Assertions.assertEquals(StaticCredentialsProvider.class,
ossStorageProperties.getAwsCredentialsProvider().getClass());
+ }
+
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index 4852eff9032..70ffa2bd0c9 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -26,6 +26,7 @@ import mockit.Mocked;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
@@ -410,4 +411,13 @@ public class S3PropertiesTest {
Assertions.assertEquals("aaa", s3Properties.accessKey);
Assertions.assertEquals("bbb", s3Properties.secretKey);
}
+
+ @Test
+ public void testS3PropertiesAwsAnonymousCredentialsProvider() {
+ Map<String, String> props = Maps.newHashMap();
+ props.put("s3.endpoint", "s3.us-west-2.amazonaws.com");
+ S3Properties s3Properties = (S3Properties)
StorageProperties.createPrimary(props);
+ AwsCredentialsProvider provider =
s3Properties.getAwsCredentialsProvider();
+ Assertions.assertEquals(AnonymousCredentialsProvider.class,
provider.getClass());
+ }
}
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_basic_type.out
b/regression-test/data/external_table_p0/hive/test_hive_basic_type.out
index cc48c4a5601..727256096a5 100644
--- a/regression-test/data/external_table_p0/hive/test_hive_basic_type.out
+++ b/regression-test/data/external_table_p0/hive/test_hive_basic_type.out
@@ -266,7 +266,7 @@ true
123.456
-- !2 --
-1 7706 1 155190 17.00 21168.23 0.04 0.02 N
O 1996-03-13 1996-02-12 1996-03-22 DELIVER IN PERSON
TRUCK egular courts above the cn beijing
+1 638 6 15635 32.00 49620.16 0.07 0.02 N
O 1996-01-30 1996-02-07 1996-02-03 DELIVER IN PERSON
MAIL arefully slyly ex cn beijing
-- !3 --
6374628540732951412 -77 -65 -70 -107 -215 65 0
-526 -1309 3750 8827 -19795 34647 57042 -1662 -138248 -890685
-228568 1633079 -2725524 6163040 -10491702 697237 74565050
127767368 93532213 -209675435 -32116110 -3624917040
-2927805617 15581947241 21893441661 24075494509 -116822110531
-59683724667 -146210393388 114424524398 1341560771667 -1638742564263
520137948334 -2927347587131 7415137351179 -7963937754617 52157548982266
140803519083304 -294675355729619 -868076759504942 181128508165910
-91 [...]
@@ -532,7 +532,7 @@ true
123.456
-- !2 --
-1 7706 1 155190 17.00 21168.23 0.04 0.02 N
O 1996-03-13 1996-02-12 1996-03-22 DELIVER IN PERSON
TRUCK egular courts above the cn beijing
+1 638 6 15635 32.00 49620.16 0.07 0.02 N
O 1996-01-30 1996-02-07 1996-02-03 DELIVER IN PERSON
MAIL arefully slyly ex cn beijing
-- !3 --
6374628540732951412 -77 -65 -70 -107 -215 65 0
-526 -1309 3750 8827 -19795 34647 57042 -1662 -138248 -890685
-228568 1633079 -2725524 6163040 -10491702 697237 74565050
127767368 93532213 -209675435 -32116110 -3624917040
-2927805617 15581947241 21893441661 24075494509 -116822110531
-59683724667 -146210393388 114424524398 1341560771667 -1638742564263
520137948334 -2927347587131 7415137351179 -7963937754617 52157548982266
140803519083304 -294675355729619 -868076759504942 181128508165910
-91 [...]
diff --git
a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
index 4c531d784d7..68f5b9a4f2f 100644
---
a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
+++
b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
@@ -167,6 +167,40 @@ suite("hive_on_hms_and_dlf",
"p2,external,new_catalog_property") {
'cos.endpoint' = '${cos_endpoint}'
"""
+ /***************OBS*******************/
+ String obs_ak = context.config.otherConfigs.get("hwYunAk")
+ String obs_sk = context.config.otherConfigs.get("hwYunSk")
+ String obs_endpoint =context.config.otherConfigs.get("hwYunEndpoint")
+ String obs_region = context.config.otherConfigs.get("hwYunRegion")
+ String obs_region_param = """
+ 'obs.region' = '${obs_region}',
+ """
+
+ String obs_parent_path = context.config.otherConfigs.get("hwYunBucket")
+
+ String obs_storage_properties = """
+ 'obs.access_key' = '${obs_ak}',
+ 'obs.secret_key' = '${obs_sk}',
+ 'obs.endpoint' = '${obs_endpoint}'
+ """
+ /***************GCS*******************/
+ String gcs_ak = context.config.otherConfigs.get("GCSAk")
+ String gcs_sk = context.config.otherConfigs.get("GCSSk")
+ String gcs_endpoint = "storage.googleapis.com"
+
+ String gcs_parent_path = "selectdb-qa-datalake-test";
+ String gcs_storage_old_properties = """
+ 'gs.access_key' = '${gcs_ak}',
+ 'gs.secret_key' = '${gcs_sk}',
+ 'gs.endpoint' = '${gcs_endpoint}'
+ """
+ String gcs_storage_new_properties = """
+ 'fs.gcs.support' = 'true',
+ 'gs.access_key' = '${gcs_ak}',
+ 'gs.secret_key' = '${gcs_sk}'
+
+ """
+
/****************HDFS*******************/
//simple
String hdfs_parent_path =
"hdfs://${externalEnvIp}:8320/user/hive/warehouse"
@@ -227,8 +261,23 @@ suite("hive_on_hms_and_dlf",
"p2,external,new_catalog_property") {
String hms_type_properties = """
"type"="hms",
"""
+ //OBS
+ String db_location = "obs://${obs_parent_path}/hive/hms/" +
System.currentTimeMillis()
+ testQueryAndInsert(hms_properties + obs_storage_properties,
"hive_hms_obs_test", db_location)
+ testQueryAndInsert(hms_properties + obs_region_param +
obs_storage_properties, "hive_hms_obs_test_region", db_location)
+ testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop +
obs_storage_properties, "hive_hms_on_obs_kerberos_old", db_location)
+ testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop +
obs_storage_properties, "hive_hms_on_obs_kerberos_new", db_location)
+ //GCS
+ if(context.config.otherConfigs.get("enableGCS")){
+ db_location = "gs://${gcs_parent_path}/hive/hms/" +
System.currentTimeMillis()
+ testQueryAndInsert(hms_properties + gcs_storage_old_properties,
"hive_hms_gcs_test", db_location)
+ testQueryAndInsert(hms_properties + gcs_storage_new_properties,
"hive_hms_gcs_test_new", db_location)
+ testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop +
gcs_storage_old_properties, "hive_hms_on_gcs_kerberos_old", db_location)
+ testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop +
gcs_storage_new_properties, "hive_hms_on_gcs_kerberos_new", db_location)
+ }
+
//COS
- String db_location = "cosn://${cos_parent_path}/hive/hms/" +
System.currentTimeMillis()
+ db_location = "cosn://${cos_parent_path}/hive/hms/" +
System.currentTimeMillis()
testQueryAndInsert(hms_properties + cos_storage_properties,
"hive_hms_cos_test", db_location)
testQueryAndInsert(hms_properties + cos_region_param +
cos_storage_properties, "hive_hms_cos_test_region", db_location)
testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop +
cos_storage_properties, "hive_hms_on_cos_kerberos_old", db_location)
diff --git
a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
index 5cc528e4ce5..5ea3354be98 100644
---
a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
+++
b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
@@ -257,7 +257,39 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
'cos.secret_key' = '${cos_sk}',
'cos.endpoint' = '${cos_endpoint}'
"""
+ /***************OBS*******************/
+ String obs_ak = context.config.otherConfigs.get("hwYunAk")
+ String obs_sk = context.config.otherConfigs.get("hwYunSk")
+ String obs_endpoint =context.config.otherConfigs.get("hwYunEndpoint")
+ String obs_region = context.config.otherConfigs.get("hwYunRegion")
+ String obs_region_param = """
+ 'obs.region' = '${obs_region}',
+ """
+ String obs_parent_path = context.config.otherConfigs.get("hwYunBucket")
+
+ String obs_storage_properties = """
+ 'obs.access_key' = '${obs_ak}',
+ 'obs.secret_key' = '${obs_sk}',
+ 'obs.endpoint' = '${obs_endpoint}'
+ """
+ /***************GCS*******************/
+ String gcs_ak = context.config.otherConfigs.get("GCSAk")
+ String gcs_sk = context.config.otherConfigs.get("GCSSk")
+ String gcs_endpoint = "storage.googleapis.com"
+
+ String gcs_parent_path = "selectdb-qa-datalake-test";
+ String gcs_storage_old_properties = """
+ 'gs.access_key' = '${gcs_ak}',
+ 'gs.secret_key' = '${gcs_sk}',
+ 'gs.endpoint' = '${gcs_endpoint}'
+ """
+ String gcs_storage_new_properties = """
+ 'fs.gcs.support' = 'true',
+ 'gs.access_key' = '${gcs_ak}',
+ 'gs.secret_key' = '${gcs_sk}'
+
+ """
/****************HDFS*******************/
//simple
String hdfs_parent_path =
"hdfs://${externalEnvIp}:8320/user/iceberg/warehouse"
@@ -328,7 +360,35 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
//new kerberos
testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_new_prop +
warehouse + oss_storage_properties, "iceberg_hms_on_oss_kerberos_new")
+ /*--------HMS on OBS-----------*/
+ warehouse = """
+ 'warehouse' =
'obs://${obs_parent_path}/iceberg-hms-obs-warehouse',
+ """
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop
+ + warehouse + obs_storage_properties, "iceberg_hms_on_obs")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop+
+ warehouse + obs_region_param + obs_storage_properties,
"iceberg_hms_on_obs")
+ //old kerberos
+ testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_old_prop +
warehouse + obs_storage_properties, "iceberg_hms_on_obs_kerberos_old")
+ //new kerberos
+ testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_new_prop +
warehouse + obs_storage_properties, "iceberg_hms_on_obs_kerberos_new")
+ /*--------HMS on GCS-----------*/
+ if(context.config.otherConfigs.get("enableGCS")){
+ warehouse = """
+ 'warehouse' =
'gs://${gcs_parent_path}/gcs/iceberg-hms-gcs-warehouse',
+ """
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop
+ + warehouse + gcs_storage_old_properties,
"iceberg_hms_on_gcs_old")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop+
+ warehouse + gcs_storage_new_properties,
"iceberg_hms_on_gcs_new")
+
+ //new kerberos
+ testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_new_prop +
warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_kerberos_new")
+ //old kerberos
+ testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_old_prop +
warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_kerberos_old")
+ }
+
/*--------HMS on COS-----------*/
warehouse = """
'warehouse' =
'cosn://${cos_parent_path}/iceberg-hms-cos-warehouse',
@@ -383,6 +443,31 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
'type'='iceberg',
'iceberg.catalog.type'='hadoop',
"""
+ /** OBS **/
+ warehouse = """
+ 'warehouse' = 'obs://${obs_parent_path}/iceberg-fs-obs-warehouse',
+ """
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse +
obs_storage_properties, "iceberg_fs_on_obs")
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse +
obs_region_param + obs_storage_properties, "iceberg_fs_on_obs_region")
+ //s3
+ warehouse = """
+ 'warehouse' = 's3://${obs_parent_path}/iceberg-fs-obs-warehouse',
+ """
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse +
obs_storage_properties, "iceberg_fs_on_obs_s3")
+ if(context.config.otherConfigs.get("enableGCS")){
+ /** GCS **/
+ warehouse = """
+ 'warehouse' = 'gs://${gcs_parent_path}/gcs/iceberg-fs-gcs-warehouse',
+ """
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse
+ gcs_storage_old_properties, "iceberg_fs_on_gcs_old")
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse
+ gcs_storage_new_properties, "iceberg_fs_on_gcs_new")
+ //s3
+ warehouse = """
+ 'warehouse' = 's3://${gcs_parent_path}/gcs/iceberg-fs-gcs-warehouse',
+ """
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse
+ gcs_storage_old_properties, "iceberg_fs_on_gcs_old")
+
+ }
/** COS **/
warehouse = """
'warehouse' = 'cos://${cos_parent_path}/iceberg-fs-cos-warehouse',
@@ -481,4 +566,4 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
}
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
