This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new e3d752b56c6 branch-4.0: [feat](oss)Support bucket-domain-name #59755
(#60432)
e3d752b56c6 is described below
commit e3d752b56c65ad5db27548828f408563853d8d3a
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Feb 3 11:12:14 2026 +0800
branch-4.0: [feat](oss)Support bucket-domain-name #59755 (#60432)
Cherry-picked from #59755
Co-authored-by: Calvin Kirs <[email protected]>
---
.../datasource/property/storage/OSSProperties.java | 77 ++++++++++++++++++++++
.../property/storage/OSSPropertiesTest.java | 8 +++
.../test_s3_tvf_s3_storage.groovy | 4 +-
.../hive_on_hms_and_dlf.groovy | 8 +++
.../iceberg_on_hms_and_filesystem_and_dlf.groovy | 14 ++++
5 files changed, 110 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
index 0cb87f9b305..c5802496c57 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
@@ -17,9 +17,11 @@
package org.apache.doris.datasource.property.storage;
+import org.apache.doris.common.UserException;
import org.apache.doris.datasource.property.ConnectorPropertiesUtils;
import org.apache.doris.datasource.property.ConnectorProperty;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import lombok.Getter;
import lombok.Setter;
@@ -28,6 +30,8 @@ import org.apache.commons.lang3.StringUtils;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -255,6 +259,11 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
super.setEndpointIfPossible();
}
+ @Override
+ public String validateAndNormalizeUri(String uri) throws UserException {
+ return super.validateAndNormalizeUri(rewriteOssBucketIfNecessary(uri));
+ }
+
@Override
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
@@ -303,4 +312,72 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
hadoopStorageConfig.set("fs.oss.accessKeySecret", secretKey);
hadoopStorageConfig.set("fs.oss.endpoint", endpoint);
}
+
+ /**
+ * Rewrites the bucket part of an OSS URI if the bucket is specified
+ * in the form of bucket.endpoint.
https://help.aliyun.com/zh/oss/user-guide/access-oss-via-bucket-domain-name
+ *
+ * <p>This method is designed for OSS usage, but it also supports
+ * the {@code s3://} scheme since OSS URIs are sometimes written
+ * using the S3-style scheme.</p>
+ *
+ * <p>HTTP and HTTPS URIs are returned unchanged.</p>
+ *
+ * <p>Examples:
+ * <pre>
+ * oss://bucket.endpoint/path -> oss://bucket/path
+ * s3://bucket.endpoint -> s3://bucket
+ * https://bucket.endpoint -> unchanged
+ * </pre>
+ *
+ * @param uri the original URI string
+ * @return the rewritten URI string, or the original URI if no rewrite is
needed
+ */
+ @VisibleForTesting
+ protected static String rewriteOssBucketIfNecessary(String uri) {
+ if (uri == null || uri.isEmpty()) {
+ return uri;
+ }
+
+ URI parsed;
+ try {
+ parsed = URI.create(uri);
+ } catch (IllegalArgumentException e) {
+ // Invalid URI, do not rewrite
+ return uri;
+ }
+
+ String scheme = parsed.getScheme();
+ if ("http".equalsIgnoreCase(scheme) ||
"https".equalsIgnoreCase(scheme)) {
+ return uri;
+ }
+
+ // For non-standard schemes (oss / s3), authority is more reliable
than host
+ String authority = parsed.getAuthority();
+ if (authority == null || authority.isEmpty()) {
+ return uri;
+ }
+
+ // Handle bucket.endpoint format
+ int dotIndex = authority.indexOf('.');
+ if (dotIndex <= 0) {
+ return uri;
+ }
+
+ String bucket = authority.substring(0, dotIndex);
+
+ try {
+ URI rewritten = new URI(
+ scheme,
+ bucket,
+ parsed.getPath(),
+ parsed.getQuery(),
+ parsed.getFragment()
+ );
+ return rewritten.toString();
+ } catch (URISyntaxException e) {
+ // Be conservative: fallback to original URI
+ return uri;
+ }
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index dbf34751742..4be6414ae01 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -269,4 +269,12 @@ public class OSSPropertiesTest {
Assertions.assertFalse(s3Properties.hadoopStorageConfig.getBoolean("fs.oss.impl.disable.cache",
false));
}
+ @Test
+ public void testOSSBucketEndpointPathProperties() throws UserException {
+ Assertions.assertEquals("oss://my-bucket/path/to/dir/",
OSSProperties.rewriteOssBucketIfNecessary("oss://my-bucket/path/to/dir/"));
+ Assertions.assertEquals("oss://my-bucket/path/to/dir/file.txt",
OSSProperties.rewriteOssBucketIfNecessary("oss://my-bucket.oss-cn-hangzhou.aliyuncs.com/path/to/dir/file.txt"));
+ Assertions.assertEquals("s3://my-bucket/path/to/dir/file.txt",
OSSProperties.rewriteOssBucketIfNecessary("s3://my-bucket.oss-cn-hangzhou.aliyuncs.com/path/to/dir/file.txt"));
+
Assertions.assertEquals("https://bucket-name.oss-cn-hangzhou.aliyuncs.com/path/to/dir/file.txt";,
OSSProperties.rewriteOssBucketIfNecessary("https://bucket-name.oss-cn-hangzhou.aliyuncs.com/path/to/dir/file.txt";));
+ }
}
+
diff --git
a/regression-test/suites/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.groovy
b/regression-test/suites/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.groovy
index f8025733298..490e4eb48d1 100644
---
a/regression-test/suites/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.groovy
+++
b/regression-test/suites/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.groovy
@@ -176,7 +176,7 @@ suite("test_s3_tvf_s3_storage",
"p0,external,external_docker") {
ak = context.config.otherConfigs.get("aliYunAk")
sk = context.config.otherConfigs.get("aliYunSk")
s3_endpoint =
getConfigOrDefault("aliYunEndpoint","oss-cn-hongkong.aliyuncs.com")
- region = getConfigOrDefault ("aliYunRegion","oss-cn-hongkong")
+ region = getConfigOrDefault ("aliYunRegion","cn-hongkong")
bucket = getConfigOrDefault ("aliYunBucket","doris-regression-hk");
@@ -185,6 +185,8 @@ suite("test_s3_tvf_s3_storage",
"p0,external,external_docker") {
s3_tvf("http://${bucket}.${s3_endpoint}";, "", "s3.access_key",
"s3.secret_key", "region", "false");
s3_tvf("http://${bucket}.${s3_endpoint}";, "", "AWS_ACCESS_KEY",
"AWS_SECRET_KEY", "region", "false");
s3_tvf("http://${bucket}.${s3_endpoint}";, "", "s3.access_key",
"s3.secret_key", "s3.region", "false");
+ s3_tvf("oss://${bucket}.${s3_endpoint}", "", "s3.access_key",
"s3.secret_key", "s3.region", "false");
+ s3_tvf("s3://${bucket}.${s3_endpoint}", "", "s3.access_key",
"s3.secret_key", "s3.region", "false");
shouldFail {
// it's OSS
s3_tvf("http://${bucket}.${s3_endpoint}";, "", "s3.access_key",
"cos.secret_key", "region", "false");
diff --git
a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
index 2d116d07b31..12850cc1091 100644
---
a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
+++
b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
@@ -338,6 +338,7 @@ suite("hive_on_hms_and_dlf",
"p2,external,new_catalog_property") {
String oss_endpoint = context.config.otherConfigs.get("aliYunEndpoint")
String bucket = context.config.otherConfigs.get("aliYunBucket")
String oss_parent_path = "${bucket}/refactor-test"
+ String
oss_bucket_endpoint_parent_path="${bucket}.${oss_endpoint}/refactor-test"
String oss_region = context.config.otherConfigs.get("aliYunRegion")
String oss_region_param = """
'oss.region' = '${oss_region}',
@@ -519,6 +520,13 @@ suite("hive_on_hms_and_dlf",
"p2,external,new_catalog_property") {
//OSS - Insert overwrite tests
db_location = "oss://${oss_parent_path}/hive/hms/overwrite/" +
System.currentTimeMillis()
testInsertOverwrite(hms_properties + oss_storage_properties,
"hive_hms_oss_overwrite_test", db_location)
+ //OSS - Partition table tests (fix for partition path scheme mismatch)
+ db_location =
"oss://${oss_bucket_endpoint_parent_path}/hive/hms/bucket_endpoint/partition/"
+ System.currentTimeMillis()
+ testPartitionTableInsert(hms_properties + oss_storage_properties,
"hive_hms_oss_partition_test", db_location)
+ testPartitionTableInsert(hms_properties + oss_region_param +
oss_storage_properties, "hive_hms_oss_bucket_endpoint_partition_test_region",
db_location)
+ //OSS - Insert overwrite tests
+ db_location =
"oss://${oss_bucket_endpoint_parent_path}/hive/hms/bucket_endpoint/overwrite/"
+ System.currentTimeMillis()
+ testInsertOverwrite(hms_properties + oss_storage_properties,
"hive_hms_oss_bucket_endpoint_overwrite_test", db_location)
//s3
db_location =
"s3a://${s3_parent_path}/hive/hms/"+System.currentTimeMillis()
diff --git
a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
index ea98f7c4baa..e7d866796b8 100644
---
a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
+++
b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
@@ -415,6 +415,7 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
String oss_endpoint = context.config.otherConfigs.get("aliYunEndpoint")
String oss_bucket = context.config.otherConfigs.get("aliYunBucket")
String oss_parent_path = "${oss_bucket}/refactor-test"
+ String
oss_bucket_endpoint_parent_path="${oss_bucket}.${oss_endpoint}/refactor-test"
String oss_region = context.config.otherConfigs.get("aliYunRegion")
String oss_region_param = """
'oss.region' = '${oss_region}',
@@ -510,11 +511,13 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*//
DEFAULT",
+ "hive.metastore.sasl.enabled " = "true",
"hive.metastore.kerberos.principal" =
"hive/[email protected]",
"""
String hms_kerberos_old_prop_not_include_kerberos_prop = """
"hive.metastore.uris" = "thrift://${externalEnvIp}:9583",
+ "hive.metastore.sasl.enabled " = "true",
"hive.metastore.kerberos.principal" =
"hive/[email protected]",
"""
@@ -523,6 +526,7 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
"hive.metastore.client.principal"="hive/[email protected]",
"hive.metastore.client.keytab" =
"${keytab_root_dir}/hive-presto-master.keytab",
"hive.metastore.service.principal" =
"hive/[email protected]",
+ "hive.metastore.sasl.enabled " = "true",
"hive.metastore.authentication.type"="kerberos",
"hadoop.security.auth_to_local" =
"RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*//
RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
@@ -549,6 +553,10 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
hmsTestQueryAndInsert(hms_kerberos_old_prop + warehouse +
oss_storage_properties, "iceberg_hms_on_oss_kerberos_old")
//new kerberos
hmsTestQueryAndInsert(hms_kerberos_new_prop + warehouse +
oss_storage_properties, "iceberg_hms_on_oss_kerberos_new")
+ warehouse """
+ 'warehouse' =
'oss://${oss_bucket_endpoint_parent_path}/iceberg-hms-warehouse',
+ """
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop + warehouse +
oss_region_param + oss_storage_properties, "iceberg_hms_on_oss")
/*--------HMS on OBS-----------*/
warehouse = """
@@ -678,6 +686,12 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
"""
testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse +
oss_storage_properties, "iceberg_fs_on_oss")
testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse +
oss_region_param + oss_storage_properties, "iceberg_fs_on_oss_region")
+
+ warehouse = """
+ 'warehouse' =
'oss://${oss_bucket_endpoint_parent_path}/iceberg-fs-oss-warehouse',
+ """
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse +
oss_region_param + oss_storage_properties, "iceberg_fs_on_oss_region")
+
/** HDFS **/
warehouse = """
'warehouse' = '${hdfs_parent_path}/iceberg-fs-hdfs-warehouse',
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
