This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 37247e6f4fa branch-3.1: [fix](paimon-catalog)Fix OSS access when using
DLS endpoint #58099 (#58129)
37247e6f4fa is described below
commit 37247e6f4fa8964b639fed176f017301c9875058
Author: Calvin Kirs <[email protected]>
AuthorDate: Wed Nov 19 15:09:52 2025 +0800
branch-3.1: [fix](paimon-catalog)Fix OSS access when using DLS endpoint
#58099 (#58129)
picked from #58099
---
.../org/apache/doris/common/util/LocationPath.java | 3 +
.../property/storage/OSSHdfsProperties.java | 40 ++++---------
.../datasource/property/storage/OSSProperties.java | 13 ++++-
.../property/storage/StorageProperties.java | 19 ++++--
.../property/storage/OSSHdfsPropertiesTest.java | 67 +++++++++++++++++++---
.../property/storage/OSSPropertiesTest.java | 2 +-
.../external_table_p0/tvf/test_catalogs_tvf.out | 4 +-
7 files changed, 98 insertions(+), 50 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index 67aa9426d5a..d11b8121c6c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -307,6 +307,9 @@ public class LocationPath {
}
public TFileType getTFileTypeForBE() {
+ if (StringUtils.isNotBlank(normalizedLocation) &&
isHdfsOnOssEndpoint(normalizedLocation)) {
+ return TFileType.FILE_HDFS;
+ }
return SchemaTypeMapper.fromSchemaToFileType(schema);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
index e5919d85286..a461d644626 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
@@ -49,21 +49,22 @@ import java.util.regex.Pattern;
public class OSSHdfsProperties extends HdfsCompatibleProperties {
@Setter
- @ConnectorProperty(names = {"oss.hdfs.endpoint",
- "dlf.endpoint", "dlf.catalog.endpoint", "oss.endpoint"},
+ @ConnectorProperty(names = {"oss.hdfs.endpoint", "oss.endpoint",
+ "dlf.endpoint", "dlf.catalog.endpoint"},
description = "The endpoint of OSS.")
protected String endpoint = "";
- @ConnectorProperty(names = {"oss.hdfs.access_key", "dlf.access_key",
"dlf.catalog.accessKeyId", "oss.access_key"},
+ @ConnectorProperty(names = {"oss.hdfs.access_key", "oss.access_key",
"dlf.access_key", "dlf.catalog.accessKeyId"},
+ sensitive = true,
description = "The access key of OSS.")
protected String accessKey = "";
- @ConnectorProperty(names = {"oss.hdfs.secret_key", "dlf.secret_key",
"dlf.catalog.secret_key", "oss.secret_key"},
+ @ConnectorProperty(names = {"oss.hdfs.secret_key", "oss.secret_key",
"dlf.secret_key", "dlf.catalog.secret_key"},
sensitive = true,
description = "The secret key of OSS.")
protected String secretKey = "";
- @ConnectorProperty(names = {"oss.hdfs.region", "dlf.region", "oss.region"},
+ @ConnectorProperty(names = {"oss.hdfs.region", "oss.region", "dlf.region"},
required = false,
description = "The region of OSS.")
protected String region;
@@ -85,8 +86,8 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
description = "The security token of OSS.")
protected String securityToken = "";
- private static final Set<String> OSS_ENDPOINT_KEY_NAME =
ImmutableSet.of("oss.hdfs.endpoint",
- "dlf.endpoint", "dlf.catalog.endpoint", "oss.endpoint");
+ private static final Set<String> OSS_ENDPOINT_KEY_NAME =
ImmutableSet.of("oss.hdfs.endpoint", "oss.endpoint",
+ "dlf.endpoint", "dlf.catalog.endpoint");
private Map<String, String> backendConfigProperties;
@@ -110,21 +111,15 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
}
String endpoint = OSS_ENDPOINT_KEY_NAME.stream()
.map(props::get)
- .filter(StringUtils::isNotBlank)
+ .filter(ep -> StringUtils.isNotBlank(ep) &&
ep.endsWith(OSS_HDFS_ENDPOINT_SUFFIX))
.findFirst()
.orElse(null);
- if (StringUtils.isBlank(endpoint)) {
- return false;
- }
- return endpoint.endsWith(OSS_HDFS_ENDPOINT_SUFFIX) ||
endpoint.contains(DLF_ENDPOINT_KEY_WORDS);
+ return StringUtils.isNotBlank(endpoint);
}
@Override
protected void checkRequiredProperties() {
super.checkRequiredProperties();
- if (!isValidEndpoint(endpoint)) {
- throw new IllegalArgumentException("Property oss.endpoint is
required and must be a valid OSS endpoint.");
- }
}
private void convertDlfToOssEndpointIfNeeded() {
@@ -144,22 +139,9 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
return Optional.empty();
}
- public static boolean isValidEndpoint(String endpoint) {
- for (Pattern pattern : ENDPOINT_PATTERN) {
- if (pattern.matcher(endpoint.toLowerCase()).matches()) {
- return true;
- }
- }
- return false;
- }
-
@Override
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
- if (!isValidEndpoint(endpoint.toLowerCase())) {
- throw new IllegalArgumentException("The endpoint is not a valid
OSS HDFS endpoint: " + endpoint
- + ". It should match the pattern:
<region>.oss-dls.aliyuncs.com");
- }
// Extract region from the endpoint, e.g.,
"cn-shanghai.oss-dls.aliyuncs.com" -> "cn-shanghai"
if (StringUtils.isBlank(this.region)) {
Optional<String> regionOptional = extractRegion(endpoint);
@@ -178,8 +160,6 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
private static final String OSS_HDFS_ENDPOINT_SUFFIX =
".oss-dls.aliyuncs.com";
- private static final String DLF_ENDPOINT_KEY_WORDS = "dlf";
-
@Override
public Map<String, String> getBackendConfigProperties() {
return backendConfigProperties;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
index 47558497a68..bb938bdf31d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
@@ -146,7 +146,6 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
* - datalake.cn-hangzhou.aliyuncs.com => region = cn-hangzhou
*/
public static final Set<Pattern> ENDPOINT_PATTERN =
ImmutableSet.of(STANDARD_ENDPOINT_PATTERN,
-
Pattern.compile("(?:https?://)?([a-z]{2}-[a-z0-9-]+)\\.oss-dls\\.aliyuncs\\.com"),
Pattern.compile("^(?:https?://)?dlf(?:-vpc)?\\.([a-z0-9-]+)\\.aliyuncs\\.com(?:/.*)?$"),
Pattern.compile("^(?:https?://)?datalake(?:-vpc)?\\.([a-z0-9-]+)\\.aliyuncs\\.com(?:/.*)?$"));
@@ -155,6 +154,8 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
private static List<String> DLF_TYPE_KEYWORDS =
Arrays.asList("hive.metastore.type",
"iceberg.catalog.type", "paimon.catalog.type");
+ private static final String DLS_URI_KEYWORDS = "oss-dls.aliyuncs";
+
protected OSSProperties(Map<String, String> origProps) {
super(Type.OSS, origProps);
}
@@ -175,6 +176,9 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
.findFirst()
.orElse(null);
if (StringUtils.isNotBlank(value)) {
+ if (value.contains(DLS_URI_KEYWORDS)) {
+ return false;
+ }
return (value.contains("aliyuncs.com"));
}
@@ -203,6 +207,10 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
if (value == null) {
return false;
}
+ boolean isDls = value.contains(DLS_URI_KEYWORDS);
+ if (isDls) {
+ return false;
+ }
if (value.startsWith("oss://")) {
return true;
}
@@ -211,8 +219,7 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
}
boolean isAliyunOss = (value.contains("oss-"));
boolean isAmazonS3 = value.contains("s3.");
- boolean isDls = value.contains("dls");
- return isAliyunOss || isAmazonS3 || isDls;
+ return isAliyunOss || isAmazonS3;
}
private static boolean isDlfMSType(Map<String, String> params) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
index 6e4b55a5e74..cedfe8388b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
@@ -168,13 +168,22 @@ public abstract class StorageProperties extends
ConnectionProperties {
Arrays.asList(
props -> (isFsSupport(props, FS_HDFS_SUPPORT)
|| HdfsProperties.guessIsMe(props)) ? new
HdfsProperties(props) : null,
- props -> ((isFsSupport(props, FS_OSS_HDFS_SUPPORT)
- || isFsSupport(props, DEPRECATED_OSS_HDFS_SUPPORT))
- || OSSHdfsProperties.guessIsMe(props)) ? new
OSSHdfsProperties(props) : null,
+ props -> {
+ // OSS-HDFS and OSS are mutually exclusive - check
OSS-HDFS first
+ if ((isFsSupport(props, FS_OSS_HDFS_SUPPORT)
+ || isFsSupport(props,
DEPRECATED_OSS_HDFS_SUPPORT))
+ || OSSHdfsProperties.guessIsMe(props)) {
+ return new OSSHdfsProperties(props);
+ }
+ // Only check for regular OSS if OSS-HDFS is not
enabled
+ if (isFsSupport(props, FS_OSS_SUPPORT)
+ || OSSProperties.guessIsMe(props)) {
+ return new OSSProperties(props);
+ }
+ return null;
+ },
props -> (isFsSupport(props, FS_S3_SUPPORT)
|| S3Properties.guessIsMe(props)) ? new
S3Properties(props) : null,
- props -> (isFsSupport(props, FS_OSS_SUPPORT)
- || OSSProperties.guessIsMe(props)) ? new
OSSProperties(props) : null,
props -> (isFsSupport(props, FS_OBS_SUPPORT)
|| OBSProperties.guessIsMe(props)) ? new
OBSProperties(props) : null,
props -> (isFsSupport(props, FS_COS_SUPPORT)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
index 4cb9bf3588b..6b50532854e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
@@ -112,18 +112,9 @@ public class OSSHdfsPropertiesTest {
origProps.put("oss.endpoint", "cn-north-2-gov-1.oss-dls.aliyuncs.com");
props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
Assertions.assertEquals("cn-north-2-gov-1",
props.getBackendConfigProperties().get("fs.oss.region"));
- origProps.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
- props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
- Assertions.assertEquals("cn-beijing",
props.getBackendConfigProperties().get("fs.oss.region"));
origProps.remove("dlf.endpoint");
props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
Assertions.assertEquals("cn-north-2-gov-1",
props.getBackendConfigProperties().get("fs.oss.region"));
- origProps.put("dlf.endpoint", "dlf-vpc.ap-southeast-5.aliyuncs.com");
- props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
- Assertions.assertEquals("ap-southeast-5",
props.getBackendConfigProperties().get("fs.oss.region"));
- origProps.put("dlf.endpoint", "dlf.us-east-1.aliyuncs.com");
- props = (OSSHdfsProperties)
StorageProperties.createAll(origProps).get(0);
- Assertions.assertEquals("us-east-1",
props.getBackendConfigProperties().get("fs.oss.region"));
}
@Test
@@ -167,4 +158,62 @@ public class OSSHdfsPropertiesTest {
Assertions.assertEquals("oss://bucket",
props.getBackendConfigProperties().get("fs.defaultFS"));
}
+ @Test
+ public void testOssAndOssHDFS() throws UserException {
+ Map<String, String> origProps = new HashMap<>();
+ origProps.put("oss.endpoint", "cn-shanghai.oss-dls.aliyuncs.com");
+ origProps.put("oss.access_key", "testAccessKey");
+ origProps.put("oss.secret_key", "testSecretKey");
+ origProps.put("oss.hdfs.fs.defaultFS", "oss://my-bucket");
+ origProps.put("dlf.endpoint", "cn-shanghai.oss-dlf.aliyuncs.com");
+ origProps.put("dlf.access_key", "testAccessKey");
+ origProps.put("dlf.secret_key", "testSecretKey");
+ StorageProperties props = StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("OSSHDFS", props.getStorageName());
+ Assertions.assertEquals("oss://my-bucket",
props.getBackendConfigProperties().get("fs.defaultFS"));
+ Assertions.assertEquals("cn-shanghai",
props.getBackendConfigProperties().get("fs.oss.region"));
+ }
+
+ @Test
+ public void testDlfAndOssHDFS() throws UserException {
+
+ Map<String, String> origProps = new HashMap<>();
+ origProps.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
+ origProps.put("dlf.access_key", "testAccessKey");
+ origProps.put("dlf.secret_key", "testSecretKey");
+ origProps.put("oss.hdfs.enabled", "true");
+ StorageProperties props = StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("OSSHDFS", props.getStorageName());
+ }
+
+
+ @Test
+ public void testOssHDFSNewProps() throws UserException {
+ Map<String, String> origProps = new HashMap<>();
+ origProps.put("oss.hdfs.endpoint", "cn-shanghai.oss-dls.aliyuncs.com");
+ origProps.put("oss.hdfs.access_key", "testAccessKey");
+ origProps.put("oss.hdfs.secret_key", "testSecretKey");
+ StorageProperties props = StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("OSSHDFS", props.getStorageName());
+ Assertions.assertEquals("cn-shanghai",
props.getBackendConfigProperties().get("fs.oss.region"));
+ Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem",
+ props.getBackendConfigProperties().get("fs.oss.impl"));
+ origProps.put("oss.endpoint", "cn-shanghai.oss.aliyuncs.com");
+ origProps.put("oss.access_key", "testAccessKey");
+ origProps.put("oss.secret_key", "testSecretKey");
+ props = StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("OSSHDFS", props.getStorageName());
+ Assertions.assertEquals("cn-shanghai",
props.getBackendConfigProperties().get("fs.oss.region"));
+ Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem",
+ props.getBackendConfigProperties().get("fs.oss.impl"));
+ origProps.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
+ origProps.put("dlf.access_key", "testAccessKey");
+ origProps.put("dlf.secret_key", "testSecretKey");
+ props = StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("OSSHDFS", props.getStorageName());
+ Assertions.assertEquals("cn-shanghai",
props.getBackendConfigProperties().get("fs.oss.region"));
+ Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem",
+ props.getBackendConfigProperties().get("fs.oss.impl"));
+ }
+
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index 874aea6e510..0bc5e823f0e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -120,7 +120,7 @@ public class OSSPropertiesTest {
origProps.put("oss.endpoint",
"http://s3.oss-cn-hongkong.aliyuncs.com";);
Assertions.assertEquals("cn-hongkong", ((OSSProperties)
StorageProperties.createPrimary(origProps)).getRegion());
origProps.put("oss.endpoint", "https://dlf.cn-beijing.aliyuncs.com";);
- Assertions.assertEquals("cn-beijing", ((OSSProperties)
StorageProperties.createAll(origProps).get(1)).getRegion());
+ Assertions.assertEquals("cn-beijing", ((OSSProperties)
StorageProperties.createAll(origProps).get(0)).getRegion());
origProps.put("oss.endpoint", "datalake-vpc.cn-shenzhen.aliyuncs.com");
Assertions.assertEquals("cn-shenzhen", ((OSSProperties)
StorageProperties.createPrimary(origProps)).getRegion());
origProps.put("oss.endpoint",
"https://datalake-vpc.cn-shenzhen.aliyuncs.com";);
diff --git a/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out
b/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out
index 7f97c337999..4f0e6ef6101 100644
--- a/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out
+++ b/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out
@@ -15,7 +15,7 @@ catalog_tvf_test_dlf hms dlf.catalog.id 987654321
catalog_tvf_test_dlf hms dlf.secret_key *XXX
-- !test_12 --
-catalog_tvf_test_dlf hms dlf.access_key AAAAAAAAAAAAAAAAAAAAAA
+catalog_tvf_test_dlf hms dlf.access_key *XXX
-- !test_13 --
catalog_tvf_test_dlf hms dlf.uid 123456789
@@ -32,7 +32,7 @@ internal internal NULL NULL
catalog_tvf_test_dlf hms dlf.secret_key *XXX
-- !test_18 --
-catalog_tvf_test_dlf hms dlf.access_key AAAAAAAAAAAAAAAAAAAAAA
+catalog_tvf_test_dlf hms dlf.access_key *XXX
-- !test_19 --
catalog_tvf_test_dlf hms dlf.uid 123456789
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
