This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new ab3b6638005 [fix](paimon-catalog)Fix OSS access when using DLS
endpoint (#58099) (#58226)
ab3b6638005 is described below
commit ab3b66380056466052bbccb6657e57c5d04a4ba8
Author: Calvin Kirs <[email protected]>
AuthorDate: Sat Nov 22 11:06:38 2025 +0800
[fix](paimon-catalog)Fix OSS access when using DLS endpoint (#58099)
(#58226)
#58099
---
.../org/apache/doris/common/util/LocationPath.java | 3 +
.../property/storage/OSSHdfsProperties.java | 39 +++----------
.../datasource/property/storage/OSSProperties.java | 13 ++++-
.../property/storage/StorageProperties.java | 19 ++++--
.../property/storage/OSSHdfsPropertiesTest.java | 67 +++++++++++++++++++---
.../property/storage/OSSPropertiesTest.java | 2 +-
6 files changed, 95 insertions(+), 48 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index 67aa9426d5a..d11b8121c6c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -307,6 +307,9 @@ public class LocationPath {
}
public TFileType getTFileTypeForBE() {
+ if (StringUtils.isNotBlank(normalizedLocation) &&
isHdfsOnOssEndpoint(normalizedLocation)) {
+ return TFileType.FILE_HDFS;
+ }
return SchemaTypeMapper.fromSchemaToFileType(schema);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
index f2c0b8c6c0c..a461d644626 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
@@ -49,22 +49,22 @@ import java.util.regex.Pattern;
public class OSSHdfsProperties extends HdfsCompatibleProperties {
@Setter
- @ConnectorProperty(names = {"oss.hdfs.endpoint",
- "dlf.endpoint", "dlf.catalog.endpoint", "oss.endpoint"},
+ @ConnectorProperty(names = {"oss.hdfs.endpoint", "oss.endpoint",
+ "dlf.endpoint", "dlf.catalog.endpoint"},
description = "The endpoint of OSS.")
protected String endpoint = "";
- @ConnectorProperty(names = {"oss.hdfs.access_key", "dlf.access_key",
"dlf.catalog.accessKeyId", "oss.access_key"},
+ @ConnectorProperty(names = {"oss.hdfs.access_key", "oss.access_key",
"dlf.access_key", "dlf.catalog.accessKeyId"},
sensitive = true,
description = "The access key of OSS.")
protected String accessKey = "";
- @ConnectorProperty(names = {"oss.hdfs.secret_key", "dlf.secret_key",
"dlf.catalog.secret_key", "oss.secret_key"},
+ @ConnectorProperty(names = {"oss.hdfs.secret_key", "oss.secret_key",
"dlf.secret_key", "dlf.catalog.secret_key"},
sensitive = true,
description = "The secret key of OSS.")
protected String secretKey = "";
- @ConnectorProperty(names = {"oss.hdfs.region", "dlf.region", "oss.region"},
+ @ConnectorProperty(names = {"oss.hdfs.region", "oss.region", "dlf.region"},
required = false,
description = "The region of OSS.")
protected String region;
@@ -86,8 +86,8 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
description = "The security token of OSS.")
protected String securityToken = "";
- private static final Set<String> OSS_ENDPOINT_KEY_NAME =
ImmutableSet.of("oss.hdfs.endpoint",
- "dlf.endpoint", "dlf.catalog.endpoint", "oss.endpoint");
+ private static final Set<String> OSS_ENDPOINT_KEY_NAME =
ImmutableSet.of("oss.hdfs.endpoint", "oss.endpoint",
+ "dlf.endpoint", "dlf.catalog.endpoint");
private Map<String, String> backendConfigProperties;
@@ -111,21 +111,15 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
}
String endpoint = OSS_ENDPOINT_KEY_NAME.stream()
.map(props::get)
- .filter(StringUtils::isNotBlank)
+ .filter(ep -> StringUtils.isNotBlank(ep) &&
ep.endsWith(OSS_HDFS_ENDPOINT_SUFFIX))
.findFirst()
.orElse(null);
- if (StringUtils.isBlank(endpoint)) {
- return false;
- }
- return endpoint.endsWith(OSS_HDFS_ENDPOINT_SUFFIX) ||
endpoint.contains(DLF_ENDPOINT_KEY_WORDS);
+ return StringUtils.isNotBlank(endpoint);
}
@Override
protected void checkRequiredProperties() {
super.checkRequiredProperties();
- if (!isValidEndpoint(endpoint)) {
- throw new IllegalArgumentException("Property oss.endpoint is
required and must be a valid OSS endpoint.");
- }
}
private void convertDlfToOssEndpointIfNeeded() {
@@ -145,22 +139,9 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
return Optional.empty();
}
- public static boolean isValidEndpoint(String endpoint) {
- for (Pattern pattern : ENDPOINT_PATTERN) {
- if (pattern.matcher(endpoint.toLowerCase()).matches()) {
- return true;
- }
- }
- return false;
- }
-
@Override
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
- if (!isValidEndpoint(endpoint.toLowerCase())) {
- throw new IllegalArgumentException("The endpoint is not a valid
OSS HDFS endpoint: " + endpoint
- + ". It should match the pattern:
<region>.oss-dls.aliyuncs.com");
- }
// Extract region from the endpoint, e.g.,
"cn-shanghai.oss-dls.aliyuncs.com" -> "cn-shanghai"
if (StringUtils.isBlank(this.region)) {
Optional<String> regionOptional = extractRegion(endpoint);
@@ -179,8 +160,6 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
private static final String OSS_HDFS_ENDPOINT_SUFFIX =
".oss-dls.aliyuncs.com";
- private static final String DLF_ENDPOINT_KEY_WORDS = "dlf";
-
@Override
public Map<String, String> getBackendConfigProperties() {
return backendConfigProperties;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
index d1b14e87ef3..0cb87f9b305 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
@@ -147,7 +147,6 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
* - datalake.cn-hangzhou.aliyuncs.com => region = cn-hangzhou
*/
public static final Set<Pattern> ENDPOINT_PATTERN =
ImmutableSet.of(STANDARD_ENDPOINT_PATTERN,
-
Pattern.compile("(?:https?://)?([a-z]{2}-[a-z0-9-]+)\\.oss-dls\\.aliyuncs\\.com"),
Pattern.compile("^(?:https?://)?dlf(?:-vpc)?\\.([a-z0-9-]+)\\.aliyuncs\\.com(?:/.*)?$"),
Pattern.compile("^(?:https?://)?datalake(?:-vpc)?\\.([a-z0-9-]+)\\.aliyuncs\\.com(?:/.*)?$"));
@@ -156,6 +155,8 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
private static List<String> DLF_TYPE_KEYWORDS =
Arrays.asList("hive.metastore.type",
"iceberg.catalog.type", "paimon.catalog.type");
+ private static final String DLS_URI_KEYWORDS = "oss-dls.aliyuncs";
+
protected OSSProperties(Map<String, String> origProps) {
super(Type.OSS, origProps);
}
@@ -176,6 +177,9 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
.findFirst()
.orElse(null);
if (StringUtils.isNotBlank(value)) {
+ if (value.contains(DLS_URI_KEYWORDS)) {
+ return false;
+ }
return (value.contains("aliyuncs.com"));
}
@@ -204,6 +208,10 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
if (value == null) {
return false;
}
+ boolean isDls = value.contains(DLS_URI_KEYWORDS);
+ if (isDls) {
+ return false;
+ }
if (value.startsWith("oss://")) {
return true;
}
@@ -212,8 +220,7 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
}
boolean isAliyunOss = (value.contains("oss-"));
boolean isAmazonS3 = value.contains("s3.");
- boolean isDls = value.contains("dls");
- return isAliyunOss || isAmazonS3 || isDls;
+ return isAliyunOss || isAmazonS3;
}
private static boolean isDlfMSType(Map<String, String> params) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
index 8a993c46a4b..80a2d50fa63 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
@@ -163,13 +163,22 @@ public abstract class StorageProperties extends
ConnectionProperties {
Arrays.asList(
props -> (isFsSupport(props, FS_HDFS_SUPPORT)
|| HdfsProperties.guessIsMe(props)) ? new
HdfsProperties(props) : null,
- props -> ((isFsSupport(props, FS_OSS_HDFS_SUPPORT)
- || isFsSupport(props, DEPRECATED_OSS_HDFS_SUPPORT))
- || OSSHdfsProperties.guessIsMe(props)) ? new
OSSHdfsProperties(props) : null,
+ props -> {
+ // OSS-HDFS and OSS are mutually exclusive - check
OSS-HDFS first
+ if ((isFsSupport(props, FS_OSS_HDFS_SUPPORT)
+ || isFsSupport(props,
DEPRECATED_OSS_HDFS_SUPPORT))
+ || OSSHdfsProperties.guessIsMe(props)) {
+ return new OSSHdfsProperties(props);
+ }
+ // Only check for regular OSS if OSS-HDFS is not
enabled
+ if (isFsSupport(props, FS_OSS_SUPPORT)
+ || OSSProperties.guessIsMe(props)) {
+ return new OSSProperties(props);
+ }
+ return null;
+ },
props -> (isFsSupport(props, FS_S3_SUPPORT)
|| S3Properties.guessIsMe(props)) ? new
S3Properties(props) : null,
- props -> (isFsSupport(props, FS_OSS_SUPPORT)
- || OSSProperties.guessIsMe(props)) ? new
OSSProperties(props) : null,
props -> (isFsSupport(props, FS_OBS_SUPPORT)
|| OBSProperties.guessIsMe(props)) ? new
OBSProperties(props) : null,
props -> (isFsSupport(props, FS_COS_SUPPORT)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
index 4cb9bf3588b..6b50532854e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
@@ -112,18 +112,9 @@ public class OSSHdfsPropertiesTest {
origProps.put("oss.endpoint", "cn-north-2-gov-1.oss-dls.aliyuncs.com");
props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
Assertions.assertEquals("cn-north-2-gov-1",
props.getBackendConfigProperties().get("fs.oss.region"));
- origProps.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
- props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
- Assertions.assertEquals("cn-beijing",
props.getBackendConfigProperties().get("fs.oss.region"));
origProps.remove("dlf.endpoint");
props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
Assertions.assertEquals("cn-north-2-gov-1",
props.getBackendConfigProperties().get("fs.oss.region"));
- origProps.put("dlf.endpoint", "dlf-vpc.ap-southeast-5.aliyuncs.com");
- props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
- Assertions.assertEquals("ap-southeast-5",
props.getBackendConfigProperties().get("fs.oss.region"));
- origProps.put("dlf.endpoint", "dlf.us-east-1.aliyuncs.com");
- props = (OSSHdfsProperties)
StorageProperties.createAll(origProps).get(0);
- Assertions.assertEquals("us-east-1",
props.getBackendConfigProperties().get("fs.oss.region"));
}
@Test
@@ -167,4 +158,62 @@ public class OSSHdfsPropertiesTest {
Assertions.assertEquals("oss://bucket",
props.getBackendConfigProperties().get("fs.defaultFS"));
}
+ @Test
+ public void testOssAndOssHDFS() throws UserException {
+ Map<String, String> origProps = new HashMap<>();
+ origProps.put("oss.endpoint", "cn-shanghai.oss-dls.aliyuncs.com");
+ origProps.put("oss.access_key", "testAccessKey");
+ origProps.put("oss.secret_key", "testSecretKey");
+ origProps.put("oss.hdfs.fs.defaultFS", "oss://my-bucket");
+ origProps.put("dlf.endpoint", "cn-shanghai.oss-dlf.aliyuncs.com");
+ origProps.put("dlf.access_key", "testAccessKey");
+ origProps.put("dlf.secret_key", "testSecretKey");
+ StorageProperties props = StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("OSSHDFS", props.getStorageName());
+ Assertions.assertEquals("oss://my-bucket",
props.getBackendConfigProperties().get("fs.defaultFS"));
+ Assertions.assertEquals("cn-shanghai",
props.getBackendConfigProperties().get("fs.oss.region"));
+ }
+
+ @Test
+ public void testDlfAndOssHDFS() throws UserException {
+
+ Map<String, String> origProps = new HashMap<>();
+ origProps.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
+ origProps.put("dlf.access_key", "testAccessKey");
+ origProps.put("dlf.secret_key", "testSecretKey");
+ origProps.put("oss.hdfs.enabled", "true");
+ StorageProperties props = StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("OSSHDFS", props.getStorageName());
+ }
+
+
+ @Test
+ public void testOssHDFSNewProps() throws UserException {
+ Map<String, String> origProps = new HashMap<>();
+ origProps.put("oss.hdfs.endpoint", "cn-shanghai.oss-dls.aliyuncs.com");
+ origProps.put("oss.hdfs.access_key", "testAccessKey");
+ origProps.put("oss.hdfs.secret_key", "testSecretKey");
+ StorageProperties props = StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("OSSHDFS", props.getStorageName());
+ Assertions.assertEquals("cn-shanghai",
props.getBackendConfigProperties().get("fs.oss.region"));
+ Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem",
+ props.getBackendConfigProperties().get("fs.oss.impl"));
+ origProps.put("oss.endpoint", "cn-shanghai.oss.aliyuncs.com");
+ origProps.put("oss.access_key", "testAccessKey");
+ origProps.put("oss.secret_key", "testSecretKey");
+ props = StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("OSSHDFS", props.getStorageName());
+ Assertions.assertEquals("cn-shanghai",
props.getBackendConfigProperties().get("fs.oss.region"));
+ Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem",
+ props.getBackendConfigProperties().get("fs.oss.impl"));
+ origProps.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
+ origProps.put("dlf.access_key", "testAccessKey");
+ origProps.put("dlf.secret_key", "testSecretKey");
+ props = StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("OSSHDFS", props.getStorageName());
+ Assertions.assertEquals("cn-shanghai",
props.getBackendConfigProperties().get("fs.oss.region"));
+ Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem",
+ props.getBackendConfigProperties().get("fs.oss.impl"));
+ }
+
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index 874aea6e510..0bc5e823f0e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -120,7 +120,7 @@ public class OSSPropertiesTest {
origProps.put("oss.endpoint",
"http://s3.oss-cn-hongkong.aliyuncs.com";);
Assertions.assertEquals("cn-hongkong", ((OSSProperties)
StorageProperties.createPrimary(origProps)).getRegion());
origProps.put("oss.endpoint", "https://dlf.cn-beijing.aliyuncs.com";);
- Assertions.assertEquals("cn-beijing", ((OSSProperties)
StorageProperties.createAll(origProps).get(1)).getRegion());
+ Assertions.assertEquals("cn-beijing", ((OSSProperties)
StorageProperties.createAll(origProps).get(0)).getRegion());
origProps.put("oss.endpoint", "datalake-vpc.cn-shenzhen.aliyuncs.com");
Assertions.assertEquals("cn-shenzhen", ((OSSProperties)
StorageProperties.createPrimary(origProps)).getRegion());
origProps.put("oss.endpoint",
"https://datalake-vpc.cn-shenzhen.aliyuncs.com";);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
