This is an automated email from the ASF dual-hosted git repository.
zykkk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2a93eb9e340 [Fix](paramter-refactor): Pass full Hive table parameters
to BE; add OSS path detection support for OSSHDFS (#53409)
2a93eb9e340 is described below
commit 2a93eb9e34085dda909f9747241141d52361dc55
Author: Calvin Kirs <[email protected]>
AuthorDate: Sat Jul 19 20:45:30 2025 +0800
[Fix](paramter-refactor): Pass full Hive table parameters to BE; add OSS
path detection support for OSSHDFS (#53409)
#50238
### Background
1. **Incomplete Hive table parameters passed to BE** In current
implementations, Hive table parameters are not fully passed to the Backend
(BE). This leads to missing connection configuration on the BE side, and in
some cases, queries only work after a restart.
2. **OSSHDFS compatibility with OSS paths** In certain scenarios, OSSHDFS
is used as the storage system, but the table `location` still uses an OSS-style
path (e.g., `oss://bucket/path`). Our logic currently fails to detect and
handle such cases correctly.
---
### Changes
- Ensure that all restructured Hive table parameters are passed to BE to
avoid connection issues.
- Improve OSS and OSSHDFS compatibility by detecting `oss://` paths even
when OSSHDFS is configured.
---
### Impact
- Fixes issues where BE fails to query Hive tables due to missing
parameters.
- Enables proper handling of OSS paths in OSSHDFS-based configurations.
---
### Test Plan
- Validate that Hive tables can be queried immediately without requiring a
BE restart.
- Verify that OSS-style paths are correctly handled when using OSSHDFS.
---
.../org/apache/doris/common/util/LocationPath.java | 12 ++++-
.../apache/doris/datasource/CatalogProperty.java | 2 +-
.../doris/datasource/hive/HMSExternalTable.java | 8 +++-
.../property/storage/OSSHdfsProperties.java | 12 ++---
.../datasource/property/storage/OSSProperties.java | 56 +++++++++++++++++-----
.../property/storage/StorageProperties.java | 1 +
.../java/org/apache/doris/fs/SchemaTypeMapper.java | 2 +-
.../property/storage/OSSHdfsPropertiesTest.java | 10 ++--
.../property/storage/OSSPropertiesTest.java | 2 +
.../org/apache/doris/fs/SchemaTypeMapperTest.java | 3 ++
10 files changed, 82 insertions(+), 26 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index c832bff5ec4..9830e627779 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -132,7 +132,7 @@ public class LocationPath {
String schema = extractScheme(location);
String normalizedLocation = location;
StorageProperties storageProperties = null;
- StorageProperties.Type type = SchemaTypeMapper.fromSchema(schema);
+ StorageProperties.Type type = fromSchemaWithContext(location, schema);
if (StorageProperties.Type.LOCAL.equals(type)) {
normalize = false;
}
@@ -154,6 +154,13 @@ public class LocationPath {
return new LocationPath(schema, normalizedLocation, fsIdentifier,
storageProperties);
}
+ public static StorageProperties.Type fromSchemaWithContext(String
location, String schema) {
+ if (isHdfsOnOssEndpoint(location)) {
+ return StorageProperties.Type.OSS_HDFS;
+ }
+ return SchemaTypeMapper.fromSchema(schema); // fallback to default
+ }
+
public static LocationPath of(String location) {
String schema = extractScheme(location);
String encodedLocation = encodedLocation(location);
@@ -271,6 +278,9 @@ public class LocationPath {
if (StringUtils.isBlank(location)) {
return null;
}
+ if (isHdfsOnOssEndpoint(location)) {
+ return TFileType.FILE_HDFS;
+ }
LocationPath locationPath = LocationPath.of(location);
return locationPath.getTFileTypeForBE();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java
index eb018ca944c..ebb7c6f292d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java
@@ -110,7 +110,7 @@ public class CatalogProperty {
private void reInitCatalogStorageProperties() {
List<StorageProperties> storageProperties;
try {
- storageProperties = StorageProperties.createAll(this.properties);
+ storageProperties = StorageProperties.createAll(getProperties());
this.storagePropertiesMap = (storageProperties.stream()
.collect(java.util.stream.Collectors.toMap(StorageProperties::getType,
Function.identity())));
} catch (UserException e) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index b20780ec9a0..5e54673d359 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -602,7 +602,13 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
}
public Map<String, String> getHadoopProperties() {
- return catalog.getCatalogProperty().getHadoopProperties();
+ return getStoragePropertiesMap().values().stream()
+ .flatMap(m ->
m.getBackendConfigProperties().entrySet().stream())
+ .collect(Collectors.toMap(
+ Map.Entry::getKey,
+ Map.Entry::getValue,
+ (v1, v2) -> v2));
+
}
public List<ColumnStatisticsObj> getHiveTableColumnStats(List<String>
columns) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
index 8c29addaf57..3023cce09c7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
@@ -50,19 +50,19 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
@Setter
@ConnectorProperty(names = {"oss.hdfs.endpoint",
- "oss.endpoint", "dlf.endpoint", "dlf.catalog.endpoint"},
+ "dlf.endpoint", "dlf.catalog.endpoint", "oss.endpoint" },
description = "The endpoint of OSS.")
protected String endpoint = "";
- @ConnectorProperty(names = {"oss.hdfs.access_key", "oss.access_key",
"dlf.access_key", "dlf.catalog.accessKeyId"},
+ @ConnectorProperty(names = {"oss.hdfs.access_key", "dlf.access_key",
"dlf.catalog.accessKeyId", "oss.access_key"},
description = "The access key of OSS.")
protected String accessKey = "";
- @ConnectorProperty(names = {"oss.hdfs.secret_key", "oss.secret_key",
"dlf.secret_key", "dlf.catalog.secret_key"},
+ @ConnectorProperty(names = {"oss.hdfs.secret_key", "dlf.secret_key",
"dlf.catalog.secret_key", "oss.secret_key"},
description = "The secret key of OSS.")
protected String secretKey = "";
- @ConnectorProperty(names = {"oss.hdfs.region", "oss.region", "dlf.region"},
+ @ConnectorProperty(names = {"oss.hdfs.region", "dlf.region", "oss.region"},
required = false,
description = "The region of OSS.")
protected String region;
@@ -85,7 +85,7 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
protected String securityToken = "";
private static final Set<String> OSS_ENDPOINT_KEY_NAME =
ImmutableSet.of("oss.hdfs.endpoint",
- "oss.endpoint", "dlf.endpoint", "dlf.catalog.endpoint");
+ "dlf.endpoint", "dlf.catalog.endpoint", "oss.endpoint");
private Map<String, String> backendConfigProperties;
@@ -96,7 +96,7 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
private static final Set<String> supportSchema = ImmutableSet.of("oss",
"hdfs");
protected OSSHdfsProperties(Map<String, String> origProps) {
- super(Type.OSS, origProps);
+ super(Type.OSS_HDFS, origProps);
}
private static final String OSS_HDFS_PREFIX_KEY = "oss.hdfs.";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
index ff401b377ad..fe71c060cc0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
@@ -35,24 +35,28 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
@Setter
@Getter
- @ConnectorProperty(names = {"oss.endpoint", "s3.endpoint", "AWS_ENDPOINT",
"endpoint", "ENDPOINT"},
+ @ConnectorProperty(names = {"oss.endpoint", "s3.endpoint", "AWS_ENDPOINT",
"endpoint", "ENDPOINT", "dlf.endpoint",
+ "dlf.catalog.endpoint"},
required = false,
description = "The endpoint of OSS.")
protected String endpoint = "";
@Getter
- @ConnectorProperty(names = {"oss.access_key", "s3.access_key",
"AWS_ACCESS_KEY", "access_key", "ACCESS_KEY"},
+ @ConnectorProperty(names = {"oss.access_key", "s3.access_key",
"AWS_ACCESS_KEY", "access_key", "ACCESS_KEY",
+ "dlf.access_key", "dlf.catalog.accessKeyId"},
description = "The access key of OSS.")
protected String accessKey = "";
@Getter
- @ConnectorProperty(names = {"oss.secret_key", "s3.secret_key",
"AWS_SECRET_KEY", "secret_key", "SECRET_KEY"},
+ @ConnectorProperty(names = {"oss.secret_key", "s3.secret_key",
"AWS_SECRET_KEY", "secret_key", "SECRET_KEY",
+ "dlf.secret_key", "dlf.catalog.secret_key"},
description = "The secret key of OSS.")
protected String secretKey = "";
@Getter
@Setter
- @ConnectorProperty(names = {"oss.region", "s3.region", "AWS_REGION",
"region", "REGION"}, required = false,
+ @ConnectorProperty(names = {"oss.region", "s3.region", "AWS_REGION",
"region", "REGION", "dlf.region"},
+ required = false,
description = "The region of OSS.")
protected String region;
@@ -66,37 +70,67 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
* - <a href="http://oss-cn-shenzhen-internal.aliyuncs.com";>...</a> =>
region = cn-shenzhen
* <p>
* Group(1) captures the region name (e.g., cn-hangzhou).
- *<p>
+ * <p>
* Support S3 compatible endpoints:<a
href="https://help.aliyun.com/zh/oss/developer-reference/
* use-amazon-s3-sdks-to-access-oss">...</a>
* - s3.cn-hangzhou.aliyuncs.com => region = cn-hangzhou
* <p>
*/
private static final Set<Pattern> ENDPOINT_PATTERN =
ImmutableSet.of(Pattern
-
.compile("^(?:https?://)?(?:s3\\.)?oss-([a-z0-9-]+?)(?:-internal)?\\.aliyuncs\\.com$"));
+
.compile("^(?:https?://)?(?:s3\\.)?oss-([a-z0-9-]+?)(?:-internal)?\\.aliyuncs\\.com$"),
+
Pattern.compile("(?:https?://)?([a-z]{2}-[a-z0-9-]+)\\.oss-dls\\.aliyuncs\\.com"),
+
Pattern.compile("^(?:https?://)?dlf(?:-vpc)?\\.([a-z0-9-]+)\\.aliyuncs\\.com(?:/.*)?$"));
protected OSSProperties(Map<String, String> origProps) {
super(Type.OSS, origProps);
}
protected static boolean guessIsMe(Map<String, String> origProps) {
- String value = Stream.of("oss.endpoint", "s3.endpoint",
"AWS_ENDPOINT", "endpoint", "ENDPOINT")
+ String value = Stream.of("oss.endpoint", "s3.endpoint",
"AWS_ENDPOINT", "endpoint", "ENDPOINT",
+ "dlf.endpoint", "dlf.catalog.endpoint")
.map(origProps::get)
.filter(Objects::nonNull)
.findFirst()
.orElse(null);
if (!Strings.isNullOrEmpty(value)) {
- return (value.contains("aliyuncs.com") &&
!value.contains("oss-dls.aliyuncs.com"));
+ return (value.contains("aliyuncs.com"));
}
Optional<String> uriValue = origProps.entrySet().stream()
.filter(e -> e.getKey().equalsIgnoreCase("uri"))
.map(Map.Entry::getValue)
.findFirst();
- if (!uriValue.isPresent()) {
+ return
uriValue.filter(OSSProperties::isKnownObjectStorage).isPresent();
+ }
+
+ private static boolean isKnownObjectStorage(String value) {
+ if (value == null) {
return false;
}
- String uri = uriValue.get();
- return uri.contains("aliyuncs.com") &&
(!uri.contains("oss-dls.aliyuncs.com"));
+ if (!value.contains("aliyuncs.com")) {
+ return false;
+ }
+ boolean isAliyunOss = (value.contains("oss-") ||
value.contains("dlf."));
+ boolean isAmazonS3 = value.contains("s3.");
+ boolean isDls = value.contains("dls");
+ return isAliyunOss || isAmazonS3 || isDls;
+ }
+
+ @Override
+ public void initNormalizeAndCheckProps() {
+ super.initNormalizeAndCheckProps();
+ if (endpoint.contains("dlf") || endpoint.contains("oss-dls")) {
+ String publicAccess =
origProps.getOrDefault("dlf.catalog.accessPublic", "false");
+ this.endpoint = getOssEndpoint(region,
Boolean.parseBoolean(publicAccess));
+ }
+ }
+
+ private static String getOssEndpoint(String region, boolean publicAccess) {
+ String prefix = "oss-";
+ String suffix = ".aliyuncs.com";
+ if (!publicAccess) {
+ suffix = "-internal" + suffix;
+ }
+ return prefix + region + suffix;
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
index d003764bba5..3f8ed8157fd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java
@@ -54,6 +54,7 @@ public abstract class StorageProperties extends
ConnectionProperties {
OSS,
OBS,
COS,
+ OSS_HDFS,
MINIO,
AZURE,
BROKER,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/SchemaTypeMapper.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/SchemaTypeMapper.java
index ca25d5b44e2..8b54250ed59 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/SchemaTypeMapper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/SchemaTypeMapper.java
@@ -61,7 +61,7 @@ public enum SchemaTypeMapper {
JFS("jfs", StorageProperties.Type.BROKER, FileSystemType.JFS,
TFileType.FILE_BROKER),
VIEWFS("viewfs", StorageProperties.Type.HDFS, FileSystemType.HDFS,
TFileType.FILE_HDFS),
FILE("file", StorageProperties.Type.LOCAL, FileSystemType.FILE,
TFileType.FILE_LOCAL),
-
+ OSS_HDFS("oss", StorageProperties.Type.OSS_HDFS, FileSystemType.HDFS,
TFileType.FILE_HDFS),
OSS("oss", StorageProperties.Type.OSS, FileSystemType.S3,
TFileType.FILE_S3),
OBS("obs", StorageProperties.Type.OBS, FileSystemType.S3,
TFileType.FILE_S3),
COS("cos", StorageProperties.Type.COS, FileSystemType.S3,
TFileType.FILE_S3),
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
index 1ee543870bd..8ec934e5f6e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java
@@ -83,7 +83,7 @@ public class OSSHdfsPropertiesTest {
}
@Test
- public void testInvalidEndpoint() {
+ public void testInvalidEndpoint() throws UserException {
Map<String, String> origProps = new HashMap<>();
origProps.put("oss.endpoint", "invalid.aliyuncs.com");
origProps.put("oss.access_key", "testAccessKey");
@@ -114,15 +114,15 @@ public class OSSHdfsPropertiesTest {
Assertions.assertEquals("cn-north-2-gov-1",
props.getBackendConfigProperties().get("fs.oss.region"));
origProps.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
- Assertions.assertEquals("cn-north-2-gov-1",
props.getBackendConfigProperties().get("fs.oss.region"));
- origProps.remove("oss.endpoint");
- props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
Assertions.assertEquals("cn-beijing",
props.getBackendConfigProperties().get("fs.oss.region"));
+ origProps.remove("dlf.endpoint");
+ props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
+ Assertions.assertEquals("cn-north-2-gov-1",
props.getBackendConfigProperties().get("fs.oss.region"));
origProps.put("dlf.endpoint", "dlf-vpc.ap-southeast-5.aliyuncs.com");
props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
Assertions.assertEquals("ap-southeast-5",
props.getBackendConfigProperties().get("fs.oss.region"));
origProps.put("dlf.endpoint", "dlf.us-east-1.aliyuncs.com");
- props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps);
+ props = (OSSHdfsProperties)
StorageProperties.createAll(origProps).get(0);
Assertions.assertEquals("us-east-1",
props.getBackendConfigProperties().get("fs.oss.region"));
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index 68611f48a0e..3f0c4c688bd 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -115,6 +115,8 @@ public class OSSPropertiesTest {
Assertions.assertEquals("cn-hongkong", ((OSSProperties)
StorageProperties.createPrimary(origProps)).getRegion());
origProps.put("oss.endpoint",
"http://s3.oss-cn-hongkong.aliyuncs.com";);
Assertions.assertEquals("cn-hongkong", ((OSSProperties)
StorageProperties.createPrimary(origProps)).getRegion());
+ origProps.put("oss.endpoint", "https://dlf.cn-beijing.aliyuncs.com";);
+ Assertions.assertEquals("cn-beijing", ((OSSProperties)
StorageProperties.createAll(origProps).get(1)).getRegion());
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/fs/SchemaTypeMapperTest.java
b/fe/fe-core/src/test/java/org/apache/doris/fs/SchemaTypeMapperTest.java
index ccb743c3895..1dca58d63fe 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/fs/SchemaTypeMapperTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/fs/SchemaTypeMapperTest.java
@@ -27,6 +27,9 @@ public class SchemaTypeMapperTest {
@Test
public void testFromSchema_AllDefinedSchemas() {
for (SchemaTypeMapper mapper : SchemaTypeMapper.values()) {
+ if (mapper.equals(SchemaTypeMapper.OSS_HDFS)) {
+ continue;
+ }
String schema = mapper.getSchema();
StorageProperties.Type expectedType =
SchemaTypeMapper.fromSchema(schema);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
