This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ae657471b20 [refactor](oss) unify FE OSS filesystem with Jindo (#61269)
ae657471b20 is described below
commit ae657471b20667da264fc4e82ea8151fb9c6803d
Author: Calvin Kirs <[email protected]>
AuthorDate: Tue Mar 17 12:47:13 2026 +0800
[refactor](oss) unify FE OSS filesystem with Jindo (#61269)
This PR unifies the FE-side OSS Hadoop filesystem implementation to
Jindo FS and removes legacy OSS filesystem dependencies that are no
longer needed.
## Why
We currently have multiple OSS filesystem implementations on the FE
classpath, including:
- `org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem`
- `paimon-oss`
This makes OSS behavior inconsistent and increases the chance of
classpath conflicts. Since Doris already packages and uses Jindo FS, FE
should consistently use Jindo instead of mixing multiple OSS filesystem
implementations.
## Changes
- Switch `OSSProperties` to use Jindo FS:
- `fs.oss.impl = com.aliyun.jindodata.oss.JindoOssFileSystem`
- `fs.AbstractFileSystem.oss.impl = com.aliyun.jindodata.oss.JindoOSS`
- Keep `OSSHdfsProperties` aligned with the same Jindo FS constants.
- Add FE unit test coverage to verify OSS Hadoop config is initialized
with Jindo FS.
- Remove legacy OSS filesystem dependencies from FE modules:
- remove `paimon-oss` from `fe-core`
- remove `paimon-oss` from `preload-extensions`
- remove `hadoop-aliyun` from FE dependency management and `hadoop-deps`
## Scope
This PR only updates FE-side OSS filesystem wiring and FE-related
dependency cleanup.
Non-FE modules are intentionally left unchanged.
## Verification
- `run-fe-ut.sh --run
org.apache.doris.datasource.property.storage.OSSPropertiesTest,org.apache.doris.datasource.property.storage.OSSHdfsPropertiesTest`
- Full FE reactor build passed
## Notes
`aliyun-sdk-oss` is still kept because it is still used by FE cloud
storage code (`OssRemote`) and is not part of the Hadoop OSS filesystem
implementation cleanup in this PR.
---
fe/be-java-extensions/hadoop-deps/pom.xml | 6 +-----
fe/be-java-extensions/preload-extensions/pom.xml | 5 -----
fe/fe-core/pom.xml | 6 ------
.../datasource/property/storage/OSSHdfsProperties.java | 4 ++--
.../doris/datasource/property/storage/OSSProperties.java | 7 ++++++-
.../datasource/property/storage/OSSPropertiesTest.java | 14 +++++++++++++-
fe/pom.xml | 12 ------------
.../refactor_catalog_param/oss_hdfs_catalog_test.groovy | 4 ++--
8 files changed, 24 insertions(+), 34 deletions(-)
diff --git a/fe/be-java-extensions/hadoop-deps/pom.xml
b/fe/be-java-extensions/hadoop-deps/pom.xml
index 54591563091..aba07779a80 100644
--- a/fe/be-java-extensions/hadoop-deps/pom.xml
+++ b/fe/be-java-extensions/hadoop-deps/pom.xml
@@ -38,10 +38,6 @@ under the License.
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aliyun</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
@@ -135,4 +131,4 @@ under the License.
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/fe/be-java-extensions/preload-extensions/pom.xml
b/fe/be-java-extensions/preload-extensions/pom.xml
index bd6fa77e310..cca9aa1038f 100644
--- a/fe/be-java-extensions/preload-extensions/pom.xml
+++ b/fe/be-java-extensions/preload-extensions/pom.xml
@@ -84,11 +84,6 @@ under the License.
<artifactId>paimon-s3</artifactId>
<version>${paimon.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-oss</artifactId>
- <version>${paimon.version}</version>
- </dependency>
<!-- For Avro and Hudi Scanner PreLoad -->
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 91f1c413476..ebad1f71779 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -578,12 +578,6 @@ under the License.
<version>${paimon.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-oss</artifactId>
- <version>${paimon.version}</version>
- </dependency>
-
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>glue</artifactId>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
index db84a15002d..6ea01815e73 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
@@ -174,8 +174,8 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
config.put("fs.oss.accessKeyId", accessKey);
config.put("fs.oss.accessKeySecret", secretKey);
config.put("fs.oss.region", region);
- config.put("fs.oss.impl",
"com.aliyun.jindodata.oss.JindoOssFileSystem");
- config.put("fs.AbstractFileSystem.oss.impl",
"com.aliyun.jindodata.oss.JindoOSS");
+ config.put("fs.oss.impl", OSSProperties.JINDO_OSS_FILE_SYSTEM_IMPL);
+ config.put("fs.AbstractFileSystem.oss.impl",
OSSProperties.JINDO_OSS_ABSTRACT_FILE_SYSTEM_IMPL);
if (StringUtils.isNotBlank(fsDefaultFS)) {
config.put(HDFS_DEFAULT_FS_NAME, fsDefaultFS);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
index c8fca077804..86d3a9806a8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
@@ -163,6 +163,9 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
private static List<String> DLF_TYPE_KEYWORDS =
Arrays.asList("hive.metastore.type",
"iceberg.catalog.type", "paimon.catalog.type");
+ static final String JINDO_OSS_FILE_SYSTEM_IMPL =
"com.aliyun.jindodata.oss.JindoOssFileSystem";
+ static final String JINDO_OSS_ABSTRACT_FILE_SYSTEM_IMPL =
"com.aliyun.jindodata.oss.JindoOSS";
+
private static final String DLS_URI_KEYWORDS = "oss-dls.aliyuncs";
protected OSSProperties(Map<String, String> origProps) {
@@ -311,10 +314,12 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
@Override
public void initializeHadoopStorageConfig() {
super.initializeHadoopStorageConfig();
- hadoopStorageConfig.set("fs.oss.impl",
"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem");
+ hadoopStorageConfig.set("fs.oss.impl", JINDO_OSS_FILE_SYSTEM_IMPL);
+ hadoopStorageConfig.set("fs.AbstractFileSystem.oss.impl",
JINDO_OSS_ABSTRACT_FILE_SYSTEM_IMPL);
hadoopStorageConfig.set("fs.oss.accessKeyId", accessKey);
hadoopStorageConfig.set("fs.oss.accessKeySecret", secretKey);
hadoopStorageConfig.set("fs.oss.endpoint", endpoint);
+ hadoopStorageConfig.set("fs.oss.region", region);
}
/**
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index fce9934cdaa..d5c5b029031 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -272,6 +272,19 @@ public class OSSPropertiesTest {
Assertions.assertFalse(s3Properties.hadoopStorageConfig.getBoolean("fs.oss.impl.disable.cache",
false));
}
+ @Test
+ public void testUseJindoFsForHadoopStorageConfig() throws UserException {
+ Map<String, String> props = Maps.newHashMap();
+ props.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com");
+ props.put("oss.region", "cn-hangzhou");
+ OSSProperties ossProperties = (OSSProperties)
StorageProperties.createPrimary(props);
+ Assertions.assertEquals(OSSProperties.JINDO_OSS_FILE_SYSTEM_IMPL,
+ ossProperties.hadoopStorageConfig.get("fs.oss.impl"));
+
Assertions.assertEquals(OSSProperties.JINDO_OSS_ABSTRACT_FILE_SYSTEM_IMPL,
+
ossProperties.hadoopStorageConfig.get("fs.AbstractFileSystem.oss.impl"));
+ Assertions.assertEquals("cn-hangzhou",
ossProperties.hadoopStorageConfig.get("fs.oss.region"));
+ }
+
@Test
public void testOSSBucketEndpointPathProperties() throws UserException {
Assertions.assertEquals("oss://my-bucket/path/to/dir/",
OSSProperties.rewriteOssBucketIfNecessary("oss://my-bucket/path/to/dir/"));
@@ -280,4 +293,3 @@ public class OSSPropertiesTest {
Assertions.assertEquals("https://bucket-name.oss-cn-hangzhou.aliyuncs.com/path/to/dir/file.txt";,
OSSProperties.rewriteOssBucketIfNecessary("https://bucket-name.oss-cn-hangzhou.aliyuncs.com/path/to/dir/file.txt";));
}
}
-
diff --git a/fe/pom.xml b/fe/pom.xml
index 336d7592773..0b17a9a18df 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -755,18 +755,6 @@ under the License.
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aliyun</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <!-- pls confirm if this is a mandatory dependency -->
- <exclusion>
- <groupId>org.ini4j</groupId>
- <artifactId>ini4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>hive-catalog-shade</artifactId>
diff --git
a/regression-test/suites/external_table_p2/refactor_catalog_param/oss_hdfs_catalog_test.groovy
b/regression-test/suites/external_table_p2/refactor_catalog_param/oss_hdfs_catalog_test.groovy
index 3f2791cbbda..ccd6278157b 100644
---
a/regression-test/suites/external_table_p2/refactor_catalog_param/oss_hdfs_catalog_test.groovy
+++
b/regression-test/suites/external_table_p2/refactor_catalog_param/oss_hdfs_catalog_test.groovy
@@ -138,7 +138,7 @@ suite("oss_hdfs_catalog_test", "p2,external") {
'oss.hdfs.enabled'='true',
"""
String new_oss_hdfs_storage_properties = """
- 'fs.oss.support' = 'true',
+ 'fs.oss-hdfs.support' = 'true',
'oss.hdfs.access_key' = '${oss_hdfs_ak}',
'oss.hdfs.secret_key' = '${oss_hdfs_sk}',
'oss.hdfs.endpoint' = '${oss_hdfs_endpoint}',
@@ -231,4 +231,4 @@ suite("oss_hdfs_catalog_test", "p2,external") {
testQueryAndInsert(iceberg_hms_type_prop + warehouse +
new_oss_hdfs_storage_properties , "iceberg_hms_on_oss_hdfs_new",null)
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
