This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new cc6adeeb05b branch-4.0: [refactor](oss) unify FE OSS filesystem with
Jindo #61269 (#61416)
cc6adeeb05b is described below
commit cc6adeeb05b167cade3d656394eb05b18517c50b
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Mar 18 09:27:44 2026 +0800
branch-4.0: [refactor](oss) unify FE OSS filesystem with Jindo #61269
(#61416)
Cherry-picked from #61269
Co-authored-by: Calvin Kirs <[email protected]>
---
fe/be-java-extensions/hadoop-deps/pom.xml | 6 +-----
fe/be-java-extensions/preload-extensions/pom.xml | 5 -----
fe/fe-core/pom.xml | 6 ------
.../datasource/property/storage/OSSHdfsProperties.java | 4 ++--
.../doris/datasource/property/storage/OSSProperties.java | 7 ++++++-
.../datasource/property/storage/OSSPropertiesTest.java | 14 +++++++++++++-
fe/pom.xml | 12 ------------
.../refactor_catalog_param/oss_hdfs_catalog_test.groovy | 4 ++--
8 files changed, 24 insertions(+), 34 deletions(-)
diff --git a/fe/be-java-extensions/hadoop-deps/pom.xml
b/fe/be-java-extensions/hadoop-deps/pom.xml
index 54591563091..aba07779a80 100644
--- a/fe/be-java-extensions/hadoop-deps/pom.xml
+++ b/fe/be-java-extensions/hadoop-deps/pom.xml
@@ -38,10 +38,6 @@ under the License.
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aliyun</artifactId>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
@@ -135,4 +131,4 @@ under the License.
</plugins>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/fe/be-java-extensions/preload-extensions/pom.xml
b/fe/be-java-extensions/preload-extensions/pom.xml
index b7493adb579..2c66405d0bc 100644
--- a/fe/be-java-extensions/preload-extensions/pom.xml
+++ b/fe/be-java-extensions/preload-extensions/pom.xml
@@ -84,11 +84,6 @@ under the License.
<artifactId>paimon-s3</artifactId>
<version>${paimon.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-oss</artifactId>
- <version>${paimon.version}</version>
- </dependency>
<!-- For Avro and Hudi Scanner PreLoad -->
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 749343410c6..f0312a2bb92 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -547,12 +547,6 @@ under the License.
<version>${paimon.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-oss</artifactId>
- <version>${paimon.version}</version>
- </dependency>
-
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>glue</artifactId>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
index a461d644626..d8860c111be 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
@@ -174,8 +174,8 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
config.put("fs.oss.accessKeyId", accessKey);
config.put("fs.oss.accessKeySecret", secretKey);
config.put("fs.oss.region", region);
- config.put("fs.oss.impl",
"com.aliyun.jindodata.oss.JindoOssFileSystem");
- config.put("fs.AbstractFileSystem.oss.impl",
"com.aliyun.jindodata.oss.JindoOSS");
+ config.put("fs.oss.impl", OSSProperties.JINDO_OSS_FILE_SYSTEM_IMPL);
+ config.put("fs.AbstractFileSystem.oss.impl",
OSSProperties.JINDO_OSS_ABSTRACT_FILE_SYSTEM_IMPL);
if (StringUtils.isNotBlank(fsDefaultFS)) {
config.put(HDFS_DEFAULT_FS_NAME, fsDefaultFS);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
index e92a925811c..8d716001e4b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
@@ -161,6 +161,9 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
private static List<String> DLF_TYPE_KEYWORDS =
Arrays.asList("hive.metastore.type",
"iceberg.catalog.type", "paimon.catalog.type");
+ static final String JINDO_OSS_FILE_SYSTEM_IMPL =
"com.aliyun.jindodata.oss.JindoOssFileSystem";
+ static final String JINDO_OSS_ABSTRACT_FILE_SYSTEM_IMPL =
"com.aliyun.jindodata.oss.JindoOSS";
+
private static final String DLS_URI_KEYWORDS = "oss-dls.aliyuncs";
protected OSSProperties(Map<String, String> origProps) {
@@ -309,10 +312,12 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
@Override
public void initializeHadoopStorageConfig() {
super.initializeHadoopStorageConfig();
- hadoopStorageConfig.set("fs.oss.impl",
"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem");
+ hadoopStorageConfig.set("fs.oss.impl", JINDO_OSS_FILE_SYSTEM_IMPL);
+ hadoopStorageConfig.set("fs.AbstractFileSystem.oss.impl",
JINDO_OSS_ABSTRACT_FILE_SYSTEM_IMPL);
hadoopStorageConfig.set("fs.oss.accessKeyId", accessKey);
hadoopStorageConfig.set("fs.oss.accessKeySecret", secretKey);
hadoopStorageConfig.set("fs.oss.endpoint", endpoint);
+ hadoopStorageConfig.set("fs.oss.region", region);
}
/**
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index 912f20b3144..bd135ecb9ed 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -273,6 +273,19 @@ public class OSSPropertiesTest {
Assertions.assertFalse(s3Properties.hadoopStorageConfig.getBoolean("fs.oss.impl.disable.cache",
false));
}
+ @Test
+ public void testUseJindoFsForHadoopStorageConfig() throws UserException {
+ Map<String, String> props = Maps.newHashMap();
+ props.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com");
+ props.put("oss.region", "cn-hangzhou");
+ OSSProperties ossProperties = (OSSProperties)
StorageProperties.createPrimary(props);
+ Assertions.assertEquals(OSSProperties.JINDO_OSS_FILE_SYSTEM_IMPL,
+ ossProperties.hadoopStorageConfig.get("fs.oss.impl"));
+
Assertions.assertEquals(OSSProperties.JINDO_OSS_ABSTRACT_FILE_SYSTEM_IMPL,
+
ossProperties.hadoopStorageConfig.get("fs.AbstractFileSystem.oss.impl"));
+ Assertions.assertEquals("cn-hangzhou",
ossProperties.hadoopStorageConfig.get("fs.oss.region"));
+ }
+
@Test
public void testOSSBucketEndpointPathProperties() throws UserException {
Assertions.assertEquals("oss://my-bucket/path/to/dir/",
OSSProperties.rewriteOssBucketIfNecessary("oss://my-bucket/path/to/dir/"));
@@ -281,4 +294,3 @@ public class OSSPropertiesTest {
Assertions.assertEquals("https://bucket-name.oss-cn-hangzhou.aliyuncs.com/path/to/dir/file.txt";,
OSSProperties.rewriteOssBucketIfNecessary("https://bucket-name.oss-cn-hangzhou.aliyuncs.com/path/to/dir/file.txt";));
}
}
-
diff --git a/fe/pom.xml b/fe/pom.xml
index d85576e246a..dbef80ea426 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -722,18 +722,6 @@ under the License.
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aliyun</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <!-- pls confirm if this is a mandatory dependency -->
- <exclusion>
- <groupId>org.ini4j</groupId>
- <artifactId>ini4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>hive-catalog-shade</artifactId>
diff --git
a/regression-test/suites/external_table_p2/refactor_catalog_param/oss_hdfs_catalog_test.groovy
b/regression-test/suites/external_table_p2/refactor_catalog_param/oss_hdfs_catalog_test.groovy
index 9f589ab1e2d..657d3497dac 100644
---
a/regression-test/suites/external_table_p2/refactor_catalog_param/oss_hdfs_catalog_test.groovy
+++
b/regression-test/suites/external_table_p2/refactor_catalog_param/oss_hdfs_catalog_test.groovy
@@ -138,7 +138,7 @@ suite("oss_hdfs_catalog_test",
"p2,external,new_catalog_property") {
'oss.hdfs.enabled'='true',
"""
String new_oss_hdfs_storage_properties = """
- 'fs.oss.support' = 'true',
+ 'fs.oss-hdfs.support' = 'true',
'oss.hdfs.access_key' = '${oss_hdfs_ak}',
'oss.hdfs.secret_key' = '${oss_hdfs_sk}',
'oss.hdfs.endpoint' = '${oss_hdfs_endpoint}',
@@ -231,4 +231,4 @@ suite("oss_hdfs_catalog_test",
"p2,external,new_catalog_property") {
testQueryAndInsert(iceberg_hms_type_prop + warehouse +
new_oss_hdfs_storage_properties , "iceberg_hms_on_oss_hdfs_new",null)
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
