This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 8a1ebba1cc9 [Improvement](multicatalog) support read tencent dlc table
on lakefs (#36891)
8a1ebba1cc9 is described below
commit 8a1ebba1cc90b2b4d05dea596d03b66fbd1bbe79
Author: Yulei-Yang <[email protected]>
AuthorDate: Thu Jun 27 14:03:48 2024 +0800
[Improvement](multicatalog) support read tencent dlc table on lakefs
(#36891)
bp #36823
---
.../main/java/org/apache/doris/common/FeConstants.java | 1 +
.../java/org/apache/doris/common/util/LocationPath.java | 15 +++++++++++++++
.../doris/datasource/property/PropertyConverter.java | 3 ++-
.../doris/datasource/property/PropertyConverterTest.java | 2 +-
4 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
index a502d79e032..f137c4cab49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
@@ -71,6 +71,7 @@ public class FeConstants {
public static final String FS_PREFIX_BOS = "bos";
public static final String FS_PREFIX_COS = "cos";
public static final String FS_PREFIX_COSN = "cosn";
+ public static final String FS_PREFIX_LAKEFS = "lakefs";
public static final String FS_PREFIX_OBS = "obs";
public static final String FS_PREFIX_OFS = "ofs";
public static final String FS_PREFIX_GFS = "gfs";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index dd1641126bf..eccb483578a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -62,6 +62,7 @@ public class LocationPath {
COSN, // Tencent
OFS, // Tencent CHDFS
GFS, // Tencent GooseFs,
+ LAKEFS, // used by Tencent DLC
OSS, // Alibaba,
OSS_HDFS, // JindoFS on OSS
JFS, // JuiceFS,
@@ -163,6 +164,10 @@ public class LocationPath {
locationType = LocationType.COSN;
this.location = location;
break;
+ case FeConstants.FS_PREFIX_LAKEFS:
+ locationType = LocationType.COSN;
+ this.location = normalizedLakefsPath(location);
+ break;
case FeConstants.FS_PREFIX_VIEWFS:
locationType = LocationType.VIEWFS;
this.location = location;
@@ -277,6 +282,15 @@ public class LocationPath {
}
}
+ private static String normalizedLakefsPath(String location) {
+ int atIndex = location.indexOf("@dlc");
+ if (atIndex != -1) {
+ return "lakefs://" + location.substring(atIndex + 1);
+ } else {
+ return location;
+ }
+ }
+
public static Pair<FileSystemType, String> getFSIdentity(String location,
String bindBrokerName) {
LocationPath locationPath = new LocationPath(location);
FileSystemType fsType = (bindBrokerName != null) ?
FileSystemType.BROKER : locationPath.getFileSystemType();
@@ -351,6 +365,7 @@ public class LocationPath {
case GCS:
// ATTN, for COSN, on FE side, use HadoopFS to access, but on
BE, use S3 client to access.
case COSN:
+ case LAKEFS:
// now we only support S3 client for object storage on BE
return TFileType.FILE_S3;
case HDFS:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
index 425ea6cdcfe..9dea5eb3802 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -188,7 +188,7 @@ public class PropertyConverter {
return OBSFileSystem.class.getName();
} else if (fsScheme.equalsIgnoreCase("oss")) {
return AliyunOSSFileSystem.class.getName();
- } else if (fsScheme.equalsIgnoreCase("cosn")) {
+ } else if (fsScheme.equalsIgnoreCase("cosn") ||
fsScheme.equalsIgnoreCase("lakefs")) {
return CosFileSystem.class.getName();
} else {
return S3AFileSystem.class.getName();
@@ -354,6 +354,7 @@ public class PropertyConverter {
cosProperties.put(CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY,
props.get(CosProperties.ENDPOINT));
cosProperties.put("fs.cosn.impl.disable.cache", "true");
cosProperties.put("fs.cosn.impl", getHadoopFSImplByScheme("cosn"));
+ cosProperties.put("fs.lakefs.impl", getHadoopFSImplByScheme("lakefs"));
if (credential.isWhole()) {
cosProperties.put(CosNConfigKeys.COSN_USERINFO_SECRET_ID_KEY,
credential.getAccessKey());
cosProperties.put(CosNConfigKeys.COSN_USERINFO_SECRET_KEY_KEY,
credential.getSecretKey());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
index 4e4b0408641..b31b79de783 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
@@ -470,7 +470,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'cos.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName0,
CosProperties.COS_PREFIX,
- "cos.ap-beijing.myqcloud.com", query0, 12, 17);
+ "cos.ap-beijing.myqcloud.com", query0, 12, 18);
String catalogName1 = "hms_oss";
String query1 = "create catalog " + catalogName1 + " properties (\n"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
