This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 17873d00871 [Improvement](multicatalog) support read tencent dlc on
lakefs (#36807)
17873d00871 is described below
commit 17873d00871c6a517a22222af371b8d2b24b4f97
Author: Yulei-Yang <[email protected]>
AuthorDate: Thu Jun 27 12:03:16 2024 +0800
[Improvement](multicatalog) support read tencent dlc on lakefs (#36807)
---
.../main/java/org/apache/doris/common/FeConstants.java | 1 +
.../java/org/apache/doris/common/util/LocationPath.java | 15 +++++++++++++++
.../doris/datasource/property/PropertyConverter.java | 3 ++-
.../doris/datasource/property/PropertyConverterTest.java | 2 +-
4 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
index 3012f6a62e7..1afa12856f0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
@@ -73,6 +73,7 @@ public class FeConstants {
public static final String FS_PREFIX_BOS = "bos";
public static final String FS_PREFIX_COS = "cos";
public static final String FS_PREFIX_COSN = "cosn";
+ public static final String FS_PREFIX_LAKEFS = "lakefs";
public static final String FS_PREFIX_OBS = "obs";
public static final String FS_PREFIX_OFS = "ofs";
public static final String FS_PREFIX_GFS = "gfs";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index fd7da29e519..a307ff63699 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -61,6 +61,7 @@ public class LocationPath {
COSN, // Tencent
OFS, // Tencent CHDFS
GFS, // Tencent GooseFs,
+ LAKEFS, // used by Tencent DLC
OSS, // Alibaba,
OSS_HDFS, // JindoFS on OSS
JFS, // JuiceFS,
@@ -158,6 +159,10 @@ public class LocationPath {
locationType = LocationType.COSN;
this.location = location;
break;
+ case FeConstants.FS_PREFIX_LAKEFS:
+ locationType = LocationType.COSN;
+ this.location = normalizedLakefsPath(location);
+ break;
case FeConstants.FS_PREFIX_VIEWFS:
locationType = LocationType.VIEWFS;
this.location = location;
@@ -270,6 +275,15 @@ public class LocationPath {
}
}
+ private static String normalizedLakefsPath(String location) {
+ int atIndex = location.indexOf("@dlc");
+ if (atIndex != -1) {
+ return "lakefs://" + location.substring(atIndex + 1);
+ } else {
+ return location;
+ }
+ }
+
public static Pair<FileSystemType, String> getFSIdentity(String location,
String bindBrokerName) {
LocationPath locationPath = new LocationPath(location);
FileSystemType fsType = (bindBrokerName != null) ?
FileSystemType.BROKER : locationPath.getFileSystemType();
@@ -337,6 +351,7 @@ public class LocationPath {
case GCS:
// ATTN, for COSN, on FE side, use HadoopFS to access, but on
BE, use S3 client to access.
case COSN:
+ case LAKEFS:
// now we only support S3 client for object storage on BE
return TFileType.FILE_S3;
case HDFS:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
index c10594c4977..a2ea2cee47e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -185,7 +185,7 @@ public class PropertyConverter {
return OBSFileSystem.class.getName();
} else if (fsScheme.equalsIgnoreCase("oss")) {
return AliyunOSSFileSystem.class.getName();
- } else if (fsScheme.equalsIgnoreCase("cosn")) {
+ } else if (fsScheme.equalsIgnoreCase("cosn") ||
fsScheme.equalsIgnoreCase("lakefs")) {
return CosFileSystem.class.getName();
} else {
return S3AFileSystem.class.getName();
@@ -341,6 +341,7 @@ public class PropertyConverter {
cosProperties.put(CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY,
props.get(CosProperties.ENDPOINT));
cosProperties.put("fs.cosn.impl.disable.cache", "true");
cosProperties.put("fs.cosn.impl", getHadoopFSImplByScheme("cosn"));
+ cosProperties.put("fs.lakefs.impl", getHadoopFSImplByScheme("lakefs"));
if (credential.isWhole()) {
cosProperties.put(CosNConfigKeys.COSN_USERINFO_SECRET_ID_KEY,
credential.getAccessKey());
cosProperties.put(CosNConfigKeys.COSN_USERINFO_SECRET_KEY_KEY,
credential.getSecretKey());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
index c70a0773ba1..c0ba75a4d36 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
@@ -471,7 +471,7 @@ public class PropertyConverterTest extends
TestWithFeService {
+ " 'cos.secret_key' = 'skk'\n"
+ ");";
testS3CompatibleCatalogProperties(catalogName0,
CosProperties.COS_PREFIX,
- "cos.ap-beijing.myqcloud.com", query0, 11, 16);
+ "cos.ap-beijing.myqcloud.com", query0, 11, 17);
String catalogName1 = "hms_oss";
String query1 = "create catalog " + catalogName1 + " properties (\n"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
