morningman commented on code in PR #21504:
URL: https://github.com/apache/doris/pull/21504#discussion_r1255325162
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java:
##########
@@ -268,6 +274,21 @@ private static Map<String, String>
convertToOSSProperties(Map<String, String> pr
return ossProperties;
}
+ private static void rewriteHdfsOnOssProperties(Map<String, String>
ossProperties, String endpoint) {
+ if (!S3Util.isHdfsOnOssEndpoint(endpoint)) {
+ // just for robustness here, avoid wrong endpoint when oss-hdfs is
enabled.
+ // convert "oss-cn-beijing.aliyuncs.com" to
"cn-beijing.oss-dls.aliyuncs.com"
Review Comment:
Add reference link
##########
fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java:
##########
@@ -87,11 +87,20 @@ public static Path toScanRangeLocation(String location) {
if (pos == -1) {
throw new RuntimeException("No '://' found in location: " +
location);
}
- location = "s3" + location.substring(pos);
+ if (isHdfsOnOssEndpoint(location)) {
+ // if hdfs service is enabled on oss, use oss location
+ location = "oss" + location.substring(pos);
+ } else {
+ location = "s3" + location.substring(pos);
+ }
}
return new Path(location);
}
+ public static boolean isHdfsOnOssEndpoint(String location) {
+ return location.contains("oss-dls.aliyuncs");
Review Comment:
Add reference link
##########
fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java:
##########
@@ -87,11 +87,20 @@ public static Path toScanRangeLocation(String location) {
if (pos == -1) {
throw new RuntimeException("No '://' found in location: " +
location);
}
- location = "s3" + location.substring(pos);
+ if (isHdfsOnOssEndpoint(location)) {
+ // if hdfs service is enabled on oss, use oss location
+ location = "oss" + location.substring(pos);
Review Comment:
Add example
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
