This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d349c760bbb [fix](load) fix s3 load connection check failed (#56123)
d349c760bbb is described below
commit d349c760bbbf82137fd5fc599c0f889dc1544257
Author: Xin Liao <[email protected]>
AuthorDate: Fri Sep 19 10:51:21 2025 +0800
[fix](load) fix s3 load connection check failed (#56123)
### What problem does this PR solve?
PR #51219 fixed the issue of S3 load HTTPS access failure, but the
changes were accidentally dropped during the cherry-pick in PR #52479.
Now we need to add those changes back.
---
.../java/org/apache/doris/analysis/LoadStmt.java | 16 +++++++-
...t_domain_connection_and_ak_sk_correction.groovy | 43 ++++++++++++++++++++++
2 files changed, 57 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
index 183d3c11094..29d8a1ddf8f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
@@ -587,7 +587,11 @@ public class LoadStmt extends DdlStmt implements
NotFallbackInParser {
private void checkEndpoint(String endpoint) throws UserException {
HttpURLConnection connection = null;
try {
- String urlStr = "http://"; + endpoint;
+ String urlStr = endpoint;
+ // Add default protocol if not specified
+ if (!endpoint.startsWith("http://";) &&
!endpoint.startsWith("https://";)) {
+ urlStr = "http://"; + endpoint;
+ }
SecurityChecker.getInstance().startSSRFChecking(urlStr);
URL url = new URL(urlStr);
connection = (HttpURLConnection) url.openConnection();
@@ -599,7 +603,13 @@ public class LoadStmt extends DdlStmt implements
NotFallbackInParser {
if (e instanceof UserException) {
msg = ((UserException) e).getDetailMessage();
} else {
- msg = e.getMessage();
+ msg = String.format("%s: %s", e.getClass().getSimpleName(),
+ e.getMessage() != null ? e.getMessage() : "Unknown
error");
+ if (e.getCause() != null) {
+ msg += String.format(" (Caused by: %s: %s)",
+ e.getCause().getClass().getSimpleName(),
+ e.getCause().getMessage() != null ?
e.getCause().getMessage() : "Unknown cause");
+ }
}
throw new UserException(InternalErrorCode.GET_REMOTE_DATA_ERROR,
"Failed to access object storage, message=" + msg, e);
@@ -638,6 +648,8 @@ public class LoadStmt extends DdlStmt implements
NotFallbackInParser {
}
public void checkWhiteList(String endpoint) throws UserException {
+ endpoint = endpoint.replaceFirst("^http://";, "");
+ endpoint = endpoint.replaceFirst("^https://";, "");
List<String> whiteList = new
ArrayList<>(Arrays.asList(Config.s3_load_endpoint_white_list));
whiteList.removeIf(String::isEmpty);
if (!whiteList.isEmpty() && !whiteList.contains(endpoint)) {
diff --git
a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy
b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy
index 927c267718b..3403d4f3fd3 100644
---
a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy
+++
b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy
@@ -79,6 +79,49 @@ suite("test_domain_connection_and_ak_sk_correction",
"load_p0") {
"""
logger.info("the first sql result is {}", result)
+ def endpoint = getS3Endpoint().replace("http://";, "").replace("https://";,
"")
+ def httpEndpoint = "http://"; + endpoint
+ label = UUID.randomUUID().toString().replace("-", "")
+ result = sql """
+ LOAD LABEL ${label}
+ (
+ DATA
INFILE("s3://${getS3BucketName()}/regression/tpch/sf1/part.tbl")
+ INTO TABLE ${tableName}
+ COLUMNS TERMINATED BY "|"
+ (p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container,
p_retailprice, p_comment, temp)
+ )
+ WITH S3
+ (
+ "AWS_ENDPOINT" = "${httpEndpoint}",
+ "AWS_ACCESS_KEY" = "${getS3AK()}",
+ "AWS_SECRET_KEY" = "${getS3SK()}",
+ "AWS_REGION" = "${getS3Region()}",
+ "PROVIDER" = "${getS3Provider()}"
+ );
+ """
+ logger.info("the first sql result is {}", result)
+
+ def httpsEndpoint = "https://"; + endpoint
+ label = UUID.randomUUID().toString().replace("-", "")
+ result = sql """
+ LOAD LABEL ${label}
+ (
+ DATA
INFILE("s3://${getS3BucketName()}/regression/tpch/sf1/part.tbl")
+ INTO TABLE ${tableName}
+ COLUMNS TERMINATED BY "|"
+ (p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container,
p_retailprice, p_comment, temp)
+ )
+ WITH S3
+ (
+ "AWS_ENDPOINT" = "${httpsEndpoint}",
+ "AWS_ACCESS_KEY" = "${getS3AK()}",
+ "AWS_SECRET_KEY" = "${getS3SK()}",
+ "AWS_REGION" = "${getS3Region()}",
+ "PROVIDER" = "${getS3Provider()}"
+ );
+ """
+ logger.info("the first sql result is {}", result)
+
label = UUID.randomUUID().toString().replace("-", "")
try {
result = sql """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
