This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 6590e8e2098 branch-3.1: [fix](test connection) Fix the check when only
the bucket root directory is present. #58359 (#58390)
6590e8e2098 is described below
commit 6590e8e20985c8c033d24cb65bf68be8d98089a8
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Nov 27 10:26:15 2025 +0800
branch-3.1: [fix](test connection) Fix the check when only the bucket root
directory is present. #58359 (#58390)
Cherry-picked from #58359
Co-authored-by: zy-kkk <[email protected]>
---
.../connectivity/AbstractIcebergConnectivityTester.java | 13 +++++++++++++
.../AbstractS3CompatibleConnectivityTester.java | 17 ++++++++++-------
.../IcebergGlueMetaStoreConnectivityTester.java | 11 +----------
.../connectivity/IcebergRestConnectivityTester.java | 15 ---------------
4 files changed, 24 insertions(+), 32 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractIcebergConnectivityTester.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractIcebergConnectivityTester.java
index 9fd5077c82b..e18988f6af9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractIcebergConnectivityTester.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractIcebergConnectivityTester.java
@@ -19,7 +19,13 @@ package org.apache.doris.datasource.connectivity;
import
org.apache.doris.datasource.property.metastore.AbstractIcebergProperties;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.regex.Pattern;
+
public abstract class AbstractIcebergConnectivityTester implements
MetaConnectivityTester {
+
+ protected static final Pattern LOCATION_PATTERN =
Pattern.compile("^(s3|s3a|s3n)://.+");
protected final AbstractIcebergProperties properties;
protected AbstractIcebergConnectivityTester(AbstractIcebergProperties
properties) {
@@ -33,4 +39,11 @@ public abstract class AbstractIcebergConnectivityTester
implements MetaConnectiv
public String getTestLocation() {
return properties.getWarehouse();
}
+
+ protected String validateLocation(String location) {
+ if (StringUtils.isNotBlank(location) &&
LOCATION_PATTERN.matcher(location).matches()) {
+ return location;
+ }
+ return null;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java
index df9de5cb77c..551448dc767 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java
@@ -17,7 +17,6 @@
package org.apache.doris.datasource.connectivity;
-import org.apache.doris.common.util.S3URI;
import org.apache.doris.common.util.S3Util;
import
org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties;
import org.apache.doris.thrift.TStorageBackendType;
@@ -35,7 +34,14 @@ public abstract class AbstractS3CompatibleConnectivityTester
implements StorageC
public
AbstractS3CompatibleConnectivityTester(AbstractS3CompatibleProperties
properties, String testLocation) {
this.properties = properties;
- this.testLocation = testLocation;
+ // Normalize s3a:// and s3n:// schemes to s3://
+ String normalized = testLocation.replaceFirst("^s3[an]://", "s3://");
+ // If the path is just a bucket (e.g., s3://bucket or s3://bucket/),
add a test key
+ // because BE's S3URI parser requires a non-empty key
+ if (normalized.matches("^s3://[^/]+/?$")) {
+ normalized = normalized.replaceFirst("/?$", "/.connectivity_test");
+ }
+ this.testLocation = normalized;
}
@Override
@@ -52,10 +58,7 @@ public abstract class AbstractS3CompatibleConnectivityTester
implements StorageC
@Override
public void testFeConnection() throws Exception {
- S3URI s3Uri = S3URI.create(testLocation,
- Boolean.parseBoolean(properties.getUsePathStyle()),
-
Boolean.parseBoolean(properties.getForceParsingByStandardUrl()));
-
+ String bucket = URI.create(testLocation).getAuthority();
String endpoint = properties.getEndpoint();
try (S3Client client = S3Util.buildS3Client(
@@ -63,7 +66,7 @@ public abstract class AbstractS3CompatibleConnectivityTester
implements StorageC
properties.getRegion(),
Boolean.parseBoolean(properties.getUsePathStyle()),
properties.getAwsCredentialsProvider())) {
- client.headBucket(b -> b.bucket(s3Uri.getBucket()));
+ client.headBucket(b -> b.bucket(bucket));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/IcebergGlueMetaStoreConnectivityTester.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/IcebergGlueMetaStoreConnectivityTester.java
index 978fffceb73..068d8aa71b7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/IcebergGlueMetaStoreConnectivityTester.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/IcebergGlueMetaStoreConnectivityTester.java
@@ -22,10 +22,8 @@ import
org.apache.doris.datasource.property.metastore.AbstractIcebergProperties;
import org.apache.commons.lang3.StringUtils;
-import java.util.regex.Pattern;
public class IcebergGlueMetaStoreConnectivityTester extends
AbstractIcebergConnectivityTester {
- private static final Pattern S3_LOCATION_PATTERN =
Pattern.compile("^(s3|s3a)://.+");
private final AWSGlueMetaStoreBaseConnectivityTester glueTester;
public IcebergGlueMetaStoreConnectivityTester(AbstractIcebergProperties
properties,
@@ -57,18 +55,11 @@ public class IcebergGlueMetaStoreConnectivityTester extends
AbstractIcebergConne
return null;
}
- String location = validateS3Location(warehouse);
+ String location = validateLocation(warehouse);
if (location == null) {
throw new IllegalArgumentException(
"Iceberg Glue warehouse location must be in S3 format
(s3:// or s3a://), but got: " + warehouse);
}
return location;
}
-
- private String validateS3Location(String location) {
- if (StringUtils.isNotBlank(location) &&
S3_LOCATION_PATTERN.matcher(location).matches()) {
- return location;
- }
- return null;
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/IcebergRestConnectivityTester.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/IcebergRestConnectivityTester.java
index ca00fba1501..78707ebd672 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/IcebergRestConnectivityTester.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/IcebergRestConnectivityTester.java
@@ -20,17 +20,14 @@ package org.apache.doris.datasource.connectivity;
import
org.apache.doris.datasource.property.metastore.AbstractIcebergProperties;
import org.apache.doris.datasource.property.metastore.IcebergRestProperties;
-import org.apache.commons.lang3.StringUtils;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.rest.RESTSessionCatalog;
import java.util.Map;
-import java.util.regex.Pattern;
public class IcebergRestConnectivityTester extends
AbstractIcebergConnectivityTester {
// For Polaris REST catalog compatibility
private static final String DEFAULT_BASE_LOCATION =
"default-base-location";
- private static final Pattern LOCATION_PATTERN =
Pattern.compile("^(s3|s3a)://.+");
private String warehouseLocation;
@@ -76,16 +73,4 @@ public class IcebergRestConnectivityTester extends
AbstractIcebergConnectivityTe
// If configured warehouse is not valid, fallback to REST API
warehouse (already validated)
return this.warehouseLocation;
}
-
- /**
- * Validate if the given location is a valid storage URI.
- * This method is specific to IcebergRestConnectivityTester because it
needs to
- * validate warehouse locations returned from REST API.
- */
- private String validateLocation(String location) {
- if (StringUtils.isNotBlank(location) &&
LOCATION_PATTERN.matcher(location).matches()) {
- return location;
- }
- return null;
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
