This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 2cb3d84ec47 branch-4.0: [improve](glue)Refine Glue region and endpoint
initialization #57365 (#57480)
2cb3d84ec47 is described below
commit 2cb3d84ec47d8d55f97892f4dcbda9d545fd8b30
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Nov 5 00:54:49 2025 +0800
branch-4.0: [improve](glue)Refine Glue region and endpoint initialization
#57365 (#57480)
Cherry-picked from #57365
Co-authored-by: Calvin Kirs <[email protected]>
---
.../java/org/apache/doris/backup/Repository.java | 16 ++++++------
.../metastore/AWSGlueMetaStoreBaseProperties.java | 18 +++++++++-----
.../AWSGlueMetaStoreBasePropertiesTest.java | 29 +++++++++++++---------
3 files changed, 38 insertions(+), 25 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
index 269417a307a..14578cb3f2d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
@@ -29,6 +29,7 @@ import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.PrintableMap;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.fs.PersistentFileSystem;
@@ -204,14 +205,15 @@ public class Repository implements Writable,
GsonPostProcessable {
StorageProperties storageProperties =
StorageProperties.createPrimary(this.fileSystem.properties);
this.fileSystem = FileSystemFactory.get(storageProperties);
} catch (RuntimeException exception) {
- LOG.warn("Failed to create file system from properties, error msg
{}",
+ LOG.warn("File system initialization failed due to incompatible
configuration parameters. "
+ + "The system has reverted to BrokerFileSystem as
a fallback. "
+ + "However, the current configuration is not
supported by this version and"
+ + " cannot be used as is. "
+ + "Please review the configuration and update it
to match the supported parameter"
+ + " format. Root cause: {}",
ExceptionUtils.getRootCause(exception), exception);
- throw new IllegalStateException(
- "Failed to initialize file system due to incompatible
configuration with the current version. "
- + "This may be caused by a change in property
formats or deprecated settings. "
- + "Please verify your configuration and ensure it
matches the "
- + "new version requirements. error msg: "
- + ExceptionUtils.getRootCause(exception),
exception);
+ BrokerProperties brokerProperties =
BrokerProperties.of(this.fileSystem.name, this.fileSystem.properties);
+ this.fileSystem = FileSystemFactory.get(brokerProperties);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
index 2f98830adc4..762b2808423 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
@@ -95,7 +95,7 @@ public class AWSGlueMetaStoreBaseProperties {
private ParamRules buildRules() {
return new ParamRules().requireTogether(new String[]{glueAccessKey,
glueSecretKey},
- "glue.access_key and glue.secret_key must be set together")
+ "glue.access_key and glue.secret_key must be set
together")
.requireAtLeastOne(new String[]{glueAccessKey, glueIAMRole},
"At least one of glue.access_key or glue.role_arn must
be set")
.requireAtLeastOne(new String[]{glueEndpoint, glueRegion},
@@ -104,14 +104,20 @@ public class AWSGlueMetaStoreBaseProperties {
private void checkAndInit() {
buildRules().validate();
-
+ if (StringUtils.isNotBlank(glueRegion) &&
StringUtils.isNotBlank(glueEndpoint)) {
+ return;
+ }
+ if (StringUtils.isBlank(glueEndpoint) &&
StringUtils.isNotBlank(glueRegion)) {
+ return;
+ }
+ // glue region is not set, try to extract from endpoint
Matcher matcher = ENDPOINT_PATTERN.matcher(glueEndpoint.toLowerCase());
- if (!matcher.matches()) {
- throw new IllegalArgumentException("Invalid AWS Glue endpoint: " +
glueEndpoint);
+ if (matcher.matches()) {
+ this.glueRegion = extractRegionFromEndpoint(matcher);
}
-
if (StringUtils.isBlank(glueRegion)) {
- this.glueRegion = extractRegionFromEndpoint(matcher);
+ //follow aws sdk default region
+ glueRegion = "us-east-1";
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBasePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBasePropertiesTest.java
index bcb04d0da2f..b49cb8d73dc 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBasePropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBasePropertiesTest.java
@@ -42,6 +42,20 @@ public class AWSGlueMetaStoreBasePropertiesTest {
Assertions.assertEquals("us-east-1", glueProps.glueRegion);
}
+ @Test
+ void testValidPropertiesWithRegion() {
+ Map<String, String> props = baseValidProps();
+ props.put("glue.region", "us-east-1");
+ props.put("glue.endpoint", "https://glue.us-east-1.amazonaws.com.cn";);
+ AWSGlueMetaStoreBaseProperties glueProps =
AWSGlueMetaStoreBaseProperties.of(props);
+
Assertions.assertTrue("https://glue.us-east-1.amazonaws.com.cn".equals(glueProps.glueEndpoint));
+ Assertions.assertEquals("us-east-1", glueProps.glueRegion);
+ props.remove("glue.region");
+ glueProps = AWSGlueMetaStoreBaseProperties.of(props);
+
Assertions.assertTrue("https://glue.us-east-1.amazonaws.com.cn".equals(glueProps.glueEndpoint));
+ Assertions.assertEquals("us-east-1", glueProps.glueRegion);
+ }
+
@Test
void testValidPropertiesWithExplicitRegion() {
Map<String, String> props = baseValidProps();
@@ -86,27 +100,18 @@ public class AWSGlueMetaStoreBasePropertiesTest {
}
@Test
- void testInvalidEndpointThrows() {
+ void testInvalidEndpoint() {
Map<String, String> props = baseValidProps();
props.put("glue.endpoint", "http://invalid-endpoint.com";);
-
- IllegalArgumentException ex = Assertions.assertThrows(
- IllegalArgumentException.class,
- () -> AWSGlueMetaStoreBaseProperties.of(props)
- );
- Assertions.assertTrue(ex.getMessage().contains("Invalid AWS Glue
endpoint"));
+ Assertions.assertDoesNotThrow(() ->
AWSGlueMetaStoreBaseProperties.of(props));
}
@Test
void testExtractRegionFailsWhenPatternMatchesButNoRegion() {
Map<String, String> props = baseValidProps();
props.put("glue.endpoint", "glue..amazonaws.com"); // malformed
-
- IllegalArgumentException ex = Assertions.assertThrows(
- IllegalArgumentException.class,
- () -> AWSGlueMetaStoreBaseProperties.of(props)
+ Assertions.assertDoesNotThrow(() ->
AWSGlueMetaStoreBaseProperties.of(props)
);
- Assertions.assertTrue(ex.getMessage().contains("Invalid AWS Glue
endpoint"));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
