This is an automated email from the ASF dual-hosted git repository.
ashingau pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0dcbc5ee18b [fix](backup) missing use_path_style properties for minio
(#25803)
0dcbc5ee18b is described below
commit 0dcbc5ee18b1a2947f314b94ffbbf710103d5c5c
Author: Mingyu Chen <[email protected]>
AuthorDate: Thu Oct 26 01:19:28 2023 +0800
[fix](backup) missing use_path_style properties for minio (#25803)
Follow #25496.
In #25496, I fixed the issue that the aws s3 properties are invalid when
passing from FE to BE.
But it missed the `use_path_style` property, which is useful for minio
access.
This PR fix it.
---
be/src/util/s3_util.h | 2 +-
.../datasource/property/S3ClientBEProperties.java | 13 +++
.../property/constants/S3Properties.java | 2 +
.../datasource/property/PropertyConverterTest.java | 125 ++++++++++++++++++++-
4 files changed, 140 insertions(+), 2 deletions(-)
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index 1f00a82aa98..247c6e5ef28 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -64,7 +64,7 @@ const static std::string S3_REGION = "AWS_REGION";
const static std::string S3_TOKEN = "AWS_TOKEN";
const static std::string S3_MAX_CONN_SIZE = "AWS_MAX_CONN_SIZE";
const static std::string S3_REQUEST_TIMEOUT_MS = "AWS_REQUEST_TIMEOUT_MS";
-const static std::string S3_CONN_TIMEOUT_MS = "AWS_CONN_TIMEOUT_MS";
+const static std::string S3_CONN_TIMEOUT_MS = "AWS_CONNECTION_TIMEOUT_MS";
struct S3Conf {
std::string ak;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java
index fe912c10384..20da63656e2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java
@@ -23,6 +23,7 @@ import
org.apache.doris.datasource.property.constants.MinioProperties;
import org.apache.doris.datasource.property.constants.ObsProperties;
import org.apache.doris.datasource.property.constants.OssProperties;
import org.apache.doris.datasource.property.constants.S3Properties;
+import org.apache.doris.datasource.property.constants.S3Properties.Env;
import java.util.HashMap;
import java.util.Map;
@@ -71,6 +72,18 @@ public class S3ClientBEProperties {
if (properties.containsKey(S3Properties.BUCKET)) {
beProperties.put(S3Properties.Env.BUCKET,
properties.get(S3Properties.BUCKET));
}
+ if (properties.containsKey(S3Properties.MAX_CONNECTIONS)) {
+ beProperties.put(Env.MAX_CONNECTIONS,
properties.get(S3Properties.MAX_CONNECTIONS));
+ }
+ if (properties.containsKey(S3Properties.REQUEST_TIMEOUT_MS)) {
+ beProperties.put(Env.REQUEST_TIMEOUT_MS,
properties.get(S3Properties.REQUEST_TIMEOUT_MS));
+ }
+ if (properties.containsKey(S3Properties.CONNECTION_TIMEOUT_MS)) {
+ beProperties.put(Env.CONNECTION_TIMEOUT_MS,
properties.get(S3Properties.CONNECTION_TIMEOUT_MS));
+ }
+ if (properties.containsKey(PropertyConverter.USE_PATH_STYLE)) {
+ beProperties.put(PropertyConverter.USE_PATH_STYLE,
properties.get(PropertyConverter.USE_PATH_STYLE));
+ }
return beProperties;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
index a1da49257bb..ea3a8333b30 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
@@ -150,6 +150,8 @@ public class S3Properties extends BaseProperties {
} else if
(entry.getKey().startsWith(MinioProperties.MINIO_PREFIX)) {
String s3Key =
entry.getKey().replace(MinioProperties.MINIO_PREFIX, S3Properties.S3_PREFIX);
s3Properties.put(s3Key, entry.getValue());
+ } else {
+ s3Properties.put(entry.getKey(), entry.getValue());
}
}
return s3Properties;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
index bf6f7bf6b99..153f989b50b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
@@ -36,6 +36,7 @@ import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.datasource.property.constants.CosProperties;
+import org.apache.doris.datasource.property.constants.GCSProperties;
import org.apache.doris.datasource.property.constants.MinioProperties;
import org.apache.doris.datasource.property.constants.ObsProperties;
import org.apache.doris.datasource.property.constants.OssProperties;
@@ -46,6 +47,7 @@ import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.utframe.TestWithFeService;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -405,11 +407,132 @@ public class PropertyConverterTest extends
TestWithFeService {
return (HMSExternalCatalog)
Env.getCurrentEnv().getCatalogMgr().getCatalog(name);
}
-
@Test
public void testSerialization() throws Exception {
MetaContext metaContext = new MetaContext();
metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT);
metaContext.setThreadLocalInfo();
}
+
+ @Test
+ public void testS3PropertiesConvertor() {
+ // 1. AWS
+ Map<String, String> origProp = Maps.newHashMap();
+ origProp.put(S3Properties.Env.ACCESS_KEY, "ak");
+ origProp.put(S3Properties.Env.SECRET_KEY, "sk");
+ origProp.put(S3Properties.Env.ENDPOINT, "endpoint");
+ origProp.put(S3Properties.Env.REGION, "region");
+ origProp.put(PropertyConverter.USE_PATH_STYLE, "true");
+ Map<String, String> beProperties =
S3ClientBEProperties.getBeFSProperties(origProp);
+ Assertions.assertEquals(5, beProperties.size());
+ Assertions.assertEquals("ak",
beProperties.get(S3Properties.Env.ACCESS_KEY));
+ Assertions.assertEquals("sk",
beProperties.get(S3Properties.Env.SECRET_KEY));
+ Assertions.assertEquals("endpoint",
beProperties.get(S3Properties.Env.ENDPOINT));
+ Assertions.assertEquals("region",
beProperties.get(S3Properties.Env.REGION));
+ Assertions.assertEquals("true",
beProperties.get(PropertyConverter.USE_PATH_STYLE));
+
+ // 2. s3.
+ origProp = Maps.newHashMap();
+ origProp.put(S3Properties.ACCESS_KEY, "ak");
+ origProp.put(S3Properties.SECRET_KEY, "sk");
+ origProp.put(S3Properties.ENDPOINT, "endpoint");
+ origProp.put(S3Properties.REGION, "region");
+ origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
+ beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
+ Assertions.assertEquals(5, beProperties.size());
+ Assertions.assertEquals("ak",
beProperties.get(S3Properties.Env.ACCESS_KEY));
+ Assertions.assertEquals("sk",
beProperties.get(S3Properties.Env.SECRET_KEY));
+ Assertions.assertEquals("endpoint",
beProperties.get(S3Properties.Env.ENDPOINT));
+ Assertions.assertEquals("region",
beProperties.get(S3Properties.Env.REGION));
+ Assertions.assertEquals("false",
beProperties.get(PropertyConverter.USE_PATH_STYLE));
+
+ // 3. minio.
+ origProp = Maps.newHashMap();
+ origProp.put(MinioProperties.ACCESS_KEY, "ak");
+ origProp.put(MinioProperties.SECRET_KEY, "sk");
+ origProp.put(MinioProperties.ENDPOINT, "endpoint");
+ origProp.put(MinioProperties.REGION, "region");
+ origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
+ beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
+ Assertions.assertEquals(5, beProperties.size());
+ Assertions.assertEquals("ak",
beProperties.get(S3Properties.Env.ACCESS_KEY));
+ Assertions.assertEquals("sk",
beProperties.get(S3Properties.Env.SECRET_KEY));
+ Assertions.assertEquals("endpoint",
beProperties.get(S3Properties.Env.ENDPOINT));
+ Assertions.assertEquals("region",
beProperties.get(S3Properties.Env.REGION));
+ Assertions.assertEquals("false",
beProperties.get(PropertyConverter.USE_PATH_STYLE));
+
+ // 3.1 minio without region
+ origProp = Maps.newHashMap();
+ origProp.put(MinioProperties.ACCESS_KEY, "ak");
+ origProp.put(MinioProperties.SECRET_KEY, "sk");
+ origProp.put(MinioProperties.ENDPOINT, "endpoint");
+ origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
+ beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
+ Assertions.assertEquals(5, beProperties.size());
+ Assertions.assertEquals("ak",
beProperties.get(S3Properties.Env.ACCESS_KEY));
+ Assertions.assertEquals("sk",
beProperties.get(S3Properties.Env.SECRET_KEY));
+ Assertions.assertEquals("endpoint",
beProperties.get(S3Properties.Env.ENDPOINT));
+ Assertions.assertEquals(MinioProperties.DEFAULT_REGION,
beProperties.get(S3Properties.Env.REGION));
+ Assertions.assertEquals("false",
beProperties.get(PropertyConverter.USE_PATH_STYLE));
+
+ // 4. obs
+ origProp = Maps.newHashMap();
+ origProp.put(ObsProperties.ACCESS_KEY, "ak");
+ origProp.put(ObsProperties.SECRET_KEY, "sk");
+ origProp.put(ObsProperties.ENDPOINT, "endpoint");
+ origProp.put(ObsProperties.REGION, "region");
+ origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
+ beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
+ Assertions.assertEquals(5, beProperties.size());
+ Assertions.assertEquals("ak",
beProperties.get(S3Properties.Env.ACCESS_KEY));
+ Assertions.assertEquals("sk",
beProperties.get(S3Properties.Env.SECRET_KEY));
+ Assertions.assertEquals("endpoint",
beProperties.get(S3Properties.Env.ENDPOINT));
+ Assertions.assertEquals("region",
beProperties.get(S3Properties.Env.REGION));
+ Assertions.assertEquals("false",
beProperties.get(PropertyConverter.USE_PATH_STYLE));
+
+ // 4. oss
+ origProp = Maps.newHashMap();
+ origProp.put(OssProperties.ACCESS_KEY, "ak");
+ origProp.put(OssProperties.SECRET_KEY, "sk");
+ origProp.put(OssProperties.ENDPOINT, "endpoint");
+ origProp.put(OssProperties.REGION, "region");
+ origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
+ beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
+ Assertions.assertEquals(5, beProperties.size());
+ Assertions.assertEquals("ak",
beProperties.get(S3Properties.Env.ACCESS_KEY));
+ Assertions.assertEquals("sk",
beProperties.get(S3Properties.Env.SECRET_KEY));
+ Assertions.assertEquals("endpoint",
beProperties.get(S3Properties.Env.ENDPOINT));
+ Assertions.assertEquals("region",
beProperties.get(S3Properties.Env.REGION));
+ Assertions.assertEquals("false",
beProperties.get(PropertyConverter.USE_PATH_STYLE));
+
+ // 4. cos
+ origProp = Maps.newHashMap();
+ origProp.put(CosProperties.ACCESS_KEY, "ak");
+ origProp.put(CosProperties.SECRET_KEY, "sk");
+ origProp.put(CosProperties.ENDPOINT, "endpoint");
+ origProp.put(CosProperties.REGION, "region");
+ origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
+ beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
+ Assertions.assertEquals(5, beProperties.size());
+ Assertions.assertEquals("ak",
beProperties.get(S3Properties.Env.ACCESS_KEY));
+ Assertions.assertEquals("sk",
beProperties.get(S3Properties.Env.SECRET_KEY));
+ Assertions.assertEquals("endpoint",
beProperties.get(S3Properties.Env.ENDPOINT));
+ Assertions.assertEquals("region",
beProperties.get(S3Properties.Env.REGION));
+ Assertions.assertEquals("false",
beProperties.get(PropertyConverter.USE_PATH_STYLE));
+
+ // 5. gs
+ origProp = Maps.newHashMap();
+ origProp.put(GCSProperties.ACCESS_KEY, "ak");
+ origProp.put(GCSProperties.SECRET_KEY, "sk");
+ origProp.put(GCSProperties.ENDPOINT, "endpoint");
+ origProp.put(GCSProperties.REGION, "region");
+ origProp.put(PropertyConverter.USE_PATH_STYLE, "false");
+ beProperties = S3ClientBEProperties.getBeFSProperties(origProp);
+ Assertions.assertEquals(5, beProperties.size());
+ Assertions.assertEquals("ak",
beProperties.get(S3Properties.Env.ACCESS_KEY));
+ Assertions.assertEquals("sk",
beProperties.get(S3Properties.Env.SECRET_KEY));
+ Assertions.assertEquals("endpoint",
beProperties.get(S3Properties.Env.ENDPOINT));
+ Assertions.assertEquals("region",
beProperties.get(S3Properties.Env.REGION));
+ Assertions.assertEquals("false",
beProperties.get(PropertyConverter.USE_PATH_STYLE));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
