This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 83e31d90b9e [improve](s3) s3 load provider should be case insensitive
(#39957)
83e31d90b9e is described below
commit 83e31d90b9ed8f7adbf20456f81f20a04c19b233
Author: Kaijie Chen <[email protected]>
AuthorDate: Fri Sep 6 00:44:35 2024 +0800
[improve](s3) s3 load provider should be case insensitive (#39957)
## Proposed changes
Some S3 vendors strictly require upper case provider info.
Provider properties in broker load `WITH S3` statement should be case
insensitive.
---
be/src/util/s3_util.cpp | 3 ++-
fe/fe-core/src/main/java/org/apache/doris/analysis/CopyStmt.java | 3 ++-
fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java | 3 ++-
.../src/main/java/org/apache/doris/analysis/StageProperties.java | 3 ++-
.../apache/doris/datasource/property/constants/S3Properties.java | 6 ++++--
fe/fe-core/src/test/java/org/apache/doris/analysis/StageTest.java | 3 ++-
.../src/main/groovy/org/apache/doris/regression/Config.groovy | 2 +-
.../src/main/groovy/org/apache/doris/regression/suite/Suite.groovy | 2 +-
8 files changed, 16 insertions(+), 9 deletions(-)
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index ab291c7340c..b2f4cdc3ce7 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -370,7 +370,8 @@ Status S3ClientFactory::convert_properties_to_s3_conf(
}
}
if (auto it = properties.find(S3_PROVIDER); it != properties.end()) {
- if (0 == strcmp(it->second.c_str(), AZURE_PROVIDER_STRING)) {
+ // S3 Provider properties should be case insensitive.
+ if (0 == strcasecmp(it->second.c_str(), AZURE_PROVIDER_STRING)) {
s3_conf->client_conf.provider = io::ObjStorageType::AZURE;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CopyStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CopyStmt.java
index db94c106072..6bd4d3506d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CopyStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CopyStmt.java
@@ -205,7 +205,8 @@ public class CopyStmt extends DdlStmt implements
NotFallbackInParser {
}
brokerProperties.put(S3_BUCKET, objInfo.getBucket());
brokerProperties.put(S3_PREFIX, objInfo.getPrefix());
- brokerProperties.put(S3Properties.PROVIDER,
objInfo.getProvider().toString());
+ // S3 Provider properties should be case insensitive.
+ brokerProperties.put(S3Properties.PROVIDER,
objInfo.getProvider().toString().toUpperCase());
StageProperties stageProperties = new
StageProperties(stagePB.getPropertiesMap());
this.copyIntoProperties.mergeProperties(stageProperties);
this.copyIntoProperties.analyze();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
index b447bf92690..790bedda198 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java
@@ -527,7 +527,8 @@ public class LoadStmt extends DdlStmt implements
NotFallbackInParser {
Map<String, String> properties = brokerDesc.getProperties();
for (Map.Entry<String, String> entry : properties.entrySet()) {
if (entry.getKey().equalsIgnoreCase(S3Properties.PROVIDER)) {
- return entry.getValue();
+ // S3 Provider properties should be case insensitive.
+ return entry.getValue().toUpperCase();
}
}
return S3Properties.S3_PROVIDER;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/StageProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/StageProperties.java
index 2ee8628fd48..063588c5c02 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StageProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StageProperties.java
@@ -135,7 +135,8 @@ public class StageProperties extends CopyProperties {
}
properties.put(PREFIX, prefix);
// analyze provider
- String provider = properties.get(PROVIDER);
+ // S3 Provider properties should be case insensitive.
+ String provider = properties.get(PROVIDER).toUpperCase();
if (!EnumUtils.isValidEnumIgnoreCase(ObjectStoreInfoPB.Provider.class,
provider)) {
throw new AnalysisException("Property " + PROVIDER + " with
invalid value " + provider);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
index fa7604dc56b..1f7342e80b8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
@@ -197,7 +197,8 @@ public class S3Properties extends BaseProperties {
private static void checkProvider(Map<String, String> properties) throws
DdlException {
if (properties.containsKey(PROVIDER)) {
properties.put(PROVIDER, properties.get(PROVIDER).toUpperCase());
- if (!PROVIDERS.stream().anyMatch(s ->
s.equals(properties.get(PROVIDER)))) {
+ // S3 Provider properties should be case insensitive.
+ if (!PROVIDERS.stream().anyMatch(s ->
s.equals(properties.get(PROVIDER).toUpperCase()))) {
throw new DdlException("Provider must be one of OSS, OBS,
AZURE, BOS, COS, S3, GCP");
}
}
@@ -330,7 +331,8 @@ public class S3Properties extends BaseProperties {
builder.setExternalEndpoint(properties.get(S3Properties.EXTERNAL_ENDPOINT));
}
if (properties.containsKey(S3Properties.PROVIDER)) {
-
builder.setProvider(Provider.valueOf(properties.get(S3Properties.PROVIDER)));
+ // S3 Provider properties should be case insensitive.
+
builder.setProvider(Provider.valueOf(properties.get(S3Properties.PROVIDER).toUpperCase()));
}
return builder;
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/StageTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/StageTest.java
index 534248be652..7b835c191c9 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/StageTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/StageTest.java
@@ -103,7 +103,8 @@ public class StageTest extends TestWithFeService {
+ "'prefix' = 'tmp_prefix', "
+ "'provider' = 'abc', "
+ "'ak'='tmp_ak', 'sk'='tmp_sk', 'access_type'='aksk');";
- parseAndAnalyzeWithException(sql, "Property provider with invalid
value abc");
+ // S3 Provider will be converted to upper case.
+ parseAndAnalyzeWithException(sql, "Property provider with invalid
value ABC");
// test getObjectInfoPB
sql = "create stage if not exists ex_stage_1 " + OBJ_INFO + ")";
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index b97511992d5..53fd6b0415f 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -630,7 +630,7 @@ class Config {
} else if (config.s3Source == "huawei") {
s3Provider = "OBS"
} else if (config.s3Source == "azure") {
- s3Provider = "AZURE"
+ s3Provider = "Azure" // case insensitive test
} else if (config.s3Source == "gcp") {
s3Provider = "GCP"
}
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 579afda48d9..713a30e98ca 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -816,7 +816,7 @@ class Suite implements GroovyInterceptable {
String getS3Url() {
String s3BucketName = context.config.otherConfigs.get("s3BucketName");
- if (context.config.otherConfigs.get("s3Provider") == "AZURE") {
+ if (context.config.otherConfigs.get("s3Provider").toUpperCase() ==
"AZURE") {
String accountName = context.config.otherConfigs.get("ak");
String s3Url =
"http://${accountName}.blob.core.windows.net/${s3BucketName}";
return s3Url
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
