This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0112c6febdc62709d0a9689b69a1ef9b04cb7d6c Author: slothever <[email protected]> AuthorDate: Fri Aug 18 18:37:21 2023 +0800 [fix](multi-catalog)fix compability issue for s3 endpoint (#23175) --- .../datasource/property/PropertyConverter.java | 40 ++++++++++++++-------- .../property/constants/OssProperties.java | 1 + 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java index 963efb62d6..754a885f4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java @@ -123,28 +123,40 @@ public class PropertyConverter { } else if (props.containsKey(MinioProperties.ENDPOINT)) { return convertToMinioProperties(props, MinioProperties.getCredential(props)); } else if (props.containsKey(S3Properties.ENDPOINT)) { - CloudCredential credential = S3Properties.getCredential(props); + CloudCredential s3Credential = S3Properties.getCredential(props); + Map<String, String> s3Properties = convertToS3Properties(props, s3Credential); String s3CliEndpoint = props.get(S3Properties.ENDPOINT); - if (s3CliEndpoint.contains(CosProperties.COS_PREFIX)) { - props.putIfAbsent(CosProperties.ENDPOINT, s3CliEndpoint); - // CosN is not compatible with S3, when use s3 properties, will convert to cosn properties. - return convertToCOSProperties(props, credential); - } - return convertToS3Properties(props, S3Properties.getCredential(props)); + return convertToCompatibleS3Properties(props, s3CliEndpoint, s3Credential, s3Properties); } else if (props.containsKey(S3Properties.Env.ENDPOINT)) { // checkout env in the end // compatible with the s3,obs,oss,cos when they use aws client. CloudCredentialWithEndpoint envCredentials = S3Properties.getEnvironmentCredentialWithEndpoint(props); - if (envCredentials.getEndpoint().contains(CosProperties.COS_PREFIX)) { - props.putIfAbsent(CosProperties.ENDPOINT, envCredentials.getEndpoint()); - // CosN is not compatible with S3, when use s3 properties, will convert to cosn properties. - return convertToCOSProperties(props, envCredentials); - } - return convertToS3EnvProperties(props, envCredentials, false); + Map<String, String> s3Properties = convertToS3EnvProperties(props, envCredentials, false); + String s3CliEndpoint = envCredentials.getEndpoint(); + return convertToCompatibleS3Properties(props, s3CliEndpoint, envCredentials, s3Properties); } return props; } + private static Map<String, String> convertToCompatibleS3Properties(Map<String, String> props, + String s3CliEndpoint, + CloudCredential credential, + Map<String, String> s3Properties) { + Map<String, String> heteroProps = new HashMap<>(s3Properties); + if (s3CliEndpoint.contains(CosProperties.COS_PREFIX)) { + props.putIfAbsent(CosProperties.ENDPOINT, s3CliEndpoint); + // CosN is not compatible with S3, when use s3 properties, will convert to cosn properties. + heteroProps.putAll(convertToCOSProperties(props, credential)); + } else if (s3CliEndpoint.contains(ObsProperties.OBS_PREFIX)) { + props.putIfAbsent(ObsProperties.ENDPOINT, s3CliEndpoint); + heteroProps.putAll(convertToOBSProperties(props, credential)); + } else if (s3CliEndpoint.contains(OssProperties.OSS_REGION_PREFIX)) { + props.putIfAbsent(OssProperties.ENDPOINT, s3CliEndpoint); + heteroProps.putAll(convertToOSSProperties(props, credential)); + } + return heteroProps; + } + private static Map<String, String> convertToOBSProperties(Map<String, String> props, CloudCredential credential) { @@ -237,7 +249,7 @@ public class PropertyConverter { CloudCredential credential) { s3Properties.put(Constants.MAX_ERROR_RETRIES, "2"); s3Properties.put("fs.s3.impl.disable.cache", "true"); - s3Properties.put("fs.s3.impl", S3AFileSystem.class.getName()); + s3Properties.putIfAbsent("fs.s3.impl", S3AFileSystem.class.getName()); String defaultProviderList = String.join(",", S3Properties.AWS_CREDENTIALS_PROVIDERS); String credentialsProviders = s3Properties .getOrDefault(S3Properties.CREDENTIALS_PROVIDER, defaultProviderList); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java index 210bc5814a..d4fa0e1c65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java @@ -26,6 +26,7 @@ import java.util.Map; public class OssProperties extends BaseProperties { public static final String OSS_PREFIX = "oss."; + public static final String OSS_REGION_PREFIX = "oss-"; public static final String OSS_FS_PREFIX = "fs.oss"; public static final String ENDPOINT = "oss.endpoint"; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
