This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 3ae4211b948 branch-3.0: [fix](create-resource) fix potential
concurrent modification exception when creating resource (#50356, #50615)
(#50533)
3ae4211b948 is described below
commit 3ae4211b94889e568fc2be8df6cd0c600e1bce44
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Thu May 15 15:49:24 2025 +0800
branch-3.0: [fix](create-resource) fix potential concurrent modification
exception when creating resource (#50356, #50615) (#50533)
bp #50356 #50615
---------
Co-authored-by: yagagagaga <[email protected]>
---
.../apache/doris/analysis/CreateResourceStmt.java | 7 +--
.../doris/analysis/CreateStorageVaultStmt.java | 17 +++++--
.../org/apache/doris/catalog/AzureResource.java | 25 ++++++-----
.../java/org/apache/doris/catalog/EsResource.java | 3 +-
.../java/org/apache/doris/catalog/HMSResource.java | 3 +-
.../org/apache/doris/catalog/HdfsResource.java | 7 +--
.../org/apache/doris/catalog/HdfsStorageVault.java | 3 +-
.../org/apache/doris/catalog/JdbcResource.java | 7 +--
.../apache/doris/catalog/OdbcCatalogResource.java | 5 ++-
.../java/org/apache/doris/catalog/Resource.java | 3 +-
.../java/org/apache/doris/catalog/S3Resource.java | 8 ++--
.../org/apache/doris/catalog/S3StorageVault.java | 3 +-
.../org/apache/doris/catalog/SparkResource.java | 3 +-
.../org/apache/doris/catalog/StorageVault.java | 8 +---
.../doris/catalog/OdbcCatalogResourceTest.java | 14 +++---
.../org/apache/doris/catalog/S3ResourceTest.java | 52 ++++++++++++----------
.../doris/cloud/catalog/HdfsStorageVaultTest.java | 30 +++++++------
.../vault_p0/create/test_create_vault.groovy | 15 +++++++
18 files changed, 128 insertions(+), 85 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java
index 3feccbba9ba..6ea190977a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java
@@ -31,6 +31,7 @@ import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
import java.util.Map;
@@ -42,7 +43,7 @@ public class CreateResourceStmt extends DdlStmt implements
NotFallbackInParser {
private final boolean isExternal;
private final boolean ifNotExists;
private final String resourceName;
- private final Map<String, String> properties;
+ private final ImmutableMap<String, String> properties;
private ResourceType resourceType;
public CreateResourceStmt(boolean isExternal, boolean ifNotExists, String
resourceName,
@@ -50,7 +51,7 @@ public class CreateResourceStmt extends DdlStmt implements
NotFallbackInParser {
this.isExternal = isExternal;
this.ifNotExists = ifNotExists;
this.resourceName = resourceName;
- this.properties = properties;
+ this.properties = ImmutableMap.copyOf(properties);
this.resourceType = ResourceType.UNKNOWN;
}
@@ -62,7 +63,7 @@ public class CreateResourceStmt extends DdlStmt implements
NotFallbackInParser {
return resourceName;
}
- public Map<String, String> getProperties() {
+ public ImmutableMap<String, String> getProperties() {
return properties;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java
index f8a40517cdc..3a4e8ff9cb5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java
@@ -28,9 +28,12 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.PrintableMap;
+import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
+import com.google.common.collect.ImmutableMap;
+
import java.util.Map;
// CREATE STORAGE VAULT vault_name
@@ -44,7 +47,7 @@ public class CreateStorageVaultStmt extends DdlStmt
implements NotFallbackInPars
private final boolean ifNotExists;
private final String vaultName;
- private final Map<String, String> properties;
+ private ImmutableMap<String, String> properties;
private boolean setAsDefault;
private int pathVersion = 0;
private int numShard = 0;
@@ -53,7 +56,7 @@ public class CreateStorageVaultStmt extends DdlStmt
implements NotFallbackInPars
public CreateStorageVaultStmt(boolean ifNotExists, String vaultName,
Map<String, String> properties) {
this.ifNotExists = ifNotExists;
this.vaultName = vaultName;
- this.properties = properties;
+ this.properties = ImmutableMap.copyOf(properties);
this.vaultType = vaultType.UNKNOWN;
}
@@ -77,7 +80,7 @@ public class CreateStorageVaultStmt extends DdlStmt
implements NotFallbackInPars
return pathVersion;
}
- public Map<String, String> getProperties() {
+ public ImmutableMap<String, String> getProperties() {
return properties;
}
@@ -143,6 +146,14 @@ public class CreateStorageVaultStmt extends DdlStmt
implements NotFallbackInPars
}
setAsDefault =
Boolean.parseBoolean(properties.getOrDefault(SET_AS_DEFAULT, "false"));
setStorageVaultType(StorageVault.StorageVaultType.fromString(type));
+
+ if (vaultType == StorageVault.StorageVaultType.S3
+ && !properties.containsKey(PropertyConverter.USE_PATH_STYLE)) {
+ properties = ImmutableMap.<String, String>builder()
+ .putAll(properties)
+ .put(PropertyConverter.USE_PATH_STYLE, "true")
+ .build();
+ }
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java
index fb04e25ad9e..c24c26ee22c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java
@@ -27,6 +27,7 @@ import org.apache.doris.fs.obj.ObjStorage;
import org.apache.doris.fs.obj.RemoteObjects;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
@@ -52,32 +53,32 @@ public class AzureResource extends Resource {
}
@Override
- protected void setProperties(Map<String, String> newProperties) throws
DdlException {
+ protected void setProperties(ImmutableMap<String, String> newProperties)
throws DdlException {
Preconditions.checkState(newProperties != null);
+ this.properties = Maps.newHashMap(newProperties);
// check properties
- S3Properties.requiredS3PingProperties(newProperties);
+ S3Properties.requiredS3PingProperties(this.properties);
// default need check resource conf valid, so need fix ut and
regression case
- boolean needCheck = isNeedCheck(newProperties);
+ boolean needCheck = isNeedCheck(this.properties);
if (LOG.isDebugEnabled()) {
LOG.debug("azure info need check validity : {}", needCheck);
}
// the endpoint for ping need add uri scheme.
- String pingEndpoint = newProperties.get(S3Properties.ENDPOINT);
+ String pingEndpoint = this.properties.get(S3Properties.ENDPOINT);
if (!pingEndpoint.startsWith("http://";)) {
- pingEndpoint = "http://"; +
newProperties.get(S3Properties.ENDPOINT);
- newProperties.put(S3Properties.ENDPOINT, pingEndpoint);
- newProperties.put(S3Properties.Env.ENDPOINT, pingEndpoint);
+ pingEndpoint = "http://"; +
this.properties.get(S3Properties.ENDPOINT);
+ this.properties.put(S3Properties.ENDPOINT, pingEndpoint);
+ this.properties.put(S3Properties.Env.ENDPOINT, pingEndpoint);
}
if (needCheck) {
- String bucketName = newProperties.get(S3Properties.BUCKET);
- String rootPath = newProperties.get(S3Properties.ROOT_PATH);
- pingAzure(bucketName, rootPath, newProperties);
+ String bucketName = this.properties.get(S3Properties.BUCKET);
+ String rootPath = this.properties.get(S3Properties.ROOT_PATH);
+ pingAzure(bucketName, rootPath, this.properties);
}
// optional
- S3Properties.optionalS3Property(newProperties);
- this.properties = newProperties;
+ S3Properties.optionalS3Property(this.properties);
}
protected static void pingAzure(String bucketName, String rootPath,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
index 8203a051129..697bd4ac2ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java
@@ -21,6 +21,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.proc.BaseProcResult;
import org.apache.doris.datasource.es.EsUtil;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -96,7 +97,7 @@ public class EsResource extends Resource {
}
@Override
- protected void setProperties(Map<String, String> properties) throws
DdlException {
+ protected void setProperties(ImmutableMap<String, String> properties)
throws DdlException {
valid(properties, false);
this.properties = processCompatibleProperties(properties);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java
index 0a3b422fc1a..007cb4660e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java
@@ -22,6 +22,7 @@ import org.apache.doris.common.proc.BaseProcResult;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.property.constants.HMSProperties;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -63,7 +64,7 @@ public class HMSResource extends Resource {
}
@Override
- protected void setProperties(Map<String, String> properties) throws
DdlException {
+ protected void setProperties(ImmutableMap<String, String> properties)
throws DdlException {
for (String field : HMSProperties.REQUIRED_FIELDS) {
if (!properties.containsKey(field)) {
throw new DdlException("Missing [" + field + "] in
properties.");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
index c9cb77fbd93..e1b482bd1b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java
@@ -23,6 +23,7 @@ import
org.apache.doris.common.security.authentication.AuthenticationConfig;
import org.apache.doris.thrift.THdfsConf;
import org.apache.doris.thrift.THdfsParams;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -73,13 +74,13 @@ public class HdfsResource extends Resource {
}
@Override
- protected void setProperties(Map<String, String> properties) throws
DdlException {
+ protected void setProperties(ImmutableMap<String, String> newProperties)
throws DdlException {
// `dfs.client.read.shortcircuit` and `dfs.domain.socket.path` should
be both set to enable short circuit read.
// We should disable short circuit read if they are not both set
because it will cause performance down.
+ this.properties = Maps.newHashMap(newProperties);
if (!(enableShortCircuitRead(properties))) {
- properties.put(HADOOP_SHORT_CIRCUIT, "false");
+ this.properties.put(HADOOP_SHORT_CIRCUIT, "false");
}
- this.properties = properties;
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
index 743bb313b83..3b6aae6c7cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
@@ -27,6 +27,7 @@ import org.apache.doris.fs.remote.dfs.DFSFileSystem;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -97,7 +98,7 @@ public class HdfsStorageVault extends StorageVault {
}
@Override
- public void modifyProperties(Map<String, String> newProperties) throws
DdlException {
+ public void modifyProperties(ImmutableMap<String, String> newProperties)
throws DdlException {
for (Map.Entry<String, String> kv : newProperties.entrySet()) {
replaceIfEffectiveValue(this.properties, kv.getKey(),
kv.getValue());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
index f189d1e428f..bbd3e6df802 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
@@ -28,6 +28,7 @@ import org.apache.doris.datasource.ExternalCatalog;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -192,10 +193,10 @@ public class JdbcResource extends Resource {
}
@Override
- protected void setProperties(Map<String, String> properties) throws
DdlException {
+ protected void setProperties(ImmutableMap<String, String> properties)
throws DdlException {
Preconditions.checkState(properties != null);
- validateProperties(properties);
- configs = properties;
+ this.configs = Maps.newHashMap(properties);
+ validateProperties(this.configs);
applyDefaultProperties();
String currentDateTime =
LocalDateTime.now(ZoneId.systemDefault()).toString().replace("T", " ");
configs.put(CREATE_TIME, currentDateTime);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java
index dbe03ce4db0..60fb9f9555e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java
@@ -22,6 +22,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.proc.BaseProcResult;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -167,10 +168,10 @@ public class OdbcCatalogResource extends Resource {
}
@Override
- protected void setProperties(Map<String, String> properties) throws
DdlException {
+ protected void setProperties(ImmutableMap<String, String> properties)
throws DdlException {
Preconditions.checkState(properties != null);
- configs = properties;
+ configs = Maps.newHashMap(properties);
checkProperties(HOST);
checkProperties(PORT);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
index 0f0fd7d5de6..14da354f5bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java
@@ -30,6 +30,7 @@ import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
@@ -233,7 +234,7 @@ public abstract class Resource implements Writable,
GsonPostProcessable {
/**
* Set and check the properties in child resources
*/
- protected abstract void setProperties(Map<String, String> properties)
throws DdlException;
+ protected abstract void setProperties(ImmutableMap<String, String>
properties) throws DdlException;
public abstract Map<String, String> getCopiedProperties();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
index 392b73d2280..22df2c9082f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
@@ -28,6 +28,7 @@ import org.apache.doris.fs.obj.RemoteObjects;
import org.apache.doris.fs.obj.S3ObjStorage;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -85,8 +86,10 @@ public class S3Resource extends Resource {
}
@Override
- protected void setProperties(Map<String, String> properties) throws
DdlException {
- Preconditions.checkState(properties != null);
+ protected void setProperties(ImmutableMap<String, String> newProperties)
throws DdlException {
+ Preconditions.checkState(newProperties != null);
+ this.properties = Maps.newHashMap(newProperties);
+
// check properties
S3Properties.requiredS3PingProperties(properties);
// default need check resource conf valid, so need fix ut and
regression case
@@ -112,7 +115,6 @@ public class S3Resource extends Resource {
}
// optional
S3Properties.optionalS3Property(properties);
- this.properties = properties;
}
protected static void pingS3(String bucketName, String rootPath,
Map<String, String> newProperties)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
index 58729aeb7e4..f8196c7ea80 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
@@ -24,6 +24,7 @@ import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.property.constants.S3Properties;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import com.google.gson.annotations.SerializedName;
import java.util.Arrays;
@@ -89,7 +90,7 @@ public class S3StorageVault extends StorageVault {
}
@Override
- public void modifyProperties(Map<String, String> properties) throws
DdlException {
+ public void modifyProperties(ImmutableMap<String, String> properties)
throws DdlException {
resource.setProperties(properties);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java
index 704d8e512d7..59b6d16801e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java
@@ -29,6 +29,7 @@ import org.apache.doris.load.loadv2.SparkRepository;
import org.apache.doris.load.loadv2.SparkYarnConfigFiles;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
@@ -281,7 +282,7 @@ public class SparkResource extends Resource {
}
@Override
- protected void setProperties(Map<String, String> properties) throws
DdlException {
+ protected void setProperties(ImmutableMap<String, String> properties)
throws DdlException {
Preconditions.checkState(properties != null);
// get spark configs
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
index 6ef829e5f3a..c9fe0c8bcc6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
@@ -22,11 +22,11 @@ import org.apache.doris.analysis.CreateStorageVaultStmt;
import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
-import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.qe.ShowResultSetMetaData;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
import com.google.protobuf.TextFormat;
import java.util.ArrayList;
@@ -146,10 +146,6 @@ public abstract class StorageVault {
vault.modifyProperties(stmt.getProperties());
break;
case S3:
- if
(!stmt.getProperties().containsKey(PropertyConverter.USE_PATH_STYLE)) {
- stmt.getProperties().put(PropertyConverter.USE_PATH_STYLE,
"true");
- }
-
CreateResourceStmt resourceStmt =
new CreateResourceStmt(false, ifNotExists, name,
stmt.getProperties());
resourceStmt.analyzeResourceType();
@@ -177,7 +173,7 @@ public abstract class StorageVault {
* @param properties
* @throws DdlException
*/
- public abstract void modifyProperties(Map<String, String> properties)
throws DdlException;
+ public abstract void modifyProperties(ImmutableMap<String, String>
properties) throws DdlException;
/**
* Check properties in child resources
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java
index a1c2e679829..88b5d9aa26f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java
@@ -29,6 +29,7 @@ import
org.apache.doris.mysql.privilege.AccessControllerManager;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import mockit.Expectations;
import mockit.Injectable;
@@ -42,7 +43,6 @@ import java.io.DataOutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.HashMap;
import java.util.Map;
public class OdbcCatalogResourceTest {
@@ -125,11 +125,13 @@ public class OdbcCatalogResourceTest {
OdbcCatalogResource odbcCatalogResource1 = new
OdbcCatalogResource("odbc1");
odbcCatalogResource1.write(dos);
- Map<String, String> configs = new HashMap<>();
- configs.put("host", "host");
- configs.put("port", "port");
- configs.put("user", "user");
- configs.put("password", "password");
+ ImmutableMap<String, String> configs = ImmutableMap.of(
+ "host", "host",
+ "port", "port",
+ "user", "user",
+ "password", "password"
+ );
+
OdbcCatalogResource odbcCatalogResource2 = new
OdbcCatalogResource("odbc2");
odbcCatalogResource2.setProperties(configs);
odbcCatalogResource2.write(dos);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
index 5f2daf94cc3..b7d14ab7017 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
@@ -31,6 +31,7 @@ import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
import mockit.Expectations;
import mockit.Injectable;
import mockit.Mocked;
@@ -172,14 +173,15 @@ public class S3ResourceTest {
S3Resource s3Resource1 = new S3Resource("s3_1");
s3Resource1.write(s3Dos);
- Map<String, String> properties = new HashMap<>();
- properties.put("AWS_ENDPOINT", "aaa");
- properties.put("AWS_REGION", "bbb");
- properties.put("AWS_ROOT_PATH", "/path/to/root");
- properties.put("AWS_ACCESS_KEY", "xxx");
- properties.put("AWS_SECRET_KEY", "yyy");
- properties.put("AWS_BUCKET", "test-bucket");
- properties.put("s3_validity_check", "false");
+ ImmutableMap<String, String> properties = ImmutableMap.of(
+ "AWS_ENDPOINT", "aaa",
+ "AWS_REGION", "bbb",
+ "AWS_ROOT_PATH", "/path/to/root",
+ "AWS_ACCESS_KEY", "xxx",
+ "AWS_SECRET_KEY", "yyy",
+ "AWS_BUCKET", "test-bucket",
+ "s3_validity_check", "false"
+ );
S3Resource s3Resource2 = new S3Resource("s3_2");
s3Resource2.setProperties(properties);
s3Resource2.write(s3Dos);
@@ -211,14 +213,15 @@ public class S3ResourceTest {
@Test
public void testModifyProperties() throws Exception {
- Map<String, String> properties = new HashMap<>();
- properties.put("AWS_ENDPOINT", "aaa");
- properties.put("AWS_REGION", "bbb");
- properties.put("AWS_ROOT_PATH", "/path/to/root");
- properties.put("AWS_ACCESS_KEY", "xxx");
- properties.put("AWS_SECRET_KEY", "yyy");
- properties.put("AWS_BUCKET", "test-bucket");
- properties.put("s3_validity_check", "false");
+ ImmutableMap<String, String> properties = ImmutableMap.of(
+ "AWS_ENDPOINT", "aaa",
+ "AWS_REGION", "bbb",
+ "AWS_ROOT_PATH", "/path/to/root",
+ "AWS_ACCESS_KEY", "xxx",
+ "AWS_SECRET_KEY", "yyy",
+ "AWS_BUCKET", "test-bucket",
+ "s3_validity_check", "false"
+ );
S3Resource s3Resource = new S3Resource("t_source");
s3Resource.setProperties(properties);
FeConstants.runningUnitTest = true;
@@ -231,14 +234,15 @@ public class S3ResourceTest {
@Test
public void testHttpScheme() throws DdlException {
// if https:// is set, it should be replaced with http://
- Map<String, String> properties = new HashMap<>();
- properties.put("AWS_ENDPOINT", "https://aaa";);
- properties.put("AWS_REGION", "bbb");
- properties.put("AWS_ROOT_PATH", "/path/to/root");
- properties.put("AWS_ACCESS_KEY", "xxx");
- properties.put("AWS_SECRET_KEY", "yyy");
- properties.put("AWS_BUCKET", "test-bucket");
- properties.put("s3_validity_check", "false");
+ ImmutableMap<String, String> properties = ImmutableMap.of(
+ "AWS_ENDPOINT", "https://aaa";,
+ "AWS_REGION", "bbb",
+ "AWS_ROOT_PATH", "/path/to/root",
+ "AWS_ACCESS_KEY", "xxx",
+ "AWS_SECRET_KEY", "yyy",
+ "AWS_BUCKET", "test-bucket",
+ "s3_validity_check", "false"
+ );
S3Resource s3Resource = new S3Resource("s3_2");
s3Resource.setProperties(properties);
Assert.assertEquals(s3Resource.getProperty(S3Properties.ENDPOINT),
"https://aaa";);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java
b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java
index 19b54bb1b0c..6183360cd92 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java
@@ -45,7 +45,6 @@ import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -236,10 +235,11 @@ public class HdfsStorageVaultTest {
Assumptions.assumeTrue(!Strings.isNullOrEmpty(hadoopFsName),
"HADOOP_FS_NAME isNullOrEmpty.");
Assumptions.assumeTrue(!Strings.isNullOrEmpty(hadoopUser),
"HADOOP_USER isNullOrEmpty.");
- Map<String, String> properties = new HashMap<>();
- properties.put(HdfsStorageVault.PropertyKey.HADOOP_FS_NAME,
hadoopFsName);
- properties.put(HdfsStorageVault.PropertyKey.HADOOP_USER_NAME,
hadoopUser);
- properties.put(HdfsStorageVault.PropertyKey.VAULT_PATH_PREFIX,
"testCheckConnectivityUtPrefix");
+ ImmutableMap<String, String> properties = ImmutableMap.of(
+ HdfsStorageVault.PropertyKey.HADOOP_FS_NAME, hadoopFsName,
+ HdfsStorageVault.PropertyKey.HADOOP_USER_NAME, hadoopUser,
+ HdfsStorageVault.PropertyKey.VAULT_PATH_PREFIX,
"testCheckConnectivityUtPrefix"
+ );
HdfsStorageVault vault = new HdfsStorageVault("testHdfsVault",
false, false);
vault.modifyProperties(properties);
@@ -251,10 +251,11 @@ public class HdfsStorageVaultTest {
@Test
public void testCheckConnectivityException() {
- Map<String, String> properties = new HashMap<>();
- properties.put(HdfsStorageVault.PropertyKey.HADOOP_FS_NAME,
"hdfs://localhost:10000");
- properties.put(HdfsStorageVault.PropertyKey.HADOOP_USER_NAME,
"notExistUser");
- properties.put(HdfsStorageVault.PropertyKey.VAULT_PATH_PREFIX,
"testCheckConnectivityUtPrefix");
+ ImmutableMap<String, String> properties = ImmutableMap.of(
+ HdfsStorageVault.PropertyKey.HADOOP_FS_NAME,
"hdfs://localhost:10000",
+ HdfsStorageVault.PropertyKey.HADOOP_USER_NAME, "notExistUser",
+ HdfsStorageVault.PropertyKey.VAULT_PATH_PREFIX,
"testCheckConnectivityUtPrefix"
+ );
HdfsStorageVault vault = new HdfsStorageVault("testHdfsVault", false,
false);
Assertions.assertThrows(DdlException.class, () -> {
@@ -264,11 +265,12 @@ public class HdfsStorageVaultTest {
@Test
public void testIgnoreCheckConnectivity() throws DdlException {
- Map<String, String> properties = new HashMap<>();
- properties.put(HdfsStorageVault.PropertyKey.HADOOP_FS_NAME,
"hdfs://localhost:10000");
- properties.put(HdfsStorageVault.PropertyKey.HADOOP_USER_NAME,
"notExistUser");
- properties.put(HdfsStorageVault.PropertyKey.VAULT_PATH_PREFIX,
"testCheckConnectivityUtPrefix");
- properties.put(S3Properties.VALIDITY_CHECK, "false");
+ ImmutableMap<String, String> properties = ImmutableMap.of(
+ HdfsStorageVault.PropertyKey.HADOOP_FS_NAME,
"hdfs://localhost:10000",
+ HdfsStorageVault.PropertyKey.HADOOP_USER_NAME, "notExistUser",
+ HdfsStorageVault.PropertyKey.VAULT_PATH_PREFIX,
"testCheckConnectivityUtPrefix",
+ S3Properties.VALIDITY_CHECK, "false"
+ );
HdfsStorageVault vault = new HdfsStorageVault("testHdfsVault", false,
false);
vault.modifyProperties(properties);
diff --git a/regression-test/suites/vault_p0/create/test_create_vault.groovy
b/regression-test/suites/vault_p0/create/test_create_vault.groovy
index 502782a6cc1..c0128291f82 100644
--- a/regression-test/suites/vault_p0/create/test_create_vault.groovy
+++ b/regression-test/suites/vault_p0/create/test_create_vault.groovy
@@ -233,6 +233,21 @@ suite("test_create_vault", "nonConcurrent") {
);
"""
+ sql """
+ CREATE STORAGE VAULT IF NOT EXISTS ${s3VaultName}
+ PROPERTIES (
+ "type"="S3",
+ "s3.endpoint"="${getS3Endpoint()}",
+ "s3.region" = "${getS3Region()}",
+ "s3.access_key" = "${getS3AK()}",
+ "s3.secret_key" = "${getS3SK()}",
+ "s3.root.path" = "${s3VaultName}",
+ "s3.bucket" = "${getS3BucketName()}",
+ "s3.external_endpoint" = "",
+ "provider" = "${getS3Provider()}"
+ );
+ """
+
sql """
CREATE TABLE ${s3VaultName} (
C_CUSTKEY INTEGER NOT NULL,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
