This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new de8dbbcbc04 [fix](storage vault) Fix two storage vault regression
cases (#50703)
de8dbbcbc04 is described below
commit de8dbbcbc04eb43d16e2db08239dedb308cad222
Author: Lei Zhang <[email protected]>
AuthorDate: Tue May 13 22:09:59 2025 +0800
[fix](storage vault) Fix two storage vault regression cases (#50703)
* move `not set use_path_style` case from `test_create_vault` to
`test_minio_storage_vault` because aliyun oss don't allow path style,
but default is path style
* fix `test_create_vault_with_case_sensitive` case fail, indroduced by
code refactor by https://github.com/apache/doris/pull/50395
---
.../org/apache/doris/catalog/HdfsStorageVault.java | 10 +++++++
.../org/apache/doris/catalog/S3StorageVault.java | 3 ++
.../org/apache/doris/catalog/StorageVault.java | 13 ++++++---
.../vault_p0/create/test_create_vault.groovy | 15 ----------
.../create/test_minio_storage_vault.groovy | 34 +++++++++++++++++++++-
5 files changed, 55 insertions(+), 20 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
index 9701b40ec25..3b6aae6c7cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
@@ -199,12 +199,22 @@ public class HdfsStorageVault extends StorageVault {
continue;
} else {
// Get rid of copy and paste from create s3 vault stmt
+ Preconditions.checkArgument(
+
!property.getKey().toLowerCase().contains(S3StorageVault.PropertyKey.REGION),
+ "Invalid argument %s", property.getKey());
+ Preconditions.checkArgument(
+
!property.getKey().toLowerCase().contains(S3StorageVault.PropertyKey.ENDPOINT),
+ "Invalid argument %s", property.getKey());
Preconditions.checkArgument(
!property.getKey().toLowerCase().contains(S3StorageVault.PropertyKey.ROOT_PATH),
"Invalid argument %s", property.getKey());
Preconditions.checkArgument(
!property.getKey().toLowerCase().contains(S3StorageVault.PropertyKey.PROVIDER),
"Invalid argument %s", property.getKey());
+ Preconditions.checkArgument(
+
!property.getKey().toLowerCase().contains(S3StorageVault.PropertyKey.BUCKET),
+ "Invalid argument %s", property.getKey());
+
if
(!NON_HDFS_CONF_PROPERTY_KEYS.contains(property.getKey().toLowerCase())) {
Cloud.HdfsBuildConf.HdfsConfKVPair.Builder conf =
Cloud.HdfsBuildConf.HdfsConfKVPair.newBuilder();
conf.setKey(property.getKey());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
index 56522ef9136..f8196c7ea80 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java
@@ -67,6 +67,9 @@ public class S3StorageVault extends StorageVault {
public static final String USE_PATH_STYLE =
PropertyConverter.USE_PATH_STYLE;
public static final String ROOT_PATH = S3Properties.ROOT_PATH;
public static final String PROVIDER = S3Properties.PROVIDER;
+ public static final String REGION = S3Properties.REGION;
+ public static final String ENDPOINT = S3Properties.ENDPOINT;
+ public static final String BUCKET = S3Properties.BUCKET;
}
public static final HashSet<String> ALLOW_ALTER_PROPERTIES = new
HashSet<>(Arrays.asList(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
index f71560c012e..c9fe0c8bcc6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java
@@ -181,10 +181,15 @@ public abstract class StorageVault {
* @throws UserException
*/
public void checkCreationProperties(Map<String, String> properties) throws
UserException {
- Preconditions.checkArgument(
- properties.get(PropertyKey.TYPE) != null, "Missing property "
+ PropertyKey.TYPE);
- Preconditions.checkArgument(
- !properties.get(PropertyKey.TYPE).isEmpty(), "Property " +
PropertyKey.TYPE + " cannot be empty");
+ String type = null;
+ for (Map.Entry<String, String> property : properties.entrySet()) {
+ if
(property.getKey().equalsIgnoreCase(StorageVault.PropertyKey.TYPE)) {
+ type = property.getValue();
+ }
+ }
+
+ Preconditions.checkArgument(type != null, "Missing property " +
PropertyKey.TYPE);
+ Preconditions.checkArgument(!type.isEmpty(), "Property " +
PropertyKey.TYPE + " cannot be empty");
}
protected void replaceIfEffectiveValue(Map<String, String> properties,
String key, String value) {
diff --git a/regression-test/suites/vault_p0/create/test_create_vault.groovy
b/regression-test/suites/vault_p0/create/test_create_vault.groovy
index c0128291f82..502782a6cc1 100644
--- a/regression-test/suites/vault_p0/create/test_create_vault.groovy
+++ b/regression-test/suites/vault_p0/create/test_create_vault.groovy
@@ -233,21 +233,6 @@ suite("test_create_vault", "nonConcurrent") {
);
"""
- sql """
- CREATE STORAGE VAULT IF NOT EXISTS ${s3VaultName}
- PROPERTIES (
- "type"="S3",
- "s3.endpoint"="${getS3Endpoint()}",
- "s3.region" = "${getS3Region()}",
- "s3.access_key" = "${getS3AK()}",
- "s3.secret_key" = "${getS3SK()}",
- "s3.root.path" = "${s3VaultName}",
- "s3.bucket" = "${getS3BucketName()}",
- "s3.external_endpoint" = "",
- "provider" = "${getS3Provider()}"
- );
- """
-
sql """
CREATE TABLE ${s3VaultName} (
C_CUSTKEY INTEGER NOT NULL,
diff --git
a/regression-test/suites/vault_p0/create/test_minio_storage_vault.groovy
b/regression-test/suites/vault_p0/create/test_minio_storage_vault.groovy
index 582e80dcd6e..4c7010fef4a 100644
--- a/regression-test/suites/vault_p0/create/test_minio_storage_vault.groovy
+++ b/regression-test/suites/vault_p0/create/test_minio_storage_vault.groovy
@@ -67,8 +67,40 @@ suite ("test_minio_storage_vault") {
order_qt_sql "select * from user"
+ //
**********************************************************************************************
+ // * case 2: test MinIO as Storage Vault not set `use_path_style` equal
`use_path_style=true` *
+ //
**********************************************************************************************
+ String vaultNameNotSetPathStyle = "minio_not_set_path_style_vault_" +
suffix
+ multi_sql """
+ CREATE STORAGE VAULT IF NOT EXISTS ${vaultNameNotSetPathStyle}
+ PROPERTIES (
+ "type"="S3",
+ "s3.endpoint"="${extMinioHost}:${extMinioPort}",
+ "s3.access_key" = "${extMinioAk}",
+ "s3.secret_key" = "${extMinioSk}",
+ "s3.region" = "${extMinioRegion}",
+ "s3.root.path" = "test3_${suffix}",
+ "s3.bucket" = "${extMinioBucket}",
+ "provider" = "S3"
+ );
+ drop table if exists user3;
+ CREATE TABLE `user3` (
+ `id` int NULL,
+ `name` varchar(32) NULL
+ )
+ PROPERTIES (
+ "storage_vault_name" = "${vaultNameNotSetPathStyle}"
+ );
+ insert into user3 values (1,'Tom'), (2, 'Jelly'), (3, 'Spike'), (4,
'Tuffy');
+ """
+
+ def result = sql "SELECT count(*) FROM user3"
+ logger.info("result:${result}")
+ assertEquals(result[0][0], 4)
+ assertEquals(result.size(), 1)
+
// **********************************************************************
- // * case 2: test MinIO as Storage Vault using S3 Virtual Host Style *
+ // * case 3: test MinIO as Storage Vault using S3 Virtual Host Style *
// **********************************************************************
String extMinioDomain = context.config.otherConfigs.get("extMinioDomain")
cmd "echo -ne '\\n${extMinioHost} ${extMinioBucket}.${extMinioDomain}' >>
/etc/hosts"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
