This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new cd3ee9a6467 branch-3.1: [case](new-catalog-property) add paimon cases
#54555 #55034 (#55037)
cd3ee9a6467 is described below
commit cd3ee9a64677d061ec91d1c721d01cc20b21d25b
Author: zgxme <[email protected]>
AuthorDate: Wed Aug 20 15:13:17 2025 +0800
branch-3.1: [case](new-catalog-property) add paimon cases #54555 #55034
(#55037)
bp #54555 #55034
---
.../docker-compose/hive/hive-3x_settings.env | 4 +-
.../create_paimon_tables.hql | 8 ++
.../external_table_p0/paimon/test_paimon_minio.out | Bin 1206 -> 2319 bytes
.../external_table_p0/paimon/test_paimon_s3.out | Bin 0 -> 329 bytes
.../test_paimon_dlf_catalog_miss_dlf_param.out | Bin 0 -> 133 bytes
.../paimon/test_paimon_dlf_catalog_new_param.out | Bin 0 -> 133 bytes
.../paimon/test_paimon_hms_catalog.out | Bin 0 -> 2143 bytes
.../paimon/paimon_base_filesystem.groovy | 23 ++-
.../paimon/test_paimon_catalog.groovy | 2 +-
.../paimon/test_paimon_minio.groovy | 54 +++----
.../external_table_p0/paimon/test_paimon_s3.groovy | 75 ++++++++++
.../paimon/test_paimon_dlf_catalog.groovy | 6 +-
... test_paimon_dlf_catalog_miss_dlf_param.groovy} | 13 +-
...vy => test_paimon_dlf_catalog_new_param.groovy} | 19 ++-
.../paimon/test_paimon_hms_catalog.groovy | 157 +++++++++++++++++++++
15 files changed, 319 insertions(+), 42 deletions(-)
diff --git a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
index 21f9d1908c3..5d59475424f 100644
--- a/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
+++ b/docker/thirdparties/docker-compose/hive/hive-3x_settings.env
@@ -30,7 +30,7 @@ export enablePaimonHms="false"
# hms on s3/oss/obs/cos
export AWSAk="*****************"
export AWSSk="*****************"
-export AWSEndpoint="s3.ap-northeast-1.amazonaws.com"
+export AWSEndpoint="s3.ap-east-1.amazonaws.com"
export OSSAk="*****************"
export OSSSk="*****************"
export OSSEndpoint="oss-cn-beijing.aliyuncs.com"
@@ -39,4 +39,4 @@ export OBSSk="*****************"
export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com"
export COSAk="*****************"
export COSSk="*****************"
-export COSRegion="ap-beijing"
\ No newline at end of file
+export COSRegion="ap-beijing"
diff --git
a/docker/thirdparties/docker-compose/hive/scripts/create_external_paimon_scripts/create_paimon_tables.hql
b/docker/thirdparties/docker-compose/hive/scripts/create_external_paimon_scripts/create_paimon_tables.hql
index 17526476199..dab087a4bff 100644
---
a/docker/thirdparties/docker-compose/hive/scripts/create_external_paimon_scripts/create_paimon_tables.hql
+++
b/docker/thirdparties/docker-compose/hive/scripts/create_external_paimon_scripts/create_paimon_tables.hql
@@ -7,6 +7,14 @@ CREATE TABLE external_test_table(
STORED BY 'org.apache.paimon.hive.PaimonStorageHandler';
INSERT INTO external_test_table VALUES(11111111, "hdfs_db_test");
+SET
hive.metastore.warehouse.dir=s3a://selectdb-qa-datalake-test-hk/paimon_warehouse;
+CREATE DATABASE IF NOT EXISTS aws_db;
+USE aws_db;
+CREATE EXTERNAL TABLE external_test_table
+STORED BY 'org.apache.paimon.hive.PaimonStorageHandler'
+LOCATION
's3a://selectdb-qa-datalake-test-hk/paimon_warehouse/aws_db.db/hive_test_table';
+
+
SET
hive.metastore.warehouse.dir=oss://doris-regression-bj/regression/paimon_warehouse;
CREATE DATABASE ali_db;
USE ali_db;
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_minio.out
b/regression-test/data/external_table_p0/paimon/test_paimon_minio.out
index c30ea34ac34..37f9f66619c 100644
Binary files
a/regression-test/data/external_table_p0/paimon/test_paimon_minio.out and
b/regression-test/data/external_table_p0/paimon/test_paimon_minio.out differ
diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_s3.out
b/regression-test/data/external_table_p0/paimon/test_paimon_s3.out
new file mode 100644
index 00000000000..b11b8ec809f
Binary files /dev/null and
b/regression-test/data/external_table_p0/paimon/test_paimon_s3.out differ
diff --git
a/regression-test/data/external_table_p2/paimon/test_paimon_dlf_catalog_miss_dlf_param.out
b/regression-test/data/external_table_p2/paimon/test_paimon_dlf_catalog_miss_dlf_param.out
new file mode 100644
index 00000000000..6e935c87544
Binary files /dev/null and
b/regression-test/data/external_table_p2/paimon/test_paimon_dlf_catalog_miss_dlf_param.out
differ
diff --git
a/regression-test/data/external_table_p2/paimon/test_paimon_dlf_catalog_new_param.out
b/regression-test/data/external_table_p2/paimon/test_paimon_dlf_catalog_new_param.out
new file mode 100644
index 00000000000..6e935c87544
Binary files /dev/null and
b/regression-test/data/external_table_p2/paimon/test_paimon_dlf_catalog_new_param.out
differ
diff --git
a/regression-test/data/external_table_p2/paimon/test_paimon_hms_catalog.out
b/regression-test/data/external_table_p2/paimon/test_paimon_hms_catalog.out
new file mode 100644
index 00000000000..51d0f28fdf2
Binary files /dev/null and
b/regression-test/data/external_table_p2/paimon/test_paimon_hms_catalog.out
differ
diff --git
a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
index af5134a3b46..6d6e876948a 100644
---
a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
+++
b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-suite("paimon_base_filesystem",
"p0,external,doris,external_docker,external_docker_doris") {
+suite("paimon_base_filesystem",
"p0,external,doris,external_docker,external_docker_doris,new_catalog_property")
{
String enabled = context.config.otherConfigs.get("enablePaimonTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
@@ -89,6 +89,27 @@ suite("paimon_base_filesystem",
"p0,external,doris,external_docker,external_dock
logger.info("catalog " + catalog_cos + " created")
logger.info("catalog " + catalog_cosn + " created")
+ sql """ switch ${catalog_oss} """
+ sql """ show databases """
+ sql """ use ${catalog_oss}.db1 """
+ // sql """ show tables """
+
+ sql """ switch ${catalog_obs} """
+ sql """ show databases """
+ sql """ use ${catalog_obs}.db1 """
+ // sql """ show tables """
+
+ sql """ switch ${catalog_cos} """
+ sql """ show databases """
+ sql """ use ${catalog_cos}.db1 """
+ // sql """ show tables """
+
+ sql """ switch ${catalog_cosn} """
+ sql """ show databases """
+ sql """ use ${catalog_cosn}.db1 """
+ // sql """ show tables """
+
+
sql """set force_jni_scanner=false"""
qt_oss oss
qt_obs obs
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
index 35d36c9a5c3..b0270671d75 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_paimon_catalog",
"p0,external,doris,external_docker,external_docker_doris") {
+suite("test_paimon_catalog",
"p0,external,doris,external_docker,external_docker_doris,new_catalog_property")
{
String file_ctl_name = "paimon_file_catalog";
String hms_ctl_name = "paimon_hms_catalog";
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy
index fd883cb6038..22e69642763 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy
@@ -15,26 +15,25 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_paimon_minio",
"p0,external,doris,external_docker,external_docker_doris") {
+suite("test_paimon_minio",
"p0,external,doris,external_docker,external_docker_doris,new_catalog_property")
{
String enabled = context.config.otherConfigs.get("enablePaimonTest")
- if (enabled != null && enabled.equalsIgnoreCase("true")) {
- String minio_port =
context.config.otherConfigs.get("iceberg_minio_port")
- String catalog_name = "test_paimon_minio"
- String externalEnvIp =
context.config.otherConfigs.get("externalEnvIp")
- String table_name = "ts_scale_orc"
-
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String minio_port =
context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String table_name = "ts_scale_orc"
+ for (String propertyPrefix : ["s3", "minio"]) {
+ def catalog_name = "test_paimon_minio_${propertyPrefix}"
sql """drop catalog if exists ${catalog_name}"""
-
sql """
- CREATE CATALOG ${catalog_name} PROPERTIES (
- 'type' = 'paimon',
- 'warehouse' = 's3://warehouse/wh',
- 's3.endpoint' =
'http://${externalEnvIp}:${minio_port}',
- 's3.access_key' = 'admin',
- 's3.secret_key' = 'password',
- 's3.path.style.access' = 'true'
- );
- """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type' = 'paimon',
+ 'warehouse' = 's3://warehouse/wh',
+ '${propertyPrefix}.endpoint' =
'http://${externalEnvIp}:${minio_port}',
+ '${propertyPrefix}.access_key' = 'admin',
+ '${propertyPrefix}.secret_key' = 'password',
+ '${propertyPrefix}.path.style.access' = 'true'
+ );
+ """
sql """switch `${catalog_name}`"""
sql """show databases; """
sql """use `${catalog_name}`.`flink_paimon`"""
@@ -44,16 +43,16 @@ suite("test_paimon_minio",
"p0,external,doris,external_docker,external_docker_do
sql """drop catalog if exists ${catalog_name}_with_region"""
sql """
- CREATE CATALOG ${catalog_name}_with_region PROPERTIES (
- 'type' = 'paimon',
- 'warehouse' = 's3://warehouse/wh',
- 's3.endpoint' =
'http://${externalEnvIp}:${minio_port}',
- 's3.access_key' = 'admin',
- 's3.secret_key' = 'password',
- 's3.region' = 'us-west-2',
- 's3.path.style.access' = 'true'
- );
- """
+ CREATE CATALOG ${catalog_name}_with_region PROPERTIES (
+ 'type' = 'paimon',
+ 'warehouse' = 's3://warehouse/wh',
+ '${propertyPrefix}.endpoint' =
'http://${externalEnvIp}:${minio_port}',
+ '${propertyPrefix}.access_key' = 'admin',
+ '${propertyPrefix}.secret_key' = 'password',
+ '${propertyPrefix}.region' = 'us-west-2',
+ '${propertyPrefix}.path.style.access' = 'true'
+ );
+ """
sql """switch `${catalog_name}_with_region`"""
sql """show databases; """
sql """use `${catalog_name}_with_region`.`flink_paimon`"""
@@ -61,6 +60,7 @@ suite("test_paimon_minio",
"p0,external,doris,external_docker,external_docker_do
order_qt_region2 """select * from ${table_name} """
sql """drop catalog if exists ${catalog_name}_with_region"""
}
+ }
}
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_s3.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_s3.groovy
new file mode 100644
index 00000000000..ec9bdc19b8e
--- /dev/null
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_s3.groovy
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_s3",
"p0,external,doris,external_docker,external_docker_doris,new_catalog_property")
{
+ def testQuery = { String catalogProperties, String prefix, String dbName ->
+ def catalog_name = "test_paimon_on_fs_${prefix}_catalog"
+ sql """
+ DROP CATALOG IF EXISTS ${catalog_name};
+ """
+ sql """
+ CREATE CATALOG IF NOT EXISTS ${catalog_name} PROPERTIES (
+ ${catalogProperties}
+ );
+ """
+ sql """
+ switch ${catalog_name};
+ """
+ sql """
+ show databases;
+ """
+ sql """
+ use ${dbName};
+ """
+ sql """
+ show tables;
+ """
+ sql """set force_jni_scanner=false"""
+ "order_qt_${prefix}" """
+ SELECT * FROM hive_test_table;
+ """
+ sql """set force_jni_scanner=true"""
+ "order_qt_${prefix}" """
+ SELECT * FROM hive_test_table;
+ """
+
+ }
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String s3_warehouse =
"s3://selectdb-qa-datalake-test-hk/paimon_warehouse"
+ String aws_ak = context.config.otherConfigs.get("AWSAK")
+ String aws_sk = context.config.otherConfigs.get("AWSSK")
+ String aws_endpoint = "s3.ap-east-1.amazonaws.com"
+ String s3_warehouse_properties = """
+ 'warehouse' = '${s3_warehouse}',
+ """
+ String s3_storage_properties = """
+ 's3.access_key' = '${aws_ak}',
+ 's3.secret_key' = '${aws_sk}',
+ 's3.endpoint' = '${aws_endpoint}'
+ """
+ String paimon_fs_catalog_properties = """
+ 'type'='paimon',
+ "paimon.catalog.type"="filesystem",
+ """
+ testQuery(paimon_fs_catalog_properties + s3_warehouse_properties +
s3_storage_properties, "aws", "aws_db")
+
+ }
+
+}
+
+
diff --git
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
index 9a5bcf0b748..b053bb25647 100644
---
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
+++
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_paimon_dlf_catalog",
"p2,external,paimon,external_remote,external_remote_paimon") {
+suite("test_paimon_dlf_catalog",
"p2,external,paimon,external_remote,external_remote_paimon,new_catalog_property")
{
String enabled = context.config.otherConfigs.get("enablePaimonTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
return
@@ -44,8 +44,10 @@ suite("test_paimon_dlf_catalog",
"p2,external,paimon,external_remote,external_re
"dlf.secret_key" = "${secret_key}"
);
"""
-
+ sql """ switch ${catalog} """
+ sql """ show databases """
sql """ use ${catalog}.regression_paimon """
+ sql """ show tables"""
sql """set force_jni_scanner=false"""
qt_c1 """ select * from tb_simple order by id """
diff --git
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_miss_dlf_param.groovy
similarity index 81%
copy from
regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
copy to
regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_miss_dlf_param.groovy
index 9a5bcf0b748..2b7a3ed024d 100644
---
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
+++
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_miss_dlf_param.groovy
@@ -15,14 +15,14 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_paimon_dlf_catalog",
"p2,external,paimon,external_remote,external_remote_paimon") {
+suite("test_paimon_dlf_catalog_miss_dlf_param",
"p2,external,paimon,external_remote,external_remote_paimon,new_catalog_property")
{
String enabled = context.config.otherConfigs.get("enablePaimonTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
return
}
try {
- String catalog = "test_paimon_dlf_catalog"
+ String catalog = "test_paimon_dlf_catalog_miss_dlf_param"
String uid = context.config.otherConfigs.get("dlf_uid")
String region = context.config.otherConfigs.get("dlf_region")
String catalog_id = context.config.otherConfigs.get("dlf_catalog_id")
@@ -41,11 +41,16 @@ suite("test_paimon_dlf_catalog",
"p2,external,paimon,external_remote,external_re
"dlf.region" = "${region}",
"dlf.catalog.id" = "${catalog_id}",
"dlf.access_key" = "${access_key}",
- "dlf.secret_key" = "${secret_key}"
+ "dlf.secret_key" = "${secret_key}",
+ "oss.endpoint"="oss-cn-beijing.aliyuncs.com",
+ "oss.access_key" = "${access_key}",
+ "oss.secret_key" = "${secret_key}"
);
"""
-
+ sql """ switch ${catalog} """
+ sql """ show databases """
sql """ use ${catalog}.regression_paimon """
+ sql """ show tables"""
sql """set force_jni_scanner=false"""
qt_c1 """ select * from tb_simple order by id """
diff --git
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_new_param.groovy
similarity index 74%
copy from
regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
copy to
regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_new_param.groovy
index 9a5bcf0b748..a8292b7957d 100644
---
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
+++
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_new_param.groovy
@@ -1,3 +1,5 @@
+import java.sql.Array
+
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -15,20 +17,21 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_paimon_dlf_catalog",
"p2,external,paimon,external_remote,external_remote_paimon") {
+suite("test_paimon_dlf_catalog_new_param",
"p2,external,paimon,external_remote,external_remote_paimon,new_catalog_property")
{
String enabled = context.config.otherConfigs.get("enablePaimonTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
return
}
try {
- String catalog = "test_paimon_dlf_catalog"
+ String catalog = "test_paimon_dlf_catalog_new_param"
String uid = context.config.otherConfigs.get("dlf_uid")
String region = context.config.otherConfigs.get("dlf_region")
String catalog_id = context.config.otherConfigs.get("dlf_catalog_id")
String access_key = context.config.otherConfigs.get("dlf_access_key")
String secret_key = context.config.otherConfigs.get("dlf_secret_key")
-
+ String dlf_vpc_endpoint =
context.config.otherConfigs.get("dlf_vpc_endpoint")
+ String dlf_public_endpoint =
context.config.otherConfigs.get("dlf_public_endpoint")
sql """drop catalog if exists ${catalog};"""
sql """
@@ -41,11 +44,17 @@ suite("test_paimon_dlf_catalog",
"p2,external,paimon,external_remote,external_re
"dlf.region" = "${region}",
"dlf.catalog.id" = "${catalog_id}",
"dlf.access_key" = "${access_key}",
- "dlf.secret_key" = "${secret_key}"
+ "dlf.secret_key" = "${secret_key}",
+ "dlf.endpoint" = "${dlf_vpc_endpoint}",
+ "oss.endpoint"="oss-cn-beijing.aliyuncs.com",
+ "oss.access_key" = "${access_key}",
+ "oss.secret_key" = "${secret_key}"
);
"""
-
+ sql """ switch ${catalog} """
+ sql """ show databases """
sql """ use ${catalog}.regression_paimon """
+ sql """ show tables"""
sql """set force_jni_scanner=false"""
qt_c1 """ select * from tb_simple order by id """
diff --git
a/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy
b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy
new file mode 100644
index 00000000000..265332cd5b3
--- /dev/null
+++
b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_hms_catalog", "p2,external,paimon,new_catalog_property") {
+
+ def testQuery = { String catalogProperties, String prefix, String dbName ->
+ def catalog_name = "test_paimon_on_hms_${prefix}_catalog"
+ sql """
+ DROP CATALOG IF EXISTS ${catalog_name};
+ """
+ sql """
+ CREATE CATALOG IF NOT EXISTS ${catalog_name} PROPERTIES (
+ ${catalogProperties}
+ );
+ """
+ sql """
+ switch ${catalog_name};
+ """
+ sql """
+ show databases;
+ """
+ sql """
+ use ${dbName};
+ """
+ sql """
+ show tables;
+ """
+ sql """set force_jni_scanner=false"""
+ "order_qt_${prefix}" """
+ SELECT * FROM external_test_table;
+ """
+ sql """set force_jni_scanner=true"""
+ "order_qt_${prefix}" """
+ SELECT * FROM external_test_table;
+ """
+ }
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ return
+ }
+ String extHiveHmsHost = context.config.otherConfigs.get("externalEnvIp")
+ String extHiveHmsPort = context.config.otherConfigs.get("hive3HmsPort")
+ String extHiveHdfsHost = context.config.otherConfigs.get("hive3HdfsPort")
+
+ /****************OSS*******************/
+ String oss_ak = context.config.otherConfigs.get("aliYunAk")
+ String oss_sk = context.config.otherConfigs.get("aliYunSk")
+ String oss_endpoint = "oss-cn-beijing.aliyuncs.com"
+ String oss_warehouse =
"oss://doris-regression-bj/regression/paimon_warehouse"
+ String oss_warehouse_properties = """
+ 'warehouse' = '${oss_warehouse}',
+ """
+ String oss_storage_properties = """
+ 'oss.access_key' = '${oss_ak}',
+ 'oss.secret_key' = '${oss_sk}',
+ 'oss.endpoint' = '${oss_endpoint}'
+ """
+ /****************COS*******************/
+ String cos_ak = context.config.otherConfigs.get("txYunAk")
+ String cos_sk = context.config.otherConfigs.get("txYunSk")
+ String cos_endpoint = "cos.ap-beijing.myqcloud.com"
+ String cos_region = "ap-beijing"
+ String cos_warehouse =
"cosn://sdb-qa-datalake-test-1308700295/paimon_warehouse"
+ String cos_warehouse_properties = """
+ 'warehouse' = '${cos_warehouse}',
+ """
+
+ String cos_storage_properties = """
+ 'cos.access_key' = '${cos_ak}',
+ 'cos.secret_key' = '${cos_sk}',
+ 'cos.endpoint' = '${cos_endpoint}'
+ """
+
+ /****************OBS*******************/
+ String obs_ak = context.config.otherConfigs.get("hwYunAk")
+ String obs_sk = context.config.otherConfigs.get("hwYunSk")
+ String obs_endpoint = "obs.cn-north-4.myhuaweicloud.com"
+ String obs_warehouse = "obs://doris-build/regression/paimon_warehouse"
+ String obs_warehouse_properties = """
+ 'warehouse' = '${obs_warehouse}',
+ """
+
+ String obs_storage_properties = """
+ 'obs.access_key' = '${obs_ak}',
+ 'obs.secret_key' = '${obs_sk}',
+ 'obs.endpoint' = '${obs_endpoint}'
+ """
+ /****************HDFS*******************/
+ String hdfs_warehouse =
"hdfs://${extHiveHmsHost}:${extHiveHdfsHost}/user/hive/warehouse"
+ String hdfs_warehouse_properties = """
+ 'warehouse' = '${hdfs_warehouse}',
+ """
+ String hdfs_storage_properties = """
+ 'fs.defaultFS' = 'hdfs://${extHiveHmsHost}:${extHiveHdfsHost}'
+ """
+ /**************** AWS S3*******************/
+ String s3_warehouse = "s3://selectdb-qa-datalake-test-hk/paimon_warehouse"
+ String aws_ak = context.config.otherConfigs.get("AWSAK")
+ String aws_sk = context.config.otherConfigs.get("AWSSK")
+ String aws_endpoint = "s3.ap-east-1.amazonaws.com"
+ String s3_warehouse_properties = """
+ 'warehouse' = '${s3_warehouse}',
+ """
+ String s3_storage_properties = """
+ 's3.access_key' = '${aws_ak}',
+ 's3.secret_key' = '${aws_sk}',
+ 's3.endpoint' = '${aws_endpoint}'
+ """
+
+ /*--------HMS START -----------*/
+ String paimon_hms_catalog_properties = """
+ 'type'='paimon',
+ 'paimon.catalog.type'='hms',
+ 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}',
+ """
+ testQuery(paimon_hms_catalog_properties + hdfs_warehouse_properties +
hdfs_storage_properties, "hdfs", "hdfs_db")
+ testQuery(paimon_hms_catalog_properties + oss_warehouse_properties +
oss_storage_properties, "oss", "ali_db")
+ testQuery(paimon_hms_catalog_properties + obs_warehouse_properties +
obs_storage_properties, "obs", "hw_db")
+ testQuery(paimon_hms_catalog_properties + cos_warehouse_properties +
cos_storage_properties, "cos", "tx_db")
+ testQuery(paimon_hms_catalog_properties + s3_warehouse_properties +
s3_storage_properties, "s3", "aws_db")
+
+ String paimon_fs_hdfs_support = """
+ 'fs.hdfs.support' = 'true',
+ """
+ String paimon_fs_oss_support = """
+ 'fs.oss.support' = 'true',
+ """
+ String paimon_fs_obs_support = """
+ 'fs.obs.support' = 'true',
+ """
+ String paimon_fs_cos_support = """
+ 'fs.cos.support' = 'true',
+ """
+ String paimon_fs_s3_support = """
+ 'fs.s3.support' = 'true',
+ """
+ testQuery(paimon_hms_catalog_properties + paimon_fs_hdfs_support +
hdfs_warehouse_properties + hdfs_storage_properties, "support_hdfs", "hdfs_db")
+ testQuery(paimon_hms_catalog_properties + paimon_fs_oss_support +
oss_warehouse_properties + oss_storage_properties, "support_oss", "ali_db")
+ testQuery(paimon_hms_catalog_properties + paimon_fs_obs_support +
obs_warehouse_properties + obs_storage_properties, "support_obs", "hw_db")
+ testQuery(paimon_hms_catalog_properties + paimon_fs_cos_support +
cos_warehouse_properties + cos_storage_properties, "support_cos", "tx_db")
+ testQuery(paimon_hms_catalog_properties + paimon_fs_s3_support +
s3_warehouse_properties + s3_storage_properties, "support_s3", "aws_db")
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
