This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 613350f0730 branch-3.1: [test](paimon) Add more paimon cases #55893
(#56055)
613350f0730 is described below
commit 613350f0730060212417d06fe29c5105f4152a66
Author: zgxme <[email protected]>
AuthorDate: Thu Sep 25 17:00:52 2025 +0800
branch-3.1: [test](paimon) Add more paimon cases #55893 (#56055)
bp#55893
add more paimon cases
1. paimon on hms and gcs
2. paimon on gcs filesystem
3. some cross feature batch incremental/time travel/system table
4. paimon on hms and hdfs(kerberos case)
---
.licenserc.yaml | 2 +
.../kerberos/entrypoint-hive-master.sh | 7 +
.../docker-compose/kerberos/hadoop-hive.env.tpl | 7 +
.../docker-compose/kerberos/kerberos.yaml.tpl | 2 +
.../docker-compose/kerberos/kerberos1_settings.env | 7 +-
.../docker-compose/kerberos/kerberos2_settings.env | 7 +-
...-5414d3da-b984-4a28-a501-e7dc42a1760f-0.parquet | Bin 0 -> 563 bytes
...manifest-3224f4a3-e7f2-402a-b1b1-bd1fe57d57ad-0 | Bin 0 -> 2016 bytes
...est-list-7dbac4ab-1226-4c1f-8346-3b388de65ea3-0 | Bin 0 -> 884 bytes
...est-list-7dbac4ab-1226-4c1f-8346-3b388de65ea3-1 | Bin 0 -> 986 bytes
.../hdfs_db.db/external_test_table/schema/schema-0 | 29 ++
.../external_test_table/snapshot/EARLIEST | 1 +
.../hdfs_db.db/external_test_table/snapshot/LATEST | 1 +
.../external_test_table/snapshot/snapshot-1 | 19 +
.../kerberos/sql/create_paimon_hive_table.hql | 7 +
.../external_table_p0/paimon/test_paimon_gcs.out | 33 ++
.../paimon/test_paimon_hms_catalog.out | 76 ++++
.../paimon/paimon_base_filesystem.groovy | 39 +-
.../paimon/test_paimon_catalog.groovy | 491 +++++++++++----------
.../paimon/test_paimon_deletion_vector_oss.groovy | 7 +-
.../paimon/test_paimon_gcs.groovy | 84 ++++
.../paimon/test_paimon_minio.groovy | 18 +
.../external_table_p0/paimon/test_paimon_s3.groovy | 9 +
.../hive/test_cloud_accessible_oss.groovy | 9 +-
.../iceberg/iceberg_rest_s3_storage_test.groovy | 8 +-
.../paimon/test_paimon_dlf_catalog.groovy | 10 +-
.../test_paimon_dlf_catalog_miss_dlf_param.groovy | 12 +-
.../test_paimon_dlf_catalog_new_param.groovy | 14 +-
.../paimon/test_paimon_hms_catalog.groovy | 77 +++-
.../hive_on_hms_and_dlf.groovy | 11 +-
.../iceberg_on_hms_and_filesystem_and_dlf.groovy | 62 ++-
31 files changed, 744 insertions(+), 305 deletions(-)
diff --git a/.licenserc.yaml b/.licenserc.yaml
index 7b22a39381e..f327adfe973 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -80,6 +80,8 @@ header:
- "regression-test/script/README"
- "regression-test/suites/load_p0/stream_load/data"
- "docker/thirdparties/docker-compose/hive/scripts/**"
+ - "docker/thirdparties/docker-compose/kerberos/paimon_data/**"
+ - "docker/thirdparties/docker-compose/kerberos/sql/**"
- "docker/thirdparties/docker-compose/iceberg/spark-defaults.conf.tpl"
- "conf/mysql_ssl_default_certificate/*"
- "conf/mysql_ssl_default_certificate/client_certificate/ca.pem"
diff --git
a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
index 246e0edab79..b19178aefb0 100755
--- a/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
+++ b/docker/thirdparties/docker-compose/kerberos/entrypoint-hive-master.sh
@@ -54,6 +54,8 @@ curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-cos-3.1.0
curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/cos_api-bundle-5.6.244.4.jar
curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/hadoop-aws-3.2.1.jar
curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/paimon-hive-connector-3.1-1.3-SNAPSHOT.jar
+curl -O
https://s3BucketName.s3Endpoint/regression/docker/hive3/gcs-connector-hadoop3-2.2.24-shaded.jar
+
/usr/local/hadoop-run.sh &
@@ -85,5 +87,10 @@ else
exit 1
fi
hive -f /usr/local/sql/create_kerberos_hive_table.sql
+if [[ ${enablePaimonHms} == "true" ]]; then
+ echo "Creating Paimon HMS catalog and table"
+ hadoop fs -put /tmp/paimon_data/* /user/hive/warehouse/
+ hive -f /usr/local/sql/create_paimon_hive_table.hql
+fi
exec_success_hook
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
index 3c453ff0fbd..9253b5aee09 100644
--- a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
+++ b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
@@ -84,4 +84,11 @@
HIVE_SITE_CONF_fs_oss_impl=org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem
HIVE_SITE_CONF_fs_oss_accessKeyId=${OSSAk}
HIVE_SITE_CONF_fs_oss_accessKeySecret=${OSSSk}
HIVE_SITE_CONF_fs_oss_endpoint=${OSSEndpoint}
+HIVE_SITE_CONF_fs_AbstractFileSystem_gs_impl=com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS
+HIVE_SITE_CONF_fs_gs_project_id=${GCSProjectId}
+HIVE_SITE_CONF_google_cloud_auth_service_account_enable=true
+HIVE_SITE_CONF_fs_gs_auth_service_account_email=${GCSAccountEmail}
+HIVE_SITE_CONF_fs_gs_auth_service_account_private_key_id=${GCSAccountPrivateKeyId}
+HIVE_SITE_CONF_fs_gs_auth_service_account_private_key=${GCSAccountPrivateKey}
+HIVE_SITE_CONF_fs_gs_proxy_address=${GCSProxyAddress}
enablePaimonHms=${enablePaimonHms}
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl
b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl
index 9a1520b74db..c253c06f3e4 100644
--- a/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl
+++ b/docker/thirdparties/docker-compose/kerberos/kerberos.yaml.tpl
@@ -32,6 +32,7 @@ services:
- ./conf/kerberos1/my.cnf:/etc/my.cnf
- ./conf/kerberos1/kdc.conf:/var/kerberos/krb5kdc/kdc.conf
- ./conf/kerberos1/krb5.conf:/etc/krb5.conf
+ - ./paimon_data:/tmp/paimon_data
hostname: hadoop-master
entrypoint: /usr/local/entrypoint-hive-master.sh 1
healthcheck:
@@ -59,6 +60,7 @@ services:
- ./conf/kerberos2/my.cnf:/etc/my.cnf
- ./conf/kerberos2/kdc.conf:/var/kerberos/krb5kdc/kdc.conf
- ./conf/kerberos2/krb5.conf:/etc/krb5.conf
+ - ./paimon_data:/tmp/paimon_data
entrypoint: /usr/local/entrypoint-hive-master.sh 2
healthcheck:
test: ["CMD", "ls", "/tmp/SUCCESS"]
diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env
b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env
index 62de14dfafa..44f2352cf16 100644
--- a/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env
+++ b/docker/thirdparties/docker-compose/kerberos/kerberos1_settings.env
@@ -60,4 +60,9 @@ export OBSSk="*****************"
export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com"
export COSAk="*****************"
export COSSk="*****************"
-export COSRegion="ap-beijing"
\ No newline at end of file
+export COSRegion="ap-beijing"
+export GCSProjectId=""
+export GCSAccountEmail=""
+export GCSAccountPrivateKeyId=""
+export GCSAccountPrivateKey=""
+export GCSProxyAddress=""
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env
b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env
index 776fd50eb2d..49099e0f441 100644
--- a/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env
+++ b/docker/thirdparties/docker-compose/kerberos/kerberos2_settings.env
@@ -56,4 +56,9 @@ export OBSSk="*****************"
export OBSEndpoint="obs.cn-north-4.myhuaweicloud.com"
export COSAk="*****************"
export COSSk="*****************"
-export COSRegion="ap-beijing"
\ No newline at end of file
+export COSRegion="ap-beijing"
+export GCSProjectId=""
+export GCSAccountEmail=""
+export GCSAccountPrivateKeyId=""
+export GCSAccountPrivateKey=""
+export GCSProxyAddress=""
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/bucket-0/data-5414d3da-b984-4a28-a501-e7dc42a1760f-0.parquet
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/bucket-0/data-5414d3da-b984-4a28-a501-e7dc42a1760f-0.parquet
new file mode 100644
index 00000000000..4b6394700c0
Binary files /dev/null and
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/bucket-0/data-5414d3da-b984-4a28-a501-e7dc42a1760f-0.parquet
differ
diff --git
a/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/manifest/manifest-3224f4a3-e7f2-402a-b1b1-bd1fe57d57ad-0
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/manifest/manifest-3224f4a3-e7f2-402a-b1b1-bd1fe57d57ad-0
new file mode 100644
index 00000000000..abffba22beb
Binary files /dev/null and
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/manifest/manifest-3224f4a3-e7f2-402a-b1b1-bd1fe57d57ad-0
differ
diff --git
a/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/manifest/manifest-list-7dbac4ab-1226-4c1f-8346-3b388de65ea3-0
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/manifest/manifest-list-7dbac4ab-1226-4c1f-8346-3b388de65ea3-0
new file mode 100644
index 00000000000..4bdc40be568
Binary files /dev/null and
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/manifest/manifest-list-7dbac4ab-1226-4c1f-8346-3b388de65ea3-0
differ
diff --git
a/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/manifest/manifest-list-7dbac4ab-1226-4c1f-8346-3b388de65ea3-1
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/manifest/manifest-list-7dbac4ab-1226-4c1f-8346-3b388de65ea3-1
new file mode 100644
index 00000000000..0e976861286
Binary files /dev/null and
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/manifest/manifest-list-7dbac4ab-1226-4c1f-8346-3b388de65ea3-1
differ
diff --git
a/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/schema/schema-0
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/schema/schema-0
new file mode 100644
index 00000000000..b7c14c7633e
--- /dev/null
+++
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/schema/schema-0
@@ -0,0 +1,29 @@
+{
+ "version" : 3,
+ "id" : 0,
+ "fields" : [ {
+ "id" : 0,
+ "name" : "a",
+ "type" : "INT",
+ "description" : "The a field"
+ }, {
+ "id" : 1,
+ "name" : "b",
+ "type" : "STRING",
+ "description" : "The b field"
+ } ],
+ "highestFieldId" : 1,
+ "partitionKeys" : [ ],
+ "primaryKeys" : [ ],
+ "options" : {
+ "path" : "hdfs:///user/hive/warehouse/hdfs_db.db/external_test_table",
+ "totalSize" : "0",
+ "numRows" : "0",
+ "rawDataSize" : "0",
+ "COLUMN_STATS_ACCURATE" :
"{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}",
+ "numFiles" : "0",
+ "bucketing_version" : "2",
+ "storage_handler" : "org.apache.paimon.hive.PaimonStorageHandler"
+ },
+ "timeMillis" : 1757405300463
+}
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/snapshot/EARLIEST
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/snapshot/EARLIEST
new file mode 100644
index 00000000000..56a6051ca2b
--- /dev/null
+++
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/snapshot/EARLIEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/snapshot/LATEST
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/snapshot/LATEST
new file mode 100644
index 00000000000..56a6051ca2b
--- /dev/null
+++
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/snapshot/LATEST
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/snapshot/snapshot-1
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/snapshot/snapshot-1
new file mode 100644
index 00000000000..685fe29e727
--- /dev/null
+++
b/docker/thirdparties/docker-compose/kerberos/paimon_data/hdfs_db.db/external_test_table/snapshot/snapshot-1
@@ -0,0 +1,19 @@
+{
+ "version" : 3,
+ "id" : 1,
+ "schemaId" : 0,
+ "baseManifestList" : "manifest-list-7dbac4ab-1226-4c1f-8346-3b388de65ea3-0",
+ "baseManifestListSize" : 884,
+ "deltaManifestList" : "manifest-list-7dbac4ab-1226-4c1f-8346-3b388de65ea3-1",
+ "deltaManifestListSize" : 986,
+ "changelogManifestList" : null,
+ "commitUser" : "861d98af-e944-4a2f-9ef7-b79a129c3fba",
+ "commitIdentifier" : 9223372036854775807,
+ "commitKind" : "APPEND",
+ "timeMillis" : 1757405309978,
+ "logOffsets" : { },
+ "totalRecordCount" : 1,
+ "deltaRecordCount" : 1,
+ "changelogRecordCount" : 0,
+ "nextRowId" : 0
+}
\ No newline at end of file
diff --git
a/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql
b/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql
new file mode 100644
index 00000000000..65f463fef22
--- /dev/null
+++
b/docker/thirdparties/docker-compose/kerberos/sql/create_paimon_hive_table.hql
@@ -0,0 +1,7 @@
+CREATE DATABASE IF NOT EXISTS hdfs_db;
+
+USE hdfs_db;
+
+CREATE EXTERNAL TABLE external_test_table
+ STORED BY 'org.apache.paimon.hive.PaimonStorageHandler'
+LOCATION 'hdfs:///user/hive/warehouse/hdfs_db.db/external_test_table';
\ No newline at end of file
diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_gcs.out
b/regression-test/data/external_table_p0/paimon/test_paimon_gcs.out
new file mode 100644
index 00000000000..912b19449f5
--- /dev/null
+++ b/regression-test/data/external_table_p0/paimon/test_paimon_gcs.out
@@ -0,0 +1,33 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !no_region --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
+-- !region --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
+-- !no_region --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
+-- !region --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
diff --git
a/regression-test/data/external_table_p2/paimon/test_paimon_hms_catalog.out
b/regression-test/data/external_table_p2/paimon/test_paimon_hms_catalog.out
index 51d0f28fdf2..abf50d7c852 100644
--- a/regression-test/data/external_table_p2/paimon/test_paimon_hms_catalog.out
+++ b/regression-test/data/external_table_p2/paimon/test_paimon_hms_catalog.out
@@ -5,6 +5,12 @@
-- !hdfs --
11111111 hdfs_db_test
+-- !hdfs_kerberos --
+11111111 hdfs_db_test
+
+-- !hdfs_kerberos --
+11111111 hdfs_db_test
+
-- !oss --
11111111 ali_db_test
11111111 ali_db_test
@@ -61,12 +67,50 @@
11111111 aws_db_test
11111111 aws_db_test
+-- !gcs --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
+-- !gcs --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
+-- !gcs_s3 --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
+-- !gcs_s3 --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
-- !support_hdfs --
11111111 hdfs_db_test
-- !support_hdfs --
11111111 hdfs_db_test
+-- !support_hdfs_kerberos --
+11111111 hdfs_db_test
+
+-- !support_hdfs_kerberos --
+11111111 hdfs_db_test
+
-- !support_oss --
11111111 ali_db_test
11111111 ali_db_test
@@ -123,3 +167,35 @@
11111111 aws_db_test
11111111 aws_db_test
+-- !support_gcs --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
+-- !support_gcs --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
+-- !support_gcs_s3 --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
+-- !support_gcs_s3 --
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+11111111 gcs_db_test
+
diff --git
a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
index 6d6e876948a..42d764dcddd 100644
---
a/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
+++
b/regression-test/suites/external_table_p0/paimon/paimon_base_filesystem.groovy
@@ -29,6 +29,8 @@ suite("paimon_base_filesystem",
"p0,external,doris,external_docker,external_dock
String catalog_cosn = "paimon_base_filesystem_paimon_cosn"
String aliYunAk = context.config.otherConfigs.get("aliYunAk")
String aliYunSk = context.config.otherConfigs.get("aliYunSk")
+ String aliYunEndpoint =
context.config.otherConfigs.get("aliYunEndpoint")
+ String bucket = context.config.otherConfigs.get("aliYunBucket")
String hwYunAk = context.config.otherConfigs.get("hwYunAk")
String hwYunSk = context.config.otherConfigs.get("hwYunSk")
String txYunAk = context.config.otherConfigs.get("txYunAk")
@@ -68,10 +70,10 @@ suite("paimon_base_filesystem",
"p0,external,doris,external_docker,external_dock
create catalog if not exists ${catalog_oss} properties (
"type" = "paimon",
"paimon.catalog.type"="filesystem",
- "warehouse" = "oss://doris-regression-bj/regression/paimon1",
+ "warehouse" = "oss://${bucket}/regression/paimon1",
"oss.access_key"="${aliYunAk}",
"oss.secret_key"="${aliYunSk}",
- "oss.endpoint"="oss-cn-beijing.aliyuncs.com"
+ "oss.endpoint"="${aliYunEndpoint}"
);
"""
sql """
@@ -93,21 +95,54 @@ suite("paimon_base_filesystem",
"p0,external,doris,external_docker,external_dock
sql """ show databases """
sql """ use ${catalog_oss}.db1 """
// sql """ show tables """
+ // 3.1 new features
+ // batch incremental
+ sql """SELECT * FROM all_table @incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM all_table FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM all_table\$snapshots;"""
sql """ switch ${catalog_obs} """
sql """ show databases """
sql """ use ${catalog_obs}.db1 """
// sql """ show tables """
+ // batch incremental
+ sql """SELECT * FROM all_table @incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM all_table FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM all_table\$snapshots;"""
sql """ switch ${catalog_cos} """
sql """ show databases """
sql """ use ${catalog_cos}.db1 """
// sql """ show tables """
+ // batch incremental
+ sql """SELECT * FROM all_table @incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM all_table FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM all_table\$snapshots;"""
sql """ switch ${catalog_cosn} """
sql """ show databases """
sql """ use ${catalog_cosn}.db1 """
// sql """ show tables """
+ // batch incremental
+ sql """SELECT * FROM all_table @incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM all_table FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM all_table\$snapshots;"""
sql """set force_jni_scanner=false"""
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
index b0270671d75..b5ca34e5a18 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
@@ -56,271 +56,280 @@ suite("test_paimon_catalog",
"p0,external,doris,external_docker,external_docker_
sql """drop catalog ${hms_ctl_name}""";
String enabled = context.config.otherConfigs.get("enablePaimonTest")
- if (enabled != null && enabled.equalsIgnoreCase("true")) {
- def qt_all_type = { String table_name ->
- qt_all """select * from ${table_name} order by c1"""
- qt_predict_like_1 """select * from ${table_name} where c13
like '%3%' order by c1"""
- qt_predict_like_2 """select * from ${table_name} where c13
like '13%' order by c1"""
- qt_predict_like_3 """select * from ${table_name} where c13
like '13' order by c1"""
- qt_predict_like_4 """select * from ${table_name} where c13
like '130str' order by c1"""
- qt_predict_like_5 """select * from ${table_name} where c13
like '130str%' order by c1"""
- qt_c1 """select * from ${table_name} where c1=10;"""
- qt_c2 """select * from ${table_name} where c2=2;"""
- qt_c3 """select * from ${table_name} where c3=3;"""
- qt_c4 """select * from ${table_name} where c4=4;"""
- qt_c5 """select * from ${table_name} where c5=5;"""
- qt_c6 """select * from ${table_name} where c6=6;"""
- qt_c7 """select * from ${table_name} where c7=7;"""
- qt_c8 """select * from ${table_name} where c8=8;"""
- qt_c9 """select * from ${table_name} where c9<10;"""
- qt_c10 """select * from ${table_name} where c10=10.1;"""
- qt_c11 """select * from ${table_name} where c11=11.1;"""
- qt_c12 """select * from ${table_name} where
c12='2020-02-02';"""
- qt_c13 """select * from ${table_name} where c13='13str';"""
- qt_c14 """select * from ${table_name} where
c14='14varchar';"""
- qt_c15 """select * from ${table_name} where c15='a';"""
- qt_c16 """select * from ${table_name} where c16=true;"""
- qt_c18 """select * from ${table_name} where c18='2023-08-13
09:32:38.53';"""
- qt_c101 """select * from ${table_name} where c1 is not null or
c2 is not null order by c1"""
- }
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ def qt_all_type = { String table_name ->
+ qt_all """select * from ${table_name} order by c1"""
+ qt_predict_like_1 """select * from ${table_name} where c13 like
'%3%' order by c1"""
+ qt_predict_like_2 """select * from ${table_name} where c13 like
'13%' order by c1"""
+ qt_predict_like_3 """select * from ${table_name} where c13 like
'13' order by c1"""
+ qt_predict_like_4 """select * from ${table_name} where c13 like
'130str' order by c1"""
+ qt_predict_like_5 """select * from ${table_name} where c13 like
'130str%' order by c1"""
+ qt_c1 """select * from ${table_name} where c1=10;"""
+ qt_c2 """select * from ${table_name} where c2=2;"""
+ qt_c3 """select * from ${table_name} where c3=3;"""
+ qt_c4 """select * from ${table_name} where c4=4;"""
+ qt_c5 """select * from ${table_name} where c5=5;"""
+ qt_c6 """select * from ${table_name} where c6=6;"""
+ qt_c7 """select * from ${table_name} where c7=7;"""
+ qt_c8 """select * from ${table_name} where c8=8;"""
+ qt_c9 """select * from ${table_name} where c9<10;"""
+ qt_c10 """select * from ${table_name} where c10=10.1;"""
+ qt_c11 """select * from ${table_name} where c11=11.1;"""
+ qt_c12 """select * from ${table_name} where c12='2020-02-02';"""
+ qt_c13 """select * from ${table_name} where c13='13str';"""
+ qt_c14 """select * from ${table_name} where c14='14varchar';"""
+ qt_c15 """select * from ${table_name} where c15='a';"""
+ qt_c16 """select * from ${table_name} where c16=true;"""
+ qt_c18 """select * from ${table_name} where c18='2023-08-13
09:32:38.53';"""
+ qt_c101 """select * from ${table_name} where c1 is not null or c2
is not null order by c1"""
+ }
- def c19 = """select * from auto_bucket order by user_id;"""
- def c20 = """select * from auto_bucket where dt="b";"""
- def c21 = """select * from auto_bucket where dt="b" and hh="c";"""
- def c22 = """select * from auto_bucket where dt="d";"""
- def c23 = """select * from complex_tab order by c1;"""
- def c24 = """select * from complex_tab where c1=1;"""
- def c25 = """select c3['a_test'], c3['b_test'], c3['bbb'],
c3['ccc'] from complex_tab order by c3['a_test'], c3['b_test']"""
- def c26 = """select array_max(c2) c from complex_tab order by c"""
- def c27 = """select * from complex_all order by c1"""
- def c28 = """select array_min(c2) c from complex_all order by c"""
- def c29 = """select array_min(c3) c from complex_all order by c"""
- def c30= """select array_min(c4) c from complex_all order by c"""
- def c31= """select array_min(c5) c from complex_all order by c"""
- def c32= """select array_min(c6) c from complex_all order by c"""
- def c33= """select array_min(c7) c from complex_all order by c"""
- def c34= """select array_min(c8) c from complex_all order by c"""
- def c35= """select array_min(c9) c from complex_all order by c"""
- def c36= """select array_min(c10) c from complex_all order by c"""
- def c37= """select array_max(c11) c from complex_all order by c"""
- def c38= """select array_max(c12) c from complex_all order by c"""
- def c39= """select array_max(c13) c from complex_all order by c"""
- def c40= """select array_size(c14) c from complex_all order by c"""
- def c41= """select array_size(c15) c from complex_all order by c"""
- def c42= """select array_size(c16) c from complex_all order by c"""
- def c43= """select array_max(c17) c from complex_all order by c"""
- def c44= """select array_size(c18) c from complex_all order by c"""
- def c45= """select array_max(c19) c from complex_all order by c"""
+ def c19 = """select * from auto_bucket order by user_id;"""
+ def c20 = """select * from auto_bucket where dt="b";"""
+ def c21 = """select * from auto_bucket where dt="b" and hh="c";"""
+ def c22 = """select * from auto_bucket where dt="d";"""
+ def c23 = """select * from complex_tab order by c1;"""
+ def c24 = """select * from complex_tab where c1=1;"""
+ def c25 = """select c3['a_test'], c3['b_test'], c3['bbb'], c3['ccc']
from complex_tab order by c3['a_test'], c3['b_test']"""
+ def c26 = """select array_max(c2) c from complex_tab order by c"""
+ def c27 = """select * from complex_all order by c1"""
+ def c28 = """select array_min(c2) c from complex_all order by c"""
+ def c29 = """select array_min(c3) c from complex_all order by c"""
+ def c30 = """select array_min(c4) c from complex_all order by c"""
+ def c31 = """select array_min(c5) c from complex_all order by c"""
+ def c32 = """select array_min(c6) c from complex_all order by c"""
+ def c33 = """select array_min(c7) c from complex_all order by c"""
+ def c34 = """select array_min(c8) c from complex_all order by c"""
+ def c35 = """select array_min(c9) c from complex_all order by c"""
+ def c36 = """select array_min(c10) c from complex_all order by c"""
+ def c37 = """select array_max(c11) c from complex_all order by c"""
+ def c38 = """select array_max(c12) c from complex_all order by c"""
+ def c39 = """select array_max(c13) c from complex_all order by c"""
+ def c40 = """select array_size(c14) c from complex_all order by c"""
+ def c41 = """select array_size(c15) c from complex_all order by c"""
+ def c42 = """select array_size(c16) c from complex_all order by c"""
+ def c43 = """select array_max(c17) c from complex_all order by c"""
+ def c44 = """select array_size(c18) c from complex_all order by c"""
+ def c45 = """select array_max(c19) c from complex_all order by c"""
- def c46= """select c20[0] c from complex_all order by c"""
- def c47= """select c21[0] c from complex_all order by c"""
- def c48= """select c22[0] c from complex_all order by c"""
- def c49= """select c23[0] c from complex_all order by c"""
- def c50= """select c24[1] c from complex_all order by c"""
- def c51= """select c25[0] c from complex_all order by c"""
- def c52= """select c26[0] c from complex_all order by c"""
- def c53= """select c27[0] c from complex_all order by c"""
- def c54= """select c28[0] c from complex_all order by c"""
- def c55= """select c29[0] c from complex_all order by c"""
- def c56= """select c30[0] c from complex_all order by c"""
- def c57= """select c31[0] c from complex_all order by c"""
- def c58= """select c32[0] c from complex_all order by c"""
- def c59= """select c33[0] c from complex_all order by c"""
- def c60= """select c34[0] c from complex_all order by c"""
- def c61= """select c35[0] c from complex_all order by c"""
- def c62= """select c36[0] c from complex_all order by c"""
- def c63= """select c37[0] c from complex_all order by c"""
+ def c46 = """select c20[0] c from complex_all order by c"""
+ def c47 = """select c21[0] c from complex_all order by c"""
+ def c48 = """select c22[0] c from complex_all order by c"""
+ def c49 = """select c23[0] c from complex_all order by c"""
+ def c50 = """select c24[1] c from complex_all order by c"""
+ def c51 = """select c25[0] c from complex_all order by c"""
+ def c52 = """select c26[0] c from complex_all order by c"""
+ def c53 = """select c27[0] c from complex_all order by c"""
+ def c54 = """select c28[0] c from complex_all order by c"""
+ def c55 = """select c29[0] c from complex_all order by c"""
+ def c56 = """select c30[0] c from complex_all order by c"""
+ def c57 = """select c31[0] c from complex_all order by c"""
+ def c58 = """select c32[0] c from complex_all order by c"""
+ def c59 = """select c33[0] c from complex_all order by c"""
+ def c60 = """select c34[0] c from complex_all order by c"""
+ def c61 = """select c35[0] c from complex_all order by c"""
+ def c62 = """select c36[0] c from complex_all order by c"""
+ def c63 = """select c37[0] c from complex_all order by c"""
- def c64= """select c38[2] c from complex_all order by c"""
- def c65= """select c39[4] c from complex_all order by c"""
- def c66= """select c40[5] c from complex_all order by c"""
- def c67= """select c41[7] c from complex_all order by c"""
- def c68= """select c42[9] c from complex_all order by c"""
- def c69= """select c43[10] c from complex_all order by c"""
- def c70= """select c44[12] c from complex_all order by c"""
- def c71= """select c45[13] c from complex_all order by c"""
- def c72= """select c46[14] c from complex_all order by c"""
- def c73= """select c47[16] c from complex_all order by c"""
- def c74= """select c48[17] c from complex_all order by c"""
- def c75= """select c49[19] c from complex_all order by c"""
- def c76= """select c50[21] c from complex_all order by c"""
- def c77= """select c51[22] c from complex_all order by c"""
- def c78= """select c52[25] c from complex_all order by c"""
- def c79= """select c53[27] c from complex_all order by c"""
- def c80= """select c54[29] c from complex_all order by c"""
- def c81= """select c55[30] c from complex_all order by c"""
+ def c64 = """select c38[2] c from complex_all order by c"""
+ def c65 = """select c39[4] c from complex_all order by c"""
+ def c66 = """select c40[5] c from complex_all order by c"""
+ def c67 = """select c41[7] c from complex_all order by c"""
+ def c68 = """select c42[9] c from complex_all order by c"""
+ def c69 = """select c43[10] c from complex_all order by c"""
+ def c70 = """select c44[12] c from complex_all order by c"""
+ def c71 = """select c45[13] c from complex_all order by c"""
+ def c72 = """select c46[14] c from complex_all order by c"""
+ def c73 = """select c47[16] c from complex_all order by c"""
+ def c74 = """select c48[17] c from complex_all order by c"""
+ def c75 = """select c49[19] c from complex_all order by c"""
+ def c76 = """select c50[21] c from complex_all order by c"""
+ def c77 = """select c51[22] c from complex_all order by c"""
+ def c78 = """select c52[25] c from complex_all order by c"""
+ def c79 = """select c53[27] c from complex_all order by c"""
+ def c80 = """select c54[29] c from complex_all order by c"""
+ def c81 = """select c55[30] c from complex_all order by c"""
- def c82= """select c56[2] c from complex_all order by c"""
- def c83= """select c57[4] c from complex_all order by c"""
- def c84= """select c58[5] c from complex_all order by c"""
- def c85= """select c59[7] c from complex_all order by c"""
- def c86= """select c60[9] c from complex_all order by c"""
- def c87= """select c61[10] c from complex_all order by c"""
- def c88= """select c62[12] c from complex_all order by c"""
- def c89= """select c63[13] c from complex_all order by c"""
- def c90= """select c64[14] c from complex_all order by c"""
- def c91= """select c65[16] c from complex_all order by c"""
- def c92= """select c66[17] c from complex_all order by c"""
- def c93= """select c67[19] c from complex_all order by c"""
- def c94= """select c68[21] c from complex_all order by c"""
- def c95= """select c69[22] c from complex_all order by c"""
- def c96= """select c70[25] c from complex_all order by c"""
- def c97= """select c71[27] c from complex_all order by c"""
- def c98= """select c72[29] c from complex_all order by c"""
- def c99= """select c73[30] c from complex_all order by c"""
+ def c82 = """select c56[2] c from complex_all order by c"""
+ def c83 = """select c57[4] c from complex_all order by c"""
+ def c84 = """select c58[5] c from complex_all order by c"""
+ def c85 = """select c59[7] c from complex_all order by c"""
+ def c86 = """select c60[9] c from complex_all order by c"""
+ def c87 = """select c61[10] c from complex_all order by c"""
+ def c88 = """select c62[12] c from complex_all order by c"""
+ def c89 = """select c63[13] c from complex_all order by c"""
+ def c90 = """select c64[14] c from complex_all order by c"""
+ def c91 = """select c65[16] c from complex_all order by c"""
+ def c92 = """select c66[17] c from complex_all order by c"""
+ def c93 = """select c67[19] c from complex_all order by c"""
+ def c94 = """select c68[21] c from complex_all order by c"""
+ def c95 = """select c69[22] c from complex_all order by c"""
+ def c96 = """select c70[25] c from complex_all order by c"""
+ def c97 = """select c71[27] c from complex_all order by c"""
+ def c98 = """select c72[29] c from complex_all order by c"""
+ def c99 = """select c73[30] c from complex_all order by c"""
- def c100= """select * from array_nested order by c1;"""
+ def c100 = """select * from array_nested order by c1;"""
- def c102= """select * from row_native_test order by id;"""
- def c103= """select * from row_jni_test order by id;"""
+ def c102 = """select * from row_native_test order by id;"""
+ def c103 = """select * from row_jni_test order by id;"""
- def c104= """select * from deletion_vector_orc;"""
- def c105= """select * from deletion_vector_parquet;"""
- def c106= """ select * from tb_with_upper_case """
- def c107= """ select id from tb_with_upper_case where id > 1 """
- def c108= """ select id from tb_with_upper_case where id = 1 """
- def c109= """ select id from tb_with_upper_case where id < 1 """
+ def c104 = """select * from deletion_vector_orc;"""
+ def c105 = """select * from deletion_vector_parquet;"""
+ def c106 = """ select * from tb_with_upper_case """
+ def c107 = """ select id from tb_with_upper_case where id > 1 """
+ def c108 = """ select id from tb_with_upper_case where id = 1 """
+ def c109 = """ select id from tb_with_upper_case where id < 1 """
- String hdfs_port = context.config.otherConfigs.get("hive2HdfsPort")
- String catalog_name = "ctl_test_paimon_catalog"
- String externalEnvIp =
context.config.otherConfigs.get("externalEnvIp")
+ String hdfs_port = context.config.otherConfigs.get("hive2HdfsPort")
+ String catalog_name = "ctl_test_paimon_catalog"
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
- sql """drop catalog if exists ${catalog_name}"""
- sql """create catalog if not exists ${catalog_name} properties (
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog if not exists ${catalog_name} properties (
"type" = "paimon",
"paimon.catalog.type"="filesystem",
"warehouse" =
"hdfs://${externalEnvIp}:${hdfs_port}/user/doris/paimon1"
);"""
- sql """use `${catalog_name}`.`db1`"""
+ sql """use `${catalog_name}`.`db1`"""
- def test_cases = { String force, String cache ->
- sql """ set force_jni_scanner=${force} """
- sql """ set enable_file_cache=${cache} """
- qt_all_type("all_table")
- qt_all_type("all_table_with_parquet")
+ def test_cases = { String force, String cache ->
+ sql """ set force_jni_scanner=${force} """
+ sql """ set enable_file_cache=${cache} """
+ qt_all_type("all_table")
+ qt_all_type("all_table_with_parquet")
- qt_c19 c19
- qt_c20 c20
- qt_c21 c21
- qt_c22 c22
- qt_c23 c23
- qt_c24 c24
- qt_c25 c25
- qt_c26 c26
- qt_c27 c27
- qt_c28 c28
- qt_c29 c29
- qt_c30 c30
- qt_c31 c31
- qt_c32 c32
- qt_c33 c33
- qt_c34 c34
- qt_c35 c35
- qt_c36 c36
- qt_c37 c37
- qt_c38 c38
- qt_c39 c39
- qt_c40 c40
- qt_c41 c41
- qt_c42 c42
- qt_c43 c43
- qt_c44 c44
- qt_c45 c45
- qt_c46 c46
- qt_c47 c47
- qt_c48 c48
- qt_c49 c49
- qt_c50 c50
- qt_c51 c51
- qt_c52 c52
- qt_c53 c53
- qt_c54 c54
- qt_c55 c55
- qt_c56 c56
- qt_c57 c57
- qt_c58 c58
- qt_c59 c59
- qt_c60 c60
- qt_c61 c61
- qt_c62 c62
- qt_c63 c63
- qt_c64 c64
- qt_c65 c65
- qt_c66 c66
- qt_c67 c67
- qt_c68 c68
- qt_c69 c69
- qt_c70 c70
- qt_c71 c71
- qt_c72 c72
- qt_c73 c73
- qt_c74 c74
- qt_c75 c75
- qt_c76 c76
- qt_c77 c77
- qt_c78 c78
- qt_c79 c79
- qt_c80 c80
- qt_c81 c81
- qt_c82 c82
- qt_c83 c83
- qt_c84 c84
- qt_c85 c85
- qt_c85 c86
- qt_c86 c87
- qt_c86 c88
- qt_c89 c89
- qt_c90 c90
- qt_c91 c91
- qt_c92 c92
- qt_c93 c93
- qt_c94 c94
- qt_c95 c95
- qt_c96 c96
- qt_c97 c97
- qt_c98 c98
- qt_c99 c99
- qt_c100 c100
- qt_c102 c102
- qt_c103 c103
- qt_c104 c104
- qt_c105 c105
- qt_c106 c106
- qt_c107 c107
- qt_c108 c108
- qt_c109 c109
- }
+ qt_c19 c19
+ qt_c20 c20
+ qt_c21 c21
+ qt_c22 c22
+ qt_c23 c23
+ qt_c24 c24
+ qt_c25 c25
+ qt_c26 c26
+ qt_c27 c27
+ qt_c28 c28
+ qt_c29 c29
+ qt_c30 c30
+ qt_c31 c31
+ qt_c32 c32
+ qt_c33 c33
+ qt_c34 c34
+ qt_c35 c35
+ qt_c36 c36
+ qt_c37 c37
+ qt_c38 c38
+ qt_c39 c39
+ qt_c40 c40
+ qt_c41 c41
+ qt_c42 c42
+ qt_c43 c43
+ qt_c44 c44
+ qt_c45 c45
+ qt_c46 c46
+ qt_c47 c47
+ qt_c48 c48
+ qt_c49 c49
+ qt_c50 c50
+ qt_c51 c51
+ qt_c52 c52
+ qt_c53 c53
+ qt_c54 c54
+ qt_c55 c55
+ qt_c56 c56
+ qt_c57 c57
+ qt_c58 c58
+ qt_c59 c59
+ qt_c60 c60
+ qt_c61 c61
+ qt_c62 c62
+ qt_c63 c63
+ qt_c64 c64
+ qt_c65 c65
+ qt_c66 c66
+ qt_c67 c67
+ qt_c68 c68
+ qt_c69 c69
+ qt_c70 c70
+ qt_c71 c71
+ qt_c72 c72
+ qt_c73 c73
+ qt_c74 c74
+ qt_c75 c75
+ qt_c76 c76
+ qt_c77 c77
+ qt_c78 c78
+ qt_c79 c79
+ qt_c80 c80
+ qt_c81 c81
+ qt_c82 c82
+ qt_c83 c83
+ qt_c84 c84
+ qt_c85 c85
+ qt_c85 c86
+ qt_c86 c87
+ qt_c86 c88
+ qt_c89 c89
+ qt_c90 c90
+ qt_c91 c91
+ qt_c92 c92
+ qt_c93 c93
+ qt_c94 c94
+ qt_c95 c95
+ qt_c96 c96
+ qt_c97 c97
+ qt_c98 c98
+ qt_c99 c99
+ qt_c100 c100
+ qt_c102 c102
+ qt_c103 c103
+ qt_c104 c104
+ qt_c105 c105
+ qt_c106 c106
+ qt_c107 c107
+ qt_c108 c108
+ qt_c109 c109
+ }
- test_cases("false", "false")
- test_cases("false", "true")
- test_cases("true", "false")
- test_cases("true", "true")
+ test_cases("false", "false")
+ test_cases("false", "true")
+ test_cases("true", "false")
+ test_cases("true", "true")
+ // 3.1 new features
+ // batch incremental
+ sql """SELECT * FROM complex_all
@incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM complex_all FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM complex_all\$snapshots;"""
- test {
- sql """select * from dup_columns_table;"""
- exception "Duplicate column name found: id"
- }
+ test {
+ sql """select * from dup_columns_table;"""
+ exception "Duplicate column name found: id"
+ }
- sql """ set force_jni_scanner=false; """
+ sql """ set force_jni_scanner=false; """
- // test view from jion paimon
- sql """ switch internal """
- String view_db = "test_view_for_paimon"
- sql """ drop database if exists ${view_db}"""
- sql """ create database if not exists ${view_db}"""
- sql """use ${view_db}"""
- sql """ create view test_tst_1 as select * from
${catalog_name}.`db1`.all_table; """
- sql """ create view test_tst_2 as select * from
${catalog_name}.`db1`.all_table_with_parquet; """
- sql """ create view test_tst_5 as select * from
${catalog_name}.`db1`.array_nested; """
- sql """ create table test_tst_6 properties ("replication_num" =
"1") as
+ // test view from jion paimon
+ sql """ switch internal """
+ String view_db = "test_view_for_paimon"
+ sql """ drop database if exists ${view_db}"""
+ sql """ create database if not exists ${view_db}"""
+ sql """use ${view_db}"""
+ sql """ create view test_tst_1 as select * from
${catalog_name}.`db1`.all_table; """
+ sql """ create view test_tst_2 as select * from
${catalog_name}.`db1`.all_table_with_parquet; """
+ sql """ create view test_tst_5 as select * from
${catalog_name}.`db1`.array_nested; """
+ sql """ create table test_tst_6 properties ("replication_num" = "1")
as
select f.c2,f.c3,c.c1 from
(select a.c2,b.c3 from test_tst_1 a inner join
test_tst_2 b on a.c2=b.c2) f
inner join test_tst_5 c on f.c2=c.c1;
"""
- def view1 = """select * from test_tst_6 order by c1"""
+ def view1 = """select * from test_tst_6 order by c1"""
- // qt_view1 view1
- }
+ // qt_view1 view1
+ }
}
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy
index 55fc903a31b..71a4d971169 100644
---
a/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy
+++
b/regression-test/suites/external_table_p0/paimon/test_paimon_deletion_vector_oss.groovy
@@ -28,14 +28,17 @@ suite("test_paimon_deletion_vector_oss",
"p0,external,doris,external_docker,exte
String catalog_name = "test_paimon_deletion_vector_oss"
String aliYunAk = context.config.otherConfigs.get("aliYunAk")
String aliYunSk = context.config.otherConfigs.get("aliYunSk")
+ String aliYunEndpoint =
context.config.otherConfigs.get("aliYunEndpoint")
+ String bucket = context.config.otherConfigs.get("aliYunBucket")
+
sql """drop catalog if exists ${catalog_name}"""
sql """create catalog if not exists ${catalog_name} properties (
"type" = "paimon",
"paimon.catalog.type"="filesystem",
- "warehouse" = "oss://doris-regression-bj/regression/paimon1",
+ "warehouse" = "oss://${bucket}/regression/paimon1",
"oss.access_key"="${aliYunAk}",
"oss.secret_key"="${aliYunSk}",
- "oss.endpoint"="oss-cn-beijing.aliyuncs.com"
+ "oss.endpoint"="${aliYunEndpoint}"
);"""
sql """use `${catalog_name}`.`db1`"""
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_gcs.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_gcs.groovy
new file mode 100644
index 00000000000..a0de09e1a1c
--- /dev/null
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_gcs.groovy
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_gcs", "p0,external,doris,new_catalog_property") {
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String gcs_warehouse =
"s3://selectdb-qa-datalake-test/paimon_warehouse"
+ String gcs_ak = context.config.otherConfigs.get("GCSAk")
+ String gcs_sk = context.config.otherConfigs.get("GCSSk")
+ String gcs_endpoint = context.config.otherConfigs.get("GCSEndpoint")
+ String gcs_region = context.config.otherConfigs.get("GCSRegion")
+
+ String table_name = "hive_test_table"
+ for (String propertyPrefix : ["gs", "s3"]) {
+ def catalog_name = "test_paimon_gcs_${propertyPrefix}"
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type' = 'paimon',
+ 'warehouse' = '${gcs_warehouse}',
+ '${propertyPrefix}.endpoint' = '${gcs_endpoint}',
+ '${propertyPrefix}.access_key' = '${gcs_ak}',
+ '${propertyPrefix}.secret_key' = '${gcs_sk}'
+ );
+ """
+ sql """switch `${catalog_name}`"""
+ sql """show databases; """
+ sql """use `${catalog_name}`.`gcs_db`"""
+ order_qt_no_region """SELECT * FROM ${table_name} """
+ // 3.1 new features
+ // batch incremental
+ sql """SELECT * FROM ${table_name}
@incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM ${table_name} FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM ${table_name}\$snapshots;"""
+ sql """drop catalog if exists ${catalog_name}"""
+
+ sql """drop catalog if exists ${catalog_name}_with_region"""
+ sql """
+ CREATE CATALOG ${catalog_name}_with_region PROPERTIES (
+ 'type' = 'paimon',
+ 'warehouse' = '${gcs_warehouse}',
+ '${propertyPrefix}.endpoint' = '${gcs_endpoint}',
+ '${propertyPrefix}.access_key' = '${gcs_ak}',
+ '${propertyPrefix}.secret_key' = '${gcs_sk}',
+ '${propertyPrefix}.region' = '${gcs_region}'
+ );
+ """
+ sql """switch `${catalog_name}_with_region`"""
+ sql """show databases; """
+ sql """use `${catalog_name}_with_region`.`gcs_db`"""
+ order_qt_region """SELECT * FROM ${table_name} """
+ // 3.1 new features
+ // batch incremental
+ sql """SELECT * FROM ${table_name}
@incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM ${table_name} FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM ${table_name}\$snapshots;"""
+ sql """drop catalog if exists ${catalog_name}_with_region"""
+ }
+ }
+}
+
+
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy
index 22e69642763..a959a034fa9 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_minio.groovy
@@ -39,6 +39,15 @@ suite("test_paimon_minio",
"p0,external,doris,external_docker,external_docker_do
sql """use `${catalog_name}`.`flink_paimon`"""
order_qt_no_region1 """select ts1,ts19 from ${table_name} """
order_qt_no_region2 """select * from ${table_name} """
+ // 3.1 new features
+ // batch incremental
+ sql """SELECT * FROM ${table_name}
@incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM ${table_name} FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM ${table_name}\$snapshots;"""
sql """drop catalog if exists ${catalog_name}"""
sql """drop catalog if exists ${catalog_name}_with_region"""
@@ -58,6 +67,15 @@ suite("test_paimon_minio",
"p0,external,doris,external_docker,external_docker_do
sql """use `${catalog_name}_with_region`.`flink_paimon`"""
order_qt_region1 """select ts1,ts19 from ${table_name} """
order_qt_region2 """select * from ${table_name} """
+ // 3.1 new features
+ // batch incremental
+ sql """SELECT * FROM ${table_name}
@incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM ${table_name} FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM ${table_name}\$snapshots;"""
sql """drop catalog if exists ${catalog_name}_with_region"""
}
}
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_s3.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_s3.groovy
index ec9bdc19b8e..ff4943e4e66 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_s3.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_s3.groovy
@@ -46,6 +46,15 @@ suite("test_paimon_s3",
"p0,external,doris,external_docker,external_docker_doris
"order_qt_${prefix}" """
SELECT * FROM hive_test_table;
"""
+ // 3.1 new features
+ // batch incremental
+ sql """SELECT * FROM hive_test_table
@incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM hive_test_table FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM hive_test_table\$snapshots;"""
}
String enabled = context.config.otherConfigs.get("enablePaimonTest")
diff --git
a/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy
b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy
index d43ecaebaf9..c4d234cc37d 100644
---
a/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy
+++
b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy
@@ -29,6 +29,7 @@ suite("test_cloud_accessible_oss",
"p2,external,hive,external_remote,external_re
String extHiveHmsPort =
context.config.otherConfigs.get("extHiveHmsPort")
String ak = context.config.otherConfigs.get("aliYunAk")
String sk = context.config.otherConfigs.get("aliYunSk")
+ String endpoint = context.config.otherConfigs.get("aliYunEndpoint")
String hms_catalog_name = "test_cloud_accessible_oss"
sql """drop catalog if exists ${hms_catalog_name};"""
sql """
@@ -36,7 +37,7 @@ suite("test_cloud_accessible_oss",
"p2,external,hive,external_remote,external_re
PROPERTIES (
'type' = 'hms',
'hive.metastore.uris' =
'thrift://${extHiveHmsHost}:${extHiveHmsPort}',
- 'oss.endpoint' = 'oss-cn-beijing.aliyuncs.com',
+ 'oss.endpoint' = '${endpoint}',
'oss.access_key' = '${ak}',
'oss.secret_key' = '${sk}'
);
@@ -54,14 +55,16 @@ suite("test_cloud_accessible_oss",
"p2,external,hive,external_remote,external_re
// dlf case
String dlf_catalog_name = "test_cloud_accessible_dlf"
String dlf_uid = context.config.otherConfigs.get("dlfUid")
+ String dlf_endpoint = context.config.otherConfigs.get("dlf_endpoint")
+ String dlf_region = context.config.otherConfigs.get("dlf_region")
sql """drop catalog if exists ${dlf_catalog_name};"""
sql """
CREATE CATALOG IF NOT EXISTS ${dlf_catalog_name}
PROPERTIES (
"type"="hms",
"hive.metastore.type" = "dlf",
- "dlf.endpoint" = "dlf.cn-beijing.aliyuncs.com",
- "dlf.region" = "cn-beijing",
+ "dlf.endpoint" = "${dlf_endpoint}",
+ "dlf.region" = "${dlf_region}",
"dlf.proxy.mode" = "DLF_ONLY",
"dlf.uid" = "${dlf_uid}",
"dlf.access_key" = "${ak}",
diff --git
a/regression-test/suites/external_table_p2/iceberg/iceberg_rest_s3_storage_test.groovy
b/regression-test/suites/external_table_p2/iceberg/iceberg_rest_s3_storage_test.groovy
index b2e9ad2ee1f..c8944dcb597 100644
---
a/regression-test/suites/external_table_p2/iceberg/iceberg_rest_s3_storage_test.groovy
+++
b/regression-test/suites/external_table_p2/iceberg/iceberg_rest_s3_storage_test.groovy
@@ -306,9 +306,11 @@ suite("iceberg_rest_s3_storage_test",
"p2,external,iceberg,external_docker,exter
/****************OSS*******************/
String oss_ak = context.config.otherConfigs.get("aliYunAk")
String oss_sk = context.config.otherConfigs.get("aliYunSk")
- String oss_endpoint = "https://oss-cn-beijing.aliyuncs.com";
- String oss_parent_path = "doris-regression-bj"
- String oss_region = "cn-beijing"
+ String endpoint = context.config.otherConfigs.get("aliYunEndpoint")
+ String oss_endpoint = "https://${endpoint}";
+ String oss_parent_path =
context.config.otherConfigs.get("aliYunBucket")
+ String oss_region = context.config.otherConfigs.get("aliYunRegion")
+
String oss_region_param = """
'oss.region' = '${oss_region}',
"""
diff --git
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
index b053bb25647..5a68844baa8 100644
---
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
+++
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog.groovy
@@ -53,7 +53,15 @@ suite("test_paimon_dlf_catalog",
"p2,external,paimon,external_remote,external_re
qt_c1 """ select * from tb_simple order by id """
sql """set force_jni_scanner=true"""
qt_c2 """ select * from tb_simple order by id """
-
+ // 3.1 new features
+ // batch incremental
+ sql """SELECT * FROM tb_simple @incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM tb_simple FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM tb_simple\$snapshots;"""
} finally {
sql """set force_jni_scanner=false"""
}
diff --git
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_miss_dlf_param.groovy
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_miss_dlf_param.groovy
index 2b7a3ed024d..b2deafc316a 100644
---
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_miss_dlf_param.groovy
+++
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_miss_dlf_param.groovy
@@ -28,6 +28,7 @@ suite("test_paimon_dlf_catalog_miss_dlf_param",
"p2,external,paimon,external_rem
String catalog_id = context.config.otherConfigs.get("dlf_catalog_id")
String access_key = context.config.otherConfigs.get("dlf_access_key")
String secret_key = context.config.otherConfigs.get("dlf_secret_key")
+ String oss_endpoint = context.config.otherConfigs.get("s3Endpoint")
sql """drop catalog if exists ${catalog};"""
@@ -42,7 +43,7 @@ suite("test_paimon_dlf_catalog_miss_dlf_param",
"p2,external,paimon,external_rem
"dlf.catalog.id" = "${catalog_id}",
"dlf.access_key" = "${access_key}",
"dlf.secret_key" = "${secret_key}",
- "oss.endpoint"="oss-cn-beijing.aliyuncs.com",
+ "oss.endpoint"="${oss_endpoint}",
"oss.access_key" = "${access_key}",
"oss.secret_key" = "${secret_key}"
);
@@ -56,6 +57,15 @@ suite("test_paimon_dlf_catalog_miss_dlf_param",
"p2,external,paimon,external_rem
qt_c1 """ select * from tb_simple order by id """
sql """set force_jni_scanner=true"""
qt_c2 """ select * from tb_simple order by id """
+ // 3.1 new features
+ // batch incremental
+ sql """SELECT * FROM tb_simple @incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM tb_simple FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM tb_simple\$snapshots;"""
} finally {
sql """set force_jni_scanner=false"""
diff --git
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_new_param.groovy
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_new_param.groovy
index a8292b7957d..1044dc8b778 100644
---
a/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_new_param.groovy
+++
b/regression-test/suites/external_table_p2/paimon/test_paimon_dlf_catalog_new_param.groovy
@@ -1,5 +1,3 @@
-import java.sql.Array
-
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -30,6 +28,7 @@ suite("test_paimon_dlf_catalog_new_param",
"p2,external,paimon,external_remote,e
String catalog_id = context.config.otherConfigs.get("dlf_catalog_id")
String access_key = context.config.otherConfigs.get("dlf_access_key")
String secret_key = context.config.otherConfigs.get("dlf_secret_key")
+ String oss_endpoint = context.config.otherConfigs.get("s3Endpoint")
String dlf_vpc_endpoint =
context.config.otherConfigs.get("dlf_vpc_endpoint")
String dlf_public_endpoint =
context.config.otherConfigs.get("dlf_public_endpoint")
@@ -46,7 +45,7 @@ suite("test_paimon_dlf_catalog_new_param",
"p2,external,paimon,external_remote,e
"dlf.access_key" = "${access_key}",
"dlf.secret_key" = "${secret_key}",
"dlf.endpoint" = "${dlf_vpc_endpoint}",
- "oss.endpoint"="oss-cn-beijing.aliyuncs.com",
+ "oss.endpoint"="${oss_endpoint}",
"oss.access_key" = "${access_key}",
"oss.secret_key" = "${secret_key}"
);
@@ -60,6 +59,15 @@ suite("test_paimon_dlf_catalog_new_param",
"p2,external,paimon,external_remote,e
qt_c1 """ select * from tb_simple order by id """
sql """set force_jni_scanner=true"""
qt_c2 """ select * from tb_simple order by id """
+ // 3.1 new features
+ // batch incremental
+ sql """SELECT * FROM tb_simple @incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM tb_simple FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM tb_simple\$snapshots;"""
} finally {
sql """set force_jni_scanner=false"""
diff --git
a/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy
b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy
index 265332cd5b3..790f1095d4e 100644
---
a/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy
+++
b/regression-test/suites/external_table_p2/paimon/test_paimon_hms_catalog.groovy
@@ -43,10 +43,19 @@ suite("test_paimon_hms_catalog",
"p2,external,paimon,new_catalog_property") {
"order_qt_${prefix}" """
SELECT * FROM external_test_table;
"""
- sql """set force_jni_scanner=true"""
- "order_qt_${prefix}" """
+ sql """set force_jni_scanner=true"""
+ "order_qt_${prefix}" """
SELECT * FROM external_test_table;
"""
+ // 3.1 new features
+ // batch incremental
+ sql """SELECT * FROM external_test_table
@incr('startTimestamp'='876488912')"""
+ // time travel
+ sql """SELECT * FROM external_test_table FOR VERSION AS OF 1;"""
+ // branch/tag
+ // TODO(zgx): add branch/tag
+ // system table
+ sql """SELECT * FROM external_test_table\$snapshots;"""
}
String enabled = context.config.otherConfigs.get("enablePaimonTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
@@ -55,12 +64,14 @@ suite("test_paimon_hms_catalog",
"p2,external,paimon,new_catalog_property") {
String extHiveHmsHost = context.config.otherConfigs.get("externalEnvIp")
String extHiveHmsPort = context.config.otherConfigs.get("hive3HmsPort")
String extHiveHdfsHost = context.config.otherConfigs.get("hive3HdfsPort")
+ String keytab_root_dir = "/keytabs"
/****************OSS*******************/
String oss_ak = context.config.otherConfigs.get("aliYunAk")
String oss_sk = context.config.otherConfigs.get("aliYunSk")
- String oss_endpoint = "oss-cn-beijing.aliyuncs.com"
- String oss_warehouse =
"oss://doris-regression-bj/regression/paimon_warehouse"
+ String oss_endpoint = context.config.otherConfigs.get("aliYunEndpoint")
+ String oss_bucket = context.config.otherConfigs.get("aliYunBucket")
+ String oss_warehouse = "oss://${oss_bucket}/regression/paimon_warehouse"
String oss_warehouse_properties = """
'warehouse' = '${oss_warehouse}',
"""
@@ -107,6 +118,27 @@ suite("test_paimon_hms_catalog",
"p2,external,paimon,new_catalog_property") {
String hdfs_storage_properties = """
'fs.defaultFS' = 'hdfs://${extHiveHmsHost}:${extHiveHdfsHost}'
"""
+
+ // kerberos
+ String hdfs_kerberos_properties = """
+ "fs.defaultFS" = "hdfs://${extHiveHmsHost}:8520",
+ "hadoop.security.authentication" = "kerberos",
+
"hadoop.kerberos.principal"="hive/[email protected]",
+ "hadoop.kerberos.keytab" =
"${keytab_root_dir}/hive-presto-master.keytab"
+ """
+ String hms_kerberos_new_prop = """
+ "hive.metastore.uris" = "thrift://${extHiveHmsHost}:9583",
+
"hive.metastore.client.principal"="hive/[email protected]",
+ "hive.metastore.client.keytab" =
"${keytab_root_dir}/hive-presto-master.keytab",
+ "hive.metastore.service.principal" =
"hive/[email protected]",
+ "hive.metastore.sasl.enabled " = "true",
+ "hadoop.security.auth_to_local" =
"RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*//
+
RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
+
RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*//
+ DEFAULT",
+ """
+
+
/**************** AWS S3*******************/
String s3_warehouse = "s3://selectdb-qa-datalake-test-hk/paimon_warehouse"
String aws_ak = context.config.otherConfigs.get("AWSAK")
@@ -121,17 +153,41 @@ suite("test_paimon_hms_catalog",
"p2,external,paimon,new_catalog_property") {
's3.endpoint' = '${aws_endpoint}'
"""
+ /**************** GCS *******************/
+ String gcs_warehouse = "gs://selectdb-qa-datalake-test/paimon_warehouse"
+ String gcs_ak = context.config.otherConfigs.get("GCSAk")
+ String gcs_sk = context.config.otherConfigs.get("GCSSk")
+ String gcs_endpoint = context.config.otherConfigs.get("GCSEndpoint")
+ String gcs_warehouse_properties = """
+ 'warehouse' = '${gcs_warehouse}',
+ """
+ String gcs_storage_properties = """
+ 'gs.access_key' = '${gcs_ak}',
+ 'gs.secret_key' = '${gcs_sk}',
+ 'gs.endpoint' = '${gcs_endpoint}'
+ """
+ String gcs_storage_s3_properties = """
+ 's3.access_key' = '${gcs_ak}',
+ 's3.secret_key' = '${gcs_sk}',
+ 's3.endpoint' = '${gcs_endpoint}'
+ """
+
/*--------HMS START -----------*/
- String paimon_hms_catalog_properties = """
- 'type'='paimon',
- 'paimon.catalog.type'='hms',
+ String paimon_hms_type_prop = """
+ 'type'='paimon',
+ 'paimon.catalog.type'='hms',
+ """
+ String paimon_hms_catalog_properties = paimon_hms_type_prop + """
'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}',
"""
testQuery(paimon_hms_catalog_properties + hdfs_warehouse_properties +
hdfs_storage_properties, "hdfs", "hdfs_db")
+ testQuery(paimon_hms_type_prop + hdfs_warehouse_properties +
hms_kerberos_new_prop + hdfs_kerberos_properties, "hdfs_kerberos", "hdfs_db")
testQuery(paimon_hms_catalog_properties + oss_warehouse_properties +
oss_storage_properties, "oss", "ali_db")
testQuery(paimon_hms_catalog_properties + obs_warehouse_properties +
obs_storage_properties, "obs", "hw_db")
testQuery(paimon_hms_catalog_properties + cos_warehouse_properties +
cos_storage_properties, "cos", "tx_db")
testQuery(paimon_hms_catalog_properties + s3_warehouse_properties +
s3_storage_properties, "s3", "aws_db")
+ testQuery(paimon_hms_catalog_properties + gcs_warehouse_properties +
gcs_storage_properties, "gcs", "gcs_db")
+ testQuery(paimon_hms_catalog_properties + gcs_warehouse_properties +
gcs_storage_s3_properties, "gcs_s3", "gcs_db")
String paimon_fs_hdfs_support = """
'fs.hdfs.support' = 'true',
@@ -148,10 +204,17 @@ suite("test_paimon_hms_catalog",
"p2,external,paimon,new_catalog_property") {
String paimon_fs_s3_support = """
'fs.s3.support' = 'true',
"""
+ String paimon_fs_gcs_support = """
+ 'fs.gs.support' = 'true',
+ """
testQuery(paimon_hms_catalog_properties + paimon_fs_hdfs_support +
hdfs_warehouse_properties + hdfs_storage_properties, "support_hdfs", "hdfs_db")
+ testQuery(paimon_hms_type_prop + paimon_fs_hdfs_support +
hdfs_warehouse_properties + hms_kerberos_new_prop + hdfs_kerberos_properties,
"support_hdfs_kerberos", "hdfs_db")
testQuery(paimon_hms_catalog_properties + paimon_fs_oss_support +
oss_warehouse_properties + oss_storage_properties, "support_oss", "ali_db")
testQuery(paimon_hms_catalog_properties + paimon_fs_obs_support +
obs_warehouse_properties + obs_storage_properties, "support_obs", "hw_db")
testQuery(paimon_hms_catalog_properties + paimon_fs_cos_support +
cos_warehouse_properties + cos_storage_properties, "support_cos", "tx_db")
testQuery(paimon_hms_catalog_properties + paimon_fs_s3_support +
s3_warehouse_properties + s3_storage_properties, "support_s3", "aws_db")
+ testQuery(paimon_hms_catalog_properties + paimon_fs_gcs_support +
gcs_warehouse_properties + gcs_storage_properties, "support_gcs", "gcs_db")
+ testQuery(paimon_hms_catalog_properties + paimon_fs_gcs_support +
gcs_warehouse_properties + gcs_storage_s3_properties, "support_gcs_s3",
"gcs_db")
+
}
diff --git
a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
index fe6908d7680..4c531d784d7 100644
---
a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
+++
b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
@@ -138,9 +138,10 @@ suite("hive_on_hms_and_dlf",
"p2,external,new_catalog_property") {
/****************OSS*******************/
String oss_ak = context.config.otherConfigs.get("aliYunAk")
String oss_sk = context.config.otherConfigs.get("aliYunSk")
- String oss_endpoint = "oss-cn-beijing.aliyuncs.com"
- String oss_parent_path = "doris-regression-bj/refactor-test"
- String oss_region = "cn-beijing"
+ String oss_endpoint = context.config.otherConfigs.get("aliYunEndpoint")
+ String bucket = context.config.otherConfigs.get("aliYunBucket")
+ String oss_parent_path = "${bucket}/refactor-test"
+ String oss_region = context.config.otherConfigs.get("aliYunRegion")
String oss_region_param = """
'oss.region' = '${oss_region}',
"""
@@ -185,7 +186,7 @@ suite("hive_on_hms_and_dlf",
"p2,external,new_catalog_property") {
/**************** DLF *******************/
String dlf_oss_properties = """
- "oss.endpoint" = "oss-cn-beijing.aliyuncs.com",
+ "oss.endpoint" = "${oss_endpoint}",
"oss.access_key" = "${dlf_access_key}",
"oss.secret_key" = "${dlf_secret_key}",
"""
@@ -278,7 +279,7 @@ suite("hive_on_hms_and_dlf",
"p2,external,new_catalog_property") {
String dlf_region_param = """
"dlf.region" = "${dlf_region}",
"""
- String dlf_endpoint = "dlf.cn-beijing.aliyuncs.com"
+ String dlf_endpoint = context.config.otherConfigs.get("dlf_endpoint")
String dlf_endpoint_properties = """
"dlf.endpoint" = "${dlf_endpoint}",
"""
diff --git
a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
index 17a6b414461..5cc528e4ce5 100644
---
a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
+++
b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
@@ -215,13 +215,13 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
/*-----S3------*/
String s3_ak = context.config.otherConfigs.get("AWSAK")
String s3_sk = context.config.otherConfigs.get("AWSSK")
- String s3_endpoint = "http://s3.ap-east-1.amazonaws.com/";
- String s3_region = "ap-east-1"
+ String s3_endpoint = context.config.otherConfigs.get("AWSEndpoint")
+ String s3_region = context.config.otherConfigs.get("AWSRegion")
String s3_parent_path = "selectdb-qa-datalake-test-hk/refactor-test"
String s3_storage_properties = """
's3.access_key' = '${s3_ak}',
's3.secret_key' = '${s3_sk}',
- 's3.endpoint' = '${s3_endpoint}'
+ 's3.endpoint' = 'http://${s3_endpoint}'
"""
String s3_region_param = """
's3.region' = '${s3_region}',
@@ -229,9 +229,10 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
/****************OSS*******************/
String oss_ak = context.config.otherConfigs.get("aliYunAk")
String oss_sk = context.config.otherConfigs.get("aliYunSk")
- String oss_endpoint = "oss-cn-beijing.aliyuncs.com"
- String oss_parent_path = "doris-regression-bj/refactor-test"
- String oss_region = "cn-beijing"
+ String oss_endpoint = context.config.otherConfigs.get("aliYunEndpoint")
+ String oss_bucket = context.config.otherConfigs.get("aliYunBucket")
+ String oss_parent_path = "${oss_bucket}/refactor-test"
+ String oss_region = context.config.otherConfigs.get("aliYunRegion")
String oss_region_param = """
'oss.region' = '${oss_region}',
"""
@@ -318,68 +319,58 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
'warehouse' =
'oss://${oss_parent_path}/iceberg-hms-warehouse',
"""
// has endpoint not region
- testQueryAndInsert(iceberg_hms_type_prop + hms_prop
- + warehouse + oss_storage_properties, "iceberg_hms_on_oss")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop + warehouse +
oss_storage_properties, "iceberg_hms_on_oss")
- testQueryAndInsert(iceberg_hms_type_prop + hms_prop
- + warehouse + oss_region_param + oss_storage_properties,
"iceberg_hms_on_oss")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop + warehouse +
oss_region_param + oss_storage_properties, "iceberg_hms_on_oss")
//old kerberos
testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_old_prop +
warehouse + oss_storage_properties, "iceberg_hms_on_oss_kerberos_old")
//new kerberos
- testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_new_prop+
warehouse + oss_storage_properties, "iceberg_hms_on_oss_kerberos_new")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_new_prop +
warehouse + oss_storage_properties, "iceberg_hms_on_oss_kerberos_new")
/*--------HMS on COS-----------*/
warehouse = """
'warehouse' =
'cosn://${cos_parent_path}/iceberg-hms-cos-warehouse',
"""
- testQueryAndInsert(iceberg_hms_type_prop + hms_prop
- + warehouse + cos_storage_properties, "iceberg_hms_on_cos")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop + warehouse +
cos_storage_properties, "iceberg_hms_on_cos")
- testQueryAndInsert(iceberg_hms_type_prop + hms_prop
- + warehouse + cos_region_param + cos_storage_properties,
"iceberg_hms_on_cos")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop + warehouse +
cos_region_param + cos_storage_properties, "iceberg_hms_on_cos")
warehouse = """
'warehouse' = 'cos://${cos_parent_path}/iceberg-hms-cos-warehouse',
"""
- testQueryAndInsert(iceberg_hms_type_prop + hms_prop
- + warehouse + cos_storage_properties, "iceberg_hms_on_cos")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop + warehouse +
cos_storage_properties, "iceberg_hms_on_cos")
//kerberos
testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_old_prop +
warehouse + cos_storage_properties, "iceberg_hms_on_cos_kerberos_old")
testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_new_prop +
warehouse + cos_storage_properties, "iceberg_hms_on_cos_kerberos_new")
-
/*--------HMS on S3-----------*/
warehouse = """
'warehouse' = 's3a://${s3_parent_path}/iceberg-hms-s3-warehouse',
"""
- testQueryAndInsert(iceberg_hms_type_prop + hms_prop
- + warehouse + s3_storage_properties, "iceberg_hms_on_s3")
- testQueryAndInsert(iceberg_hms_type_prop + hms_prop
- + warehouse + s3_region_param + s3_storage_properties,
"iceberg_hms_on_s3")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop + warehouse +
s3_storage_properties, "iceberg_hms_on_s3")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop + warehouse +
s3_region_param + s3_storage_properties, "iceberg_hms_on_s3")
//kerberos Docker HMS c
// testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_old_prop+
warehouse + s3_storage_properties, "iceberg_hms_on_s3_kerberos_old")
// testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_new_prop +
warehouse + s3_storage_properties, "iceberg_hms_on_s3_kerberos_new")
warehouse = """
'warehouse' = 's3://${s3_parent_path}/iceberg-hms-s3-warehouse',
"""
- testQueryAndInsert(iceberg_hms_type_prop + hms_prop
- + warehouse + s3_storage_properties, "iceberg_hms_on_s3")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop + warehouse +
s3_storage_properties, "iceberg_hms_on_s3")
/*--------HMS on HDFS-----------*/
warehouse = """
'warehouse' = '${hdfs_parent_path}/iceberg-hms-hdfs-warehouse',
"""
- testQueryAndInsert(iceberg_hms_type_prop + hms_prop
- + warehouse + hdfs_properties, "iceberg_hms_on_hdfs")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop + warehouse +
hdfs_properties, "iceberg_hms_on_hdfs")
warehouse = """
'warehouse' = 'hdfs://${externalEnvIp}:8520/iceberg-hms-hdfs-warehouse',
"""
//old kerberos
- testQueryAndInsert(iceberg_hms_type_prop +
hms_kerberos_old_prop_not_include_kerberos_prop+ warehouse +
hdfs_kerberos_properties, "iceberg_hms_on_hdfs_kerberos_old")
+ testQueryAndInsert(iceberg_hms_type_prop +
hms_kerberos_old_prop_not_include_kerberos_prop + warehouse +
hdfs_kerberos_properties, "iceberg_hms_on_hdfs_kerberos_old")
//new kerberos
testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_new_prop +
warehouse + hdfs_kerberos_properties, "iceberg_hms_on_hdfs_kerberos_hdfs")
@@ -437,7 +428,8 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
/*--------FILESYSTEM END-----------*/
-
+ String dlf_uid = context.config.otherConfigs.get("dlf_uid")
+ String dlf_catalog_id = context.config.otherConfigs("dlf_catalog_id")
String dlf_access_key = context.config.otherConfigs.get("dlf_access_key")
String dlf_secret_key = context.config.otherConfigs.get("dlf_secret_key")
/**************** DLF *******************/
@@ -456,12 +448,12 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
String dlf_region_param = """
"dlf.region" = "${dlf_region}",
"""
- String dlf_endpoint = "dlf.cn-beijing.aliyuncs.com"
+ String dlf_endpoint = context.config.otherConfigs.get("dlf_endpoint")
String dlf_endpoint_properties = """
"dlf.endpoint" = "${dlf_endpoint}",
"""
String dlf_oss_properties = """
- "oss.endpoint" = "oss-cn-beijing.aliyuncs.com",
+ "oss.endpoint" = "${oss_endpoint}",
"oss.access_key" = "${dlf_access_key}",
"oss.secret_key" = "${dlf_secret_key}",
"""
@@ -477,15 +469,15 @@ suite("iceberg_on_hms_and_filesystem_and_dlf",
"p2,external,new_catalog_property
'iceberg.catalog.type'='dlf',
"""
- testQuery(iceberg_dlf_type_properties + dlf_region_param +
dlf_catalog_base_properties, "iceberg_dlf_catalog", "regression_iceberg",
"tb_simple", 2)
+ testQuery(iceberg_dlf_type_properties + dlf_region_param +
dlf_catalog_base_properties, "iceberg_dlf_catalog", "regression_iceberg",
"tb_simple", 2)
-
testQuery(iceberg_dlf_type_properties+dlf_region_param+dlf_endpoint_properties
+ dlf_catalog_base_properties, "iceberg_dlf_catalog", "regression_iceberg",
"tb_simple", 2)
- testQuery(iceberg_dlf_type_properties+dlf_oss_properties +
dlf_region_param + dlf_catalog_base_properties, "iceberg_dlf_catalog",
"regression_iceberg", "tb_simple", 2)
- testQuery(iceberg_dlf_type_properties_no_warehouse+dlf_oss_properties +
dlf_region_param + dlf_catalog_base_properties, "iceberg_dlf_catalog",
"regression_iceberg", "tb_simple", 2)
+ testQuery(iceberg_dlf_type_properties + dlf_region_param +
dlf_endpoint_properties + dlf_catalog_base_properties, "iceberg_dlf_catalog",
"regression_iceberg", "tb_simple", 2)
+ testQuery(iceberg_dlf_type_properties + dlf_oss_properties +
dlf_region_param + dlf_catalog_base_properties, "iceberg_dlf_catalog",
"regression_iceberg", "tb_simple", 2)
+ testQuery(iceberg_dlf_type_properties_no_warehouse + dlf_oss_properties +
dlf_region_param + dlf_catalog_base_properties, "iceberg_dlf_catalog",
"regression_iceberg", "tb_simple", 2)
// region is required
shouldFail {
- testQuery(iceberg_dlf_type_properties+dlf_catalog_base_properties,
"iceberg_dlf_catalog", "regression_iceberg", "tb_simple", 2)
+ testQuery(iceberg_dlf_type_properties + dlf_catalog_base_properties,
"iceberg_dlf_catalog", "regression_iceberg", "tb_simple", 2)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
