This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 16f488257c1 [case][fix](iceberg)move rest cases from p2 to p0 and fix
iceberg version issue (#37898)
16f488257c1 is described below
commit 16f488257c1b714ea66aef53e78a8c2c6a6f455d
Author: wuwenchi <[email protected]>
AuthorDate: Tue Jul 23 11:08:19 2024 +0800
[case][fix](iceberg)move rest cases from p2 to p0 and fix iceberg version
issue (#37898)
## Proposed changes
move cases from p2 to p0
---
.../docker-compose/iceberg/iceberg.yaml.tpl | 1 +
.../docker-compose/iceberg/spark-init-iceberg.sql | 25 ++-
.../datasource/iceberg/source/IcebergScanNode.java | 9 +-
.../iceberg/test_gen_iceberg_by_api.out | 10 ++
.../iceberg/test_iceberg_equality_delete.out | 41 +++++
.../iceberg/test_iceberg_position_delete.out} | 0
.../iceberg/test_iceberg_time_travel.out | 46 +++++
.../test_iceberg_upper_case_column_name.out} | 44 -----
.../hive/test_upper_case_column_name.out | 44 -----
.../iceberg/iceberg_equality_delete.out | 109 ------------
...est_external_catalog_iceberg_hadoop_catalog.out | 26 ---
.../test_external_catalog_icebergv2_nereids.out | 74 --------
.../hive/test_upper_case_column_name.groovy | 32 ----
.../iceberg/test_gen_iceberg_by_api.groovy | 192 +++++++++++++++++++++
.../iceberg/test_iceberg_equality_delete.groovy | 152 ++++++++++++++++
.../iceberg/test_iceberg_position_delete.groovy} | 42 +++--
.../iceberg/test_iceberg_time_travel.groovy | 55 ++++++
.../test_iceberg_upper_case_column_name.groovy} | 75 +++-----
.../hive/test_upper_case_column_name.groovy | 32 ----
.../iceberg/iceberg_equality_delete.groovy | 57 ------
..._external_catalog_iceberg_hadoop_catalog.groovy | 49 ------
.../test_external_catalog_icebergv2_nereids.groovy | 84 ---------
22 files changed, 574 insertions(+), 625 deletions(-)
diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
index 8f71d6c5087..10c86d742e6 100644
--- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
+++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
@@ -26,6 +26,7 @@ services:
depends_on:
- rest
- minio
+ - mc
volumes:
- ./data/output/spark-warehouse:/home/iceberg/warehouse
- ./data/output/spark-notebooks:/home/iceberg/notebooks/notebooks
diff --git a/docker/thirdparties/docker-compose/iceberg/spark-init-iceberg.sql
b/docker/thirdparties/docker-compose/iceberg/spark-init-iceberg.sql
index 84d8d054d7c..36c416b7deb 100644
--- a/docker/thirdparties/docker-compose/iceberg/spark-init-iceberg.sql
+++ b/docker/thirdparties/docker-compose/iceberg/spark-init-iceberg.sql
@@ -1,4 +1,6 @@
create database if not exists demo.test_db;
+use demo.test_db;
+
drop table if exists demo.test_db.location_s3a_table;
create table demo.test_db.location_s3a_table (
id int,
@@ -25,5 +27,26 @@ tblproperties (
insert into demo.test_db.location_s3_table values (1,'a');
update demo.test_db.location_s3_table set val='b' where id=1;
+drop table if exists demo.test_db.tb_ts_ntz_filter;
create table demo.test_db.tb_ts_ntz_filter (ts timestamp_ntz) using iceberg;
-insert into demo.test_db.tb_ts_ntz_filter values (timestamp_ntz '2024-06-11
12:34:56.123456');
\ No newline at end of file
+insert into demo.test_db.tb_ts_ntz_filter values (timestamp_ntz '2024-06-11
12:34:56.123456');
+
+drop table if exists iceberg_upper_case_parquet;
+CREATE TABLE iceberg_upper_case_parquet (
+ `ID` BIGINT,
+ `NAME` STRING)
+USING iceberg
+TBLPROPERTIES(
+ 'write.format.default' = 'parquet');
+
+insert into iceberg_upper_case_parquet values (1, 'name');
+
+drop table if exists iceberg_upper_case_orc;
+CREATE TABLE iceberg_upper_case_orc (
+ `ID` BIGINT,
+ `NAME` STRING)
+USING iceberg
+TBLPROPERTIES(
+ 'write.format.default' = 'orc');
+
+insert into iceberg_upper_case_orc values (1, 'name');
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
index 85778d0ae40..a79256f7f0d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java
@@ -74,7 +74,6 @@ import org.apache.iceberg.util.SnapshotUtil;
import org.apache.iceberg.util.TableScanUtil;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -299,11 +298,11 @@ public class IcebergScanNode extends FileQueryScanNode {
List<IcebergDeleteFileFilter> filters = new ArrayList<>();
for (DeleteFile delete : spitTask.deletes()) {
if (delete.content() == FileContent.POSITION_DELETES) {
- ByteBuffer lowerBoundBytes =
delete.lowerBounds().get(MetadataColumns.DELETE_FILE_POS.fieldId());
- Optional<Long> positionLowerBound =
Optional.ofNullable(lowerBoundBytes)
+ Optional<Long> positionLowerBound =
Optional.ofNullable(delete.lowerBounds())
+ .map(m ->
m.get(MetadataColumns.DELETE_FILE_POS.fieldId()))
.map(bytes ->
Conversions.fromByteBuffer(MetadataColumns.DELETE_FILE_POS.type(), bytes));
- ByteBuffer upperBoundBytes =
delete.upperBounds().get(MetadataColumns.DELETE_FILE_POS.fieldId());
- Optional<Long> positionUpperBound =
Optional.ofNullable(upperBoundBytes)
+ Optional<Long> positionUpperBound =
Optional.ofNullable(delete.upperBounds())
+ .map(m ->
m.get(MetadataColumns.DELETE_FILE_POS.fieldId()))
.map(bytes ->
Conversions.fromByteBuffer(MetadataColumns.DELETE_FILE_POS.type(), bytes));
filters.add(IcebergDeleteFileFilter.createPositionDelete(delete.path().toString(),
positionLowerBound.orElse(-1L),
positionUpperBound.orElse(-1L)));
diff --git
a/regression-test/data/external_table_p0/iceberg/test_gen_iceberg_by_api.out
b/regression-test/data/external_table_p0/iceberg/test_gen_iceberg_by_api.out
new file mode 100644
index 00000000000..b3d42dfcd19
--- /dev/null
+++ b/regression-test/data/external_table_p0/iceberg/test_gen_iceberg_by_api.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !q01 --
+1 1970-01-03T09:02:03.000001 a
+1 1970-01-03T09:02:03.000001 b
+2 1970-01-03T09:02:04.000001 c
+2 1970-01-03T09:02:04.000001 d
+
+-- !q02 --
+463870
+
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_equality_delete.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_equality_delete.out
new file mode 100644
index 00000000000..3a21417f1a7
--- /dev/null
+++
b/regression-test/data/external_table_p0/iceberg/test_iceberg_equality_delete.out
@@ -0,0 +1,41 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !one_delete_column --
+1 b
+2 c
+
+-- !one_delete_column_orc --
+1 b
+2 c
+
+-- !count1 --
+2
+
+-- !count1_orc --
+2
+
+-- !max1 --
+c
+
+-- !max1_orc --
+c
+
+-- !one_delete_column --
+1 id2 1 b
+2 id2 1 c
+
+-- !one_delete_column_orc --
+1 id2 1 b
+2 id2 1 c
+
+-- !count3 --
+2
+
+-- !count3_orc --
+2
+
+-- !max3 --
+c
+
+-- !max3_orc --
+c
+
diff --git
a/regression-test/data/external_table_p2/iceberg/iceberg_position_delete.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_position_delete.out
similarity index 100%
rename from
regression-test/data/external_table_p2/iceberg/iceberg_position_delete.out
rename to
regression-test/data/external_table_p0/iceberg/test_iceberg_position_delete.out
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_time_travel.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_time_travel.out
new file mode 100644
index 00000000000..c888cedbdf0
--- /dev/null
+++
b/regression-test/data/external_table_p0/iceberg/test_iceberg_time_travel.out
@@ -0,0 +1,46 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !q1 --
+5632
+
+-- !q2 --
+30
+
+-- !q3 --
+2 select xxxxxxxxx
+2 select xxxxxxxxx
+2 select xxxxxxxxx
+2 select xxxxxxxxx
+2 select xxxxxxxxx
+2 select xxxxxxxxx
+2 select xxxxxxxxx
+2 select xxxxxxxxx
+2 select xxxxxxxxx
+2 select xxxxxxxxx
+
+-- !q4 --
+1 hello world
+1 hello world
+2 select xxxxxxxxx
+2 select xxxxxxxxx
+3 example xxxx
+3 example xxxx
+4 more dataxxx
+4 more dataxxx
+5 another examplexxx
+5 another examplexxx
+
+-- !q5 --
+480
+
+-- !q6 --
+1 hello world
+1 hello world
+1 hello world
+1 hello world
+1 hello world
+1 hello world
+1 hello world
+1 hello world
+1 hello world
+1 hello world
+
diff --git
a/regression-test/data/external_table_p2/hive/test_upper_case_column_name.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_upper_case_column_name.out
similarity index 58%
copy from
regression-test/data/external_table_p2/hive/test_upper_case_column_name.out
copy to
regression-test/data/external_table_p0/iceberg/test_iceberg_upper_case_column_name.out
index 1b39ef27718..dc7acbbe880 100644
---
a/regression-test/data/external_table_p2/hive/test_upper_case_column_name.out
+++
b/regression-test/data/external_table_p0/iceberg/test_iceberg_upper_case_column_name.out
@@ -1,48 +1,4 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
--- !hiveParquet1 --
-1 name
-
--- !hiveParquet2 --
-1 name
-
--- !hiveParquet3 --
-
--- !hiveParquet4 --
-1 name
-
--- !hiveParquet5 --
-
--- !hiveParquet6 --
-1
-
--- !hiveParquet7 --
-name
-
--- !hiveParquet8 --
-1 name
-
--- !hiveOrc1 --
-1 name
-
--- !hiveOrc2 --
-1 name
-
--- !hiveOrc3 --
-
--- !hiveOrc4 --
-1 name
-
--- !hiveOrc5 --
-
--- !hiveOrc6 --
-1
-
--- !hiveOrc7 --
-name
-
--- !hiveOrc8 --
-1 name
-
-- !icebergParquet1 --
1 name
diff --git
a/regression-test/data/external_table_p2/hive/test_upper_case_column_name.out
b/regression-test/data/external_table_p2/hive/test_upper_case_column_name.out
index 1b39ef27718..fa5838be4dd 100644
---
a/regression-test/data/external_table_p2/hive/test_upper_case_column_name.out
+++
b/regression-test/data/external_table_p2/hive/test_upper_case_column_name.out
@@ -43,47 +43,3 @@ name
-- !hiveOrc8 --
1 name
--- !icebergParquet1 --
-1 name
-
--- !icebergParquet2 --
-1 name
-
--- !icebergParquet3 --
-
--- !icebergParquet4 --
-1 name
-
--- !icebergParquet5 --
-
--- !icebergParquet6 --
-1
-
--- !icebergParquet7 --
-name
-
--- !icebergParquet8 --
-1 name
-
--- !icebergOrc1 --
-1 name
-
--- !icebergOrc2 --
-1 name
-
--- !icebergOrc3 --
-
--- !icebergOrc4 --
-1 name
-
--- !icebergOrc5 --
-
--- !icebergOrc6 --
-1
-
--- !icebergOrc7 --
-name
-
--- !icebergOrc8 --
-1 name
-
diff --git
a/regression-test/data/external_table_p2/iceberg/iceberg_equality_delete.out
b/regression-test/data/external_table_p2/iceberg/iceberg_equality_delete.out
deleted file mode 100644
index 44cf1b7b2bf..00000000000
--- a/regression-test/data/external_table_p2/iceberg/iceberg_equality_delete.out
+++ /dev/null
@@ -1,109 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !one_delete_column --
-1 Customer#000000001 IVhzIApeRb ot,c,E 151 update-phone-1
711.56 BUILDING update-comment-1
-2 Customer#000000002 XSTf4,NCwDVaWNe6tEgvwfmRchLXak 13
23-768-687-3665 121.65 AUTOMOBILE l accounts. blithely ironic theodolites
integrate boldly: caref
-3 Customer#000000003 MG9kdTD2WBHm 1 11-719-748-3364 7498.12
AUTOMOBILE deposits eat slyly ironic, even instructions. express foxes
detect slyly. blithely even accounts abov
-4 Customer#000000004 XxVSJsLAGtn 47 update-phone-4 15.39
MACHINERY update-comment-2
-6 Customer#000000006 sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn 20
30-114-968-4951 7638.57 AUTOMOBILE tions. even deposits boost according to
the slyly bold packages. final accounts cajole requests. furious
-7 Customer#000000007 TcGe5gaZNgVePxU5kRrvXBfkasDTea 18
28-190-982-9759 9561.95 AUTOMOBILE ainst the ironic, express theodolites.
express, even pinto beans among the exp
-8 Customer#000000008 I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 17
27-147-574-9335 6819.74 BUILDING among the slyly regular theodolites
kindle blithely courts. carefully even theodolites haggle slyly along the ide
-9 Customer#000000009 xKiAFTjUsCuxfeleNqefumTrjS 8
18-338-906-3675 8324.07 FURNITURE r theodolites according to the requests
wake thinly excuses: pending requests haggle furiousl
-10 Customer#000000010 6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2 5
15-741-346-9870 2753.54 HOUSEHOLD es regular deposits haggle. fur
-11 Customer#000000011 PkWS 3HlXqwTuzrKg633BEi 23 33-464-151-3439
-272.60 BUILDING ckages. requests sleep slyly. quickly even pinto beans
promise above the slyly regular pinto beans.
-12 Customer#000000012 9PWKuhzT4Zr1Q 13 23-791-276-1263 3396.49
HOUSEHOLD to the carefully final braids. blithely regular requests nag.
ironic theodolites boost quickly along
-13 Customer#000000013 nsXQu0oVjD7PM659uC3SRSp 3 13-761-547-5974
3857.34 BUILDING ounts sleep carefully after the close frays. carefully
bold notornis use ironic requests. blithely
-14 Customer#000000014 KXkletMlL2JQEA 1 11-845-129-3851 5266.30
FURNITURE , ironic packages across the unus
-15 Customer#000000015 YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn 23
33-687-542-7601 2788.52 HOUSEHOLD platelets. regular deposits detect
asymptotes. blithely unusual packages nag slyly at the fluf
-16 Customer#000000016 cYiaeMLZSMAOQ2 d0W, 10 20-781-609-3107
4681.03 FURNITURE kly silent courts. thinly regular theodolites sleep
fluffily after
-17 Customer#000000017 izrh 6jdqtp2eqdtbkswDD8SG4SzXruMfIXyR7 2
12-970-682-3487 6.34 AUTOMOBILE packages wake! blithely even pint
-18 Customer#000000018 3txGO AiuFux3zT0Z9NYaFRnZt 6
16-155-215-1315 5494.43 BUILDING s sleep. carefully even instructions
nag furiously alongside of t
-19 Customer#000000019 uc,3bHIx84H,wdrmLOjVsiqXCq2tr 18
28-396-526-5053 8914.71 HOUSEHOLD nag. furiously careful packages are
slyly at the accounts. furiously regular in
-20 Customer#000000020 JrPk8Pqplj4Ne 22 32-957-234-8742 7603.40
FURNITURE g alongside of the special excuses-- fluffily enticing packages
wake
-
--- !one_delete_column_orc --
-1 Customer#000000001 IVhzIApeRb ot,c,E 151 update-phone-1
711.56 BUILDING update-comment-1
-2 Customer#000000002 XSTf4,NCwDVaWNe6tEgvwfmRchLXak 13
23-768-687-3665 121.65 AUTOMOBILE l accounts. blithely ironic theodolites
integrate boldly: caref
-3 Customer#000000003 MG9kdTD2WBHm 1 11-719-748-3364 7498.12
AUTOMOBILE deposits eat slyly ironic, even instructions. express foxes
detect slyly. blithely even accounts abov
-4 Customer#000000004 XxVSJsLAGtn 47 update-phone-4 15.39
MACHINERY update-comment-2
-6 Customer#000000006 sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn 20
30-114-968-4951 7638.57 AUTOMOBILE tions. even deposits boost according to
the slyly bold packages. final accounts cajole requests. furious
-7 Customer#000000007 TcGe5gaZNgVePxU5kRrvXBfkasDTea 18
28-190-982-9759 9561.95 AUTOMOBILE ainst the ironic, express theodolites.
express, even pinto beans among the exp
-8 Customer#000000008 I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 17
27-147-574-9335 6819.74 BUILDING among the slyly regular theodolites
kindle blithely courts. carefully even theodolites haggle slyly along the ide
-9 Customer#000000009 xKiAFTjUsCuxfeleNqefumTrjS 8
18-338-906-3675 8324.07 FURNITURE r theodolites according to the requests
wake thinly excuses: pending requests haggle furiousl
-10 Customer#000000010 6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2 5
15-741-346-9870 2753.54 HOUSEHOLD es regular deposits haggle. fur
-11 Customer#000000011 PkWS 3HlXqwTuzrKg633BEi 23 33-464-151-3439
-272.60 BUILDING ckages. requests sleep slyly. quickly even pinto beans
promise above the slyly regular pinto beans.
-12 Customer#000000012 9PWKuhzT4Zr1Q 13 23-791-276-1263 3396.49
HOUSEHOLD to the carefully final braids. blithely regular requests nag.
ironic theodolites boost quickly along
-13 Customer#000000013 nsXQu0oVjD7PM659uC3SRSp 3 13-761-547-5974
3857.34 BUILDING ounts sleep carefully after the close frays. carefully
bold notornis use ironic requests. blithely
-14 Customer#000000014 KXkletMlL2JQEA 1 11-845-129-3851 5266.30
FURNITURE , ironic packages across the unus
-15 Customer#000000015 YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn 23
33-687-542-7601 2788.52 HOUSEHOLD platelets. regular deposits detect
asymptotes. blithely unusual packages nag slyly at the fluf
-16 Customer#000000016 cYiaeMLZSMAOQ2 d0W, 10 20-781-609-3107
4681.03 FURNITURE kly silent courts. thinly regular theodolites sleep
fluffily after
-17 Customer#000000017 izrh 6jdqtp2eqdtbkswDD8SG4SzXruMfIXyR7 2
12-970-682-3487 6.34 AUTOMOBILE packages wake! blithely even pint
-18 Customer#000000018 3txGO AiuFux3zT0Z9NYaFRnZt 6
16-155-215-1315 5494.43 BUILDING s sleep. carefully even instructions
nag furiously alongside of t
-19 Customer#000000019 uc,3bHIx84H,wdrmLOjVsiqXCq2tr 18
28-396-526-5053 8914.71 HOUSEHOLD nag. furiously careful packages are
slyly at the accounts. furiously regular in
-20 Customer#000000020 JrPk8Pqplj4Ne 22 32-957-234-8742 7603.40
FURNITURE g alongside of the special excuses-- fluffily enticing packages
wake
-
--- !count1 --
-19
-
--- !count1_orc --
-19
-
--- !max1 --
-update-comment-2
-
--- !max1_orc --
-update-comment-2
-
--- !one_delete_column --
-1 Customer#000000001 IVhzIApeRb ot,c,E 151 update-phone-1
711.56 BUILDING update-comment-1
-2 Customer#000000002 XSTf4,NCwDVaWNe6tEgvwfmRchLXak 13
23-768-687-3665 121.65 AUTOMOBILE l accounts. blithely ironic theodolites
integrate boldly: caref
-3 Customer#000000003 MG9kdTD2WBHm 1 11-719-748-3364 7498.12
AUTOMOBILE deposits eat slyly ironic, even instructions. express foxes
detect slyly. blithely even accounts abov
-4 Customer#000000004 XxVSJsLAGtn 47 update-phone-4 15.39
MACHINERY update-comment-2
-6 Customer#000000006 sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn 20
30-114-968-4951 7638.57 AUTOMOBILE tions. even deposits boost according to
the slyly bold packages. final accounts cajole requests. furious
-7 Customer#000000007 TcGe5gaZNgVePxU5kRrvXBfkasDTea 18
28-190-982-9759 9561.95 AUTOMOBILE ainst the ironic, express theodolites.
express, even pinto beans among the exp
-8 Customer#000000008 I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 17
27-147-574-9335 6819.74 BUILDING among the slyly regular theodolites
kindle blithely courts. carefully even theodolites haggle slyly along the ide
-9 Customer#000000009 xKiAFTjUsCuxfeleNqefumTrjS 8
18-338-906-3675 8324.07 FURNITURE r theodolites according to the requests
wake thinly excuses: pending requests haggle furiousl
-10 Customer#000000010 6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2 5
15-741-346-9870 2753.54 HOUSEHOLD es regular deposits haggle. fur
-11 Customer#000000011 PkWS 3HlXqwTuzrKg633BEi 23 33-464-151-3439
-272.60 BUILDING ckages. requests sleep slyly. quickly even pinto beans
promise above the slyly regular pinto beans.
-12 Customer#000000012 9PWKuhzT4Zr1Q 13 23-791-276-1263 3396.49
HOUSEHOLD to the carefully final braids. blithely regular requests nag.
ironic theodolites boost quickly along
-13 Customer#000000013 nsXQu0oVjD7PM659uC3SRSp 3 13-761-547-5974
3857.34 BUILDING ounts sleep carefully after the close frays. carefully
bold notornis use ironic requests. blithely
-14 Customer#000000014 KXkletMlL2JQEA 1 11-845-129-3851 5266.30
FURNITURE , ironic packages across the unus
-15 Customer#000000015 YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn 23
33-687-542-7601 2788.52 HOUSEHOLD platelets. regular deposits detect
asymptotes. blithely unusual packages nag slyly at the fluf
-16 Customer#000000016 cYiaeMLZSMAOQ2 d0W, 10 20-781-609-3107
4681.03 FURNITURE kly silent courts. thinly regular theodolites sleep
fluffily after
-17 Customer#000000017 izrh 6jdqtp2eqdtbkswDD8SG4SzXruMfIXyR7 2
12-970-682-3487 6.34 AUTOMOBILE packages wake! blithely even pint
-18 Customer#000000018 3txGO AiuFux3zT0Z9NYaFRnZt 6
16-155-215-1315 5494.43 BUILDING s sleep. carefully even instructions
nag furiously alongside of t
-19 Customer#000000019 uc,3bHIx84H,wdrmLOjVsiqXCq2tr 18
28-396-526-5053 8914.71 HOUSEHOLD nag. furiously careful packages are
slyly at the accounts. furiously regular in
-20 Customer#000000020 JrPk8Pqplj4Ne 22 32-957-234-8742 7603.40
FURNITURE g alongside of the special excuses-- fluffily enticing packages
wake
-
--- !one_delete_column_orc --
-1 Customer#000000001 IVhzIApeRb ot,c,E 151 update-phone-1
711.56 BUILDING update-comment-1
-2 Customer#000000002 XSTf4,NCwDVaWNe6tEgvwfmRchLXak 13
23-768-687-3665 121.65 AUTOMOBILE l accounts. blithely ironic theodolites
integrate boldly: caref
-3 Customer#000000003 MG9kdTD2WBHm 1 11-719-748-3364 7498.12
AUTOMOBILE deposits eat slyly ironic, even instructions. express foxes
detect slyly. blithely even accounts abov
-4 Customer#000000004 XxVSJsLAGtn 47 update-phone-4 15.39
MACHINERY update-comment-2
-6 Customer#000000006 sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn 20
30-114-968-4951 7638.57 AUTOMOBILE tions. even deposits boost according to
the slyly bold packages. final accounts cajole requests. furious
-7 Customer#000000007 TcGe5gaZNgVePxU5kRrvXBfkasDTea 18
28-190-982-9759 9561.95 AUTOMOBILE ainst the ironic, express theodolites.
express, even pinto beans among the exp
-8 Customer#000000008 I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5 17
27-147-574-9335 6819.74 BUILDING among the slyly regular theodolites
kindle blithely courts. carefully even theodolites haggle slyly along the ide
-9 Customer#000000009 xKiAFTjUsCuxfeleNqefumTrjS 8
18-338-906-3675 8324.07 FURNITURE r theodolites according to the requests
wake thinly excuses: pending requests haggle furiousl
-10 Customer#000000010 6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2 5
15-741-346-9870 2753.54 HOUSEHOLD es regular deposits haggle. fur
-11 Customer#000000011 PkWS 3HlXqwTuzrKg633BEi 23 33-464-151-3439
-272.60 BUILDING ckages. requests sleep slyly. quickly even pinto beans
promise above the slyly regular pinto beans.
-12 Customer#000000012 9PWKuhzT4Zr1Q 13 23-791-276-1263 3396.49
HOUSEHOLD to the carefully final braids. blithely regular requests nag.
ironic theodolites boost quickly along
-13 Customer#000000013 nsXQu0oVjD7PM659uC3SRSp 3 13-761-547-5974
3857.34 BUILDING ounts sleep carefully after the close frays. carefully
bold notornis use ironic requests. blithely
-14 Customer#000000014 KXkletMlL2JQEA 1 11-845-129-3851 5266.30
FURNITURE , ironic packages across the unus
-15 Customer#000000015 YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn 23
33-687-542-7601 2788.52 HOUSEHOLD platelets. regular deposits detect
asymptotes. blithely unusual packages nag slyly at the fluf
-16 Customer#000000016 cYiaeMLZSMAOQ2 d0W, 10 20-781-609-3107
4681.03 FURNITURE kly silent courts. thinly regular theodolites sleep
fluffily after
-17 Customer#000000017 izrh 6jdqtp2eqdtbkswDD8SG4SzXruMfIXyR7 2
12-970-682-3487 6.34 AUTOMOBILE packages wake! blithely even pint
-18 Customer#000000018 3txGO AiuFux3zT0Z9NYaFRnZt 6
16-155-215-1315 5494.43 BUILDING s sleep. carefully even instructions
nag furiously alongside of t
-19 Customer#000000019 uc,3bHIx84H,wdrmLOjVsiqXCq2tr 18
28-396-526-5053 8914.71 HOUSEHOLD nag. furiously careful packages are
slyly at the accounts. furiously regular in
-20 Customer#000000020 JrPk8Pqplj4Ne 22 32-957-234-8742 7603.40
FURNITURE g alongside of the special excuses-- fluffily enticing packages
wake
-
--- !count3 --
-19
-
--- !count3_orc --
-19
-
--- !max3 --
-update-comment-2
-
--- !max3_orc --
-update-comment-2
-
diff --git
a/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_hadoop_catalog.out
b/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_hadoop_catalog.out
deleted file mode 100644
index aaa9037977d..00000000000
---
a/regression-test/data/external_table_p2/iceberg/test_external_catalog_iceberg_hadoop_catalog.out
+++ /dev/null
@@ -1,26 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !q01 --
-2879562
-
--- !q02 --
-1
-3
-5
-6
-7
-8
-11
-
--- !q03 --
-1 Customer#000000001 j5JsirBM9P MOROCCO 0 MOROCCO AFRICA
25-989-741-2988 BUILDING
-3 Customer#000000003 fkRGN8n ARGENTINA7 ARGENTINA AMERICA
11-719-748-3364 AUTOMOBILE
-5 Customer#000000005 hwBtxkoBF qSW4KrI CANADA 5 CANADA
AMERICA 13-750-942-6364 HOUSEHOLD
-
--- !q04 --
-1 1970-01-03 09:02:03.000001 a
-1 1970-01-03 09:02:03.000001 b
-2 1970-01-03 09:02:04.000001 c
-2 1970-01-03 09:02:04.000001 d
-
--- !q05 --
-463870
diff --git
a/regression-test/data/external_table_p2/iceberg/test_external_catalog_icebergv2_nereids.out
b/regression-test/data/external_table_p2/iceberg/test_external_catalog_icebergv2_nereids.out
deleted file mode 100644
index a6e68d2f621..00000000000
---
a/regression-test/data/external_table_p2/iceberg/test_external_catalog_icebergv2_nereids.out
+++ /dev/null
@@ -1,74 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !q01 --
-149988
-
--- !q02 --
-1
-3
-4
-7
-
--- !q03 --
-8242263
-
--- !q04 --
-0
-
--- !q05 --
-1 Customer#000000001 IVhzIApeRb ot,c,E 15 25-989-741-2988
711.56 BUILDING to the even, regular platelets. regular, ironic
epitaphs nag e
-3 Customer#000000003 MG9kdTD2WBHm 1 11-719-748-3364 7498.12
AUTOMOBILE deposits eat slyly ironic, even instructions. express foxes
detect slyly. blithely even accounts abov
-4 Customer#000000004 XxVSJsLAGtn 4 14-128-190-5944 2866.83
MACHINERY requests. final, regular ideas sleep final accou
-
--- !q06 --
-604519555
-604519557
-604519558
-
--- !q07 --
-12979.65
-219204.52
-5908.20
-
--- !q08 --
-120001848
-
--- !q09 --
-1
-2
-3
-
--- !q10 --
-150000000
-149999999
-149999996
-
--- !q11 --
-1
-2
-3
-
--- !q12 --
-150000000
-149999999
-149999996
-
--- !q13 --
-1
-4
-7
-
--- !q14 --
-Customer#000000004
-Customer#000000007
-
--- !q15 --
-150000
-
--- !q16 --
-150000
-
--- !q17 --
-150000
-
--- !q18 --
-150000
diff --git
a/regression-test/suites/external_table_p0/hive/test_upper_case_column_name.groovy
b/regression-test/suites/external_table_p0/hive/test_upper_case_column_name.groovy
index 76246398613..a43af56a12e 100644
---
a/regression-test/suites/external_table_p0/hive/test_upper_case_column_name.groovy
+++
b/regression-test/suites/external_table_p0/hive/test_upper_case_column_name.groovy
@@ -32,22 +32,6 @@ suite("test_upper_case_column_name",
"p0,external,hive,external_docker,external_
def hiveOrc6 = """select id from hive_upper_case_orc where id=1;"""
def hiveOrc7 = """select name from hive_upper_case_orc where id=1;"""
def hiveOrc8 = """select id, name from hive_upper_case_orc where id=1;"""
- def icebergParquet1 = """select * from iceberg_upper_case_parquet;"""
- def icebergParquet2 = """select * from iceberg_upper_case_parquet where
id=1;"""
- def icebergParquet3 = """select * from iceberg_upper_case_parquet where
id>1;"""
- def icebergParquet4 = """select * from iceberg_upper_case_parquet where
name='name';"""
- def icebergParquet5 = """select * from iceberg_upper_case_parquet where
name!='name';"""
- def icebergParquet6 = """select id from iceberg_upper_case_parquet where
id=1;"""
- def icebergParquet7 = """select name from iceberg_upper_case_parquet where
id=1;"""
- def icebergParquet8 = """select id, name from iceberg_upper_case_parquet
where id=1;"""
- def icebergOrc1 = """select * from iceberg_upper_case_orc;"""
- def icebergOrc2 = """select * from iceberg_upper_case_orc where id=1;"""
- def icebergOrc3 = """select * from iceberg_upper_case_orc where id>1;"""
- def icebergOrc4 = """select * from iceberg_upper_case_orc where
name='name';"""
- def icebergOrc5 = """select * from iceberg_upper_case_orc where
name!='name';"""
- def icebergOrc6 = """select id from iceberg_upper_case_orc where id=1;"""
- def icebergOrc7 = """select name from iceberg_upper_case_orc where id=1;"""
- def icebergOrc8 = """select id, name from iceberg_upper_case_orc where
id=1;"""
String enabled = context.config.otherConfigs.get("enableHiveTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
@@ -82,22 +66,6 @@ suite("test_upper_case_column_name",
"p0,external,hive,external_docker,external_
qt_hiveOrc6 hiveOrc6
qt_hiveOrc7 hiveOrc7
qt_hiveOrc8 hiveOrc8
- // qt_icebergParquet1 icebergParquet1
- // qt_icebergParquet2 icebergParquet2
- // qt_icebergParquet3 icebergParquet3
- // qt_icebergParquet4 icebergParquet4
- // qt_icebergParquet5 icebergParquet5
- // qt_icebergParquet6 icebergParquet6
- // qt_icebergParquet7 icebergParquet7
- // qt_icebergParquet8 icebergParquet8
- // qt_icebergOrc1 icebergOrc1
- // qt_icebergOrc2 icebergOrc2
- // qt_icebergOrc3 icebergOrc3
- // qt_icebergOrc4 icebergOrc4
- // qt_icebergOrc5 icebergOrc5
- // qt_icebergOrc6 icebergOrc6
- // qt_icebergOrc7 icebergOrc7
- // qt_icebergOrc8 icebergOrc8
}
}
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_gen_iceberg_by_api.groovy
b/regression-test/suites/external_table_p0/iceberg/test_gen_iceberg_by_api.groovy
new file mode 100644
index 00000000000..4fcc180bd02
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_gen_iceberg_by_api.groovy
@@ -0,0 +1,192 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_gen_iceberg_by_api",
"p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test.")
+ return
+ }
+
+ String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalog_name = "test_gen_iceberg_by_api"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
+ "s3.region" = "us-east-1"
+ );"""
+
+ sql """switch ${catalog_name};"""
+ sql """ use `test_db`; """
+
+ def q01 = {
+ qt_q01 """ select * from multi_partition2 order by val """
+ qt_q02 """ select count(*) from table_with_append_file where MAN_ID is
not null """
+ }
+
+ q01()
+}
+
+/*
+
+// 打包后，在iceberg-spark的docker中运行
+
+package org.example;
+
+
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.types.Types;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+public class CreateTable {
+ public static void main(String[] args) throws Exception {
+ CreateTable createTable = new CreateTable();
+ createTable.create();
+ }
+
+ public void create() {
+ HashMap<String, String> prop = new HashMap<>();
+ prop.put("uri", "http://172.21.0.101:18181";);
+ prop.put("io-impl", "org.apache.iceberg.aws.s3.S3FileIO");
+ prop.put("type", "rest");
+ prop.put("AWS_ACCESS_KEY_ID", "admin");
+ prop.put("AWS_SECRET_ACCESS_KEY", "password");
+ prop.put("AWS_REGION", "us-east-1");
+ prop.put("s3.endpoint", "http://172.21.0.101:19001";);
+ // RESTCatalog catalog = new RESTCatalog();
+ Catalog catalog = CatalogUtil.buildIcebergCatalog("df", prop, null);
+ catalog.initialize("ddf", prop);
+ List<TableIdentifier> test_db =
catalog.listTables(Namespace.of("test_db"));
+ System.out.println(test_db);
+
+ table1(catalog);
+ table2(catalog);
+
+ }
+
+ public void table1(Catalog catalog) {
+
+ TableIdentifier of = TableIdentifier.of("test_db", "multi_partition2");
+ boolean exists = catalog.tableExists(of);
+ Table table;
+ if (exists) {
+ catalog.dropTable(of);
+ }
+ Schema schema = new Schema(
+ Types.NestedField.required(1, "id",
Types.IntegerType.get()),
+ Types.NestedField.required(2, "ts",
Types.TimestampType.withZone()),
+ Types.NestedField.required(3, "val",
Types.StringType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema)
+ .identity("id")
+ .hour("ts")
+ .build();
+ table = catalog.createTable(of, schema, spec);
+
+ DataFile build = DataFiles.builder(table.spec())
+ .withPartitionValues(new
ArrayList<String>(){{add("1");add("1000");}})
+
.withInputFile(table.io().newInputFile("s3://warehouse/data/multi_partition2/00000-0-f309508c-953a-468f-8bcf-d910b2c7a1e5-00001.parquet"))
+ .withFileSizeInBytes(884)
+ .withRecordCount(1)//read the record count
+ .withFormat(FileFormat.PARQUET)
+ .build();
+ DataFile build2 = DataFiles.builder(table.spec())
+ .withPartitionValues(new
ArrayList<String>(){{add("1");add("1000");}})
+
.withInputFile(table.io().newInputFile("s3://warehouse/data/multi_partition2/00000-1-4acbae74-1265-4c3b-a8d1-d773037e8b42-00001.parquet"))
+ .withFileSizeInBytes(884)
+ .withRecordCount(1)//read the record count
+ .withFormat(FileFormat.PARQUET)
+ .build();
+ DataFile build3 = DataFiles.builder(table.spec())
+ .withPartitionValues(new
ArrayList<String>(){{add("2");add("2000");}})
+
.withInputFile(table.io().newInputFile("s3://warehouse/data/multi_partition2/00000-2-9c8199d3-9bc5-4e57-84d7-ad1cedcfbe94-00001.parquet"))
+ .withFileSizeInBytes(884)
+ .withRecordCount(1)//read the record count
+ .withFormat(FileFormat.PARQUET)
+ .build();
+ DataFile build4 = DataFiles.builder(table.spec())
+ .withPartitionValues(new
ArrayList<String>(){{add("2");add("2000");}})
+
.withInputFile(table.io().newInputFile("s3://warehouse/data/multi_partition2/00000-3-c042039c-f716-41a1-a857-eedf23f7be92-00001.parquet"))
+ .withFileSizeInBytes(884)
+ .withRecordCount(1)//read the record count
+ .withFormat(FileFormat.PARQUET)
+ .build();
+
+ table.newAppend()
+ .appendFile(build)
+ .appendFile(build2)
+ .appendFile(build3)
+ .appendFile(build4)
+ .commit();
+ }
+
+ public void table2(Catalog catalog) {
+ TableIdentifier of = TableIdentifier.of("test_db",
"table_with_append_file");
+ boolean exists = catalog.tableExists(of);
+ if (exists) {
+ catalog.dropTable(of);
+ }
+ Schema schema = new Schema(
+ Types.NestedField.required(1, "START_TIME",
Types.LongType.get()),
+ Types.NestedField.required(2, "ENT_TIME",
Types.LongType.get()),
+ Types.NestedField.required(3, "MAN_ID",
Types.StringType.get()),
+ Types.NestedField.required(4, "END_TIME_MICROS",
Types.TimestampType.withZone()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema)
+ .hour("END_TIME_MICROS")
+ .build();
+ Table table = catalog.createTable(of, schema, spec);
+
+ InputFile inputFile = table.io()
+
.newInputFile("s3://warehouse/data/table_with_append_file/sample-data.snappy.parquet");
+
+ DataFile build = DataFiles.builder(table.spec())
+ .withPartitionValues(new ArrayList<String>(){{add("3");}})
+ .withInputFile(inputFile)
+ .withFileSizeInBytes(inputFile.getLength())
+ .withRecordCount(1)//read the record count
+ .withFormat(FileFormat.PARQUET)
+ .build();
+
+ table.newAppend()
+ .appendFile(build)
+ .commit();
+ }
+}
+
+
+*/
\ No newline at end of file
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete.groovy
new file mode 100644
index 00000000000..74a3dc953eb
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_equality_delete.groovy
@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_equality_delete",
"p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test.")
+ return
+ }
+
+ String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalog_name = "test_iceberg_equality_delete"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
+ "s3.region" = "us-east-1"
+ );"""
+
+ logger.info("catalog " + catalog_name + " created")
+ sql """switch ${catalog_name};"""
+ logger.info("switched to catalog " + catalog_name)
+ sql """ use test_db;"""
+
+ // one delete column
+ qt_one_delete_column """select * from customer_flink_one order by id"""
+ qt_one_delete_column_orc """select * from customer_flink_one_orc order
by id"""
+ qt_count1 """select count(*) from customer_flink_one"""
+ qt_count1_orc """select count(*) from customer_flink_one_orc"""
+ qt_max1 """select max(val) from customer_flink_one"""
+ qt_max1_orc """select max(val) from customer_flink_one_orc"""
+
+ // three delete columns
+ qt_one_delete_column """select * from customer_flink_three order by
id1"""
+ qt_one_delete_column_orc """select * from customer_flink_three_orc
order by id1"""
+ qt_count3 """select count(*) from customer_flink_three"""
+ qt_count3_orc """select count(*) from customer_flink_three_orc"""
+ qt_max3 """select max(val) from customer_flink_three"""
+ qt_max3_orc """select max(val) from customer_flink_three_orc"""
+
+ sql """drop catalog ${catalog_name}"""
+}
+
+/*
+
+---- flink sql:
+
+export AWS_ACCESS_KEY_ID=admin
+export AWS_SECRET_ACCESS_KEY=password
+export AWS_REGION=us-east-1
+export AWS_DEFAULT_REGION=us-east-1
+export S3_PATH_STYLE_ACCESS=true
+export S3_ENDPOINT=http://172.21.0.101:19001
+
+CREATE CATALOG iceberg_rest WITH (
+ 'type' = 'iceberg',
+ 'catalog-type' = 'rest',
+ 'uri'='http://172.21.0.101:18181',
+ 'warehouse'='s3a://warehouse/wh',
+ 's3.endpoint' = 'http://172.21.0.101:19001'
+);
+
+use iceberg_rest.test_db;
+
+create table customer_flink_one (
+ id int,
+ val string,
+ PRIMARY KEY(`id`) NOT ENFORCED
+) WITH (
+ 'format-version'='2',
+ 'write-format'='parquet',
+ 'upsert-enabled'='true'
+);
+
+insert into customer_flink_one values (1,'a');
+insert into customer_flink_one values (2,'b');
+insert into customer_flink_one values (1,'b');
+insert into customer_flink_one values (2, 'a'),(2, 'c');
+
+
+create table customer_flink_one_orc (
+ id int,
+ val string,
+ PRIMARY KEY(`id`) NOT ENFORCED
+) WITH (
+ 'format-version'='2',
+ 'write-format'='orc',
+ 'upsert-enabled'='true'
+);
+
+insert into customer_flink_one_orc values (1,'a');
+insert into customer_flink_one_orc values (2,'b');
+insert into customer_flink_one_orc values (1,'b');
+insert into customer_flink_one_orc values (2, 'a'),(2, 'c');
+
+
+create table customer_flink_three (
+ id1 int,
+ id2 string,
+ id3 int,
+ val string,
+ PRIMARY KEY(`id1`,`id2`,`id3`) NOT ENFORCED
+) WITH (
+ 'format-version'='2',
+ 'write-format'='parquet',
+ 'upsert-enabled'='true'
+);
+insert into customer_flink_three values (1,'id2',1,'a');
+insert into customer_flink_three values (1,'id2',1,'b');
+insert into customer_flink_three values (2,'id2',1,'b');
+insert into customer_flink_three values (2,'id2',1,'a'),(2,'id2',1,'c');
+
+create table customer_flink_three_orc (
+ id1 int,
+ id2 string,
+ id3 int,
+ val string,
+ PRIMARY KEY(`id1`,`id2`,`id3`) NOT ENFORCED
+) WITH (
+ 'format-version'='2',
+ 'write-format'='orc',
+ 'upsert-enabled'='true'
+);
+
+insert into customer_flink_three_orc values (1,'id2',1,'a');
+insert into customer_flink_three_orc values (1,'id2',1,'b');
+insert into customer_flink_three_orc values (2,'id2',1,'b');
+insert into customer_flink_three_orc values (2,'id2',1,'a'),(2,'id2',1,'c');
+
+*/
diff --git
a/regression-test/suites/external_table_p2/iceberg/iceberg_position_delete.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_delete.groovy
similarity index 92%
rename from
regression-test/suites/external_table_p2/iceberg/iceberg_position_delete.groovy
rename to
regression-test/suites/external_table_p0/iceberg/test_iceberg_position_delete.groovy
index 8f62aaba50d..6a14712a2af 100644
---
a/regression-test/suites/external_table_p2/iceberg/iceberg_position_delete.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_position_delete.groovy
@@ -15,27 +15,34 @@
// specific language governing permissions and limitations
// under the License.
-suite("iceberg_position_delete",
"p2,external,iceberg,external_remote,external_remote_iceberg") {
-
- String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
- if (enabled != null && enabled.equalsIgnoreCase("true")) {
-
- String catalog_name = "test_external_iceberg_position_delete"
- String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
- String extHdfsPort = context.config.otherConfigs.get("extHdfsPort")
- sql """drop catalog if exists ${catalog_name};"""
- sql """
- create catalog if not exists ${catalog_name} properties (
- 'type'='iceberg',
- 'iceberg.catalog.type'='hadoop',
- 'warehouse' =
'hdfs://${extHiveHmsHost}:${extHdfsPort}/usr/hive/warehouse/hadoop_catalog'
- );
- """
+suite("test_iceberg_position_delete",
"p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test.")
+ return
+ }
+
+ String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalog_name = "test_iceberg_position_delete"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
+ "s3.region" = "us-east-1"
+ );"""
logger.info("catalog " + catalog_name + " created")
sql """switch ${catalog_name};"""
logger.info("switched to catalog " + catalog_name)
- sql """ use multi_catalog;"""
+ sql """ use test_db;"""
qt_gen_data_1 """ select * from iceberg_position_gen_data where name
= 'xyzxxxxxx' and id != 9;"""
qt_gen_data_2 """ select * from iceberg_position_gen_data where id =
1; """
@@ -158,7 +165,6 @@ suite("iceberg_position_delete",
"p2,external,iceberg,external_remote,external_r
assertTrue(iceberg_position_gen_7.size() == 5632)
sql """drop catalog ${catalog_name}"""
- }
}
/*
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_time_travel.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_time_travel.groovy
new file mode 100644
index 00000000000..5f4ffcef643
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_time_travel.groovy
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_time_travel",
"p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test.")
+ return
+ }
+
+ String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalog_name = "test_iceberg_time_travel"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
+ "s3.region" = "us-east-1"
+ );"""
+
+ sql """ use ${catalog_name}.test_db """
+
+ // test for 'FOR TIME AS OF' and 'FOR VERSION AS OF'
+ def q01 = {
+ qt_q1 """ select count(*) from iceberg_position_gen_data """ //
5632
+ qt_q2 """ select count(*) from iceberg_position_gen_data FOR TIME
AS OF '2024-07-14 14:17:01' """// 120
+ qt_q3 """ select * from iceberg_position_gen_data order by id
limit 10"""
+ qt_q4 """ select * from iceberg_position_gen_data FOR TIME AS OF
'2024-07-14 14:17:01' order by id limit 10"""
+ qt_q5 """ select count(*) from iceberg_position_gen_data FOR
VERSION AS OF 3106988132043095748 """ // 240
+ qt_q6 """ select * from iceberg_position_gen_data FOR VERSION AS
OF 3106988132043095748 order by id limit 10"""
+ }
+
+ q01()
+}
diff --git
a/regression-test/suites/external_table_p2/hive/test_upper_case_column_name.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_upper_case_column_name.groovy
similarity index 50%
copy from
regression-test/suites/external_table_p2/hive/test_upper_case_column_name.groovy
copy to
regression-test/suites/external_table_p0/iceberg/test_iceberg_upper_case_column_name.groovy
index 28d4e6742e0..79e51a18ed1 100644
---
a/regression-test/suites/external_table_p2/hive/test_upper_case_column_name.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_upper_case_column_name.groovy
@@ -15,23 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-suite("upper_case_column_name",
"p2,external,hive,external_remote,external_remote_hive") {
- def hiveParquet1 = """select * from hive_upper_case_parquet;"""
- def hiveParquet2 = """select * from hive_upper_case_parquet where id=1;"""
- def hiveParquet3 = """select * from hive_upper_case_parquet where id>1;"""
- def hiveParquet4 = """select * from hive_upper_case_parquet where
name='name';"""
- def hiveParquet5 = """select * from hive_upper_case_parquet where
name!='name';"""
- def hiveParquet6 = """select id from hive_upper_case_parquet where id=1;"""
- def hiveParquet7 = """select name from hive_upper_case_parquet where
id=1;"""
- def hiveParquet8 = """select id, name from hive_upper_case_parquet where
id=1;"""
- def hiveOrc1 = """select * from hive_upper_case_orc;"""
- def hiveOrc2 = """select * from hive_upper_case_orc where id=1;"""
- def hiveOrc3 = """select * from hive_upper_case_orc where id>1;"""
- def hiveOrc4 = """select * from hive_upper_case_orc where name='name';"""
- def hiveOrc5 = """select * from hive_upper_case_orc where name!='name';"""
- def hiveOrc6 = """select id from hive_upper_case_orc where id=1;"""
- def hiveOrc7 = """select name from hive_upper_case_orc where id=1;"""
- def hiveOrc8 = """select id, name from hive_upper_case_orc where id=1;"""
+suite("test_iceberg_upper_case_column_name",
"p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test.")
+ return
+ }
+
def icebergParquet1 = """select * from iceberg_upper_case_parquet;"""
def icebergParquet2 = """select * from iceberg_upper_case_parquet where
id=1;"""
def icebergParquet3 = """select * from iceberg_upper_case_parquet where
id>1;"""
@@ -49,39 +39,25 @@ suite("upper_case_column_name",
"p2,external,hive,external_remote,external_remot
def icebergOrc7 = """select name from iceberg_upper_case_orc where id=1;"""
def icebergOrc8 = """select id, name from iceberg_upper_case_orc where
id=1;"""
+ String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port = context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalog_name = "test_iceberg_upper_case_column_name"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """
+ CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
+ "s3.region" = "us-east-1"
+ );"""
+
+ sql """use `${catalog_name}`.`test_db`"""
- String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
- if (enabled != null && enabled.equalsIgnoreCase("true")) {
- String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
- String extHiveHmsPort =
context.config.otherConfigs.get("extHiveHmsPort")
- String catalog_name = "upper_case"
- sql """drop catalog if exists ${catalog_name};"""
- sql """
- create catalog if not exists ${catalog_name} properties (
- 'type'='hms',
- 'hive.metastore.uris' =
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
- );
- """
- logger.info("catalog " + catalog_name + " created")
- sql """switch ${catalog_name};"""
- logger.info("switched to catalog " + catalog_name)
- sql """use multi_catalog;"""
- qt_hiveParquet1 hiveParquet1
- qt_hiveParquet2 hiveParquet2
- qt_hiveParquet3 hiveParquet3
- qt_hiveParquet4 hiveParquet4
- qt_hiveParquet5 hiveParquet5
- qt_hiveParquet6 hiveParquet6
- qt_hiveParquet7 hiveParquet7
- qt_hiveParquet8 hiveParquet8
- qt_hiveOrc1 hiveOrc1
- qt_hiveOrc2 hiveOrc2
- qt_hiveOrc3 hiveOrc3
- qt_hiveOrc4 hiveOrc4
- qt_hiveOrc5 hiveOrc5
- qt_hiveOrc6 hiveOrc6
- qt_hiveOrc7 hiveOrc7
- qt_hiveOrc8 hiveOrc8
qt_icebergParquet1 icebergParquet1
qt_icebergParquet2 icebergParquet2
qt_icebergParquet3 icebergParquet3
@@ -98,6 +74,5 @@ suite("upper_case_column_name",
"p2,external,hive,external_remote,external_remot
qt_icebergOrc6 icebergOrc6
qt_icebergOrc7 icebergOrc7
qt_icebergOrc8 icebergOrc8
- }
}
diff --git
a/regression-test/suites/external_table_p2/hive/test_upper_case_column_name.groovy
b/regression-test/suites/external_table_p2/hive/test_upper_case_column_name.groovy
index 28d4e6742e0..0033e0dca24 100644
---
a/regression-test/suites/external_table_p2/hive/test_upper_case_column_name.groovy
+++
b/regression-test/suites/external_table_p2/hive/test_upper_case_column_name.groovy
@@ -32,22 +32,6 @@ suite("upper_case_column_name",
"p2,external,hive,external_remote,external_remot
def hiveOrc6 = """select id from hive_upper_case_orc where id=1;"""
def hiveOrc7 = """select name from hive_upper_case_orc where id=1;"""
def hiveOrc8 = """select id, name from hive_upper_case_orc where id=1;"""
- def icebergParquet1 = """select * from iceberg_upper_case_parquet;"""
- def icebergParquet2 = """select * from iceberg_upper_case_parquet where
id=1;"""
- def icebergParquet3 = """select * from iceberg_upper_case_parquet where
id>1;"""
- def icebergParquet4 = """select * from iceberg_upper_case_parquet where
name='name';"""
- def icebergParquet5 = """select * from iceberg_upper_case_parquet where
name!='name';"""
- def icebergParquet6 = """select id from iceberg_upper_case_parquet where
id=1;"""
- def icebergParquet7 = """select name from iceberg_upper_case_parquet where
id=1;"""
- def icebergParquet8 = """select id, name from iceberg_upper_case_parquet
where id=1;"""
- def icebergOrc1 = """select * from iceberg_upper_case_orc;"""
- def icebergOrc2 = """select * from iceberg_upper_case_orc where id=1;"""
- def icebergOrc3 = """select * from iceberg_upper_case_orc where id>1;"""
- def icebergOrc4 = """select * from iceberg_upper_case_orc where
name='name';"""
- def icebergOrc5 = """select * from iceberg_upper_case_orc where
name!='name';"""
- def icebergOrc6 = """select id from iceberg_upper_case_orc where id=1;"""
- def icebergOrc7 = """select name from iceberg_upper_case_orc where id=1;"""
- def icebergOrc8 = """select id, name from iceberg_upper_case_orc where
id=1;"""
String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
@@ -82,22 +66,6 @@ suite("upper_case_column_name",
"p2,external,hive,external_remote,external_remot
qt_hiveOrc6 hiveOrc6
qt_hiveOrc7 hiveOrc7
qt_hiveOrc8 hiveOrc8
- qt_icebergParquet1 icebergParquet1
- qt_icebergParquet2 icebergParquet2
- qt_icebergParquet3 icebergParquet3
- qt_icebergParquet4 icebergParquet4
- qt_icebergParquet5 icebergParquet5
- qt_icebergParquet6 icebergParquet6
- qt_icebergParquet7 icebergParquet7
- qt_icebergParquet8 icebergParquet8
- qt_icebergOrc1 icebergOrc1
- qt_icebergOrc2 icebergOrc2
- qt_icebergOrc3 icebergOrc3
- qt_icebergOrc4 icebergOrc4
- qt_icebergOrc5 icebergOrc5
- qt_icebergOrc6 icebergOrc6
- qt_icebergOrc7 icebergOrc7
- qt_icebergOrc8 icebergOrc8
}
}
diff --git
a/regression-test/suites/external_table_p2/iceberg/iceberg_equality_delete.groovy
b/regression-test/suites/external_table_p2/iceberg/iceberg_equality_delete.groovy
deleted file mode 100644
index 984181ac2f3..00000000000
---
a/regression-test/suites/external_table_p2/iceberg/iceberg_equality_delete.groovy
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("iceberg_equality_delete",
"p2,external,iceberg,external_remote,external_remote_iceberg") {
-
- String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
- if (enabled != null && enabled.equalsIgnoreCase("true")) {
-
- String catalog_name = "test_external_iceberg_equality_delete"
- String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
- String extHdfsPort = context.config.otherConfigs.get("extHdfsPort")
- sql """drop catalog if exists ${catalog_name};"""
- sql """
- create catalog if not exists ${catalog_name} properties (
- 'type'='iceberg',
- 'iceberg.catalog.type'='hadoop',
- 'warehouse' =
'hdfs://${extHiveHmsHost}:${extHdfsPort}/usr/hive/warehouse/hadoop_catalog'
- );
- """
-
- logger.info("catalog " + catalog_name + " created")
- sql """switch ${catalog_name};"""
- logger.info("switched to catalog " + catalog_name)
- sql """ use multi_catalog;"""
-
- // one delete column
- qt_one_delete_column """select * from customer_flink_one order by
c_custkey"""
- qt_one_delete_column_orc """select * from customer_flink_one_orc order
by c_custkey"""
- qt_count1 """select count(*) from customer_flink_one"""
- qt_count1_orc """select count(*) from customer_flink_one_orc"""
- qt_max1 """select max(c_comment) from customer_flink_one"""
- qt_max1_orc """select max(c_comment) from customer_flink_one_orc"""
- // three delete columns
- qt_one_delete_column """select * from customer_flink_three order by
c_custkey"""
- qt_one_delete_column_orc """select * from customer_flink_three_orc
order by c_custkey"""
- qt_count3 """select count(*) from customer_flink_three"""
- qt_count3_orc """select count(*) from customer_flink_three_orc"""
- qt_max3 """select max(c_comment) from customer_flink_three"""
- qt_max3_orc """select max(c_comment) from customer_flink_three_orc"""
-
- sql """drop catalog ${catalog_name}"""
- }
-}
diff --git
a/regression-test/suites/external_table_p2/iceberg/test_external_catalog_iceberg_hadoop_catalog.groovy
b/regression-test/suites/external_table_p2/iceberg/test_external_catalog_iceberg_hadoop_catalog.groovy
deleted file mode 100644
index 12ec14992ec..00000000000
---
a/regression-test/suites/external_table_p2/iceberg/test_external_catalog_iceberg_hadoop_catalog.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_external_catalog_iceberg_hadoop_catalog",
"p2,external,iceberg,external_remote,external_remote_iceberg") {
- String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
- if (enabled != null && enabled.equalsIgnoreCase("true")) {
- String iceberg_catalog_name = "test_external_iceberg_catalog_hadoop"
- String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
- String extHdfsPort = context.config.otherConfigs.get("extHdfsPort")
- sql """drop catalog if exists ${iceberg_catalog_name};"""
- sql """
- create catalog if not exists ${iceberg_catalog_name} properties (
- 'type'='iceberg',
- 'iceberg.catalog.type'='hadoop',
- 'warehouse' =
'hdfs://${extHiveHmsHost}:${extHdfsPort}/usr/hive/warehouse/hadoop_catalog'
- );
- """
-
- sql """switch ${iceberg_catalog_name};"""
- def q01 = {
- qt_q01 """ select count(*) from iceberg_hadoop_catalog """
- qt_q02 """ select c_custkey from iceberg_hadoop_catalog group by
c_custkey order by c_custkey limit 7 """
- qt_q03 """ select * from iceberg_hadoop_catalog order by c_custkey
limit 3 """
- }
-
- def q02 = {
- qt_q04 """ select * from multi_partition2 order by val """
- qt_q05 """ select count(*) from table_with_append_file where
MAN_ID is not null """
- }
-
- sql """ use `multi_catalog`; """
- q01()
- q02()
- }
-}
diff --git
a/regression-test/suites/external_table_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy
b/regression-test/suites/external_table_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy
deleted file mode 100644
index 38385b3c458..00000000000
---
a/regression-test/suites/external_table_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy
+++ /dev/null
@@ -1,84 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_external_catalog_icebergv2_nereids",
"p2,external,iceberg,external_remote,external_remote_iceberg") {
- String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
- if (enabled != null && enabled.equalsIgnoreCase("true")) {
- String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
- String extHiveHmsPort =
context.config.otherConfigs.get("extHiveHmsPort")
- String hms_catalog_name = "test_external_hms_catalog_iceberg_nereids"
- String iceberg_catalog_name = "test_external_iceberg_catalog_nereids"
-
- sql """drop catalog if exists ${hms_catalog_name};"""
- sql """
- create catalog if not exists ${hms_catalog_name} properties (
- 'type'='hms',
- 'hive.metastore.uris' =
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
- );
- """
-
- sql """drop catalog if exists ${iceberg_catalog_name};"""
- sql """
- create catalog if not exists ${iceberg_catalog_name} properties (
- 'type'='iceberg',
- 'iceberg.catalog.type'='hms',
- 'hive.metastore.uris' =
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
- );
- """
- sql """set enable_nereids_planner=true;"""
- sql """set enable_fallback_to_original_planner=false;"""
-
- sql """switch ${hms_catalog_name};"""
- // test parquet format format
- def q01 = {
- qt_q01 """ select count(1) as c from customer_small """
- qt_q02 """ select c_custkey from customer_small group by c_custkey
order by c_custkey limit 4 """
- qt_q03 """ select count(1) from orders_small """
- qt_q04 """ select count(1) from customer_small where c_name =
'Customer#000000005' or c_name = 'Customer#000000006' """
- qt_q05 """ select * from customer_small order by c_custkey limit 3
"""
- qt_q06 """ select o_orderkey from orders_small where o_orderkey >
652566 order by o_orderkey limit 3 """
- qt_q07 """ select o_totalprice from orders_small where o_custkey <
3357 order by o_custkey limit 3 """
- qt_q08 """ select count(1) as c from customer """
- }
- // test time travel stmt
- def q02 = {
- qt_q09 """ select c_custkey from customer for time as of
'2022-12-27 10:21:36' order by c_custkey limit 3 """
- qt_q10 """ select c_custkey from customer for time as of
'2022-12-28 10:21:36' order by c_custkey desc limit 3 """
- qt_q11 """ select c_custkey from customer for version as of
906874575350293177 order by c_custkey limit 3 """
- qt_q12 """ select c_custkey from customer for version as of
6352416983354893547 order by c_custkey desc limit 3 """
- }
- // in predicate
- def q03 = {
- qt_q13 """ select c_custkey from customer_small where c_custkey in
(1, 2, 4, 7) order by c_custkey """
- qt_q14 """ select c_name from customer_small where c_name in
('Customer#000000004', 'Customer#000000007') order by c_custkey """
- }
-
- // test for 'FOR TIME AS OF' and 'FOR VERSION AS OF'
- def q04 = {
- qt_q15 """ select count(*) from
${hms_catalog_name}.tpch_1000_icebergv2.customer_small FOR TIME AS OF
'2022-12-22 02:29:30' """
- qt_q16 """ select count(*) from
${hms_catalog_name}.tpch_1000_icebergv2.customer_small FOR VERSION AS OF
6113938156088124425 """
- qt_q17 """ select count(*) from
${iceberg_catalog_name}.tpch_1000_icebergv2.customer_small FOR TIME AS OF
'2022-12-22 02:29:30' """
- qt_q18 """ select count(*) from
${iceberg_catalog_name}.tpch_1000_icebergv2.customer_small FOR VERSION AS OF
6113938156088124425 """
- }
-
- sql """ use `tpch_1000_icebergv2`; """
- q01()
- q02()
- q03()
- q04()
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
