This is an automated email from the ASF dual-hosted git repository. boroknagyz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 814acf3bbf015ca305603d33dada8f4233d929d0 Author: Zoltan Borok-Nagy <[email protected]> AuthorDate: Mon Mar 11 15:02:30 2024 +0100 IMPALA-12893: (part 1) Specify 'format-version' explicitly in Iceberg tests This CR is the first step to upgrade to Iceberg 1.4.3. The biggest change in behavior in Iceberg 1.4.3 is that Iceberg V2 tables are the default. Because of this we update some test files to explicitly create V1/V2 tables. We also introduce test files that create Iceberg tables without explicitly specifying the format version, these tests have the name *-default.test. The latter tests will need to be updated when we actually upgrade to Iceberg 1.4.3. Change-Id: Ieb4f6c1b206d1d4fd878f07ea5f1436dcae560cd Reviewed-on: http://gerrit.cloudera.org:8080/21167 Tested-by: Impala Public Jenkins <[email protected]> Reviewed-by: Andrew Sherman <[email protected]> --- ...eberg-alter.test => iceberg-alter-default.test} | 2 +- .../{iceberg-alter.test => iceberg-alter-v1.test} | 99 ++-------------------- .../{iceberg-alter.test => iceberg-alter-v2.test} | 26 +++--- ...est => iceberg-partitioned-insert-default.test} | 0 ...ert.test => iceberg-partitioned-insert-v1.test} | 27 ++++-- ...ert.test => iceberg-partitioned-insert-v2.test} | 45 ++++++---- tests/query_test/test_iceberg.py | 51 +++++++---- 7 files changed, 105 insertions(+), 145 deletions(-) diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-default.test similarity index 99% copy from testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test copy to testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-default.test index 5d20f8660..c1ef2185e 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test +++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-default.test @@ -492,4 +492,4 @@ DESCRIBE ice_alter_cols; 'd','decimal(22,3)','','true' ---- TYPES STRING,STRING,STRING,STRING -==== \ No newline at end of file +==== diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v1.test similarity index 77% copy from testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test copy to testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v1.test index 5d20f8660..2310f669f 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test +++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v1.test @@ -4,7 +4,7 @@ CREATE TABLE iceberg_hadoop_tables( level STRING ) STORED AS ICEBERG -TBLPROPERTIES('iceberg.catalog'='hadoop.tables'); +TBLPROPERTIES('format-version'='1', 'iceberg.catalog'='hadoop.tables'); ALTER TABLE iceberg_hadoop_tables ADD COLUMNS(event_time TIMESTAMP, register_time DATE); ALTER TABLE iceberg_hadoop_tables ADD COLUMNS(message STRING, price DECIMAL(8,1)); ALTER TABLE iceberg_hadoop_tables ADD COLUMNS(map_test MAP <STRING, array <STRING>>, struct_test STRUCT <f1: BIGINT, f2: BIGINT>); @@ -73,7 +73,7 @@ CREATE TABLE iceberg_hadoop_catalog( level STRING ) STORED AS ICEBERG -TBLPROPERTIES('iceberg.catalog'='hadoop.catalog', +TBLPROPERTIES('format-version'='1', 'iceberg.catalog'='hadoop.catalog', 'iceberg.catalog_location'='$WAREHOUSE_LOCATION_PREFIX/other/$DATABASE/hadoop_catalog_test'); ALTER TABLE iceberg_hadoop_catalog ADD COLUMNS(event_time TIMESTAMP, register_time DATE); ALTER TABLE iceberg_hadoop_catalog ADD COLUMNS(message STRING, price DECIMAL(8,1)); @@ -132,7 +132,7 @@ STRING, BIGINT, BIGINT ---- QUERY CREATE TABLE iceberg_rename (i int) STORED AS ICEBERG -TBLPROPERTIES('iceberg.catalog'='hive.catalog'); +TBLPROPERTIES('format-version'='1', 'iceberg.catalog'='hive.catalog'); INSERT INTO iceberg_rename values (42); ALTER TABLE iceberg_rename RENAME TO iceberg_rename2; SELECT * FROM iceberg_rename2; @@ -149,7 +149,7 @@ Could not resolve table reference: 'iceberg_rename' ---- QUERY CREATE TABLE iceberg_changing_fileformats (i int) STORED AS ICEBERG -TBLPROPERTIES('write.format.default'='orc'); +TBLPROPERTIES('format-version'='1', 'write.format.default'='orc'); DESCRIBE FORMATTED iceberg_changing_fileformats; ---- RESULTS: VERIFY_IS_SUBSET '','write.format.default','orc ' @@ -186,7 +186,8 @@ string, string, string ==== ---- QUERY CREATE TABLE ice_alter_cols (f float) -STORED AS ICEBERG; +STORED AS ICEBERG +TBLPROPERTIES('format-version'='1'); INSERT INTO ice_alter_cols VALUES (2.718); SELECT * FROM ice_alter_cols; ---- RESULTS @@ -315,6 +316,7 @@ STRING,STRING,STRING,STRING CREATE TABLE iceberg_changing_parq_tblprops (i int) STORED AS ICEBERG TBLPROPERTIES ( + 'format-version'='1', 'write.parquet.row-group-size-bytes'='134217728', 'write.parquet.compression-codec'='zstd', 'write.parquet.compression-level'='12', @@ -352,91 +354,6 @@ DESCRIBE FORMATTED iceberg_changing_parq_tblprops; string, string, string ==== ---- QUERY -CREATE TABLE iceberg_upgrade_v2_no_write_mode (i INT) STORED AS ICEBERG; -DESCRIBE FORMATTED iceberg_upgrade_v2_no_write_mode; ----- RESULTS: VERIFY_IS_NOT_IN -'','write.delete.mode ','merge-on-read ' -'','write.update.mode ','merge-on-read ' -'','write.merge.mode ','merge-on-read ' ----- TYPES -string, string, string -==== ----- QUERY -# Setting format-version to 1 doesn't add write modes. -ALTER TABLE iceberg_upgrade_v2_no_write_mode SET TBLPROPERTIES('format-version'='1'); -DESCRIBE FORMATTED iceberg_upgrade_v2_no_write_mode; ----- RESULTS: VERIFY_IS_NOT_IN -'','write.delete.mode ','merge-on-read ' -'','write.update.mode ','merge-on-read ' -'','write.merge.mode ','merge-on-read ' ----- TYPES -string, string, string -==== ----- QUERY -ALTER TABLE iceberg_upgrade_v2_no_write_mode SET TBLPROPERTIES('format-version'='2'); -DESCRIBE FORMATTED iceberg_upgrade_v2_no_write_mode; ----- RESULTS: VERIFY_IS_SUBSET -'','write.delete.mode ','merge-on-read ' -'','write.update.mode ','merge-on-read ' -'','write.merge.mode ','merge-on-read ' ----- TYPES -string, string, string -==== ----- QUERY -CREATE TABLE iceberg_upgrade_v2_delete_mode (i INT) STORED AS ICEBERG; -ALTER TABLE iceberg_upgrade_v2_delete_mode -SET TBLPROPERTIES('format-version'='2', 'write.delete.mode'='copy-on-write'); -DESCRIBE FORMATTED iceberg_upgrade_v2_delete_mode; ----- RESULTS: VERIFY_IS_SUBSET -'','write.delete.mode ','copy-on-write ' ----- TYPES -string, string, string -==== ----- QUERY -DESCRIBE FORMATTED iceberg_upgrade_v2_delete_mode; ----- RESULTS: VERIFY_IS_NOT_IN -'','write.update.mode ','merge-on-read ' -'','write.merge.mode ','merge-on-read ' ----- TYPES -string, string, string -==== ----- QUERY -CREATE TABLE iceberg_upgrade_v2_update_mode (i INT) STORED AS ICEBERG; -ALTER TABLE iceberg_upgrade_v2_update_mode -SET TBLPROPERTIES('format-version'='2', 'write.update.mode'='copy-on-write'); -DESCRIBE FORMATTED iceberg_upgrade_v2_update_mode; ----- RESULTS: VERIFY_IS_SUBSET -'','write.update.mode ','copy-on-write ' ----- TYPES -string, string, string -==== ----- QUERY -DESCRIBE FORMATTED iceberg_upgrade_v2_update_mode; ----- RESULTS: VERIFY_IS_NOT_IN -'','write.delete.mode ','merge-on-read ' -'','write.merge.mode ','merge-on-read ' ----- TYPES -string, string, string -==== ----- QUERY -CREATE TABLE iceberg_upgrade_v2_merge_mode (i INT) STORED AS ICEBERG; -ALTER TABLE iceberg_upgrade_v2_merge_mode -SET TBLPROPERTIES('format-version'='2', 'write.merge.mode'='merge-on-read'); -DESCRIBE FORMATTED iceberg_upgrade_v2_merge_mode; ----- RESULTS: VERIFY_IS_SUBSET -'','write.merge.mode ','merge-on-read ' ----- TYPES -string, string, string -==== ----- QUERY -DESCRIBE FORMATTED iceberg_upgrade_v2_merge_mode; ----- RESULTS: VERIFY_IS_NOT_IN -'','write.update.mode ','merge-on-read ' -'','write.delete.mode ','merge-on-read ' ----- TYPES -string, string, string -==== ----- QUERY # Add a column that already exists and a new column that does not exist with # "if not exists" clause. ALTER TABLE ice_alter_cols ADD IF NOT EXISTS COLUMNS (a bigint, d bigint) @@ -492,4 +409,4 @@ DESCRIBE ice_alter_cols; 'd','decimal(22,3)','','true' ---- TYPES STRING,STRING,STRING,STRING -==== \ No newline at end of file +==== diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v2.test similarity index 96% rename from testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test rename to testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v2.test index 5d20f8660..e99c4a5ae 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test +++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v2.test @@ -4,7 +4,7 @@ CREATE TABLE iceberg_hadoop_tables( level STRING ) STORED AS ICEBERG -TBLPROPERTIES('iceberg.catalog'='hadoop.tables'); +TBLPROPERTIES('format-version'='2', 'iceberg.catalog'='hadoop.tables'); ALTER TABLE iceberg_hadoop_tables ADD COLUMNS(event_time TIMESTAMP, register_time DATE); ALTER TABLE iceberg_hadoop_tables ADD COLUMNS(message STRING, price DECIMAL(8,1)); ALTER TABLE iceberg_hadoop_tables ADD COLUMNS(map_test MAP <STRING, array <STRING>>, struct_test STRUCT <f1: BIGINT, f2: BIGINT>); @@ -73,7 +73,7 @@ CREATE TABLE iceberg_hadoop_catalog( level STRING ) STORED AS ICEBERG -TBLPROPERTIES('iceberg.catalog'='hadoop.catalog', +TBLPROPERTIES('format-version'='2', 'iceberg.catalog'='hadoop.catalog', 'iceberg.catalog_location'='$WAREHOUSE_LOCATION_PREFIX/other/$DATABASE/hadoop_catalog_test'); ALTER TABLE iceberg_hadoop_catalog ADD COLUMNS(event_time TIMESTAMP, register_time DATE); ALTER TABLE iceberg_hadoop_catalog ADD COLUMNS(message STRING, price DECIMAL(8,1)); @@ -132,7 +132,7 @@ STRING, BIGINT, BIGINT ---- QUERY CREATE TABLE iceberg_rename (i int) STORED AS ICEBERG -TBLPROPERTIES('iceberg.catalog'='hive.catalog'); +TBLPROPERTIES('format-version'='2', 'iceberg.catalog'='hive.catalog'); INSERT INTO iceberg_rename values (42); ALTER TABLE iceberg_rename RENAME TO iceberg_rename2; SELECT * FROM iceberg_rename2; @@ -149,7 +149,7 @@ Could not resolve table reference: 'iceberg_rename' ---- QUERY CREATE TABLE iceberg_changing_fileformats (i int) STORED AS ICEBERG -TBLPROPERTIES('write.format.default'='orc'); +TBLPROPERTIES('format-version'='2', 'write.format.default'='orc'); DESCRIBE FORMATTED iceberg_changing_fileformats; ---- RESULTS: VERIFY_IS_SUBSET '','write.format.default','orc ' @@ -186,7 +186,8 @@ string, string, string ==== ---- QUERY CREATE TABLE ice_alter_cols (f float) -STORED AS ICEBERG; +STORED AS ICEBERG +TBLPROPERTIES('format-version'='2'); INSERT INTO ice_alter_cols VALUES (2.718); SELECT * FROM ice_alter_cols; ---- RESULTS @@ -315,6 +316,7 @@ STRING,STRING,STRING,STRING CREATE TABLE iceberg_changing_parq_tblprops (i int) STORED AS ICEBERG TBLPROPERTIES ( + 'format-version'='2', 'write.parquet.row-group-size-bytes'='134217728', 'write.parquet.compression-codec'='zstd', 'write.parquet.compression-level'='12', @@ -352,7 +354,8 @@ DESCRIBE FORMATTED iceberg_changing_parq_tblprops; string, string, string ==== ---- QUERY -CREATE TABLE iceberg_upgrade_v2_no_write_mode (i INT) STORED AS ICEBERG; +CREATE TABLE iceberg_upgrade_v2_no_write_mode (i INT) STORED AS ICEBERG +TBLPROPERTIES ('format-version'='1'); DESCRIBE FORMATTED iceberg_upgrade_v2_no_write_mode; ---- RESULTS: VERIFY_IS_NOT_IN '','write.delete.mode ','merge-on-read ' @@ -383,7 +386,8 @@ DESCRIBE FORMATTED iceberg_upgrade_v2_no_write_mode; string, string, string ==== ---- QUERY -CREATE TABLE iceberg_upgrade_v2_delete_mode (i INT) STORED AS ICEBERG; +CREATE TABLE iceberg_upgrade_v2_delete_mode (i INT) STORED AS ICEBERG +TBLPROPERTIES('format-version'='1'); ALTER TABLE iceberg_upgrade_v2_delete_mode SET TBLPROPERTIES('format-version'='2', 'write.delete.mode'='copy-on-write'); DESCRIBE FORMATTED iceberg_upgrade_v2_delete_mode; @@ -401,7 +405,8 @@ DESCRIBE FORMATTED iceberg_upgrade_v2_delete_mode; string, string, string ==== ---- QUERY -CREATE TABLE iceberg_upgrade_v2_update_mode (i INT) STORED AS ICEBERG; +CREATE TABLE iceberg_upgrade_v2_update_mode (i INT) STORED AS ICEBERG +TBLPROPERTIES('format-version'='1'); ALTER TABLE iceberg_upgrade_v2_update_mode SET TBLPROPERTIES('format-version'='2', 'write.update.mode'='copy-on-write'); DESCRIBE FORMATTED iceberg_upgrade_v2_update_mode; @@ -419,7 +424,8 @@ DESCRIBE FORMATTED iceberg_upgrade_v2_update_mode; string, string, string ==== ---- QUERY -CREATE TABLE iceberg_upgrade_v2_merge_mode (i INT) STORED AS ICEBERG; +CREATE TABLE iceberg_upgrade_v2_merge_mode (i INT) STORED AS ICEBERG +TBLPROPERTIES('format-version'='1'); ALTER TABLE iceberg_upgrade_v2_merge_mode SET TBLPROPERTIES('format-version'='2', 'write.merge.mode'='merge-on-read'); DESCRIBE FORMATTED iceberg_upgrade_v2_merge_mode; @@ -492,4 +498,4 @@ DESCRIBE ice_alter_cols; 'd','decimal(22,3)','','true' ---- TYPES STRING,STRING,STRING,STRING -==== \ No newline at end of file +==== diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert-default.test similarity index 100% copy from testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test copy to testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert-default.test diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert-v1.test similarity index 98% copy from testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test copy to testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert-v1.test index da70926fa..6c35c4818 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test +++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert-v1.test @@ -4,7 +4,8 @@ # the partitioned column. create table ice_only_part (i int) partitioned by spec (i) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='1'); ---- RESULTS 'Table has been created.' ==== @@ -61,7 +62,8 @@ aggregation(SUM, RowsRead): 1 # Test inserts with multple partition columns. create table ice_multi_part (i int, d date, s string) partitioned by spec(i, d) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='1'); ---- RESULTS 'Table has been created.' ==== @@ -157,7 +159,8 @@ aggregation(SUM, RowsRead): 3 # Test that Impala only writes one file per partitions. create table ice_bigints (i BIGINT, j BIGINT, k BIGINT) partitioned by spec (i, j) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='1'); ---- RESULTS 'Table has been created.' ==== @@ -216,7 +219,8 @@ partitioned by spec ( double_col, date_col, string_col) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='1'); ---- RESULTS 'Table has been created.' ==== @@ -403,7 +407,8 @@ STRING, STRING, STRING, STRING # Test inserts with multple partition columns. create table ice_part_non_order (i int, d date, s string) partitioned by spec(s, d) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='1'); ---- RESULTS 'Table has been created.' ==== @@ -439,7 +444,8 @@ STRING, BIGINT, BIGINT ---- QUERY # Test inserts with different partition specs. create table ice_alter_part (i int, d date, s string) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='1'); ---- RESULTS 'Table has been created.' ==== @@ -516,7 +522,8 @@ STRING, BIGINT, BIGINT ---- QUERY create table ice_void (i int, s string, d date) partitioned by spec (void(i), truncate(1, s), year(d)) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='1'); insert into ice_void values (1, 'one', '2001-01-01'), (11,'other','2001-01-11'), (2, 'two', '2002-02-02'), @@ -668,7 +675,8 @@ SHOW PARTITIONS ice_void; STRING, BIGINT, BIGINT ==== ---- QUERY -create table store_sales partitioned by spec (ss_sold_date_sk) stored as iceberg +create table store_sales partitioned by spec (ss_sold_date_sk) +stored as iceberg tblproperties ('format-version'='1') as select * from tpcds_parquet.store_sales; select count(*) from store_sales; ---- RESULTS @@ -688,7 +696,8 @@ BIGINT # Check that result strings are not URL encoded. create table special_char_partitions (i int, s string, s2 string) partitioned by spec (i, s, truncate(4, s)) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='1'); insert into special_char_partitions values (1, '11/14/31', '44/1'), (2, '11"14"31', '43"3'), (3, '11=14=31', '65=2'), (4, '', 'a'), (5, cast(null as string), 'b'); select * from special_char_partitions; diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert-v2.test similarity index 96% rename from testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test rename to testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert-v2.test index da70926fa..c65326d17 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert.test +++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-partitioned-insert-v2.test @@ -4,7 +4,8 @@ # the partitioned column. create table ice_only_part (i int) partitioned by spec (i) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='2'); ---- RESULTS 'Table has been created.' ==== @@ -61,7 +62,8 @@ aggregation(SUM, RowsRead): 1 # Test inserts with multple partition columns. create table ice_multi_part (i int, d date, s string) partitioned by spec(i, d) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='2'); ---- RESULTS 'Table has been created.' ==== @@ -157,7 +159,8 @@ aggregation(SUM, RowsRead): 3 # Test that Impala only writes one file per partitions. create table ice_bigints (i BIGINT, j BIGINT, k BIGINT) partitioned by spec (i, j) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='2'); ---- RESULTS 'Table has been created.' ==== @@ -216,7 +219,8 @@ partitioned by spec ( double_col, date_col, string_col) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='2'); ---- RESULTS 'Table has been created.' ==== @@ -403,7 +407,8 @@ STRING, STRING, STRING, STRING # Test inserts with multple partition columns. create table ice_part_non_order (i int, d date, s string) partitioned by spec(s, d) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='2'); ---- RESULTS 'Table has been created.' ==== @@ -439,7 +444,8 @@ STRING, BIGINT, BIGINT ---- QUERY # Test inserts with different partition specs. create table ice_alter_part (i int, d date, s string) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='2'); ---- RESULTS 'Table has been created.' ==== @@ -507,8 +513,8 @@ SHOW PARTITIONS ice_alter_part; ---- RESULTS '{"i":"3","d":"18605"}',1,1 '{"i":"4","d":"18606"}',1,1 -'{"i":"5","d":null,"d_year":"50","s_bucket_5":"2"}',1,1 -'{"i":"6","d":null,"d_year":"50","s_bucket_5":"0"}',1,1 +'{"i":"5","d_year":"50","s_bucket_5":"2"}',1,1 +'{"i":"6","d_year":"50","s_bucket_5":"0"}',1,1 '{}',2,1 ---- TYPES STRING, BIGINT, BIGINT @@ -516,7 +522,8 @@ STRING, BIGINT, BIGINT ---- QUERY create table ice_void (i int, s string, d date) partitioned by spec (void(i), truncate(1, s), year(d)) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='2'); insert into ice_void values (1, 'one', '2001-01-01'), (11,'other','2001-01-11'), (2, 'two', '2002-02-02'), @@ -578,10 +585,10 @@ STRING, STRING, STRING, STRING ---- QUERY SHOW PARTITIONS ice_void; ---- RESULTS +'{"d_year":"34","i":"4","s_null":null}',1,1 '{"i_null":null,"s_trunc":"o","d_year":"31"}',2,1 '{"i_null":null,"s_trunc":"t","d_year":"32"}',1,1 '{"i_null":null,"s_trunc":"t","d_year":"33"}',1,1 -'{"i_null":null,"s_trunc":null,"d_year":"34","i":"4","s_null":null}',1,1 ---- TYPES STRING, BIGINT, BIGINT ==== @@ -616,11 +623,11 @@ STRING, STRING, STRING, STRING ---- QUERY SHOW PARTITIONS ice_void; ---- RESULTS +'{"d_year":"34","i":"4","s_null":null}',1,1 +'{"i":"5","s_null":null,"d_null":null}',2,1 '{"i_null":null,"s_trunc":"o","d_year":"31"}',2,1 '{"i_null":null,"s_trunc":"t","d_year":"32"}',1,1 '{"i_null":null,"s_trunc":"t","d_year":"33"}',1,1 -'{"i_null":null,"s_trunc":null,"d_year":"34","i":"4","s_null":null}',1,1 -'{"i_null":null,"s_trunc":null,"d_year":null,"i":"5","s_null":null,"d_null":null}',2,1 ---- TYPES STRING, BIGINT, BIGINT ==== @@ -658,17 +665,18 @@ STRING, STRING, STRING, STRING ---- QUERY SHOW PARTITIONS ice_void; ---- RESULTS +'{"d_year":"34","i":"4","s_null":null}',1,1 +'{"i":"5","s_null":null,"d_null":null}',2,1 '{"i_null":null,"s_trunc":"o","d_year":"31"}',2,1 '{"i_null":null,"s_trunc":"t","d_year":"32"}',1,1 '{"i_null":null,"s_trunc":"t","d_year":"33"}',1,1 -'{"i_null":null,"s_trunc":null,"d_year":"34","i":"4","s_null":null}',1,1 -'{"i_null":null,"s_trunc":null,"d_year":null,"i":"5","s_null":null,"d_null":null}',2,1 -'{"i_null":null,"s_trunc":null,"d_year":null,"i":null,"s_null":null,"d_null":null}',2,1 +'{"s_null":null,"d_null":null,"i_null":null}',2,1 ---- TYPES STRING, BIGINT, BIGINT ==== ---- QUERY -create table store_sales partitioned by spec (ss_sold_date_sk) stored as iceberg +create table store_sales partitioned by spec (ss_sold_date_sk) +stored as iceberg tblproperties ('format-version'='2') as select * from tpcds_parquet.store_sales; select count(*) from store_sales; ---- RESULTS @@ -688,7 +696,8 @@ BIGINT # Check that result strings are not URL encoded. create table special_char_partitions (i int, s string, s2 string) partitioned by spec (i, s, truncate(4, s)) -stored as iceberg; +stored as iceberg +tblproperties ('format-version'='2'); insert into special_char_partitions values (1, '11/14/31', '44/1'), (2, '11"14"31', '43"3'), (3, '11=14=31', '65=2'), (4, '', 'a'), (5, cast(null as string), 'b'); select * from special_char_partitions; @@ -790,7 +799,7 @@ show partitions special_char_partitions; '{"i":"3","s":"11=14=31","s_trunc":"11=1"}',1,1 '{"i":"4","s":"","s_trunc":""}',1,1 '{"i":"5","s":null,"s_trunc":null}',1,1 -'{"i":null,"s":null,"s_trunc":null,"s2":"98\\/22"}',1,1 +'{"s2":"98\\/22"}',1,1 ---- TYPES STRING,BIGINT,BIGINT ==== diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py index a6cca7ffc..ce57ae725 100644 --- a/tests/query_test/test_iceberg.py +++ b/tests/query_test/test_iceberg.py @@ -69,8 +69,14 @@ class TestIcebergTable(IcebergTestSuite): def test_create_iceberg_tables(self, vector, unique_database): self.run_test_case('QueryTest/iceberg-create', vector, use_db=unique_database) - def test_alter_iceberg_tables(self, vector, unique_database): - self.run_test_case('QueryTest/iceberg-alter', vector, use_db=unique_database) + def test_alter_iceberg_tables_v1(self, vector, unique_database): + self.run_test_case('QueryTest/iceberg-alter-v1', vector, use_db=unique_database) + + def test_alter_iceberg_tables_v2(self, vector, unique_database): + self.run_test_case('QueryTest/iceberg-alter-v2', vector, use_db=unique_database) + + def test_alter_iceberg_tables_default(self, vector, unique_database): + self.run_test_case('QueryTest/iceberg-alter-default', vector, use_db=unique_database) def test_external_iceberg_tables(self, vector, unique_database): self.run_test_case('QueryTest/iceberg-external', vector, unique_database) @@ -179,8 +185,16 @@ class TestIcebergTable(IcebergTestSuite): def test_insert(self, vector, unique_database): self.run_test_case('QueryTest/iceberg-insert', vector, use_db=unique_database) - def test_partitioned_insert(self, vector, unique_database): - self.run_test_case('QueryTest/iceberg-partitioned-insert', vector, + def test_partitioned_insert_v1(self, vector, unique_database): + self.run_test_case('QueryTest/iceberg-partitioned-insert-v1', vector, + use_db=unique_database) + + def test_partitioned_insert_v2(self, vector, unique_database): + self.run_test_case('QueryTest/iceberg-partitioned-insert-v2', vector, + use_db=unique_database) + + def test_partitioned_insert_default(self, vector, unique_database): + self.run_test_case('QueryTest/iceberg-partitioned-insert-default', vector, use_db=unique_database) def test_insert_overwrite(self, vector, unique_database): @@ -783,10 +797,10 @@ class TestIcebergTable(IcebergTestSuite): # Create table table_name = "ice_part" qualified_table_name = "%s.%s" % (unique_database, table_name) - create_table = 'create table %s ' \ - '(s string, i int) partitioned by spec(truncate(5, s), identity(i)) ' \ - 'stored as iceberg' \ - % qualified_table_name + create_table = """create table {} + (s string, i int) partitioned by spec(truncate(5, s), identity(i)) + stored as iceberg + tblproperties ('format-version'='1')""".format(qualified_table_name) self.client.execute(create_table) partition_spec = self.get_current_partition_spec(unique_database, table_name) @@ -878,17 +892,22 @@ class TestIcebergTable(IcebergTestSuite): assert truncate_s['field-id'] == 1004 @SkipIf.not_dfs - def test_writing_metrics_to_metadata(self, vector, unique_database): + def test_writing_metrics_to_metadata_v1(self, vector, unique_database): + self._test_writing_metrics_to_metadata_impl(unique_database, 'ice_stats_v1', '1') + + @SkipIf.not_dfs + def test_writing_metrics_to_metadata_v2(self, vector, unique_database): + self._test_writing_metrics_to_metadata_impl(unique_database, 'ice_stats_v2', '2') + + def _test_writing_metrics_to_metadata_impl(self, unique_database, table_name, version): # Create table - table_name = "ice_stats" qualified_table_name = "%s.%s" % (unique_database, table_name) - query = 'create table %s ' \ - '(s string, i int, b boolean, bi bigint, ts timestamp, dt date, ' \ - 'dc decimal(10, 3)) ' \ - 'stored as iceberg' \ - % qualified_table_name + query = """create table {} + (s string, i int, b boolean, bi bigint, ts timestamp, dt date, + dc decimal(10, 3)) + stored as iceberg + tblproperties ('format-version'='{}')""".format(qualified_table_name, version) self.client.execute(query) - # Insert data # 1st data file: query = 'insert into %s values ' \
