This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 3d22c7fe0 IMPALA-12209: Always include format-version in DESCRIBE
FORMATTED and SHOW CREATE TABLE for Iceberg tables
3d22c7fe0 is described below
commit 3d22c7fe052010c6b4012a5cc664a5ca1d8a0b03
Author: Daniel Vanko <[email protected]>
AuthorDate: Tue Oct 7 16:07:59 2025 +0200
IMPALA-12209: Always include format-version in DESCRIBE FORMATTED and SHOW
CREATE TABLE for Iceberg tables
HiveCatalog does not include format-version for Iceberg tables in the
table's parameters, therefore the output of SHOW CREATE TABLE may not
replicate the original table.
This patch makes sure to add it to both the SHOW CREATE TABLE and
DESCRIBE FORMATTED/EXTENDED output.
Additionally, adds ICEBERG_DEFAULT_FORMAT_VERSION variable to E2E
tests, deducting from IMPALA_ICEBERG_VERSION environment variable.
If Iceberg version is at least 1.4, default format-version is 2, before
1.4 it's 1. This way tests can work with multiple Iceberg versions.
Testing:
* updated show-create-table.test and show-create-table-with-stats.test
for Iceberg tables
* added format-version checks to multiple DESCRIBE FORMATTED tests
Change-Id: I991edf408b24fa73e8a8abe64ac24929aeb8e2f8
Reviewed-on: http://gerrit.cloudera.org:8080/23514
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
.../org/apache/impala/analysis/ToSqlUtils.java | 8 +-
.../org/apache/impala/catalog/IcebergTable.java | 3 +
.../impala/service/DescribeResultFactory.java | 8 +-
.../queries/QueryTest/iceberg-alter-default.test | 2 +
.../queries/QueryTest/iceberg-alter-v1.test | 2 +
.../queries/QueryTest/iceberg-alter-v2.test | 6 +
.../queries/QueryTest/iceberg-catalogs.test | 4 +
.../QueryTest/iceberg-create-table-like-table.test | 8 +
.../queries/QueryTest/iceberg-create.test | 15 ++
.../queries/QueryTest/iceberg-external.test | 2 +
.../iceberg-migrate-from-external-hdfs-tables.test | 4 +
.../QueryTest/iceberg-missing-data-files.test | 1 +
.../queries/QueryTest/iceberg-old-fileformat.test | 1 +
.../queries/QueryTest/iceberg-query.test | 5 +
.../queries/QueryTest/iceberg-rest-catalog.test | 1 +
.../queries/QueryTest/iceberg-rest-fgac.test | 1 +
.../QueryTest/show-create-table-with-stats.test | 4 +-
.../queries/QueryTest/show-create-table.test | 193 +++++++++++++++------
tests/common/environ.py | 18 ++
tests/common/impala_test_suite.py | 10 +-
tests/metadata/test_show_create_table.py | 27 +--
21 files changed, 243 insertions(+), 80 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java
b/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java
index 8c0b917da..1785a8c45 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ToSqlUtils.java
@@ -189,10 +189,10 @@ public class ToSqlUtils {
commonProps.remove(KuduTable.KEY_TABLE_ID);
} else if (table instanceof FeIcebergTable) {
FeIcebergTable feIcebergTable = (FeIcebergTable) table;
- if (feIcebergTable.getFormatVersion() == IcebergTable.ICEBERG_FORMAT_V1)
{
- commonProps.put(TableProperties.FORMAT_VERSION,
- String.valueOf(IcebergTable.ICEBERG_FORMAT_V1));
- }
+ // Add "format-version" property if it's not already present.
+ commonProps.putIfAbsent(IcebergTable.FORMAT_VERSION,
+ Integer.toString(feIcebergTable.getFormatVersion()));
+
// Hide Iceberg internal metadata properties
removeHiddenIcebergTableProperties(commonProps);
} else if (table instanceof FePaimonTable) {
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
index f8da3623e..579e049b3 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
@@ -164,6 +164,9 @@ public class IcebergTable extends Table implements
FeIcebergTable {
// Internal Iceberg table property that specifies the UUID of the table.
public static final String UUID = "uuid";
+ // Internal Iceberg table property that specifies the table format version.
+ public static final String FORMAT_VERSION = "format-version";
+
// Parquet compression codec and compression level table properties.
public static final String PARQUET_COMPRESSION_CODEC =
"write.parquet.compression-codec";
diff --git
a/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
b/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
index 8880da3d4..b40bf4a9e 100644
--- a/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
+++ b/fe/src/main/java/org/apache/impala/service/DescribeResultFactory.java
@@ -30,6 +30,7 @@ import org.apache.impala.catalog.FeDb;
import org.apache.impala.catalog.FeIcebergTable;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.IcebergColumn;
+import org.apache.impala.catalog.IcebergTable;
import org.apache.impala.catalog.KuduColumn;
import org.apache.impala.catalog.StructField;
import org.apache.impala.catalog.StructType;
@@ -236,8 +237,13 @@ public class DescribeResultFactory {
sb.append(MetastoreShim.getAllColumnsInformation(msTable.getSd().getCols(),
msTable.getPartitionKeys(), true, false, true));
if (table instanceof FeIcebergTable) {
+ FeIcebergTable feIcebergTable = (FeIcebergTable) table;
sb.append(MetastoreShim.getPartitionTransformInformation(
- FeIcebergTable.Utils.getPartitionTransformKeys((FeIcebergTable)
table)));
+ FeIcebergTable.Utils.getPartitionTransformKeys(feIcebergTable)));
+
+ // msTable is a deep copy hence we can add the "format-version"
parameter to it
+ msTable.getParameters().putIfAbsent(IcebergTable.FORMAT_VERSION,
+ Integer.toString(feIcebergTable.getFormatVersion()));
}
// Add the extended table metadata information.
sb.append(MetastoreShim.getTableInformation(msTable));
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-default.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-default.test
index 74b25189c..bdbbf89b1 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-default.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-default.test
@@ -323,6 +323,7 @@ TBLPROPERTIES (
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','134217728 '
'','write.parquet.compression-codec','zstd '
@@ -342,6 +343,7 @@ ALTER TABLE iceberg_changing_parq_tblprops set
TBLPROPERTIES(
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','268435456 '
'','write.parquet.compression-codec','snappy '
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v1.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v1.test
index 1fe2c3525..ac42be9de 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v1.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v1.test
@@ -325,6 +325,7 @@ TBLPROPERTIES (
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','1 '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','134217728 '
'','write.parquet.compression-codec','zstd '
@@ -344,6 +345,7 @@ ALTER TABLE iceberg_changing_parq_tblprops set
TBLPROPERTIES(
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','1 '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','268435456 '
'','write.parquet.compression-codec','snappy '
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v2.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v2.test
index 024b1a9a7..695310c9e 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v2.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter-v2.test
@@ -325,6 +325,7 @@ TBLPROPERTIES (
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','2 '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','134217728 '
'','write.parquet.compression-codec','zstd '
@@ -344,6 +345,7 @@ ALTER TABLE iceberg_changing_parq_tblprops set
TBLPROPERTIES(
);
DESCRIBE FORMATTED iceberg_changing_parq_tblprops;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','2 '
'','write.format.default','parquet '
'','write.parquet.row-group-size-bytes','268435456 '
'','write.parquet.compression-codec','snappy '
@@ -379,6 +381,7 @@ string, string, string
ALTER TABLE iceberg_upgrade_v2_no_write_mode SET
TBLPROPERTIES('format-version'='2');
DESCRIBE FORMATTED iceberg_upgrade_v2_no_write_mode;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','2 '
'','write.delete.mode ','merge-on-read '
'','write.update.mode ','merge-on-read '
'','write.merge.mode ','merge-on-read '
@@ -392,6 +395,7 @@ ALTER TABLE iceberg_upgrade_v2_delete_mode
SET TBLPROPERTIES('format-version'='2', 'write.delete.mode'='copy-on-write');
DESCRIBE FORMATTED iceberg_upgrade_v2_delete_mode;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','2 '
'','write.delete.mode ','copy-on-write '
---- TYPES
string, string, string
@@ -411,6 +415,7 @@ ALTER TABLE iceberg_upgrade_v2_update_mode
SET TBLPROPERTIES('format-version'='2', 'write.update.mode'='copy-on-write');
DESCRIBE FORMATTED iceberg_upgrade_v2_update_mode;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','2 '
'','write.update.mode ','copy-on-write '
---- TYPES
string, string, string
@@ -430,6 +435,7 @@ ALTER TABLE iceberg_upgrade_v2_merge_mode
SET TBLPROPERTIES('format-version'='2', 'write.merge.mode'='merge-on-read');
DESCRIBE FORMATTED iceberg_upgrade_v2_merge_mode;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','2 '
'','write.merge.mode ','merge-on-read '
---- TYPES
string, string, string
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-catalogs.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-catalogs.test
index d5593a2d1..1f40c1bf4 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-catalogs.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-catalogs.test
@@ -14,6 +14,7 @@ DESCRIBE FORMATTED iceberg_hadoop_catalogs;
'Location:
','$NAMENODE/test-warehouse/ice_hadoop_cat/$DATABASE/iceberg_hadoop_catalogs','NULL'
'','write.format.default','parquet '
'','iceberg.catalog ','ice_hadoop_cat '
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
---- TYPES
string, string, string
====
@@ -33,6 +34,7 @@ DESCRIBE FORMATTED iceberg_hadoop_catalogs_with_id;
'','iceberg.catalog ','ice_hadoop_cat '
'','iceberg.table_identifier','org.$DATABASE.tbl'
'','name ','org.$DATABASE.tbl'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
---- TYPES
string, string, string
====
@@ -67,6 +69,7 @@ DESCRIBE FORMATTED iceberg_hadoop_cat_with_id_ext;
'','iceberg.catalog ','ice_hadoop_cat '
'','iceberg.table_identifier','org.$DATABASE.tbl'
'','name ','org.$DATABASE.tbl'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
---- TYPES
string, string, string
====
@@ -104,6 +107,7 @@ DESCRIBE FORMATTED iceberg_hive_catalogs;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/iceberg_hive_catalogs','NULL'
'','write.format.default','parquet '
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
---- TYPES
string, string, string
====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create-table-like-table.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create-table-like-table.test
index eb67cee4d..74e903983 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create-table-like-table.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create-table-like-table.test
@@ -48,6 +48,7 @@ describe formatted ice_hadoop_tbl_no_part_clone;
'col_array','array<string>','NULL'
'col_map','map<string,array<string>>','NULL'
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/ice_hadoop_tbl_no_part_clone','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','hadoop.tables '
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
'','write.format.default','parquet '
@@ -109,6 +110,7 @@ describe formatted ice_hadoop_tbl_clone;
'# col_name ','transform_type ','NULL'
'id','BUCKET[3]','NULL'
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/ice_hadoop_tbl_clone','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','hadoop.tables '
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
'','write.format.default','parquet '
@@ -171,6 +173,7 @@ describe formatted ice_hadoop_catalog_no_part_clone;
'col_array','array<string>','NULL'
'col_map','map<string,array<string>>','NULL'
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/cat_loc/$DATABASE/ice_hadoop_catalog_no_part_clone','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','hadoop.catalog '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE.db/cat_loc'
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
@@ -237,6 +240,7 @@ describe formatted ice_hadoop_catalog_clone;
'# col_name ','transform_type ','NULL'
'id','BUCKET[3]','NULL'
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/cat_loc/$DATABASE/ice_hadoop_catalog_clone','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','hadoop.catalog '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE.db/cat_loc'
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
@@ -299,6 +303,7 @@ describe formatted ice_hive_catalog_no_part_clone;
'col_array','array<string>','NULL'
'col_map','map<string,array<string>>','NULL'
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/ice_hive_catalog_no_part_clone','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.table_identifier','$DATABASE.ice_hive_catalog_no_part_clone'
row_regex: '','metadata_location
','$NAMENODE/test-warehouse/$DATABASE.db/ice_hive_catalog_no_part_clone/metadata/.*.metadata.json'
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
@@ -365,6 +370,7 @@ describe formatted ice_hive_catalog_clone;
'# col_name ','transform_type ','NULL'
'id','BUCKET[3]','NULL'
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/ice_hive_catalog_clone','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.table_identifier','$DATABASE.ice_hive_catalog_clone'
row_regex: '','metadata_location
','$NAMENODE/test-warehouse/$DATABASE.db/ice_hive_catalog_clone/metadata/.*.metadata.json'
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
@@ -428,6 +434,7 @@ describe formatted ice_hadoop_cat_no_part_clone;
'col_array','array<string>','NULL'
'col_map','map<string,array<string>>','NULL'
'Location:
','$NAMENODE/test-warehouse/ice_hadoop_cat/$DATABASE/ice_hadoop_cat_no_part_clone','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','ice_hadoop_cat '
'','iceberg.table_identifier','$DATABASE.ice_hadoop_cat_no_part_clone'
'','name ','$DATABASE.ice_hadoop_cat_no_part_clone'
@@ -494,6 +501,7 @@ describe formatted ice_hadoop_cat_clone;
'# col_name ','transform_type ','NULL'
'id','BUCKET[3]','NULL'
'Location:
','$NAMENODE/test-warehouse/ice_hadoop_cat/$DATABASE/ice_hadoop_cat_clone','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','ice_hadoop_cat '
'','iceberg.table_identifier','$DATABASE.ice_hadoop_cat_clone'
'','name ','$DATABASE.ice_hadoop_cat_clone'
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
index 49b84ac7b..0b34a4e00 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
@@ -217,6 +217,7 @@ string
DESCRIBE FORMATTED iceberg_hadoop_cat_query;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test/$DATABASE/iceberg_hadoop_cat_query','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test'
'','write.format.default','parquet '
'','iceberg.catalog ','hadoop.catalog '
@@ -234,6 +235,7 @@ TBLPROPERTIES('iceberg.catalog'='hadoop.catalog',
DESCRIBE FORMATTED iceberg_hadoop_cat_with_ident;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test/org/db/tbl','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test'
'','write.format.default','parquet '
'','iceberg.catalog ','hadoop.catalog '
@@ -256,6 +258,7 @@ TBLPROPERTIES('iceberg.catalog'='hadoop.catalog',
DESCRIBE FORMATTED iceberg_hadoop_cat_with_ident_ext;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test/org/db/tbl','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/$DATABASE/hadoop_catalog_test/iceberg_test'
'','write.format.default','parquet '
'','iceberg.catalog ','hadoop.catalog '
@@ -366,6 +369,7 @@ STRING,STRING,STRING,STRING
DESCRIBE FORMATTED iceberg_part_hive_cat;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/iceberg_part_hive_cat','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'','metadata_location
','$NAMENODE/test-warehouse/$DATABASE.db/iceberg_part_hive_cat/metadata/.*.metadata.json'
'','table_type ','ICEBERG '
'','iceberg.catalog ','hive.catalog '
@@ -568,6 +572,7 @@ create table iceberg_stored_by (i int, s string, ts
timestamp, d date) stored by
---- QUERY
describe formatted iceberg_stored_by;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','table_type ','ICEBERG '
---- TYPES
string, string, string
@@ -580,6 +585,7 @@ create table ice_tbl (i int) stored as iceberg
tblproperties('external.table.pur
---- QUERY
describe formatted ice_tbl;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','FALSE '
---- TYPES
string, string, string
@@ -595,6 +601,7 @@ create table ice_hive_cat_tbl (i int) stored as iceberg
tblproperties(
---- QUERY
describe formatted ice_hive_cat_tbl;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','FALSE '
---- TYPES
string, string, string
@@ -612,6 +619,7 @@ The table property 'external.table.purge' will be set to
'TRUE' on newly created
---- QUERY
describe formatted ice_hadoop_tbl;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','TRUE '
---- TYPES
string, string, string
@@ -623,6 +631,7 @@ create table ice_tbl_with_pk
stored as iceberg;
describe formatted ice_tbl_with_pk;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'','current-schema.*','.*identifier-field-ids\\\\":\[1,3\].*'
---- TYPES
string, string, string
@@ -635,6 +644,7 @@ create table ice_tbl_with_pk_partitioned
stored as iceberg;
describe formatted ice_tbl_with_pk_partitioned;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'','current-schema.*','.*identifier-field-ids\\\\":\[1,2\].*'
---- TYPES
string, string, string
@@ -647,6 +657,7 @@ create table ice_tbl_with_pk_partition_transform
stored as iceberg;
describe formatted ice_tbl_with_pk_partition_transform;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'','current-schema.*','.*identifier-field-ids\\\\":\[2,3\].*'
---- TYPES
string, string, string
@@ -659,6 +670,7 @@ create table ctas_pk_in_source_not_used
as select * from ice_tbl_with_pk;
describe formatted ctas_pk_in_source_not_used;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'.*','current-schema.*','.*'
---- RESULTS: VERIFY_IS_NOT_IN
row_regex:'.*','current-schema.*','.*identifier-field-ids.*'
@@ -673,6 +685,7 @@ create table ctas_pk_unpartitioned
as select * from ice_tbl_with_pk;
describe formatted ctas_pk_unpartitioned;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'.*','current-schema.*','.*identifier-field-ids\\\\":\[3\].*'
---- TYPES
STRING,STRING,STRING
@@ -686,6 +699,7 @@ create table ctas_pk_partitioned
as select * from ice_tbl_with_pk;
describe formatted ctas_pk_partitioned;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'.*','current-schema.*','.*identifier-field-ids\\\\":\[2,3\].*'
---- TYPES
STRING,STRING,STRING
@@ -695,6 +709,7 @@ STRING,STRING,STRING
create table ice_like_pk like ice_tbl_with_pk_partitioned;
describe formatted ice_like_pk;
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
row_regex:'.*','current-schema.*','.*identifier-field-ids\\\\":\[1,2\].*'
---- TYPES
STRING,STRING,STRING
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-external.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-external.test
index e57dc2a19..69519d85c 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-external.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-external.test
@@ -30,6 +30,7 @@ alter table ice_hive_ext set tblproperties
('external.table.purge'='true');
insert into ice_hive_ext values (6,6);
describe formatted ice_hive_ext
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','true '
---- TYPES
STRING,STRING,STRING
@@ -65,6 +66,7 @@ alter table ice_hadoop_tables_ext set tblproperties
('external.table.purge'='tru
insert into ice_hadoop_tables_ext values (6,6);
describe formatted ice_hadoop_tables_ext
---- RESULTS: VERIFY_IS_SUBSET
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','true '
---- TYPES
STRING,STRING,STRING
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test
index 087ebc35c..396ea0bbd 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test
@@ -90,6 +90,7 @@ aggregation(SUM, NumRowGroups): 0
describe formatted parquet_partitioned;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/parquet_partitioned','NULL'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','iceberg.catalog ','hadoop.tables '
'','external.table.purge','true '
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
@@ -177,6 +178,7 @@ describe formatted parquet_nopartitioned;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/parquet_nopartitioned','NULL'
row_regex: '','metadata_location
','$NAMENODE/test-warehouse/$DATABASE.db/parquet_nopartitioned/metadata/.*.metadata.json'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','true '
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
'','write.format.default','parquet '
@@ -237,6 +239,7 @@ describe formatted decimal_tbl;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/decimal_tbl','NULL'
row_regex: '','metadata_location
','$NAMENODE/test-warehouse/$DATABASE.db/decimal_tbl/metadata/.*.metadata.json'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','true '
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
'','write.format.default','parquet '
@@ -281,6 +284,7 @@ describe formatted table_at_random_location;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/$DATABASE.db/random_location','NULL'
row_regex: '','metadata_location
','$NAMENODE/test-warehouse/$DATABASE.db/random_location/metadata/.*.metadata.json'
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
'','external.table.purge','true '
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
'','write.format.default','parquet '
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-missing-data-files.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-missing-data-files.test
index efbf0f8e2..4264bb7bb 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-missing-data-files.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-missing-data-files.test
@@ -72,6 +72,7 @@ describe formatted missing_files_part
'','default-partition-spec','{\\"spec-id\\":0,\\"fields\\":[{\\"name\\":\\"p\\",\\"transform\\":\\"identity\\",\\"source-id\\":2,\\"field-id\\":1000}]}'
'','engine.hive.enabled ','true '
'','external.table.purge','TRUE '
+'','format-version ','2 '
'','numFiles ','2 '
'','numRows ','2 '
'','snapshot-count ','2 '
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-old-fileformat.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-old-fileformat.test
index 641faae5f..4f5c5765a 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-old-fileformat.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-old-fileformat.test
@@ -17,6 +17,7 @@ DESCRIBE FORMATTED
iceberg_partitioned_orc_external_old_fileformat;
'','iceberg.table_identifier','functional_parquet.iceberg_partitioned_orc'
'','iceberg.file_format','orc '
'','iceberg.catalog ','hadoop.catalog '
+'','format-version ','$ICEBERG_DEFAULT_FORMAT_VERSION '
---- RESULTS: VERIFY_IS_NOT_IN
'','write.format.default','orc '
---- TYPES
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
index 45d4a715f..49e6c100c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-query.test
@@ -368,6 +368,7 @@ STRING,STRING,STRING,STRING
describe formatted iceberg_partitioned;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/iceberg_test/iceberg_partitioned','NULL'
+'','format-version ','1 '
'','write.format.default','parquet '
'','iceberg.catalog ','hadoop.tables '
---- TYPES
@@ -377,6 +378,7 @@ string, string, string
describe formatted iceberg_non_partitioned;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/iceberg_test/iceberg_non_partitioned','NULL'
+'','format-version ','1 '
'','write.format.default','parquet '
'','iceberg.catalog ','hadoop.tables '
---- TYPES
@@ -386,6 +388,7 @@ string, string, string
describe formatted hadoop_catalog_test_external;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/hadoop_catalog_test/functional_parquet/hadoop_catalog_test','NULL'
+'','format-version ','1 '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/iceberg_test/hadoop_catalog/hadoop_catalog_test'
'','iceberg.table_identifier','functional_parquet.hadoop_catalog_test'
'','write.format.default','parquet '
@@ -397,6 +400,7 @@ string, string, string
describe formatted iceberg_partitioned_orc_external;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc/functional_parquet/iceberg_partitioned_orc','NULL'
+'','format-version ','1 '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/iceberg_test/hadoop_catalog/iceberg_partitioned_orc'
'','iceberg.table_identifier','functional_parquet.iceberg_partitioned_orc'
'','write.format.default','orc '
@@ -467,6 +471,7 @@ bigint,bigint
describe formatted iceberg_resolution_test_external;
---- RESULTS: VERIFY_IS_SUBSET
'Location:
','$NAMENODE/test-warehouse/iceberg_test/hadoop_catalog/iceberg_resolution_test/functional_parquet/iceberg_resolution_test','NULL'
+'','format-version ','1 '
'','iceberg.catalog_location','$WAREHOUSE_LOCATION_PREFIX/test-warehouse/iceberg_test/hadoop_catalog/iceberg_resolution_test'
'','iceberg.table_identifier','functional_parquet.iceberg_resolution_test'
'','write.format.default','parquet '
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-rest-catalog.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-rest-catalog.test
index efb48e393..554d545fb 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-rest-catalog.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-rest-catalog.test
@@ -64,6 +64,7 @@ row_regex:'Erasure Coding Policy:','$ERASURECODE_POLICY
*','NULL'
'','EXTERNAL ','TRUE '
'','bucketing_version ','2 '
'','engine.hive.enabled ','true '
+'','format-version ','1 '
'','gc.enabled ','TRUE '
'','numFiles ','1 '
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-rest-fgac.test
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-rest-fgac.test
index 58ed14528..f8c9dc446 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/iceberg-rest-fgac.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/iceberg-rest-fgac.test
@@ -75,6 +75,7 @@ row_regex:'Erasure Coding Policy:','$ERASURECODE_POLICY
*','NULL'
'','EXTERNAL ','TRUE '
'','bucketing_version ','2 '
'','engine.hive.enabled ','true '
+'','format-version ','1 '
'','gc.enabled ','TRUE '
'','numFiles ','1 '
'','storage_handler
','org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/show-create-table-with-stats.test
b/testdata/workloads/functional-query/queries/QueryTest/show-create-table-with-stats.test
index 3283893ae..f44885624 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/show-create-table-with-stats.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/show-create-table-with-stats.test
@@ -65,8 +65,8 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.ice_with_stats (
)
STORED AS ICEBERG
LOCATION '$$location_uri$$'
-TBLPROPERTIES ('OBJCAPABILITIES'='EXTREAD,EXTWRITE',
'engine.hive.enabled'='true', 'external.table.purge'='TRUE',
'impala.computeStatsSnapshotIds'='<NUM>',
'impala.lastComputeStatsTime'='<NUM>', 'table_type'='ICEBERG',
'write.delete.mode'='merge-on-read', 'write.format.default'='parquet',
'write.merge.mode'='merge-on-read', 'write.parquet.compression-codec'='snappy',
'write.update.mode'='merge-on-read');
-ALTER TABLE show_create_table_test_db.ice_with_stats SET TBLPROPERTIES
('EXTERNAL'='TRUE', 'OBJCAPABILITIES'='EXTREAD,EXTWRITE',
'engine.hive.enabled'='true', 'external.table.purge'='TRUE',
'impala.computeStatsSnapshotIds'='<NUM>',
'impala.events.catalogServiceId'='<NUM>',
'impala.events.catalogVersion'='<NUM>', 'impala.lastComputeStatsTime'='<NUM>',
'numFiles'='<NUM>', 'numRows'='<NUM>', 'table_type'='ICEBERG',
'totalSize'='<NUM>', 'write.delete.mode'='merge-on-read', 'write.format.defa
[...]
+TBLPROPERTIES ('OBJCAPABILITIES'='EXTREAD,EXTWRITE',
'engine.hive.enabled'='true', 'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'impala.computeStatsSnapshotIds'='<NUM>',
'impala.lastComputeStatsTime'='<NUM>', 'table_type'='ICEBERG',
'write.delete.mode'='merge-on-read', 'write.format.default'='parquet',
'write.merge.mode'='merge-on-read', 'write.parquet.compression-codec'='snappy',
'write.update.mode'='merge-on-read');
+ALTER TABLE show_create_table_test_db.ice_with_stats SET TBLPROPERTIES
('EXTERNAL'='TRUE', 'OBJCAPABILITIES'='EXTREAD,EXTWRITE',
'engine.hive.enabled'='true', 'external.table.purge'='TRUE',
'format-version'='$$iceberg_default_format_version$$',
'impala.computeStatsSnapshotIds'='<NUM>',
'impala.events.catalogServiceId'='<NUM>',
'impala.events.catalogVersion'='<NUM>', 'impala.lastComputeStatsTime'='<NUM>',
'numFiles'='<NUM>', 'numRows'='<NUM>', 'table_type'='ICEBERG',
'totalSize'='<NUM>', [...]
ALTER TABLE show_create_table_test_db.ice_with_stats SET COLUMN STATS i
('numDVs'='1', 'numNulls'='0', 'numTrues'='-1', 'numFalses'='-1');
ALTER TABLE show_create_table_test_db.ice_with_stats SET COLUMN STATS s
('numDVs'='1', 'numNulls'='0', 'maxSize'='1', 'avgSize'='1', 'numTrues'='-1',
'numFalses'='-1');
====
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
index c47ea1a0a..fd1b72433 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/show-create-table.test
@@ -612,6 +612,7 @@ STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
+ 'format-version'='$$iceberg_default_format_version$$',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
@@ -645,6 +646,7 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.iceberg_test2 (
STORED AS ICEBERG
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
+ 'format-version'='$$iceberg_default_format_version$$',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
@@ -680,6 +682,7 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.iceberg_test3 (
STORED AS ICEBERG
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
+ 'format-version'='$$iceberg_default_format_version$$',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
@@ -741,6 +744,7 @@ PARTITIONED BY SPEC (
STORED AS ICEBERG
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
+ 'format-version'='$$iceberg_default_format_version$$',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
@@ -768,10 +772,14 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.iceberg_test_orc (
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
-'external.table.purge'='TRUE', 'write.format.default'='orc',
-'engine.hive.enabled'='true', 'iceberg.catalog'='hadoop.tables',
-'write.delete.mode'='merge-on-read', 'write.update.mode'='merge-on-read',
-'write.merge.mode'='merge-on-read')
+ 'external.table.purge'='TRUE',
+ 'format-version'='$$iceberg_default_format_version$$',
+ 'write.format.default'='orc',
+ 'engine.hive.enabled'='true',
+ 'iceberg.catalog'='hadoop.tables',
+ 'write.delete.mode'='merge-on-read',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
# Default Iceberg table
@@ -785,10 +793,15 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.iceberg_default_tbl (
)
STORED AS ICEBERG
LOCATION '$$location_uri$$'
-TBLPROPERTIES ('external.table.purge'='TRUE', 'write.format.default'='parquet',
- 'engine.hive.enabled'='true', 'table_type'='ICEBERG',
'write.delete.mode'='merge-on-read',
- 'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read',
- 'write.parquet.compression-codec'='snappy')
+TBLPROPERTIES ('write.format.default'='parquet',
+ 'write.parquet.compression-codec'='snappy',
+ 'engine.hive.enabled'='true',
+ 'external.table.purge'='TRUE',
+ 'format-version'='$$iceberg_default_format_version$$',
+ 'table_type'='ICEBERG',
+ 'write.delete.mode'='merge-on-read',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
# Default Iceberg table with ORC format
@@ -805,10 +818,15 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.iceberg_default_tbl_orc (
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
-'write.format.default'='orc', 'write.parquet.compression-codec'='snappy',
-'engine.hive.enabled'='true', 'external.table.purge'='TRUE',
'table_type'='ICEBERG',
-'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read',
-'write.delete.mode'='merge-on-read')
+ 'write.format.default'='orc',
+ 'write.parquet.compression-codec'='snappy',
+ 'engine.hive.enabled'='true',
+ 'external.table.purge'='TRUE',
+ 'format-version'='$$iceberg_default_format_version$$',
+ 'table_type'='ICEBERG',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read',
+ 'write.delete.mode'='merge-on-read')
====
---- CREATE_TABLE
# Iceberg table in HiveCatalog
@@ -825,10 +843,16 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.iceberg_hive_cat_explicit (
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
-'write.format.default'='parquet', 'iceberg.catalog'='hive.catalog',
-'engine.hive.enabled'='true', 'external.table.purge'='TRUE',
'table_type'='ICEBERG',
-'write.parquet.compression-codec'='snappy',
'write.delete.mode'='merge-on-read',
-'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read')
+ 'write.format.default'='parquet',
+ 'iceberg.catalog'='hive.catalog',
+ 'engine.hive.enabled'='true',
+ 'external.table.purge'='TRUE',
+ 'format-version'='$$iceberg_default_format_version$$',
+ 'table_type'='ICEBERG',
+ 'write.parquet.compression-codec'='snappy',
+ 'write.delete.mode'='merge-on-read',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_nullable_test (
@@ -855,6 +879,7 @@ STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
'external.table.purge'='TRUE',
+ 'format-version'='$$iceberg_default_format_version$$',
'write.delete.mode'='merge-on-read',
'write.update.mode'='merge-on-read',
'write.merge.mode'='merge-on-read',
@@ -880,10 +905,15 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.iceberg_old_style_partitions (
PARTITIONED BY SPEC (p, d)
STORED AS ICEBERG
LOCATION '$$location_uri$$'
-TBLPROPERTIES ('external.table.purge'='TRUE', 'write.format.default'='parquet',
-'engine.hive.enabled'='true', 'table_type'='ICEBERG',
'write.parquet.compression-codec'='snappy',
-'write.delete.mode'='merge-on-read', 'write.update.mode'='merge-on-read',
-'write.merge.mode'='merge-on-read')
+TBLPROPERTIES ('external.table.purge'='TRUE',
+ 'write.format.default'='parquet',
+ 'engine.hive.enabled'='true',
+ 'format-version'='$$iceberg_default_format_version$$',
+ 'table_type'='ICEBERG',
+ 'write.parquet.compression-codec'='snappy',
+ 'write.delete.mode'='merge-on-read',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_ctas
@@ -899,10 +929,15 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.iceberg_ctas (
PARTITIONED BY SPEC (BUCKET(5, id))
STORED AS ICEBERG
LOCATION '$$location_uri$$'
-TBLPROPERTIES ('external.table.purge'='TRUE', 'write.format.default'='parquet',
-'engine.hive.enabled'='true', 'table_type'='ICEBERG',
'write.parquet.compression-codec'='snappy',
-'write.delete.mode'='merge-on-read', 'write.update.mode'='merge-on-read',
-'write.merge.mode'='merge-on-read')
+TBLPROPERTIES ('external.table.purge'='TRUE',
+ 'write.format.default'='parquet',
+ 'engine.hive.enabled'='true',
+ 'format-version'='$$iceberg_default_format_version$$',
+ 'table_type'='ICEBERG',
+ 'write.parquet.compression-codec'='snappy',
+ 'write.delete.mode'='merge-on-read',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_ctas_ht
@@ -921,9 +956,14 @@ PARTITIONED BY SPEC (BUCKET(5, id))
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
-'external.table.purge'='TRUE', 'write.format.default'='parquet',
-'engine.hive.enabled'='true', 'iceberg.catalog'='hadoop.tables',
'write.delete.mode'='merge-on-read',
-'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read')
+ 'external.table.purge'='TRUE',
+ 'write.format.default'='parquet',
+ 'engine.hive.enabled'='true',
+ 'format-version'='$$iceberg_default_format_version$$',
+ 'iceberg.catalog'='hadoop.tables',
+ 'write.delete.mode'='merge-on-read',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_catalogs_hive (i int)
@@ -937,10 +977,15 @@ PARTITIONED BY SPEC (BUCKET(3, i))
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
-'external.table.purge'='TRUE', 'write.format.default'='parquet',
-'engine.hive.enabled'='true', 'table_type'='ICEBERG',
'write.parquet.compression-codec'='snappy',
-'write.delete.mode'='merge-on-read', 'write.update.mode'='merge-on-read',
-'write.merge.mode'='merge-on-read')
+ 'external.table.purge'='TRUE',
+ 'write.format.default'='parquet',
+ 'engine.hive.enabled'='true',
+ 'format-version'='$$iceberg_default_format_version$$',
+ 'table_type'='ICEBERG',
+ 'write.parquet.compression-codec'='snappy',
+ 'write.delete.mode'='merge-on-read',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_catalogs_hadoop (i int)
@@ -954,9 +999,14 @@ PARTITIONED BY SPEC (BUCKET(3, i))
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
-'external.table.purge'='TRUE', 'write.format.default'='parquet',
-'engine.hive.enabled'='true', 'iceberg.catalog'='ice_hadoop_cat',
'write.delete.mode'='merge-on-read',
-'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read')
+ 'external.table.purge'='TRUE',
+ 'write.format.default'='parquet',
+ 'engine.hive.enabled'='true',
+ 'format-version'='$$iceberg_default_format_version$$',
+ 'iceberg.catalog'='ice_hadoop_cat',
+ 'write.delete.mode'='merge-on-read',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
CREATE TABLE iceberg_void_transform (i int, j int)
@@ -967,10 +1017,15 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.iceberg_void_transform (i INT NU
PARTITIONED BY SPEC (VOID(i), VOID(j))
STORED AS ICEBERG
LOCATION '$$location_uri$$'
-TBLPROPERTIES ('external.table.purge'='TRUE', 'write.format.default'='parquet',
-'engine.hive.enabled'='true', 'table_type'='ICEBERG',
'write.parquet.compression-codec'='snappy',
-'write.delete.mode'='merge-on-read', 'write.update.mode'='merge-on-read',
-'write.merge.mode'='merge-on-read')
+TBLPROPERTIES ('external.table.purge'='TRUE',
+ 'write.format.default'='parquet',
+ 'engine.hive.enabled'='true',
+ 'format-version'='$$iceberg_default_format_version$$',
+ 'table_type'='ICEBERG',
+ 'write.parquet.compression-codec'='snappy',
+ 'write.delete.mode'='merge-on-read',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read')
====
---- CREATE_TABLE
# Creating V1 tables explicitly should not set 'merge-on-read' write modes if
no write mode is
@@ -983,9 +1038,12 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.ice_explicit_v1 (i INT NULL)
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
-'format-version'='1', 'engine.hive.enabled'='true',
'external.table.purge'='TRUE',
-'table_type'='ICEBERG',
-'write.format.default'='parquet', 'write.parquet.compression-codec'='snappy')
+ 'engine.hive.enabled'='true',
+ 'external.table.purge'='TRUE',
+ 'format-version'='1',
+ 'table_type'='ICEBERG',
+ 'write.format.default'='parquet',
+ 'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Creating V2 tables should set 'merge-on-read' write modes if no write mode
is specified.
@@ -997,10 +1055,15 @@ CREATE EXTERNAL TABLE show_create_table_test_db.ice_v2
(i INT NULL)
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
- 'external.table.purge'='TRUE', 'write.format.default'='parquet',
- 'engine.hive.enabled'='true', 'table_type'='ICEBERG',
'write.delete.mode'='merge-on-read',
- 'write.update.mode'='merge-on-read', 'write.merge.mode'='merge-on-read',
- 'write.parquet.compression-codec'='snappy')
+ 'external.table.purge'='TRUE',
+ 'write.format.default'='parquet',
+ 'engine.hive.enabled'='true',
+ 'format-version'='2',
+ 'table_type'='ICEBERG',
+ 'write.delete.mode'='merge-on-read',
+ 'write.update.mode'='merge-on-read',
+ 'write.merge.mode'='merge-on-read',
+ 'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Creating V2 tables should not set write mode if user specified any of it to
any value.
@@ -1013,9 +1076,13 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.ice_v2_explicit_delete (i INT NU
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
- 'external.table.purge'='TRUE', 'write.format.default'='parquet',
- 'engine.hive.enabled'='true', 'table_type'='ICEBERG',
'write.delete.mode'='merge-on-read',
- 'write.parquet.compression-codec'='snappy')
+ 'external.table.purge'='TRUE',
+ 'write.format.default'='parquet',
+ 'engine.hive.enabled'='true',
+ 'format-version'='2',
+ 'table_type'='ICEBERG',
+ 'write.delete.mode'='merge-on-read',
+ 'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Creating V2 tables should not set write mode if user specified any of it to
any value.
@@ -1028,9 +1095,13 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.ice_v2_explicit_delete_2 (i INT
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
- 'external.table.purge'='TRUE', 'write.format.default'='parquet',
- 'engine.hive.enabled'='true', 'table_type'='ICEBERG',
'write.delete.mode'='copy-on-write',
- 'write.parquet.compression-codec'='snappy')
+ 'external.table.purge'='TRUE',
+ 'write.format.default'='parquet',
+ 'engine.hive.enabled'='true',
+ 'format-version'='2',
+ 'table_type'='ICEBERG',
+ 'write.delete.mode'='copy-on-write',
+ 'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Creating V2 tables should not set write mode if user specified any of it to
any value.
@@ -1043,9 +1114,13 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.ice_v2_explicit_update (i INT NU
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
- 'external.table.purge'='TRUE', 'write.format.default'='parquet',
- 'engine.hive.enabled'='true', 'table_type'='ICEBERG',
'write.update.mode'='copy-on-write',
- 'write.parquet.compression-codec'='snappy')
+ 'external.table.purge'='TRUE',
+ 'write.format.default'='parquet',
+ 'engine.hive.enabled'='true',
+ 'format-version'='2',
+ 'table_type'='ICEBERG',
+ 'write.update.mode'='copy-on-write',
+ 'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Creating V2 tables should not set write mode if user specified any of it to
any value.
@@ -1058,9 +1133,13 @@ CREATE EXTERNAL TABLE
show_create_table_test_db.ice_v2_explicit_merge (i INT NUL
STORED AS ICEBERG
LOCATION '$$location_uri$$'
TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE',
- 'external.table.purge'='TRUE', 'write.format.default'='parquet',
- 'engine.hive.enabled'='true', 'table_type'='ICEBERG',
'write.merge.mode'='copy-on-write',
- 'write.parquet.compression-codec'='snappy')
+ 'external.table.purge'='TRUE',
+ 'write.format.default'='parquet',
+ 'engine.hive.enabled'='true',
+ 'format-version'='2',
+ 'table_type'='ICEBERG',
+ 'write.merge.mode'='copy-on-write',
+ 'write.parquet.compression-codec'='snappy')
====
---- CREATE_TABLE
# Test create Bucketed Table
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 9869bf8da..a754001e2 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -77,6 +77,24 @@ else:
MANAGED_WAREHOUSE_DIR = 'test-warehouse'
EXTERNAL_WAREHOUSE_DIR = 'test-warehouse'
+# Set Iceberg's default format version based on version, if <1.4 it's 1, if
>=1.4 then 2
+impala_iceberg_version = os.environ.get("IMPALA_ICEBERG_VERSION")
+
+ICEBERG_DEFAULT_FORMAT_VERSION = '1'
+if impala_iceberg_version:
+ # Extract the major and minor version numbers
+ version_parts = impala_iceberg_version.split('.')
+ if len(version_parts) >= 2:
+ try:
+ major = int(version_parts[0])
+ minor = int(version_parts[1])
+ # If version is 1.4 or higher, use format version 2
+ if major > 1 or (major == 1 and minor >= 4):
+ ICEBERG_DEFAULT_FORMAT_VERSION = '2'
+ except ValueError:
+ # If we can't parse the version, default to format version 1
+ pass
+
IS_APACHE_HIVE = os.environ.get("USE_APACHE_HIVE", False) == 'true'
# Resolve any symlinks in the path.
diff --git a/tests/common/impala_test_suite.py
b/tests/common/impala_test_suite.py
index c2d7744dd..f59b4185d 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -46,7 +46,8 @@ from tests.common.base_test_suite import BaseTestSuite
from tests.common.environ import (
HIVE_MAJOR_VERSION,
MANAGED_WAREHOUSE_DIR,
- EXTERNAL_WAREHOUSE_DIR)
+ EXTERNAL_WAREHOUSE_DIR,
+ ICEBERG_DEFAULT_FORMAT_VERSION)
from tests.common.errors import Timeout
from tests.common.impala_connection import create_connection
from tests.common.impala_service import ImpaladService
@@ -672,7 +673,8 @@ class ImpalaTestSuite(BaseTestSuite):
"INTERNAL_LISTEN_HOST",
"INTERNAL_LISTEN_IP",
"MANAGED_WAREHOUSE_DIR",
- "EXTERNAL_WAREHOUSE_DIR"])
+ "EXTERNAL_WAREHOUSE_DIR",
+ "ICEBERG_DEFAULT_FORMAT_VERSION"])
repl.update({
'$ERASURECODE_POLICY': os.getenv("ERASURECODE_POLICY", "NONE"),
'$SECONDARY_FILESYSTEM': os.getenv("SECONDARY_FILESYSTEM", ""),
@@ -740,7 +742,9 @@ class ImpalaTestSuite(BaseTestSuite):
.replace('$MANAGED_WAREHOUSE_DIR',
MANAGED_WAREHOUSE_DIR) \
.replace('$EXTERNAL_WAREHOUSE_DIR',
- EXTERNAL_WAREHOUSE_DIR)
+ EXTERNAL_WAREHOUSE_DIR) \
+
.replace('$ICEBERG_DEFAULT_FORMAT_VERSION',
+ ICEBERG_DEFAULT_FORMAT_VERSION)
if use_db:
test_section[section_name] = test_section[section_name].replace(
'$DATABASE', use_db)
diff --git a/tests/metadata/test_show_create_table.py
b/tests/metadata/test_show_create_table.py
index c248c1794..c4be7af2c 100644
--- a/tests/metadata/test_show_create_table.py
+++ b/tests/metadata/test_show_create_table.py
@@ -24,7 +24,7 @@ from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIf, SkipIfFS, SkipIfHive2
from tests.common.test_dimensions import create_uncompressed_text_dimension
from tests.util.test_file_parser import QueryTestSectionReader, remove_comments
-from tests.common.environ import HIVE_MAJOR_VERSION
+from tests.common.environ import HIVE_MAJOR_VERSION,
ICEBERG_DEFAULT_FORMAT_VERSION
from tests.util.filesystem_utils import WAREHOUSE
@@ -139,7 +139,7 @@ class TestShowCreateTable(ImpalaTestSuite):
if not test_case.existing_table:
# create table in Impala
- self.__exec(self.__replace_warehouse(test_case.create_table_sql))
+ self.__exec(self.__replace_variables(test_case.create_table_sql))
# execute "SHOW CREATE TABLE ..."
result = self.__exec(test_case.show_create_table_sql)
create_table_result = self.__normalize(result.data[0])
@@ -149,7 +149,7 @@ class TestShowCreateTable(ImpalaTestSuite):
self.__exec(test_case.drop_table_sql)
# check the result matches the expected result
- expected_result =
self.__normalize(self.__replace_warehouse(self.__replace_uri(
+ expected_result =
self.__normalize(self.__replace_variables(self.__replace_uri(
test_case.expected_result,
self.__get_location_uri(create_table_result))))
self.__compare_result(expected_result, create_table_result)
@@ -176,7 +176,7 @@ class TestShowCreateTable(ImpalaTestSuite):
test_case = ShowCreateTableTestCase(test_section, test_file_name,
unique_db_name)
if not test_case.existing_table:
# create table in Impala (support multiple setup statements)
- setup_sql = self.__replace_warehouse(test_case.create_table_sql)
+ setup_sql = self.__replace_variables(test_case.create_table_sql)
for stmt in re.split(r";\s*", setup_sql.strip()):
if not stmt:
continue
@@ -212,7 +212,7 @@ class TestShowCreateTable(ImpalaTestSuite):
self.__exec(test_case.drop_table_sql)
# Build expected statements list and compare per-statement
- expected_sql = self.__replace_warehouse(self.__replace_uri(
+ expected_sql = self.__replace_variables(self.__replace_uri(
test_case.expected_result,
self.__get_location_uri(location_source)
))
@@ -328,18 +328,18 @@ class TestShowCreateTable(ImpalaTestSuite):
"""
s = ' '.join(s.split())
for k in self.FILTER_TBL_PROPERTIES:
- kv_regex = "'%s'\s*=\s*'[^\']+'\s*,?" % (k)
+ kv_regex = r"'%s'\s*=\s*'[^\']+'\s*,?" % (k)
s = re.sub(kv_regex, "", s)
# If we removed the last property, there will be a dangling comma that is
not valid
# e.g. 'k1'='v1', ) -> 'k1'='v1')
- s = re.sub(",\s*\)", ")", s)
+ s = re.sub(r",\s*\)", ")", s)
# Need to remove any whitespace after left parens and before right parens
- s = re.sub("\(\s+", "(", s)
- s = re.sub("\s+\)", ")", s)
+ s = re.sub(r"\(\s+", "(", s)
+ s = re.sub(r"\s+\)", ")", s)
# If the only properties were removed, the properties sections may be
empty, which
# is not valid
- s = re.sub("TBLPROPERTIES\s*\(\s*\)", "", s)
- s = re.sub("SERDEPROPERTIES\s*\(\s*\)", "", s)
+ s = re.sub(r"TBLPROPERTIES\s*\(\s*\)", "", s)
+ s = re.sub(r"SERDEPROPERTIES\s*\(\s*\)", "", s)
# By removing properties in the middle we might ended up having extra
whitespaces,
# let's remove them.
s = ' '.join(s.split())
@@ -357,8 +357,9 @@ class TestShowCreateTable(ImpalaTestSuite):
def __replace_uri(self, s, uri):
return s if uri is None else s.replace("$$location_uri$$", uri)
- def __replace_warehouse(self, s):
- return s.replace("$$warehouse$$", WAREHOUSE)
+ def __replace_variables(self, s):
+ return s.replace("$$warehouse$$", WAREHOUSE)\
+ .replace("$$iceberg_default_format_version$$",
ICEBERG_DEFAULT_FORMAT_VERSION)
# Represents one show-create-table test case. Performs validation of the test
sections
