This is an automated email from the ASF dual-hosted git repository.
mbod pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a550003 HIVE-25658: Addendum: Fix regex for masking totalSize table
properties in Iceberg q-tests (Marton Bod, reviewed by Peter Vary)
a550003 is described below
commit a5500035c12d94de9094d880822776d276817439
Author: Marton Bod <[email protected]>
AuthorDate: Fri Nov 19 23:41:17 2021 +0100
HIVE-25658: Addendum: Fix regex for masking totalSize table properties in
Iceberg q-tests (Marton Bod, reviewed by Peter Vary)
---
.../test/queries/positive/create_iceberg_table.q | 3 +++
.../create_iceberg_table_stored_as_fileformat.q | 3 +++
.../create_iceberg_table_stored_by_iceberg.q | 3 +++
...g_table_stored_by_iceberg_with_serdeproperties.q | 3 +++
.../positive/describe_iceberg_metadata_tables.q | 3 +++
.../test/queries/positive/describe_iceberg_table.q | 3 +++
.../query_iceberg_metadata_of_partitioned_table.q | 4 ++++
.../query_iceberg_metadata_of_unpartitioned_table.q | 4 ++++
.../positive/query_iceberg_metadata_with_joins.q | 3 +++
.../queries/positive/truncate_force_iceberg_table.q | 4 ++--
.../test/queries/positive/truncate_iceberg_table.q | 4 ++--
.../results/positive/create_iceberg_table.q.out | 2 +-
.../create_iceberg_table_stored_as_fileformat.q.out | 10 +++++-----
.../create_iceberg_table_stored_by_iceberg.q.out | 2 +-
...ble_stored_by_iceberg_with_serdeproperties.q.out | 2 +-
.../results/positive/describe_iceberg_table.q.out | 8 ++++----
...ry_iceberg_metadata_of_unpartitioned_table.q.out | Bin 29346 -> 29566 bytes
.../positive/truncate_force_iceberg_table.q.out | 2 +-
.../results/positive/truncate_iceberg_table.q.out | 4 ++--
19 files changed, 48 insertions(+), 19 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table.q
b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table.q
index a86d642..880b92b 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table.q
@@ -1,3 +1,6 @@
+-- Mask the totalSize value as it can have slight variability, causing test
flakiness
+--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
+
set hive.vectorized.execution.enabled=false;
CREATE EXTERNAL TABLE ice_t (i int, s string, ts timestamp, d date) STORED BY
ICEBERG;
DESCRIBE FORMATTED ice_t;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_as_fileformat.q
b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_as_fileformat.q
index bc6adb8..ab1a309 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_as_fileformat.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_as_fileformat.q
@@ -1,3 +1,6 @@
+-- Mask the totalSize value as it can have slight variability, causing test
flakiness
+--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
+
set hive.vectorized.execution.enabled=false;
DROP TABLE IF EXISTS ice_orc;
CREATE EXTERNAL TABLE ice_orc (i int, s string, ts timestamp, d date) STORED
BY ICEBERG STORED AS ORC;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg.q
b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg.q
index f932901..7b6f12a 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg.q
@@ -1,3 +1,6 @@
+-- Mask the totalSize value as it can have slight variability, causing test
flakiness
+--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
+
set hive.vectorized.execution.enabled=false;
CREATE EXTERNAL TABLE ice_t (i int, s string, ts timestamp, d date) STORED BY
ICEBERG;
DESCRIBE FORMATTED ice_t;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q
b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q
index 7c44eb6..47c1f52 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q
@@ -1,3 +1,6 @@
+-- Mask the totalSize value as it can have slight variability, causing test
flakiness
+--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
+
set hive.vectorized.execution.enabled=false;
CREATE EXTERNAL TABLE ice_t (i int, s string, ts timestamp, d date) STORED BY
ICEBERG WITH SERDEPROPERTIES('write.format.default'='orc');
DESCRIBE FORMATTED ice_t;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q
b/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q
index 6fd7daa..fbab1c9 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q
@@ -1,3 +1,6 @@
+-- Mask the totalSize value as it can have slight variability, causing test
flakiness
+--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
+
drop table if exists ice_meta_desc;
create external table ice_meta_desc (id int, value string) stored by iceberg
stored as orc;
insert into ice_meta_desc values (1,
'one'),(2,'two'),(3,'three'),(4,'four'),(5,'five');
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_table.q
b/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_table.q
index 53ee461..371c41c 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_table.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_table.q
@@ -1,3 +1,6 @@
+-- Mask the totalSize value as it can have slight variability, causing test
flakiness
+--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
+
DROP TABLE IF EXISTS ice_t;
CREATE EXTERNAL TABLE ice_t (i int, s string, ts timestamp, d date) STORED BY
ICEBERG;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_partitioned_table.q
b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_partitioned_table.q
index 2f456d3..1c1ef7a 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_partitioned_table.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_partitioned_table.q
@@ -1,4 +1,8 @@
-- SORT_QUERY_RESULTS
+-- Mask the file size values as it can have slight variability, causing test
flakiness
+--! qt:replace:/("file_size_in_bytes":)\d+/$1#Masked#/
+--! qt:replace:/("total-files-size":)\d+/$1#Masked#/
+--! qt:replace:/((ORC|PARQUET|AVRO)\s+\d+\s+)\d+/$1#Masked#/
set hive.vectorized.execution.enabled = false;
set
tez.mrreader.config.update.properties=hive.io.file.readcolumn.names,hive.io.file.readcolumn.ids;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_unpartitioned_table.q
b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_unpartitioned_table.q
index f6d0ed0..2272417 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_unpartitioned_table.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_of_unpartitioned_table.q
@@ -1,4 +1,8 @@
-- SORT_QUERY_RESULTS
+-- Mask the file size values as it can have slight variability, causing test
flakiness
+--! qt:replace:/("file_size_in_bytes":)\d+/$1#Masked#/
+--! qt:replace:/("total-files-size":)\d+/$1#Masked#/
+--! qt:replace:/((ORC|PARQUET|AVRO)\s+\d+\s+)\d+/$1#Masked#/
set hive.vectorized.execution.enabled = false;
set
tez.mrreader.config.update.properties=hive.io.file.readcolumn.names,hive.io.file.readcolumn.ids;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_with_joins.q
b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_with_joins.q
index ef745e8..3c6d83c 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_with_joins.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_metadata_with_joins.q
@@ -1,4 +1,7 @@
-- SORT_QUERY_RESULTS
+-- Mask the file size values as it can have slight variability, causing test
flakiness
+--! qt:replace:/("file_size_in_bytes":)\d+/$1#Masked#/
+--! qt:replace:/("total-files-size":)\d+/$1#Masked#/
set hive.vectorized.execution.enabled = false;
set
tez.mrreader.config.update.properties=hive.io.file.readcolumn.names,hive.io.file.readcolumn.ids;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q
b/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q
index 77e394d..b1c002e 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q
@@ -1,6 +1,6 @@
-- SORT_QUERY_RESULTS
--- Mask the totalSize value as it can change whenever the file format (e.g.
ORC) version changes
---! qt:replace:/(\s+totalSize\s+)[\S+](\s+)/$1#Masked#$2/
+-- Mask the totalSize value as it can have slight variability, causing test
flakiness
+--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
set hive.vectorized.execution.enabled=false;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q
b/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q
index 277f1c1..a6a0e4e 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q
@@ -1,6 +1,6 @@
-- SORT_QUERY_RESULTS
--- Mask the totalSize value as it can change whenever the file format (e.g.
ORC) version changes
---! qt:replace:/(\s+totalSize\s+)[\S+](\s+)/$1#Masked#$2/
+-- Mask the totalSize value as it can have slight variability, causing test
flakiness
+--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/
set hive.vectorized.execution.enabled=false;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out
b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out
index eedb423..42a347a 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out
@@ -37,7 +37,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out
b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out
index 78f4205..c7393a6 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out
@@ -41,7 +41,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default ORC
@@ -104,7 +104,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default PARQUET
@@ -167,7 +167,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default AVRO
@@ -230,7 +230,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default AVRO
@@ -290,7 +290,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default ORC
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out
b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out
index eedb423..42a347a 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out
@@ -37,7 +37,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out
b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out
index d7b43ed..d4a62cd 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out
@@ -37,7 +37,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default orc
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
index 6bfe2f6..a20fec6 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
@@ -77,7 +77,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
@@ -132,7 +132,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
@@ -188,7 +188,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
@@ -232,7 +232,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_unpartitioned_table.q.out
b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_unpartitioned_table.q.out
index d65def6..e06df09 100644
Binary files
a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_unpartitioned_table.q.out
and
b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_metadata_of_unpartitioned_table.q.out
differ
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
index 9ad42d5..1fd52a5 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
@@ -99,7 +99,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 1998
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default parquet
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
index a170985..af44a6d 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
@@ -99,7 +99,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 1096
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default orc
@@ -229,7 +229,7 @@ Table Parameters:
serialization.format 1
storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 371
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default orc
