This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 72732da9d IMPALA-12609: Implement SHOW METADATA TABLES IN statement to
list Iceberg Metadata tables
new 18b9c08c5 IMPALA-12600: Schema evolution with equality delete files
new 63f52807f IMPALA-12611: Add support to MAP type Iceberg Metadata table
columns
new effc9df93 IMPALA-12782: Show info of the event processing in /events
webUI
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/catalog/catalog-server.cc | 79 +++++++-
.../iceberg-metadata/iceberg-metadata-scanner.cc | 96 +++++++---
.../iceberg-metadata/iceberg-metadata-scanner.h | 68 +++++--
be/src/exec/iceberg-metadata/iceberg-row-reader.cc | 197 ++++++++++++++------
be/src/exec/iceberg-metadata/iceberg-row-reader.h | 46 +++--
be/src/util/json-util.h | 21 +++
common/thrift/JniCatalog.thrift | 25 +++
.../impala/catalog/events/MetastoreEvents.java | 2 +
.../catalog/events/MetastoreEventsProcessor.java | 70 ++++++-
.../java/org/apache/impala/util/DebugUtils.java | 3 +
.../apache/impala/util/IcebergMetadataScanner.java | 29 ++-
testdata/data/README | 23 +++
...47c78-9a7a-4d68-81d9-ab22e44a3630-00002.parquet | Bin 0 -> 671 bytes
...845c344-9698e59400000000_1309071497_data.0.parq | Bin 0 -> 836 bytes
...47c78-9a7a-4d68-81d9-ab22e44a3630-00004.parquet | Bin 0 -> 671 bytes
...caf13-6a91-4fd5-b509-54ec8b16864f-00001.parquet | Bin 0 -> 1118 bytes
...caf13-6a91-4fd5-b509-54ec8b16864f-00002.parquet | Bin 0 -> 671 bytes
...845c344-9698e59400000000_1656341410_data.0.parq | Bin 0 -> 836 bytes
...845c344-9698e59400000000_1634747934_data.0.parq | Bin 0 -> 824 bytes
.../94efa501-9664-420d-a524-30535c11d363-m0.avro} | Bin 3848 -> 4386 bytes
.../bf0f2c96-954e-4b3c-a686-8b06e9fd56e8-m0.avro} | Bin 3848 -> 4097 bytes
.../fb7a4022-ee0a-4540-87d8-b8fa8e4c8596-m0.avro} | Bin 3848 -> 3865 bytes
.../fb7a4022-ee0a-4540-87d8-b8fa8e4c8596-m1.avro} | Bin 3827 -> 3833 bytes
...69-1-bf0f2c96-954e-4b3c-a686-8b06e9fd56e8.avro} | Bin 2312 -> 2379 bytes
...84-1-fb7a4022-ee0a-4540-87d8-b8fa8e4c8596.avro} | Bin 2696 -> 2753 bytes
...1-fb7a4022-ee0a-4540-87d8-b8fa8e4c8596.avro_tmp | 116 ++++++++++++
...92-1-94efa501-9664-420d-a524-30535c11d363.avro} | Bin 2162 -> 2193 bytes
.../metadata/v1.metadata.json | 30 ++-
.../metadata/v2.metadata.json | 59 +++---
.../metadata/v3.metadata.json | 121 +++++++++++++
.../metadata/v4.metadata.json | 144 +++++++++++++++
.../metadata/v5.metadata.json | 172 ++++++++++++++++++
.../metadata/v6.metadata.json | 201 +++++++++++++++++++++
.../metadata/version-hint.text | 0
.../functional/functional_schema_template.sql | 15 ++
.../datasets/functional/schema_constraints.csv | 1 +
.../queries/PlannerTest/iceberg-v2-tables.test | 59 ++++++
.../queries/QueryTest/iceberg-metadata-tables.test | 90 ++++++++-
.../iceberg-v2-read-equality-deletes.test | 23 +++
tests/custom_cluster/test_web_pages.py | 66 +++++++
www/events.tmpl | 65 ++++++-
41 files changed, 1642 insertions(+), 179 deletions(-)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/data/d=2024-03-20/00000-10-e4b47c78-9a7a-4d68-81d9-ab22e44a3630-00002.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/data/d=2024-03-20/3645a3085845c344-9698e59400000000_1309071497_data.0.parq
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/data/d=2024-03-21/00000-10-e4b47c78-9a7a-4d68-81d9-ab22e44a3630-00004.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/data/d=2024-03-21/00000-11-a72caf13-6a91-4fd5-b509-54ec8b16864f-00001.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/data/d=2024-03-21/00000-11-a72caf13-6a91-4fd5-b509-54ec8b16864f-00002.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/data/d=2024-03-21/3645a3085845c344-9698e59400000000_1656341410_data.0.parq
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/data/d=2024-03-22/3645a3085845c344-9698e59400000000_1634747934_data.0.parq
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_equality_partition_evolution/metadata/807d7d2a-0557-4bbe-9f07-9467de72598a-m0.avro
=>
iceberg_v2_equality_delete_schema_evolution/metadata/94efa501-9664-420d-a524-30535c11d363-m0.avro}
(80%)
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_equality_partition_evolution/metadata/807d7d2a-0557-4bbe-9f07-9467de72598a-m0.avro
=>
iceberg_v2_equality_delete_schema_evolution/metadata/bf0f2c96-954e-4b3c-a686-8b06e9fd56e8-m0.avro}
(85%)
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_equality_partition_evolution/metadata/807d7d2a-0557-4bbe-9f07-9467de72598a-m0.avro
=>
iceberg_v2_equality_delete_schema_evolution/metadata/fb7a4022-ee0a-4540-87d8-b8fa8e4c8596-m0.avro}
(91%)
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_equality_partitioned/metadata/09ca0acc-c19e-4073-80f7-b476a6e568c7-m1.avro
=>
iceberg_v2_equality_delete_schema_evolution/metadata/fb7a4022-ee0a-4540-87d8-b8fa8e4c8596-m1.avro}
(91%)
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_equality/metadata/snap-361598097944440235-1-22029492-dcdb-4ba1-a447-9b9f7200f9e4.avro
=>
iceberg_v2_equality_delete_schema_evolution/metadata/snap-3986738438831924669-1-bf0f2c96-954e-4b3c-a686-8b06e9fd56e8.avro}
(80%)
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_spark_compaction_with_dangling_delete/metadata/snap-1208327814823543579-1-52100098-3c71-4111-8d7e-1c02e8343a0e.avro
=>
iceberg_v2_equality_delete_schema_evolution/metadata/snap-5816823095034839884-1-fb7a4022-ee0a-4540-87d8-b8fa8e4c8596.avro}
(69%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/metadata/snap-5816823095034839884-1-fb7a4022-ee0a-4540-87d8-b8fa8e4c8596.avro_tmp
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_positional/metadata/snap-6816997371555012807-1-8cbef400-daea-478a-858a-2baf2438f644.avro
=>
iceberg_v2_equality_delete_schema_evolution/metadata/snap-7131747670101362192-1-94efa501-9664-420d-a524-30535c11d363.avro}
(87%)
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_equality_multi_eq_ids
=> iceberg_v2_equality_delete_schema_evolution}/metadata/v1.metadata.json (69%)
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_equality_nulls
=> iceberg_v2_equality_delete_schema_evolution}/metadata/v2.metadata.json (54%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/metadata/v3.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/metadata/v4.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/metadata/v5.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_equality_delete_schema_evolution/metadata/v6.metadata.json
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_multiple_storage_locations
=> iceberg_v2_equality_delete_schema_evolution}/metadata/version-hint.text
(100%)
