This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from b81368a22 IMPALA-12858: [DOCS] Correct idle_client_poll_period_s docs
new 65094a74f IMPALA-12598: Allow multiple equality field id lists for
Iceberg tables
new cc63757c1 IMPALA-12838: Adds exec_options parameter to
single_node_perf_run.py
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
bin/run-workload.py | 16 +-
bin/single_node_perf_run.py | 12 +-
common/fbs/IcebergObjects.fbs | 1 +
common/thrift/CatalogObjects.thrift | 7 +-
.../org/apache/impala/catalog/FeIcebergTable.java | 3 +-
.../impala/catalog/IcebergContentFileStore.java | 31 +--
.../impala/catalog/IcebergEqualityDeleteTable.java | 2 +-
.../apache/impala/planner/IcebergScanPlanner.java | 205 ++++++++++-----
.../java/org/apache/impala/util/IcebergUtil.java | 11 +
.../org/apache/impala/planner/PlannerTest.java | 24 ++
testdata/data/README | 56 +++-
...3256830-d9bd9e2f00000000_1372039299_data.0.parq | Bin 616 -> 0 bytes
...f44b347b-e035009600000001_138281890_data.0.parq | Bin 1630 -> 0 bytes
...836ebfcc-95c9ce7a00000000_909175610_data.0.parq | Bin 1630 -> 0 bytes
.../2d3fafd7-bce6-483f-be82-e0ccce9203fc-m0.avro | Bin 6716 -> 0 bytes
.../57a963d3-0e4e-4540-8080-a57afd51ba99-m0.avro | Bin 6696 -> 0 bytes
.../8bd425d8-25fb-4603-8cc7-aeb5ad2a3917-m0.avro | Bin 6691 -> 0 bytes
...726-1-2d3fafd7-bce6-483f-be82-e0ccce9203fc.avro | Bin 4303 -> 0 bytes
...078-1-57a963d3-0e4e-4540-8080-a57afd51ba99.avro | Bin 4430 -> 0 bytes
...958-1-8bd425d8-25fb-4603-8cc7-aeb5ad2a3917.avro | Bin 4383 -> 0 bytes
.../metadata/v4.metadata.json | 137 ----------
.../metadata/version-hint.text | 1 -
...63f85-c950-4d32-a9fd-0abd5229d768-00001.parquet | Bin 0 -> 655 bytes
...5d982-ca94-4e1d-9b86-2f973c000cf0-00001.parquet | Bin 0 -> 712 bytes
...5d982-ca94-4e1d-9b86-2f973c000cf0-00002.parquet | Bin 0 -> 444 bytes
...478ff-204b-4eb4-9a71-9876a31daf64-00001.parquet | Bin 0 -> 673 bytes
...478ff-204b-4eb4-9a71-9876a31daf64-00002.parquet | Bin 0 -> 473 bytes
...4a344-4216-4d37-9e9b-091de336e3fa-00001.parquet | Bin 0 -> 713 bytes
...4a344-4216-4d37-9e9b-091de336e3fa-00002.parquet | Bin 0 -> 726 bytes
.../2eb78a39-190a-4eab-b24a-df07f32f2cc0-m0.avro | Bin 0 -> 3714 bytes
.../2eb78a39-190a-4eab-b24a-df07f32f2cc0-m1.avro | Bin 0 -> 3706 bytes
.../58a6295d-0076-4e3e-ac77-84ce48c406cf-m0.avro | Bin 0 -> 3733 bytes
.../58a6295d-0076-4e3e-ac77-84ce48c406cf-m1.avro | Bin 0 -> 3724 bytes
.../b8fe0a34-e755-4ba0-92f1-c72bef85a82a-m0.avro | Bin 0 -> 3729 bytes
.../b8fe0a34-e755-4ba0-92f1-c72bef85a82a-m1.avro | Bin 0 -> 3698 bytes
.../ea25da34-c91b-4a0f-a003-3958e87caffd-m0.avro | Bin 0 -> 3712 bytes
...430-1-ea25da34-c91b-4a0f-a003-3958e87caffd.avro | Bin 0 -> 2172 bytes
...290-1-58a6295d-0076-4e3e-ac77-84ce48c406cf.avro | Bin 0 -> 3172 bytes
...861-1-2eb78a39-190a-4eab-b24a-df07f32f2cc0.avro | Bin 0 -> 2840 bytes
...049-1-b8fe0a34-e755-4ba0-92f1-c72bef85a82a.avro | Bin 0 -> 2506 bytes
.../metadata/v1.metadata.json | 15 +-
.../metadata/v2.metadata.json | 39 +--
.../metadata/v3.metadata.json | 65 ++---
.../metadata/v4.metadata.json | 142 ++++++++++
.../metadata/v5.metadata.json | 171 ++++++++++++
.../metadata/version-hint.text | 0
...8dcf5-a880-466d-ae9d-2dd332f98412-00001.parquet | Bin 0 -> 853 bytes
...8dcf5-a880-466d-ae9d-2dd332f98412-00002.parquet | Bin 0 -> 663 bytes
...90527-66f7-41de-bd3a-a6ef952918fc-00001.parquet | Bin 0 -> 853 bytes
...90527-66f7-41de-bd3a-a6ef952918fc-00002.parquet | Bin 0 -> 656 bytes
...b89d3-fcf6-4847-8fd1-68e5b33d0ad6-00001.parquet | Bin 0 -> 863 bytes
...b89d3-fcf6-4847-8fd1-68e5b33d0ad6-00002.parquet | Bin 0 -> 662 bytes
...fc20aca4-23ae231a00000001_738940911_data.0.parq | Bin 0 -> 1633 bytes
.../103b5b20-fb15-41bb-a97d-1e2ddc147650-m0.avro | Bin 0 -> 4053 bytes
.../c0500e2e-00c0-48fb-9c29-31bbafc91d57-m0.avro | Bin 0 -> 3728 bytes
.../c0500e2e-00c0-48fb-9c29-31bbafc91d57-m1.avro | Bin 0 -> 3714 bytes
.../d7fa3972-f84c-4b35-aa37-2079458ccea8-m0.avro | Bin 0 -> 3729 bytes
.../d7fa3972-f84c-4b35-aa37-2079458ccea8-m1.avro | Bin 0 -> 3715 bytes
.../f9fa006c-0078-4caf-8eaf-f9d499fc6939-m0.avro | Bin 0 -> 3728 bytes
.../f9fa006c-0078-4caf-8eaf-f9d499fc6939-m1.avro | Bin 0 -> 3714 bytes
...153-1-c0500e2e-00c0-48fb-9c29-31bbafc91d57.avro | Bin 0 -> 3164 bytes
...762-1-f9fa006c-0078-4caf-8eaf-f9d499fc6939.avro | Bin 0 -> 2336 bytes
...116-1-103b5b20-fb15-41bb-a97d-1e2ddc147650.avro | Bin 0 -> 2834 bytes
...435-1-d7fa3972-f84c-4b35-aa37-2079458ccea8.avro | Bin 0 -> 2668 bytes
.../metadata/v1.metadata.json | 32 ++-
.../metadata/v2.metadata.json | 87 +++++++
.../metadata/v3.metadata.json | 119 +++++++++
.../metadata/v4.metadata.json | 146 +++++++++++
.../metadata/v5.metadata.json | 178 +++++++++++++
.../metadata/version-hint.text | 0
.../functional/functional_schema_template.sql | 42 ++-
.../datasets/functional/schema_constraints.csv | 4 +-
.../queries/PlannerTest/iceberg-v2-tables.test | 288 +++++++++++++++++++++
.../iceberg-v2-read-equality-deletes.test | 62 +++++
tests/query_test/test_iceberg.py | 40 ---
75 files changed, 1562 insertions(+), 374 deletions(-)
delete mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/af4e128ee3256830-d9bd9e2f00000000_1372039299_data.0.parq
delete mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/delete-41417e7df44b347b-e035009600000001_138281890_data.0.parq
delete mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/delete-61438487836ebfcc-95c9ce7a00000000_909175610_data.0.parq
delete mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/2d3fafd7-bce6-483f-be82-e0ccce9203fc-m0.avro
delete mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/57a963d3-0e4e-4540-8080-a57afd51ba99-m0.avro
delete mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/8bd425d8-25fb-4603-8cc7-aeb5ad2a3917-m0.avro
delete mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-397031335297740726-1-2d3fafd7-bce6-483f-be82-e0ccce9203fc.avro
delete mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-6117850509763739078-1-57a963d3-0e4e-4540-8080-a57afd51ba99.avro
delete mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-8494861454990126958-1-8bd425d8-25fb-4603-8cc7-aeb5ad2a3917.avro
delete mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v4.metadata.json
delete mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/version-hint.text
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/data/00000-22-09b63f85-c950-4d32-a9fd-0abd5229d768-00001.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/data/00000-23-1365d982-ca94-4e1d-9b86-2f973c000cf0-00001.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/data/00000-23-1365d982-ca94-4e1d-9b86-2f973c000cf0-00002.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/data/00000-24-94d478ff-204b-4eb4-9a71-9876a31daf64-00001.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/data/00000-24-94d478ff-204b-4eb4-9a71-9876a31daf64-00002.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/data/00000-25-8814a344-4216-4d37-9e9b-091de336e3fa-00001.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/data/00000-25-8814a344-4216-4d37-9e9b-091de336e3fa-00002.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/2eb78a39-190a-4eab-b24a-df07f32f2cc0-m0.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/2eb78a39-190a-4eab-b24a-df07f32f2cc0-m1.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/58a6295d-0076-4e3e-ac77-84ce48c406cf-m0.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/58a6295d-0076-4e3e-ac77-84ce48c406cf-m1.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/b8fe0a34-e755-4ba0-92f1-c72bef85a82a-m0.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/b8fe0a34-e755-4ba0-92f1-c72bef85a82a-m1.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/ea25da34-c91b-4a0f-a003-3958e87caffd-m0.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/snap-2374780975027972430-1-ea25da34-c91b-4a0f-a003-3958e87caffd.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/snap-4077234998626563290-1-58a6295d-0076-4e3e-ac77-84ce48c406cf.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/snap-5777805847908928861-1-2eb78a39-190a-4eab-b24a-df07f32f2cc0.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/snap-8127619959873391049-1-b8fe0a34-e755-4ba0-92f1-c72bef85a82a.avro
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_different_equality_ids
=> iceberg_v2_delete_equality_multi_eq_ids}/metadata/v1.metadata.json (80%)
rename
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_different_equality_ids
=> iceberg_v2_delete_equality_multi_eq_ids}/metadata/v2.metadata.json (61%)
rename
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_different_equality_ids
=> iceberg_v2_delete_equality_multi_eq_ids}/metadata/v3.metadata.json (52%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/v4.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_multi_eq_ids/metadata/v5.metadata.json
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_equality_partitioned
=> iceberg_v2_delete_equality_multi_eq_ids}/metadata/version-hint.text (100%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/data/00000-0-7788dcf5-a880-466d-ae9d-2dd332f98412-00001.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/data/00000-0-7788dcf5-a880-466d-ae9d-2dd332f98412-00002.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/data/00000-0-ddf90527-66f7-41de-bd3a-a6ef952918fc-00001.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/data/00000-0-ddf90527-66f7-41de-bd3a-a6ef952918fc-00002.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/data/00000-0-e93b89d3-fcf6-4847-8fd1-68e5b33d0ad6-00001.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/data/00000-0-e93b89d3-fcf6-4847-8fd1-68e5b33d0ad6-00002.parquet
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/data/delete-3e480099fc20aca4-23ae231a00000001_738940911_data.0.parq
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/103b5b20-fb15-41bb-a97d-1e2ddc147650-m0.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/c0500e2e-00c0-48fb-9c29-31bbafc91d57-m0.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/c0500e2e-00c0-48fb-9c29-31bbafc91d57-m1.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/d7fa3972-f84c-4b35-aa37-2079458ccea8-m0.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/d7fa3972-f84c-4b35-aa37-2079458ccea8-m1.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/f9fa006c-0078-4caf-8eaf-f9d499fc6939-m0.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/f9fa006c-0078-4caf-8eaf-f9d499fc6939-m1.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/snap-152862018760071153-1-c0500e2e-00c0-48fb-9c29-31bbafc91d57.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/snap-2066775081852432762-1-f9fa006c-0078-4caf-8eaf-f9d499fc6939.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/snap-6283211732171745116-1-103b5b20-fb15-41bb-a97d-1e2ddc147650.avro
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/snap-7591397613223797435-1-d7fa3972-f84c-4b35-aa37-2079458ccea8.avro
rename
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_different_equality_ids
=> iceberg_v2_delete_pos_and_multi_eq_ids}/metadata/v1.metadata.json (50%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/v2.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/v3.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/v4.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_pos_and_multi_eq_ids/metadata/v5.metadata.json
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{iceberg_v2_delete_equality_partitioned
=> iceberg_v2_delete_pos_and_multi_eq_ids}/metadata/version-hint.text (100%)
