This is an automated email from the ASF dual-hosted git repository.
boroknagyz pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 7450c96 IMPALA-11133 (Addendum): Encode a string in utf8 before
printing it
new c10e951 IMPALA-11053: Impala should be able to read migrated
partitioned Iceberg tables
new 2d6300a IMPALA-10999 Flakiness in TestAsyncLoadData.test_async_load
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/CMakeLists.txt | 1 +
be/src/exec/file-metadata-utils.cc | 143 ++++++++++++++++
be/src/exec/file-metadata-utils.h | 56 ++++++
be/src/exec/hdfs-orc-scanner.cc | 3 +-
be/src/exec/hdfs-scan-node-base.cc | 7 +
be/src/exec/hdfs-scan-node-base.h | 7 +
be/src/exec/hdfs-scanner.cc | 13 +-
be/src/exec/hdfs-scanner.h | 4 +
be/src/exec/orc-column-readers.cc | 14 +-
be/src/exec/parquet/hdfs-parquet-scanner.cc | 3 +-
be/src/exec/parquet/parquet-metadata-utils.h | 3 +
be/src/runtime/dml-exec-state.cc | 2 +-
be/src/scheduling/scheduler.cc | 6 +
common/fbs/CatalogObjects.fbs | 8 +
common/fbs/IcebergObjects.fbs | 33 +++-
common/protobuf/planner.proto | 1 +
common/thrift/CatalogObjects.thrift | 4 +
common/thrift/PlanNodes.thrift | 1 +
.../org/apache/impala/catalog/FeIcebergTable.java | 14 +-
.../org/apache/impala/catalog/HdfsPartition.java | 32 +++-
.../java/org/apache/impala/catalog/HdfsTable.java | 5 +-
.../org/apache/impala/planner/HdfsScanNode.java | 3 +
.../org/apache/impala/planner/IcebergScanNode.java | 3 +
.../java/org/apache/impala/util/IcebergUtil.java | 131 +++++++++++++-
testdata/data/README | 23 ++-
.../283c54cb-5a45-4a2c-bca8-4bfa0e61cdbd-m0.avro | Bin 0 -> 3926 bytes
...21-1-283c54cb-5a45-4a2c-bca8-4bfa0e61cdbd.avro} | Bin 1874 -> 1999 bytes
.../metadata/v1.metadata.json | 164 ++++++++++++++++++
.../metadata/v2.metadata.json | 188 +++++++++++++++++++++
.../p_date=2022-02-22/p_string=impala/000000_0 | Bin 0 -> 433 bytes
.../db72fbf2-f9f6-4985-8a5f-fd9f632f2c77-m0.avro | Bin 0 -> 3925 bytes
...30-1-db72fbf2-f9f6-4985-8a5f-fd9f632f2c77.avro} | Bin 1874 -> 2003 bytes
.../metadata/v1.metadata.json | 164 ++++++++++++++++++
.../metadata/v2.metadata.json | 188 +++++++++++++++++++++
.../metadata/version-hint.text | 0
.../p_date=2022-02-22/p_string=impala/000000_0 | Bin 0 -> 189 bytes
.../2d05a7d4-c229-44c3-860e-e77e46e71a19-m0.avro | Bin 0 -> 3691 bytes
...86-1-2d05a7d4-c229-44c3-860e-e77e46e71a19.avro} | Bin 1874 -> 1938 bytes
.../metadata/v1.metadata.json | 89 ++++++++++
.../metadata/v2.metadata.json | 116 +++++++++++++
.../metadata/version-hint.text | 0
.../p_float_double=1.1/p_dec_dec=2.718/000000_0 | Bin 0 -> 429 bytes
.../p_float_double=1.1/p_dec_dec=3.141/000000_0 | Bin 0 -> 429 bytes
.../8db62f0e-38e5-434b-94dc-c84210302ad8-m0.avro | Bin 0 -> 3687 bytes
...46-1-8db62f0e-38e5-434b-94dc-c84210302ad8.avro} | Bin 1874 -> 1941 bytes
.../metadata/v1.metadata.json | 89 ++++++++++
.../metadata/v2.metadata.json | 116 +++++++++++++
.../metadata/version-hint.text | 0
.../p_float_double=1.1/p_dec_dec=2.718/000000_0 | Bin 0 -> 189 bytes
.../p_float_double=1.1/p_dec_dec=3.141/000000_0 | Bin 0 -> 189 bytes
.../functional/functional_schema_template.sql | 56 ++++++
.../datasets/functional/schema_constraints.csv | 4 +
.../queries/QueryTest/iceberg-migrated-tables.test | 106 ++++++++++++
tests/metadata/test_load.py | 23 ++-
tests/query_test/test_iceberg.py | 3 +
55 files changed, 1791 insertions(+), 35 deletions(-)
create mode 100644 be/src/exec/file-metadata-utils.cc
create mode 100644 be/src/exec/file-metadata-utils.h
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_alltypes_part/metadata/283c54cb-5a45-4a2c-bca8-4bfa0e61cdbd-m0.avro
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{airports_orc/metadata/snap-4990977953383402321-1-1ebf435e-7da7-41e7-bebf-eb3ebf1b1002.avro
=>
iceberg_alltypes_part/metadata/snap-6167994413873848621-1-283c54cb-5a45-4a2c-bca8-4bfa0e61cdbd.avro}
(83%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_alltypes_part/metadata/v1.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_alltypes_part/metadata/v2.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_alltypes_part/p_bool=true/p_int=1/p_bigint=11/p_float=1.1/p_double=2.222/p_decimal=123.321/p_date=2022-02-22/p_string=impala/000000_0
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_alltypes_part_orc/metadata/db72fbf2-f9f6-4985-8a5f-fd9f632f2c77-m0.avro
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{airports_orc/metadata/snap-4990977953383402321-1-1ebf435e-7da7-41e7-bebf-eb3ebf1b1002.avro
=>
iceberg_alltypes_part_orc/metadata/snap-7569365419257304230-1-db72fbf2-f9f6-4985-8a5f-fd9f632f2c77.avro}
(83%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_alltypes_part_orc/metadata/v1.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_alltypes_part_orc/metadata/v2.metadata.json
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{complextypestbl_iceberg_orc =>
iceberg_alltypes_part_orc}/metadata/version-hint.text (100%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_alltypes_part_orc/p_bool=true/p_int=1/p_bigint=11/p_float=1.1/p_double=2.222/p_decimal=123.321/p_date=2022-02-22/p_string=impala/000000_0
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_legacy_partition_schema_evolution/metadata/2d05a7d4-c229-44c3-860e-e77e46e71a19-m0.avro
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{airports_orc/metadata/snap-4990977953383402321-1-1ebf435e-7da7-41e7-bebf-eb3ebf1b1002.avro
=>
iceberg_legacy_partition_schema_evolution/metadata/snap-6654673546382518186-1-2d05a7d4-c229-44c3-860e-e77e46e71a19.avro}
(85%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_legacy_partition_schema_evolution/metadata/v1.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_legacy_partition_schema_evolution/metadata/v2.metadata.json
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{complextypestbl_iceberg_orc =>
iceberg_legacy_partition_schema_evolution}/metadata/version-hint.text (100%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_legacy_partition_schema_evolution/p_int_long=1/p_float_double=1.1/p_dec_dec=2.718/000000_0
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_legacy_partition_schema_evolution/p_int_long=1/p_float_double=1.1/p_dec_dec=3.141/000000_0
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_legacy_partition_schema_evolution_orc/metadata/8db62f0e-38e5-434b-94dc-c84210302ad8-m0.avro
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{airports_orc/metadata/snap-4990977953383402321-1-1ebf435e-7da7-41e7-bebf-eb3ebf1b1002.avro
=>
iceberg_legacy_partition_schema_evolution_orc/metadata/snap-888589552112488046-1-8db62f0e-38e5-434b-94dc-c84210302ad8.avro}
(85%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_legacy_partition_schema_evolution_orc/metadata/v1.metadata.json
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_legacy_partition_schema_evolution_orc/metadata/v2.metadata.json
copy
testdata/data/iceberg_test/hadoop_catalog/ice/{complextypestbl_iceberg_orc =>
iceberg_legacy_partition_schema_evolution_orc}/metadata/version-hint.text (100%)
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_legacy_partition_schema_evolution_orc/p_int_long=1/p_float_double=1.1/p_dec_dec=2.718/000000_0
create mode 100644
testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_legacy_partition_schema_evolution_orc/p_int_long=1/p_float_double=1.1/p_dec_dec=3.141/000000_0
create mode 100644
testdata/workloads/functional-query/queries/QueryTest/iceberg-migrated-tables.test
