This is an automated email from the ASF dual-hosted git repository.
SteNicholas pushed a change to branch release-0.3
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
from d863d3b fix(release): make source archives reproducible (#177) (#178)
new 17056a4 fix: support nullable map key schemas and reject null key
data (#176)
new c0b8780 fix(parquet): reuse RowGroupPageIndexReader in
FileReaderWrapper layer (#166)
new 32f095f fix(parquet): allow reading nested list/map columns whose
leaf types differ only in representation (#172)
new c8ad743 feat: update commit message to version 12 (#179)
new fcad678 fix(parquet): avoid seeking into the middle of a row group
when prefetch and page index filtering are both enabled (#185)
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/source/user_guide/data_types.rst | 7 +-
docs/source/user_guide/schema.rst | 32 +---
.../reader/prefetch_file_batch_reader_impl.cpp | 25 ++-
.../reader/prefetch_file_batch_reader_impl.h | 5 +
src/paimon/common/types/data_type_json_parser.cpp | 14 +-
.../common/types/data_type_json_parser_test.cpp | 4 +-
src/paimon/core/index/global_index_meta.cpp | 39 +++-
src/paimon/core/index/global_index_meta.h | 8 +-
.../core/index/index_file_meta_serializer_test.cpp | 31 ++++
...ializer.h => index_file_meta_v4_deserializer.h} | 51 +++++-
src/paimon/core/schema/arrow_schema_validator.cpp | 4 -
src/paimon/core/schema/schema_validation_test.cpp | 10 +-
src/paimon/core/schema/table_schema_test.cpp | 17 +-
.../core/table/sink/commit_message_serializer.cpp | 8 +-
src/paimon/core/table/sink/commit_message_test.cpp | 35 +++-
src/paimon/format/avro/avro_file_batch_reader.cpp | 1 +
src/paimon/format/orc/orc_adapter.cpp | 1 +
src/paimon/format/parquet/column_index_filter.cpp | 12 +-
src/paimon/format/parquet/column_index_filter.h | 8 +-
.../format/parquet/column_index_filter_test.cpp | 49 +++--
src/paimon/format/parquet/file_reader_wrapper.cpp | 49 +++--
src/paimon/format/parquet/file_reader_wrapper.h | 9 +
.../format/parquet/file_reader_wrapper_test.cpp | 53 ++++++
.../parquet/page_filtered_row_group_reader.cpp | 24 +--
.../parquet/page_filtered_row_group_reader.h | 5 +-
.../page_filtered_row_group_reader_test.cpp | 58 ++++--
.../format/parquet/parquet_file_batch_reader.cpp | 92 ++++++++--
.../format/parquet/parquet_file_batch_reader.h | 7 +-
.../parquet/parquet_file_batch_reader_test.cpp | 150 ++++++++++++++++
test/inte/scan_and_read_inte_test.cpp | 44 +++++
test/inte/write_and_read_inte_test.cpp | 199 +++++++++++++++++++++
.../append_multiple/schema/schema-0 | 2 +-
.../append_simple.db/append_simple/schema/schema-0 | 2 +-
.../append_with_multiple_map/schema/schema-0 | 18 +-
.../pk_with_multiple_type/schema/schema-0 | 2 +-
.../schema/schema-0 | 2 +-
.../append_table_with_nested_type/schema/schema-0 | 2 +-
.../nullable_map_key.db/nullable_map_key/README | 13 ++
...data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc | Bin 0 -> 562 bytes
...anifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0} | Bin 2086 -> 2089 bytes
...st-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0} | Bin 1006 -> 1006 bytes
...st-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1} | Bin 1006 -> 1110 bytes
.../nullable_map_key}/schema/schema-0 | 15 +-
.../nullable_map_key}/snapshot/EARLIEST | 0
.../nullable_map_key}/snapshot/LATEST | 0
.../nullable_map_key/snapshot/snapshot-1 | 17 ++
.../pk_btree_source_meta/README | 32 ++++
...data-022e32ce-1d75-4b9a-8496-a3311c44bad0-0.orc | Bin 0 -> 694 bytes
...data-293d6705-bd19-4986-9aa0-3a24443e7e8e-0.orc | Bin 0 -> 722 bytes
...data-b4949ff6-78f4-4e58-86ad-33eb3e5e6749-0.orc | Bin 0 -> 694 bytes
...data-b5896b79-1c4d-4359-920a-ce719ea303af-0.orc | Bin 0 -> 713 bytes
...data-c2e3ff2f-a3e1-48a1-8bb1-f868477c796a-0.orc | Bin 0 -> 722 bytes
.../commit_messages/commit_messages-01 | Bin 0 -> 1633 bytes
.../index-05388eac-7c9d-48cf-9053-f68b800ebe9e-0 | Bin 0 -> 112 bytes
.../index-5d1ef2e5-c136-4059-8380-fe5d60344f83-0 | Bin 0 -> 112 bytes
.../index-759d0483-a0e7-417b-8c3d-288f36679770-0 | Bin 0 -> 96 bytes
...manifest-0216d085-645d-486b-9ebb-e48df26b8aa7-0 | Bin 0 -> 1970 bytes
...manifest-5ba1f8c6-fcde-45f8-b59b-a948ef3013be-0 | Bin 0 -> 1970 bytes
...manifest-d347fbd7-d7cc-4e5e-9f4a-efa8d0ce8651-0 | Bin 0 -> 1961 bytes
...manifest-180aa8f5-0932-462f-8f02-a1000b585552-0 | Bin 0 -> 3083 bytes
...manifest-e108640e-b3a6-499d-9821-fe36afde58d6-0 | Bin 0 -> 2815 bytes
...manifest-e108640e-b3a6-499d-9821-fe36afde58d6-1 | Bin 0 -> 3078 bytes
...manifest-fbd7c824-c952-4f2b-ad6c-0227a823a745-0 | Bin 0 -> 2833 bytes
...manifest-fbd7c824-c952-4f2b-ad6c-0227a823a745-1 | Bin 0 -> 3184 bytes
...est-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-0 | Bin 0 -> 392 bytes
...est-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-1 | Bin 0 -> 1533 bytes
...est-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-2 | Bin 0 -> 1533 bytes
...est-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-3 | Bin 0 -> 1543 bytes
...est-list-6678a969-de46-4cb3-b744-36a17ac35371-0 | Bin 0 -> 1603 bytes
...est-list-6678a969-de46-4cb3-b744-36a17ac35371-1 | Bin 0 -> 1529 bytes
...est-list-6678a969-de46-4cb3-b744-36a17ac35371-2 | Bin 0 -> 1713 bytes
...est-list-6678a969-de46-4cb3-b744-36a17ac35371-3 | Bin 0 -> 1548 bytes
...est-list-8e8a1e2e-9fbe-4fb9-920b-82edaed23f41-0 | Bin 0 -> 1723 bytes
...est-list-8e8a1e2e-9fbe-4fb9-920b-82edaed23f41-1 | Bin 0 -> 1554 bytes
.../pk_btree_source_meta/schema/schema-0 | 30 ++++
.../pk_btree_source_meta}/snapshot/EARLIEST | 0
.../pk_btree_source_meta}/snapshot/LATEST | 0
.../pk_btree_source_meta/snapshot/snapshot-1 | 17 ++
.../pk_btree_source_meta/snapshot/snapshot-2 | 18 ++
.../pk_btree_source_meta/snapshot/snapshot-3 | 18 ++
.../pk_btree_source_meta/snapshot/snapshot-4 | 18 ++
.../pk_btree_source_meta/snapshot/snapshot-5 | 18 ++
.../pk_table_nested_type/schema/schema-0 | 2 +-
.../schema/schema-0 | 2 +-
.../parquet_append_table/schema/schema-0 | 2 +-
.../pk_table_nested_type/schema/schema-0 | 2 +-
86 files changed, 1087 insertions(+), 196 deletions(-)
copy src/paimon/core/index/{index_file_meta_v3_deserializer.h =>
index_file_meta_v4_deserializer.h} (56%)
create mode 100644
test/test_data/orc/nullable_map_key.db/nullable_map_key/README
create mode 100644
test/test_data/orc/nullable_map_key.db/nullable_map_key/bucket-0/data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc
copy
test/test_data/{avro/append_with_multiple_map.db/append_with_multiple_map/manifest/manifest-3c01ce68-8b9e-4bc8-8a56-b481d26faab6-0
=>
orc/nullable_map_key.db/nullable_map_key/manifest/manifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0}
(91%)
copy
test/test_data/{avro/append_with_multiple_map.db/append_with_multiple_map/manifest/manifest-list-6a56727e-83dc-4b80-a420-86a036632e80-0
=>
orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0}
(95%)
copy
test/test_data/{avro/append_simple.db/append_simple/manifest/manifest-list-3602dcaf-7dc8-4879-9dce-9215f674e866-0
=>
orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1}
(86%)
copy
test/test_data/orc/{append_table_with_append_pt_branch.db/append_table_with_append_pt_branch
=> nullable_map_key.db/nullable_map_key}/schema/schema-0 (54%)
copy
test/test_data/{append_table_with_multiple_file_format.db/append_table_with_multiple_file_format
=> orc/nullable_map_key.db/nullable_map_key}/snapshot/EARLIEST (100%)
copy
test/test_data/{append_with_bitmap_multi_stripes.db/append_with_bitmap_multi_stripes
=> orc/nullable_map_key.db/nullable_map_key}/snapshot/LATEST (100%)
create mode 100644
test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/snapshot-1
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/README
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-022e32ce-1d75-4b9a-8496-a3311c44bad0-0.orc
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-293d6705-bd19-4986-9aa0-3a24443e7e8e-0.orc
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-b4949ff6-78f4-4e58-86ad-33eb3e5e6749-0.orc
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-b5896b79-1c4d-4359-920a-ce719ea303af-0.orc
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/bucket-0/data-c2e3ff2f-a3e1-48a1-8bb1-f868477c796a-0.orc
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/commit_messages/commit_messages-01
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-05388eac-7c9d-48cf-9053-f68b800ebe9e-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-5d1ef2e5-c136-4059-8380-fe5d60344f83-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/index/index-759d0483-a0e7-417b-8c3d-288f36679770-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-0216d085-645d-486b-9ebb-e48df26b8aa7-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-5ba1f8c6-fcde-45f8-b59b-a948ef3013be-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/index-manifest-d347fbd7-d7cc-4e5e-9f4a-efa8d0ce8651-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-180aa8f5-0932-462f-8f02-a1000b585552-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-e108640e-b3a6-499d-9821-fe36afde58d6-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-e108640e-b3a6-499d-9821-fe36afde58d6-1
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-fbd7c824-c952-4f2b-ad6c-0227a823a745-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-fbd7c824-c952-4f2b-ad6c-0227a823a745-1
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-1
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-2
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-3fe1dd64-b6ac-46dc-b968-1740e05383a2-3
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-6678a969-de46-4cb3-b744-36a17ac35371-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-6678a969-de46-4cb3-b744-36a17ac35371-1
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-6678a969-de46-4cb3-b744-36a17ac35371-2
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-6678a969-de46-4cb3-b744-36a17ac35371-3
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-8e8a1e2e-9fbe-4fb9-920b-82edaed23f41-0
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/manifest/manifest-list-8e8a1e2e-9fbe-4fb9-920b-82edaed23f41-1
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/schema/schema-0
copy
test/test_data/{append_table_with_multiple_file_format.db/append_table_with_multiple_file_format
=> orc/pk_btree_source_meta.db/pk_btree_source_meta}/snapshot/EARLIEST (100%)
copy test/test_data/orc/{append_09.db/append_09 =>
pk_btree_source_meta.db/pk_btree_source_meta}/snapshot/LATEST (100%)
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/snapshot-1
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/snapshot-2
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/snapshot-3
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/snapshot-4
create mode 100644
test/test_data/orc/pk_btree_source_meta.db/pk_btree_source_meta/snapshot/snapshot-5
