This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git
from 05ad2a058a [core] Fix that cannot read binlog table with projection
(#6417)
new a214e7176e [python] support blob type and blob write and read (#6390)
new 0e05b7eb15 [Python] Enable field merge read in row-tracking table
(#6399)
new 8bda95e154 [Python] Introduce incremental-between read by timestamp
(#6391)
new fe68d21324 [core]Python: fix blob write when blob_as_descriptor is
true (#6404)
new 462c4203ac [python] Support blob read && write (#6420)
new 82fe60a684 [python] Blob type more test for descriptor (#6422)
new e036dec7e2 [python] Make FileStoreWrite.max_seq_numbers lazied (#6418)
new 3d1b30f003 [python] Filter manifest files by partition predicate in
scan (#6419)
new fa4671c59c [python] Filter manifest entry by advance to reduce memory
(#6428)
new d45886ba40 [Python] optimize codes related to push_down_utils (#6430)
new 59c6ae019a [flink] disable clustering during writing if incremental
clustering enabled (#6432)
new 71128279c9 [python] Drop stats for manifest entries reading (#6429)
new 139f65cc8f [Python] clean code for pypaimon (#6433)
new 11360cf31c [python] Refactor dicts to static fields to improve
performance (#6436)
new 94644ac287 [flink] Produce real random id in SourceSplitGenerator
(#6441)
new 10a71b1d07 [Python] SimpleStats supports BinaryRow (#6444)
new 445b217d1e [Python] Refactor BinaryRow to reuse keys and key fields
(#6445)
new 8ef5214160 [Python] filter_manifest_entry should not evolution primary
keys
new 80ac5e2b45 [Python] Remove useless TODO in SimpleStats
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/content/program-api/python-api.md | 192 ++-
.../org/apache/paimon/table/source/DataSplit.java | 16 +
.../paimon/flink/sink/FlinkTableSinkBase.java | 15 +-
.../paimon/flink/source/FileStoreSourceSplit.java | 13 +
.../source/FileStoreSourceSplitGenerator.java | 37 +-
.../source/FileStoreSourceSplitGeneratorTest.java | 32 +-
paimon-python/pypaimon/__init__.py | 2 +-
paimon-python/pypaimon/common/config.py | 1 +
paimon-python/pypaimon/common/core_options.py | 9 +
.../pypaimon/common/delta_varint_compressor.py | 125 ++
paimon-python/pypaimon/common/file_io.py | 63 +-
paimon-python/pypaimon/common/predicate.py | 432 ++++---
paimon-python/pypaimon/common/predicate_builder.py | 14 +-
paimon-python/pypaimon/common/uri_reader.py | 171 +++
paimon-python/pypaimon/filesystem/pvfs.py | 2 +-
.../pypaimon/manifest/manifest_file_manager.py | 49 +-
.../pypaimon/manifest/manifest_list_manager.py | 15 +-
.../pypaimon/manifest/schema/data_file_meta.py | 87 +-
.../pypaimon/manifest/schema/manifest_entry.py | 30 +
.../pypaimon/manifest/schema/simple_stats.py | 20 +-
.../pypaimon/manifest/simple_stats_evolution.py | 123 ++
.../pypaimon/manifest/simple_stats_evolutions.py | 75 ++
paimon-python/pypaimon/read/push_down_utils.py | 76 +-
paimon-python/pypaimon/read/read_builder.py | 3 +-
.../pypaimon/read/reader/concat_batch_reader.py | 68 ++
.../read/reader/data_evolution_merge_reader.py | 85 ++
..._record_reader.py => data_file_batch_reader.py} | 0
paimon-python/pypaimon/read/reader/field_bunch.py | 120 ++
.../pypaimon/read/reader/format_blob_reader.py | 189 +++
.../pypaimon/{catalog => read/scanner}/__init__.py | 0
.../scanner/empty_starting_scanner.py} | 11 +-
.../full_starting_scanner.py} | 284 +++--
.../read/scanner/incremental_starting_scanner.py | 69 ++
.../scanner/starting_scanner.py} | 10 +-
paimon-python/pypaimon/read/split_read.py | 253 +++-
paimon-python/pypaimon/read/table_read.py | 32 +-
paimon-python/pypaimon/read/table_scan.py | 361 +-----
paimon-python/pypaimon/schema/data_types.py | 6 +
paimon-python/pypaimon/schema/schema.py | 36 +-
paimon-python/pypaimon/schema/schema_manager.py | 4 +-
paimon-python/pypaimon/schema/table_schema.py | 25 -
paimon-python/pypaimon/snapshot/snapshot.py | 1 +
.../pypaimon/snapshot/snapshot_manager.py | 55 +
paimon-python/pypaimon/table/file_store_table.py | 23 +
paimon-python/pypaimon/table/row/binary_row.py | 58 +
paimon-python/pypaimon/table/row/blob.py | 250 ++++
paimon-python/pypaimon/table/row/generic_row.py | 66 +-
paimon-python/pypaimon/table/row/internal_row.py | 6 -
paimon-python/pypaimon/table/row/offset_row.py | 15 +-
paimon-python/pypaimon/table/row/projected_row.py | 76 ++
paimon-python/pypaimon/tests/binary_row_test.py | 334 +++++
paimon-python/pypaimon/tests/blob_table_test.py | 1277 ++++++++++++++++++++
paimon-python/pypaimon/tests/blob_test.py | 1066 ++++++++++++++++
.../pypaimon/tests/data_evolution_test.py | 483 ++++++++
.../pypaimon/tests/delta_varint_compressor_test.py | 379 ++++++
.../pypaimon/tests/file_store_commit_test.py | 24 +-
.../pypaimon/tests/filesystem_catalog_test.py | 3 +-
paimon-python/pypaimon/tests/predicates_test.py | 26 +-
.../pypaimon/tests/py36/ao_predicate_test.py | 8 +-
.../pypaimon/tests/py36/ao_simple_test.py | 6 +-
.../pypaimon/tests/py36/data_evolution_test.py | 483 ++++++++
.../pypaimon/tests/py36/reader_predicate_test.py | 93 ++
.../pypaimon/tests/py36/rest_ao_read_write_test.py | 89 +-
.../pypaimon/tests/reader_append_only_test.py | 71 +-
paimon-python/pypaimon/tests/reader_base_test.py | 71 +-
.../pypaimon/tests/reader_predicate_test.py | 93 ++
.../pypaimon/tests/reader_primary_key_test.py | 79 +-
.../pypaimon/tests/rest/rest_base_test.py | 3 +-
.../pypaimon/tests/rest/rest_read_write_test.py | 2 +-
.../pypaimon/tests/uri_reader_factory_test.py | 227 ++++
paimon-python/pypaimon/write/batch_table_write.py | 25 +-
paimon-python/pypaimon/write/blob_format_writer.py | 107 ++
paimon-python/pypaimon/write/file_store_commit.py | 90 +-
paimon-python/pypaimon/write/file_store_write.py | 77 +-
.../writer/blob_writer.py} | 40 +-
.../pypaimon/write/writer/data_blob_writer.py | 321 +++++
paimon-python/pypaimon/write/writer/data_writer.py | 90 +-
.../pypaimon/write/writer/key_value_data_writer.py | 8 +-
78 files changed, 8283 insertions(+), 997 deletions(-)
create mode 100644 paimon-python/pypaimon/common/delta_varint_compressor.py
create mode 100644 paimon-python/pypaimon/common/uri_reader.py
create mode 100644 paimon-python/pypaimon/manifest/simple_stats_evolution.py
create mode 100644 paimon-python/pypaimon/manifest/simple_stats_evolutions.py
create mode 100644
paimon-python/pypaimon/read/reader/data_evolution_merge_reader.py
rename paimon-python/pypaimon/read/reader/{data_file_record_reader.py =>
data_file_batch_reader.py} (100%)
create mode 100644 paimon-python/pypaimon/read/reader/field_bunch.py
create mode 100644 paimon-python/pypaimon/read/reader/format_blob_reader.py
copy paimon-python/pypaimon/{catalog => read/scanner}/__init__.py (100%)
copy paimon-python/pypaimon/{catalog/catalog_loader.py =>
read/scanner/empty_starting_scanner.py} (79%)
copy paimon-python/pypaimon/read/{table_scan.py =>
scanner/full_starting_scanner.py} (59%)
create mode 100644
paimon-python/pypaimon/read/scanner/incremental_starting_scanner.py
copy paimon-python/pypaimon/{catalog/catalog_loader.py =>
read/scanner/starting_scanner.py} (81%)
create mode 100644 paimon-python/pypaimon/table/row/binary_row.py
create mode 100644 paimon-python/pypaimon/table/row/blob.py
create mode 100644 paimon-python/pypaimon/table/row/projected_row.py
create mode 100644 paimon-python/pypaimon/tests/binary_row_test.py
create mode 100644 paimon-python/pypaimon/tests/blob_table_test.py
create mode 100644 paimon-python/pypaimon/tests/blob_test.py
create mode 100644 paimon-python/pypaimon/tests/data_evolution_test.py
create mode 100644 paimon-python/pypaimon/tests/delta_varint_compressor_test.py
create mode 100644 paimon-python/pypaimon/tests/py36/data_evolution_test.py
create mode 100644 paimon-python/pypaimon/tests/py36/reader_predicate_test.py
create mode 100644 paimon-python/pypaimon/tests/reader_predicate_test.py
create mode 100644 paimon-python/pypaimon/tests/uri_reader_factory_test.py
create mode 100644 paimon-python/pypaimon/write/blob_format_writer.py
copy paimon-python/pypaimon/{read/partition_info.py =>
write/writer/blob_writer.py} (51%)
create mode 100644 paimon-python/pypaimon/write/writer/data_blob_writer.py
