This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a change to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
from eb43b419ce [python] Execute hybrid search routes concurrently (#9797)
new a535078fe6 [python] Preserve declared primary-key order in merge reads
(#9801)
new 237cde76ad [python] Reuse gathered window values across aggregations
(#9804)
new 41a4dd6d74 [python] Stream batch vector refinement candidates (#9806)
new 880fb02af0 [vortex] Do not push nanosecond timestamp literals that
overflow int64 (#9810)
new b540c00762 [common] Fix inferred variant shredding failing on decimals
with trailing zeros (#9811)
new 35a8e3dd79 [filesystem][s3] Serve s3a:// paths with the S3 plugin
(#9814)
new c4dff62194 [common] Read a shredded variant decimal as the same string
as an unshredded one (#9818)
new ffae3040b4 [python] Skip empty batches when advancing merge readers
(#9820)
new 900a760f6d [format] Validate ORC stripe metadata before reading (#9828)
new 9555f0e79c [python] Group LeRobot video frame decoding by payload
(#9799)
new d716a5e160 [python] Support selected-key projection for
shared-shredding MAP (#9789)
new 0c198a8c09 [python] Align native scan planning with Rust (#9825)
new 6e21f12d10 [python] Add block-based manifest Avro reader (#9826)
new 117e8070d0 [core] Expose Avro header and physical block locations
(#9835)
new 8790fdca94 [python] Align batch scan semantics with Java and extend
native planning (#9839)
new 2b29dfa1e7 [rest] Add generic entity labels to REST Catalog (#9838)
new bad0c3e011 [python] Decode LeRobot cameras in parallel (#9832)
new 99c6fdf00e [core] Support input streams in Avro block readers (#9842)
new 4def22901a [python] Expand native planning and align incremental reads
with Java (#9846)
new a5f3dd4b99 [rest] Add semantic view definition management to REST
Catalog (#9848)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci-python.yml | 6 +
docs/docs/concepts/rest/management-api.md | 108 ++++-
docs/docs/concepts/rest/rest-api.md | 10 +-
docs/docs/concepts/rest/semantic-views.md | 168 +++++++
docs/docs/maintenance/filesystems.mdx | 2 +-
docs/docs/program-api/rest-api.mdx | 33 ++
docs/docs/pypaimon/reading.md | 33 +-
docs/scripts/validate-rest-openapi.js | 110 +++++
docs/static/rest-catalog-open-api.yaml | 269 ++++++++++-
docs/static/rest-management-open-api.yaml | 225 ++++++++-
.../java/org/apache/paimon/management/Label.java | 58 +++
.../apache/paimon/management/LabelManagement.java | 68 +++
.../paimon/management/SemanticViewManagement.java | 58 +++
.../main/java/org/apache/paimon/rest/RESTApi.java | 149 ++++++
.../apache/paimon/rest/RESTLabelManagement.java | 78 ++++
.../paimon/rest/RESTSemanticViewManagement.java | 66 +++
.../java/org/apache/paimon/rest/ResourcePaths.java | 43 ++
...kSchemaRequest.java => UpsertLabelRequest.java} | 29 +-
...Request.java => UpsertSemanticViewRequest.java} | 28 +-
.../paimon/rest/responses/ErrorResponse.java | 3 +
.../paimon/rest/responses/GetLabelResponse.java | 83 ++++
.../GetSemanticViewResponse.java} | 38 +-
...liciesResponse.java => ListLabelsResponse.java} | 39 +-
...esponse.java => ListSemanticViewsResponse.java} | 33 +-
.../PolicyType.java => view/SemanticView.java} | 27 +-
.../apache/paimon/view/SemanticViewDefinition.java | 84 ++++
.../org/apache/paimon/rest/RESTApiLabelTest.java | 392 ++++++++++++++++
.../paimon/rest/RESTApiSemanticViewTest.java | 420 +++++++++++++++++
.../requests/RequestJacksonCompatibilityTest.java | 8 +-
.../paimon/data/variant/BaseVariantReader.java | 7 +-
.../data/variant/InferVariantShreddingSchema.java | 17 +-
.../org/apache/paimon/data/variant/VariantGet.java | 37 +-
.../apache/paimon/fs/ByteArraySeekableStream.java | 2 +-
.../variant/InferVariantShreddingSchemaTest.java | 32 ++
.../data/variant/PaimonShreddingUtilsTest.java | 49 ++
.../paimon/fs/ByteArraySeekableStreamTest.java | 17 +-
.../apache/paimon/manifest/ManifestAvroReader.java | 18 +
.../java/org/apache/paimon/rest/RESTCatalog.java | 13 +
.../apache/paimon/manifest/ManifestFileTest.java | 22 +-
.../rest/RESTCatalogLabelManagementTest.java | 236 ++++++++++
.../RESTCatalogSemanticViewManagementTest.java | 156 +++++++
paimon-filesystems/paimon-s3/pom.xml | 14 +
.../main/java/org/apache/paimon/s3/S3ALoader.java | 21 +-
.../services/org.apache.paimon.fs.FileIOLoader | 1 +
.../java/org/apache/paimon/s3/S3ASchemeTest.java | 151 ++++++
.../java/org/apache/avro/file/RawBlockReader.java | 86 +++-
.../java/org/apache/orc/impl/RecordReaderImpl.java | 44 ++
.../apache/paimon/format/avro/AvroBlockReader.java | 18 +
.../org/apache/orc/impl/RecordReaderImplTest.java | 56 +++
.../paimon/format/avro/AvroBlockReaderTest.java | 221 +++++++++
.../parquet/reader/VariantShreddingReadTest.java | 52 +++
.../writer/InferVariantShreddingWriteTest.java | 33 ++
paimon-python/README.md | 71 +++
paimon-python/conftest.py | 5 +-
paimon-python/dev/run_mixed_tests.sh | 3 +-
.../pypaimon/common/options/core_options.py | 42 +-
.../pypaimon/data/map_shared_shredding.py | 262 ++++++++++-
.../pypaimon/globalindex/indexed_split.py | 16 +
.../pypaimon/manifest/manifest_file_manager.py | 107 ++++-
.../pypaimon/multimodal/lerobot/dataset.py | 36 +-
paimon-python/pypaimon/multimodal/temporal.py | 40 +-
paimon-python/pypaimon/multimodal/video.py | 38 +-
paimon-python/pypaimon/read/interval_partition.py | 7 +
paimon-python/pypaimon/read/native_plan.py | 63 ++-
paimon-python/pypaimon/read/query_auth_split.py | 4 +
paimon-python/pypaimon/read/read_builder.py | 180 ++++++--
.../pypaimon/read/reader/data_file_batch_reader.py | 35 ++
.../pypaimon/read/reader/format_pyarrow_reader.py | 102 ++++-
.../read/reader/nested_leaf_batch_reader.py | 17 +-
.../pypaimon/read/reader/sort_merge_reader.py | 23 +-
paimon-python/pypaimon/read/scan_distribution.py | 72 +++
.../read/scanner/append_table_split_generator.py | 128 +-----
.../read/scanner/data_evolution_split_generator.py | 16 +-
.../pypaimon/read/scanner/file_scanner.py | 87 ++--
.../pypaimon/read/scanner/split_generator.py | 24 +-
paimon-python/pypaimon/read/sliced_split.py | 10 +
paimon-python/pypaimon/read/split.py | 7 +-
paimon-python/pypaimon/read/split_read.py | 68 ++-
paimon-python/pypaimon/read/split_serializer.py | 3 +-
.../pypaimon/read/streaming_table_scan.py | 24 +-
paimon-python/pypaimon/read/table_read.py | 78 +++-
paimon-python/pypaimon/read/table_scan.py | 192 +++++---
.../pypaimon/snapshot/time_travel_util.py | 19 +
paimon-python/pypaimon/table/file_store_table.py | 5 +-
.../source/primary_key_sorted_index_result.py | 2 +
.../pypaimon/table/source/vector_search_read.py | 69 ++-
.../pypaimon/tests/batch_scan_level_filter_test.py | 154 +++++++
.../tests/batch_vector_streaming_refine_test.py | 266 +++++++++++
.../pypaimon/tests/data_evolution_formats_test.py | 111 +++++
.../tests/data_evolution_split_generator_test.py | 7 +
.../pypaimon/tests/deletion_vector_path_test.py | 457 +++++++++++++++++++
.../pypaimon/tests/e2e/java_py_read_write_test.py | 26 ++
.../format_pyarrow_shared_shredding_map_test.py | 295 ++++++++++--
paimon-python/pypaimon/tests/global_index_test.py | 5 +
.../pypaimon/tests/interval_partition_test.py | 86 ++++
.../tests/manifest/manifest_manager_test.py | 135 +++++-
.../tests/map_selected_key_projection_test.py | 506 +++++++++++++++++++++
.../pypaimon/tests/multimodal_lerobot_test.py | 56 +++
.../pypaimon/tests/multimodal_temporal_test.py | 68 ++-
.../pypaimon/tests/multimodal_video_test.py | 80 ++++
.../tests/native_plan_capabilities_test.py | 354 ++++++++++++++
.../tests/native_plan_distribution_test.py | 364 +++++++++++++++
.../tests/native_plan_dynamic_bucket_test.py | 101 ++++
.../pypaimon/tests/native_plan_expanded_test.py | 279 ++++++++++++
.../pypaimon/tests/native_plan_incremental_test.py | 358 +++++++++++++++
.../pypaimon/tests/native_plan_integration_test.py | 90 +++-
paimon-python/pypaimon/tests/native_plan_test.py | 299 +++++++++---
.../pypaimon/tests/reader_primary_key_test.py | 60 ++-
.../tests/schema_evolution_pk_read_test.py | 36 +-
.../pypaimon/tests/split_serializer_test.py | 13 +
.../pypaimon/tests/table_query_auth_test.py | 1 +
paimon-python/pypaimon/tests/test_first_row_e2e.py | 9 +-
.../pypaimon/tests/test_nested_projection_e2e.py | 56 ++-
.../pypaimon/tests/test_partial_update_e2e.py | 4 +-
.../tests/test_read_builder_nested_projection.py | 24 +-
.../pypaimon/tests/test_sort_merge_reader.py | 85 ++++
.../pypaimon/utils/file_store_path_factory.py | 179 +++++++-
paimon-python/pypaimon/utils/projection.py | 59 ++-
paimon-python/pypaimon/write/file_store_commit.py | 16 +-
paimon-python/pypaimon/write/table_delete.py | 13 +-
.../apache/paimon/spark/sql/VariantTestBase.scala | 30 ++
.../format/vortex/VortexPredicateConverter.java | 18 +-
.../vortex/VortexPredicateConverterTest.java | 21 +
123 files changed, 9817 insertions(+), 730 deletions(-)
create mode 100644 docs/docs/concepts/rest/semantic-views.md
create mode 100644
paimon-api/src/main/java/org/apache/paimon/management/Label.java
create mode 100644
paimon-api/src/main/java/org/apache/paimon/management/LabelManagement.java
create mode 100644
paimon-api/src/main/java/org/apache/paimon/management/SemanticViewManagement.java
create mode 100644
paimon-api/src/main/java/org/apache/paimon/rest/RESTLabelManagement.java
create mode 100644
paimon-api/src/main/java/org/apache/paimon/rest/RESTSemanticViewManagement.java
copy
paimon-api/src/main/java/org/apache/paimon/rest/requests/{RollbackSchemaRequest.java
=> UpsertLabelRequest.java} (64%)
copy
paimon-api/src/main/java/org/apache/paimon/rest/requests/{RollbackSchemaRequest.java
=> UpsertSemanticViewRequest.java} (61%)
create mode 100644
paimon-api/src/main/java/org/apache/paimon/rest/responses/GetLabelResponse.java
copy
paimon-api/src/main/java/org/apache/paimon/rest/{requests/CreateDatabaseRequest.java
=> responses/GetSemanticViewResponse.java} (57%)
copy
paimon-api/src/main/java/org/apache/paimon/rest/responses/{ListPoliciesResponse.java
=> ListLabelsResponse.java} (68%)
copy
paimon-api/src/main/java/org/apache/paimon/rest/responses/{ListPermissionsResponse.java
=> ListSemanticViewsResponse.java} (67%)
copy paimon-api/src/main/java/org/apache/paimon/{management/PolicyType.java =>
view/SemanticView.java} (63%)
create mode 100644
paimon-api/src/main/java/org/apache/paimon/view/SemanticViewDefinition.java
create mode 100644
paimon-api/src/test/java/org/apache/paimon/rest/RESTApiLabelTest.java
create mode 100644
paimon-api/src/test/java/org/apache/paimon/rest/RESTApiSemanticViewTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogLabelManagementTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogSemanticViewManagementTest.java
copy
paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIOLoader.java
=>
paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3ALoader.java
(72%)
create mode 100644
paimon-filesystems/paimon-s3/src/test/java/org/apache/paimon/s3/S3ASchemeTest.java
create mode 100644
paimon-format/src/test/java/org/apache/paimon/format/avro/AvroBlockReaderTest.java
create mode 100644 paimon-python/pypaimon/read/scan_distribution.py
create mode 100644 paimon-python/pypaimon/tests/batch_scan_level_filter_test.py
create mode 100644
paimon-python/pypaimon/tests/batch_vector_streaming_refine_test.py
create mode 100644 paimon-python/pypaimon/tests/deletion_vector_path_test.py
create mode 100644 paimon-python/pypaimon/tests/interval_partition_test.py
create mode 100644
paimon-python/pypaimon/tests/map_selected_key_projection_test.py
create mode 100644
paimon-python/pypaimon/tests/native_plan_capabilities_test.py
create mode 100644
paimon-python/pypaimon/tests/native_plan_distribution_test.py
create mode 100644
paimon-python/pypaimon/tests/native_plan_dynamic_bucket_test.py
create mode 100644 paimon-python/pypaimon/tests/native_plan_expanded_test.py
create mode 100644 paimon-python/pypaimon/tests/native_plan_incremental_test.py
create mode 100644 paimon-python/pypaimon/tests/test_sort_merge_reader.py
