This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a change to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
from eaf7c8744b [core][python] Add optional Bloom filters to BTree indexes
(#9946)
new 7590416537 [core][spark][docs] Support row filters in full-text search
(#9855)
new 0a5678bfbe [core] Use a single BTree scan for bounded range predicates
(#9844)
new 48d2fc60ef [flink] Fix rollback_to procedure throwing
NullPointerException when neither snapshot_id nor tag is set (#9944)
new fa8ae00346 [core] Reject a non-parquet per-level or changelog format
for variant columns at DDL time (#9942)
new 512f285d1c [common] Infer a variant shredding schema regardless of the
writer's key order (#9940)
new 23633c3c44 [common] Convert variant timestamps with the requested zone
instead of the JVM default (#9939)
new 1e9c720f96 [python] Share final lookups across batch vector queries
(#9935)
new b9e8d12f6d [python] Keep batch vector scoring vectorized for null rows
(#9934)
new 0952d147bd [core] Preserve query authorization when narrowing chain
table splits (#9929)
new c0398c9d7e [python] Support distributed vector search with Ray (#9930)
new 2af7567e5f [core] Apply deletion vectors during data evolution file
index pushdown (#9922)
new a27b548472 [format] Use buffered encoder for Avro writes (#9928)
new f97545a0a6 [api] Sign x-acs-action in DLFOpenApiV4Signer (#9937)
new f8c9bb2dc8 [format] Simplify buffered Avro encoder selection
new 93b899b6b2 [core] Separate commit history bounds from strict mode
(#9960)
new e4f49ec969 [core] Request only top-k candidates from data-evolution
full-text indexes (#9955)
new 9d93f1bac8 [core] Fix OOM error handling in MergeTreeWriter and
improve resource cleanup (#9933)
new c2b6a0435c [python] Distribute vector search refinement with Ray
(#9951)
new a74d86960b [python] Add opt-in Rust native data reads (#9954)
new 6bfd8418dc [core] Optimize tag reads during snapshot expiration (#9967)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci-python.yml | 2 +-
docs/docs/concepts/rest/dlf.md | 6 +-
docs/docs/flink/procedures/snapshots.md | 6 +-
.../multimodal-table/global-index/full-text.mdx | 62 ++
.../global-index/hybrid-search.mdx | 12 +
docs/docs/primary-key-table/global-index.mdx | 4 +-
docs/docs/pypaimon/multimodal-search.md | 71 +++
docs/generated/core_configuration.html | 14 +-
.../main/java/org/apache/paimon/CoreOptions.java | 41 +-
.../apache/paimon/rest/auth/DLFOpenApiActions.java | 184 ++++++
.../paimon/rest/auth/DLFOpenApiV4Signer.java | 13 +
.../paimon/rest/auth/DLFOpenApiActionsTest.java | 141 +++++
.../paimon/rest/auth/DLFOpenApiV4SignerTest.java | 92 +++
.../paimon/data/variant/BaseVariantReader.java | 3 +-
.../data/variant/InferVariantShreddingSchema.java | 24 +-
.../org/apache/paimon/data/variant/VariantGet.java | 88 ++-
.../paimon/globalindex/GlobalIndexEvaluator.java | 69 ++-
.../paimon/globalindex/GlobalIndexReader.java | 22 +
.../globalindex/OffsetGlobalIndexReader.java | 7 +
.../globalindex/SortedFileGlobalIndexReader.java | 13 +-
.../paimon/globalindex/UnionGlobalIndexReader.java | 7 +
.../paimon/globalindex/btree/BTreeIndexReader.java | 5 +
.../globalindex/btree/LazyFilteredBTreeReader.java | 10 +
.../paimon/data/variant/GenericVariantTest.java | 81 +++
.../variant/InferVariantShreddingSchemaTest.java | 47 ++
.../data/variant/PaimonShreddingUtilsTest.java | 44 +-
.../globalindex/GlobalIndexEvaluatorTest.java | 48 ++
.../btree/LazyFilteredBTreeIndexReaderTest.java | 90 +++
.../TestFullTextGlobalIndexReader.java | 5 +
.../paimon/catalog/TableQueryAuthResult.java | 33 +
.../org/apache/paimon/io/FileIndexEvaluator.java | 47 +-
.../apache/paimon/io/RollingFileWriterImpl.java | 11 +-
.../apache/paimon/manifest/ManifestAvroReader.java | 12 +
.../org/apache/paimon/manifest/ManifestFile.java | 33 +-
.../paimon/operation/DataEvolutionSplitRead.java | 25 +-
.../apache/paimon/operation/FileDeletionBase.java | 145 ++++-
.../paimon/operation/FileStoreCommitImpl.java | 9 +-
.../apache/paimon/operation/SnapshotDeletion.java | 6 +-
.../paimon/operation/commit/StrictModeChecker.java | 18 +-
.../org/apache/paimon/schema/SchemaValidation.java | 46 ++
.../apache/paimon/table/ChainGroupReadTable.java | 93 ++-
.../paimon/table/ChainTableFileStoreTable.java | 5 +-
.../apache/paimon/table/ChainTableStreamScan.java | 49 +-
.../apache/paimon/table/ExpireSnapshotsImpl.java | 81 ++-
.../org/apache/paimon/table/PostponeUtils.java | 4 +-
.../paimon/table/source/AbstractDataTableScan.java | 22 +-
.../table/source/DataEvolutionFullTextRead.java | 221 ++++++-
.../table/source/DataEvolutionFullTextScan.java | 88 ++-
.../apache/paimon/table/source/DataTableScan.java | 12 +
.../paimon/table/source/FilteredRowIdReader.java | 132 ++++
.../paimon/table/source/FullTextSearchBuilder.java | 11 +
.../table/source/FullTextSearchBuilderImpl.java | 42 +-
.../table/source/HybridSearchBuilderImpl.java | 13 +-
.../table/source/IndexFullTextSearchSplit.java | 46 +-
.../apache/paimon/table/source/QueryAuthSplit.java | 10 +
.../paimon/table/source/RawFullTextReadImpl.java | 33 +-
.../java/org/apache/paimon/CoreOptionsTest.java | 32 +
.../paimon/catalog/TableQueryAuthResultTest.java | 85 +++
.../apache/paimon/io/FileIndexEvaluatorTest.java | 83 ++-
.../apache/paimon/io/RollingFileWriterTest.java | 57 ++
.../apache/paimon/manifest/ManifestFileTest.java | 44 ++
.../paimon/operation/ExpireSnapshotsTest.java | 153 ++++-
.../paimon/operation/FileStoreCommitTest.java | 39 ++
.../apache/paimon/schema/SchemaValidationTest.java | 63 ++
.../paimon/table/ChainTableFileStoreTableTest.java | 314 +++++++++-
.../paimon/table/DataEvolutionFileIndexTest.java | 68 +-
.../org/apache/paimon/table/PostponeUtilsTest.java | 2 +-
.../apache/paimon/table/sink/TableCommitTest.java | 67 +-
.../table/source/FullTextSearchBuilderTest.java | 687 ++++++++++++++++++++-
.../table/source/PrimaryKeyFullTextSearchTest.java | 32 +
.../flink/action/DataEvolutionMergeIntoAction.java | 2 +-
.../apache/paimon/flink/action/RescaleAction.java | 3 +-
.../dataevolution/DataEvolutionDeleteSink.java | 3 +-
.../flink/procedure/RollbackToProcedure.java | 7 +-
.../action/DeleteActionDataEvolutionITCase.java | 2 +-
.../flink/procedure/RollbackProcedureITCase.java | 74 +++
.../apache/paimon/format/avro/AvroFileFormat.java | 19 +-
.../paimon/format/avro/AvroFileFormatTest.java | 153 +++++
.../parquet/reader/VariantShreddingReadTest.java | 62 ++
.../writer/InferVariantShreddingWriteTest.java | 41 ++
.../index/NativeFullTextRowFilterTest.java | 517 ++++++++++++++++
paimon-python/README.md | 21 +
.../pypaimon/api/auth/dlf_openapi_actions.py | 103 +++
paimon-python/pypaimon/api/auth/dlf_signer.py | 8 +
.../pypaimon/common/options/core_options.py | 12 +
.../lumina/lumina_vector_global_index_reader.py | 2 +-
.../vindex/vindex_vector_global_index_reader.py | 2 +-
paimon-python/pypaimon/multimodal/query.py | 67 +-
paimon-python/pypaimon/ray/vector_search.py | 176 ++++++
paimon-python/pypaimon/read/native_plan.py | 81 ++-
paimon-python/pypaimon/read/table_read.py | 321 +++++++++-
paimon-python/pypaimon/read/table_scan.py | 13 +-
.../pypaimon/table/source/vector_search_read.py | 31 +-
.../pypaimon/tests/batch_vector_lookup_test.py | 181 ++++++
.../pypaimon/tests/data_evolution_formats_test.py | 4 +-
.../pypaimon/tests/data_evolution_test.py | 4 +-
.../pypaimon/tests/native_plan_integration_test.py | 373 ++++++++++-
paimon-python/pypaimon/tests/native_plan_test.py | 30 +-
paimon-python/pypaimon/tests/native_read_test.py | 349 +++++++++++
.../pypaimon/tests/ray_vector_refinement_test.py | 134 ++++
.../pypaimon/tests/ray_vector_search_test.py | 409 ++++++++++++
.../pypaimon/tests/rest/dlf_signer_test.py | 127 ++++
paimon-python/pypaimon/tests/table_update_test.py | 3 +-
.../tests/vector_metric_consistency_test.py | 23 +-
.../pypaimon/tests/vector_scoring_test.py | 50 ++
paimon-python/pypaimon/tests/vindex_input_test.py | 11 +-
.../paimon/spark/procedure/RescaleProcedure.java | 2 +-
.../org/apache/paimon/spark/PaimonBaseScan.scala | 4 +-
.../spark/SparkPostponeStagedCommitter.scala | 4 +-
.../paimon/spark/commands/PaimonSparkWriter.scala | 4 +-
.../paimon/spark/sql/FullTextSearchTest.scala | 228 +++++++
.../apache/paimon/spark/sql/HybridSearchTest.scala | 57 +-
.../apache/paimon/spark/sql/VariantTestBase.scala | 47 ++
113 files changed, 7558 insertions(+), 390 deletions(-)
create mode 100644
paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java
create mode 100644
paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/FilteredRowIdReader.java
create mode 100644
paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/NativeFullTextRowFilterTest.java
create mode 100644 paimon-python/pypaimon/api/auth/dlf_openapi_actions.py
create mode 100644 paimon-python/pypaimon/ray/vector_search.py
create mode 100644 paimon-python/pypaimon/tests/batch_vector_lookup_test.py
create mode 100644 paimon-python/pypaimon/tests/native_read_test.py
create mode 100644 paimon-python/pypaimon/tests/ray_vector_refinement_test.py
create mode 100644 paimon-python/pypaimon/tests/ray_vector_search_test.py
