This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a change to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
from b6d8a70499 [core] Fix the failing DataEvolutionUtilsTest (#9911)
new e0233b7ffc [core] Skip BTree index reads for all-matching predicates
(#9834)
new c9193cabcf [python] Add manifest sidecar codec and block-pruned reads
(#9908)
new ef9423f555 [format] Drop filters on unprojected columns before Parquet
conversion (#9858)
new b946d37d8a [python] Keep local search and lookup on one snapshot
(#9843)
new 6a9c019e2e [core][flink][spark] Add API for top partitions (#9914)
new 66e400f321 [python] Preserve manifest sidecar references on commit
retries (#9921)
new ae04d95ffd [python] Clean up normal files when composite writer
metadata creation fails (#9917)
new 4750a774fe [format] Support file.block-size for Avro data file (#9910)
new bcd1681e74 [core] Add option to disable overflow exception in
sum/product agg (#9874)
new 19f0453a09 [python] Support allow_exact_matches in temporal as-of
joins (#9915)
new 93fa0a5b4f [core] Fall back to raw vector search when scalar index
cannot evaluate filter (#9909)
new eccfcb4386 [common] Keep blank variant object keys unshredded during
schema inference (#9813)
new b2a8ab824c [spark] Support remove_orphan_blobs procedure. (#9812)
new e69c4c8689 [common] Reject out-of-range integral casts when extracting
a variant scalar (#9809)
new dd4837d595 [flink] Scope scan.parallelism to source operators (#9803)
new cc21db42e8 [python] Build global index shards concurrently (#9798)
new 917601f87c [api] Add an opt-in DLFOpenApiV4Signer for DlfNext
endpoints (#9918)
new b09fafcd1f [core] Preserve query authorization when wrapping data
evolution index splits (#9925)
new 127c26452d [core] Reuse sidecar block cache for unfiltered scans
(#9924)
new eaf7c8744b [core][python] Add optional Bloom filters to BTree indexes
(#9946)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/docs/concepts/rest/dlf.md | 9 +-
docs/docs/concepts/spec/manifest.md | 15 +-
docs/docs/multimodal-table/global-index/btree.mdx | 1 +
.../global-index/manage-indexes.mdx | 1 +
docs/docs/primary-key-table/blob-storage.md | 4 +-
.../primary-key-table/merge-engine/aggregation.mdx | 6 +
docs/docs/pypaimon/multimodal-reading.md | 15 +-
docs/docs/pypaimon/multimodal-search.md | 8 +
docs/docs/spark/procedures.md | 1 +
docs/docs/spark/procedures/maintenance.md | 37 +
docs/generated/core_configuration.html | 2 +-
.../main/java/org/apache/paimon/CoreOptions.java | 17 +-
.../org/apache/paimon/rest/RESTCatalogOptions.java | 3 +-
.../apache/paimon/rest/auth/DLFAuthProvider.java | 6 +-
.../paimon/rest/auth/DLFOpenApiV4Signer.java | 330 +++++++++
.../paimon/rest/auth/DLFOpenApiV4SignerTest.java | 290 ++++++++
.../paimon/data/variant/BaseVariantReader.java | 7 +-
.../data/variant/InferVariantShreddingSchema.java | 8 +-
.../org/apache/paimon/data/variant/VariantGet.java | 88 ++-
.../java/org/apache/paimon/format/FileFormat.java | 5 +-
.../paimon/globalindex/GlobalIndexIOMeta.java | 14 +-
.../globalindex/btree/BTreeGlobalIndexer.java | 8 +
.../globalindex/btree/BTreeIndexOptions.java | 7 +
.../paimon/globalindex/btree/BTreeIndexReader.java | 15 +-
.../paimon/globalindex/btree/BTreeIndexWriter.java | 17 +-
.../globalindex/btree/LazyFilteredBTreeReader.java | 111 +++
.../paimon/data/variant/GenericVariantTest.java | 37 +
.../variant/InferVariantShreddingSchemaTest.java | 28 +
.../globalindex/btree/AbstractIndexReaderTest.java | 1 +
.../globalindex/btree/BTreeBloomFilterTest.java | 239 ++++++
.../globalindex/btree/BTreeIndexOptionsTest.java | 5 +
.../globalindex/btree/BTreeThreadSafetyTest.java | 1 +
.../btree/LazyFilteredBTreeIndexReaderTest.java | 128 +++-
.../paimon/globalindex/DataEvolutionBatchScan.java | 28 +-
.../DataEvolutionGlobalIndexScanner.java | 3 +-
.../iceberg/manifest/IcebergManifestFile.java | 3 +-
.../iceberg/manifest/IcebergManifestList.java | 3 +-
.../org/apache/paimon/manifest/ManifestFile.java | 50 +-
.../apache/paimon/manifest/ManifestSidecar.java | 50 +-
.../compact/aggregate/FieldProductAgg.java | 53 +-
.../mergetree/compact/aggregate/FieldSumAgg.java | 77 +-
.../aggregate/factory/FieldProductAggFactory.java | 11 +-
.../aggregate/factory/FieldSumAggFactory.java | 10 +-
.../paimon/table/FallbackReadFileStoreTable.java | 10 +
.../paimon/table/format/FormatTableScan.java | 56 ++
.../table/source/AbstractBatchTableScan.java | 7 +
.../source/AbstractDataEvolutionVectorRead.java | 57 +-
.../table/source/DataEvolutionBatchVectorRead.java | 2 +-
.../table/source/DataEvolutionVectorRead.java | 2 +-
.../paimon/table/source/PartitionTopNUtils.java | 127 ++++
.../org/apache/paimon/table/source/TableScan.java | 23 +
.../apache/paimon/table/system/AuditLogTable.java | 5 +
.../globalindex/DataEvolutionBatchScanTest.java | 68 ++
.../manifest/IcebergManifestBlockSizeTest.java | 182 +++++
.../apache/paimon/manifest/ManifestFileTest.java | 107 ++-
.../LookupChangelogMergeFunctionWrapperTest.java | 3 +-
.../aggregate/FieldAggregatorRetractNullTest.java | 5 +-
.../compact/aggregate/FieldAggregatorTest.java | 151 ++--
.../paimon/table/AppendOnlySimpleTableTest.java | 47 ++
.../paimon/table/BtreeGlobalIndexTableTest.java | 75 ++
.../paimon/table/format/FormatTableScanTest.java | 59 ++
.../table/source/TableScanListPartitionsTest.java | 47 ++
.../table/source/VectorSearchBuilderTest.java | 101 +++
.../paimon/flink/PaimonDataStreamScanProvider.java | 21 +-
.../flink/lookup/DynamicPartitionLevelLoader.java | 63 +-
.../flink/lookup/DynamicPartitionNumberLoader.java | 14 +-
.../paimon/flink/source/BaseDataTableSource.java | 4 +-
.../paimon/flink/source/DataTableSource.java | 3 +-
.../paimon/flink/source/SystemTableSource.java | 5 +-
.../vectorsearch/FlinkDataEvolutionVectorRead.java | 2 +-
.../apache/paimon/flink/ReadWriteTableITCase.java | 46 +-
.../paimon/flink/source/DataTableSourceTest.java | 43 ++
.../apache/paimon/format/avro/AvroFileFormat.java | 6 +
.../format/parquet/ParquetReaderFactory.java | 21 +-
.../paimon/format/avro/AvroFileFormatTest.java | 100 +++
.../format/parquet/ParquetReadWriteTest.java | 51 ++
.../parquet/reader/VariantShreddingReadTest.java | 50 ++
.../writer/InferVariantShreddingWriteTest.java | 31 +
paimon-python/pypaimon/api/auth/__init__.py | 2 +
paimon-python/pypaimon/api/auth/dlf_provider.py | 5 +-
paimon-python/pypaimon/api/auth/dlf_signer.py | 228 +++++-
paimon-python/pypaimon/api/auth/factory.py | 1 +
paimon-python/pypaimon/common/options/config.py | 3 +-
.../pypaimon/common/options/core_options.py | 47 ++
.../pypaimon/globalindex/btree/bloom_filter.py | 151 ++++
.../globalindex/btree/btree_index_reader.py | 17 +-
.../globalindex/btree/btree_index_writer.py | 34 +-
.../pypaimon/globalindex/btree/sst_file_reader.py | 42 ++
.../pypaimon/globalindex/create_global_index.py | 216 ++++--
.../data_evolution_global_index_scanner.py | 2 +
.../pypaimon/manifest/manifest_file_manager.py | 35 +-
.../pypaimon/manifest/manifest_sidecar.py | 533 ++++++++++++++
paimon-python/pypaimon/multimodal/query.py | 23 +-
paimon-python/pypaimon/multimodal/temporal.py | 47 +-
.../pypaimon/read/scanner/file_scanner.py | 3 +-
paimon-python/pypaimon/read/table_scan.py | 6 +-
.../pypaimon/snapshot/time_travel_util.py | 10 +
paimon-python/pypaimon/table/file_store_table.py | 25 +-
.../pypaimon/table/source/full_text_read.py | 6 +
.../pypaimon/table/source/full_text_scan.py | 17 +-
.../table/source/global_index_live_row_filter.py | 13 +-
.../pypaimon/table/source/hybrid_search_builder.py | 8 +-
.../table/source/primary_key_full_text_scan.py | 20 +-
.../table/source/primary_key_vector_scan.py | 20 +-
.../pypaimon/table/source/vector_search_scan.py | 9 +-
.../pypaimon/tests/btree_bloom_filter_test.py | 176 +++++
.../pypaimon/tests/file_store_commit_test.py | 52 ++
paimon-python/pypaimon/tests/file_type_test.py | 3 +
.../pypaimon/tests/global_index_build_test.py | 214 +++++-
.../tests/manifest/manifest_block_index_test.py | 542 ++++++++++++++
.../tests/manifest/manifest_sidecar_test.py | 801 +++++++++++++++++++++
.../pypaimon/tests/multimodal_table_test.py | 38 +-
.../pypaimon/tests/multimodal_temporal_test.py | 128 ++++
.../tests/primary_key_index_definitions_test.py | 12 +-
.../pypaimon/tests/reader_append_only_test.py | 6 +-
.../pypaimon/tests/rest/dlf_signer_test.py | 192 ++++-
.../pypaimon/tests/search_snapshot_test.py | 233 ++++++
.../pypaimon/tests/vector_search_filter_test.py | 13 +-
.../pypaimon/tests/vindex_batch_write_test.py | 2 +
.../tests/write/writer_file_cleanup_test.py | 58 ++
paimon-python/pypaimon/utils/file_type.py | 4 +-
paimon-python/pypaimon/write/file_store_commit.py | 6 +
.../pypaimon/write/writer/data_vector_writer.py | 103 +--
.../write/writer/dedicated_format_writer.py | 51 +-
.../org/apache/paimon/spark/SparkProcedures.java | 2 +
...cedure.java => RemoveOrphanBlobsProcedure.java} | 43 +-
.../SparkManagedBlobOrphanFilesCleanBase.java | 27 +-
.../spark/read/SparkDataEvolutionVectorRead.java | 2 +-
.../spark/catalog/functions/PaimonFunctions.scala | 2 +-
.../catalyst/analysis/ReplacePaimonFunctions.scala | 55 +-
.../SparkManagedBlobOrphanFilesClean.scala | 365 ++++++++++
.../procedure/RemoveOrphanBlobsProcedureTest.scala | 691 ++++++++++++++++++
.../apache/paimon/spark/sql/VariantTestBase.scala | 28 +
.../paimon/spark/table/PaimonFormatTableTest.scala | 15 +-
134 files changed, 8101 insertions(+), 632 deletions(-)
create mode 100644
paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiV4Signer.java
create mode 100644
paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiV4SignerTest.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/globalindex/btree/BTreeBloomFilterTest.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/source/PartitionTopNUtils.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergManifestBlockSizeTest.java
create mode 100644 paimon-python/pypaimon/globalindex/btree/bloom_filter.py
create mode 100644 paimon-python/pypaimon/manifest/manifest_sidecar.py
create mode 100644 paimon-python/pypaimon/tests/btree_bloom_filter_test.py
create mode 100644
paimon-python/pypaimon/tests/manifest/manifest_block_index_test.py
create mode 100644
paimon-python/pypaimon/tests/manifest/manifest_sidecar_test.py
create mode 100644 paimon-python/pypaimon/tests/search_snapshot_test.py
create mode 100644
paimon-python/pypaimon/tests/write/writer_file_cleanup_test.py
copy
paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/{RemoveOrphanFilesProcedure.java
=> RemoveOrphanBlobsProcedure.java} (81%)
copy
paimon-flink/paimon-flink-1.18/src/main/java/org/apache/paimon/flink/procedure/PurgeFilesProcedure.java
=>
paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/SparkManagedBlobOrphanFilesCleanBase.java
(52%)
create mode 100644
paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/procedure/SparkManagedBlobOrphanFilesClean.scala
create mode 100644
paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/RemoveOrphanBlobsProcedureTest.scala
