This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a change to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
from 2a5cbe906c [common] Fix JSON round-trip of temporal and decimal
predicate literals (#9640)
new b0bdd36514 [common] Forward newTwoPhaseOutputStream through
ResolvingFileIO and resolve it at commit (#9655)
new 2376ae0472 [python] Stream training samples into native vector index
trainers (#9758)
new 187f8ee9c7 [core] Allow custom primary-key compaction rewriters (#9762)
new 2e937fa45b [common] Reject out-of-range range-bitmap chunk-size option
(#9765)
new 8f97c1fb71 [api] Require field ids for schema fields, auto-assign them
for parameter lists (#9767)
new 01eae245f4 [common] Make quoting mean literal in array and row string
cast rules (#9769)
new e18c0bc621 [common] Support VECTOR elements in InternalArray accessors
(#9773)
new 1fe8b5dac1 [common] Widen hilbert index bytes beyond 8 dimensions
(#9777)
new 355c245de8 [fs] Delegate skipFully to hadoop's IOUtils (#9779)
new 34718382ab [python] Support custom dataset readers in
PaimonLeRobotDataset (#9785)
new 9764bc1e19 [common] Bound the file-size memo in both local cache
managers (#9775)
new 925499a271 [python] Fix bucket registration for dynamic bucket
overflow (#9788)
new 84c93e4fa7 [spark] Compile the shared Spark test sources against 4.0.3
for paimon-spark-4.0 (#9794)
new 1cca8fa1ed [python] Align manifest metadata and prune manifest files
by bucket (#9807)
new a638ead92b [core] Support manifest sorting for non-partitioned
bucketed tables (#9808)
new 4ab5532bf4 [core] Make compact_manifest perform a full manifest sort
(#9802)
new e847186065 [core] Fix nullable file records in the manifest Avro
reader (#9816)
new 680fa0e560 [python] Run native planning CI against paimon-rust main
(#9817)
new 4e2596674d [python] Vectorize batch raw vector scoring (#9796)
new eb43b419ce [python] Execute hybrid search routes concurrently (#9797)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci-python.yml | 77 ++-
.github/workflows/publish-snapshot-spark4.yml | 2 +-
docs/docs/flink/procedures/compaction.md | 4 +
docs/docs/program-api/java-writing.md | 29 +
docs/docs/pypaimon/lerobot.md | 27 +
docs/docs/spark/procedures/maintenance.md | 3 +
docs/generated/core_configuration.html | 4 +-
.../main/java/org/apache/paimon/CoreOptions.java | 11 +-
.../apache/paimon/types/DataTypeJsonParser.java | 8 +-
.../paimon/types/DataTypeJsonParserTest.java | 93 ++++
.../paimon/casting/StringToArrayCastRule.java | 49 +-
.../apache/paimon/casting/StringToRowCastRule.java | 57 +-
.../org/apache/paimon/casting/TokenSplitter.java | 141 +++++
.../java/org/apache/paimon/data/BinaryArray.java | 4 +-
.../java/org/apache/paimon/data/InternalArray.java | 3 +
.../java/org/apache/paimon/data/InternalRow.java | 2 +
.../rangebitmap/RangeBitmapFileIndex.java | 13 +-
.../paimon/fs/BaseMultiPartUploadCommitter.java | 12 +-
.../java/org/apache/paimon/fs/ResolvingFileIO.java | 11 +-
.../org/apache/paimon/fs/cache/FileSizeMemo.java | 68 +++
.../paimon/fs/cache/LocalDiskCacheManager.java | 12 +-
.../paimon/fs/cache/LocalMemoryCacheManager.java | 19 +-
.../org/apache/paimon/fs/hadoop/HadoopFileIO.java | 7 +-
.../apache/paimon/sort/hilbert/HilbertIndexer.java | 9 +-
.../org/apache/paimon/utils/ParameterUtils.java | 17 +-
.../apache/paimon/casting/CastExecutorTest.java | 169 ++++++
.../paimon/data/InternalArrayVectorGetterTest.java | 104 ++++
.../rangebitmap/RangeBitmapFileIndexTest.java | 29 +
.../fs/BaseMultiPartUploadCommitterTest.java | 97 ++++
.../org/apache/paimon/fs/ResolvingFileIOTest.java | 18 +
.../apache/paimon/fs/cache/CachingFileIOTest.java | 38 ++
.../apache/paimon/fs/cache/FileSizeMemoTest.java | 115 ++++
.../fs/hadoop/HadoopFileIOSkipFullyTest.java | 97 ++++
.../paimon/sort/hilbert/HilbertIndexerTest.java | 108 ++++
.../apache/paimon/utils/ParameterUtilsTest.java | 63 +++
.../apache/paimon/manifest/ManifestAvroReader.java | 5 +-
.../mergetree/compact/CompactRewriterFactory.java | 46 ++
.../compact/KvCompactionManagerFactory.java | 4 +
.../compact/MergeTreeCompactManagerFactory.java | 45 +-
.../paimon/operation/FileStoreCommitImpl.java | 20 +-
.../apache/paimon/operation/FileStoreWrite.java | 6 +
.../paimon/operation/KeyValueFileStoreWrite.java | 7 +
.../operation/ManifestAdjacentSortedRun.java | 8 +-
.../paimon/operation/ManifestCompactDryRun.java | 4 +-
.../paimon/operation/ManifestFileMerger.java | 26 +-
.../paimon/operation/ManifestFileSorter.java | 160 ++++--
.../org/apache/paimon/schema/SchemaValidation.java | 6 +-
.../org/apache/paimon/table/sink/TableWrite.java | 11 +
.../apache/paimon/table/sink/TableWriteImpl.java | 7 +
.../paimon/manifest/ManifestFileMetaTest.java | 309 +++++++++++
.../apache/paimon/manifest/ManifestFileTest.java | 115 +++-
.../manifest/NoPartitionManifestFileMetaTest.java | 128 +++++
.../MergeTreeCompactManagerFactoryTest.java | 45 ++
.../paimon/operation/FileStoreCommitTest.java | 12 +-
.../paimon/operation/ManifestFileMergerTest.java | 4 +-
.../operation/ManifestFileMergerTestUtils.java | 33 +-
.../apache/paimon/schema/SchemaValidationTest.java | 29 +-
.../table/sink/CompactRewriterFactoryTest.java | 230 ++++++++
.../apache/paimon/azure/HadoopCompliantFileIO.java | 7 +-
.../apache/paimon/cosn/HadoopCompliantFileIO.java | 7 +-
.../apache/paimon/gs/HadoopCompliantFileIO.java | 7 +-
.../apache/paimon/jindo/HadoopCompliantFileIO.java | 7 +-
.../apache/paimon/obs/HadoopCompliantFileIO.java | 7 +-
.../apache/paimon/oss/HadoopCompliantFileIO.java | 7 +-
.../apache/paimon/s3/HadoopCompliantFileIO.java | 7 +-
.../org/apache/paimon/s3/S3MultiPartUpload.java | 32 +-
.../paimon/s3/S3ResolvingTwoPhaseCommitITCase.java | 135 +++++
.../procedure/CompactManifestProcedureITCase.java | 2 +-
.../paimon/format/avro/AvroRecordDecoder.java | 10 +
.../paimon/format/avro/AvroFileFormatTest.java | 69 +++
paimon-python/README.md | 10 +
.../vindex/vindex_vector_index_writer.py | 53 +-
.../pypaimon/manifest/manifest_file_manager.py | 12 +
.../pypaimon/manifest/manifest_list_manager.py | 10 +
.../pypaimon/manifest/schema/manifest_file_meta.py | 12 +
paimon-python/pypaimon/multimodal/__init__.py | 6 +-
.../pypaimon/multimodal/lerobot/__init__.py | 6 +-
.../pypaimon/multimodal/lerobot/dataset.py | 616 +++++++++++++--------
.../pypaimon/multimodal/lerobot/reader.py | 187 +++++++
.../read/scanner/bucket_select_converter.py | 18 +-
.../pypaimon/read/scanner/file_scanner.py | 12 +
paimon-python/pypaimon/read/split_serializer.py | 24 +-
.../pypaimon/table/source/hybrid_search_builder.py | 35 +-
.../pypaimon/table/source/vector_search_read.py | 113 +++-
.../pypaimon/tests/batch_vector_raw_scan_test.py | 12 +-
.../pypaimon/tests/global_index_build_test.py | 30 +-
.../pypaimon/tests/hybrid_search_execution_test.py | 142 +++++
.../tests/manifest/manifest_manager_test.py | 86 ++-
.../tests/manifest/manifest_schema_test.py | 10 +-
.../pypaimon/tests/multimodal_lerobot_test.py | 161 +++++-
.../pypaimon/tests/pushdown_bucket_test.py | 66 +++
.../pypaimon/tests/split_serializer_test.py | 68 ++-
.../pypaimon/tests/vector_scoring_test.py | 26 +-
.../pypaimon/tests/vindex_training_test.py | 115 ++++
.../write/simple_hash_bucket_assigner_test.py | 39 ++
paimon-python/pypaimon/write/row_key_extractor.py | 2 +-
paimon-spark/paimon-spark-4.0/pom.xml | 6 +-
.../StreamTestCheckAnswerWithTimeoutStub.scala | 63 ---
.../pom.xml | 186 ++++---
pom.xml | 1 +
100 files changed, 4415 insertions(+), 789 deletions(-)
create mode 100644
paimon-api/src/test/java/org/apache/paimon/types/DataTypeJsonParserTest.java
create mode 100644
paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java
create mode 100644
paimon-common/src/main/java/org/apache/paimon/fs/cache/FileSizeMemo.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/data/InternalArrayVectorGetterTest.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/fs/BaseMultiPartUploadCommitterTest.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/fs/cache/FileSizeMemoTest.java
create mode 100644
paimon-common/src/test/java/org/apache/paimon/fs/hadoop/HadoopFileIOSkipFullyTest.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/mergetree/compact/CompactRewriterFactory.java
copy
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/source/aggregate/PushedAggregateResult.java
=>
paimon-core/src/test/java/org/apache/paimon/operation/ManifestFileMergerTestUtils.java
(57%)
create mode 100644
paimon-core/src/test/java/org/apache/paimon/table/sink/CompactRewriterFactoryTest.java
create mode 100644
paimon-filesystems/paimon-s3-impl/src/test/java/org/apache/paimon/s3/S3ResolvingTwoPhaseCommitITCase.java
create mode 100644 paimon-python/pypaimon/multimodal/lerobot/reader.py
create mode 100644 paimon-python/pypaimon/tests/hybrid_search_execution_test.py
create mode 100644 paimon-python/pypaimon/tests/vindex_training_test.py
delete mode 100644
paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/spark/sql/streaming/StreamTestCheckAnswerWithTimeoutStub.scala
copy paimon-spark/{paimon-spark-4.0 => paimon-spark-ut-4.0}/pom.xml (52%)
