This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a change to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
from 1fc0f17ad2 [python] Extend ContiguousWindowDataset for frame-level
training (#9875)
new 7a7544a3f1 [core] Introduce a vector read planner for data evolution
(#9877)
new 81e68d1522 [common] Delegate listFilesIterative through the remaining
FileIO wrappers (#9865)
new 80c8f1f781 [core] Fix FileStore tag manager ignoring the table branch
(#9868)
new a01b000d73 [common] Stop answering value predicates from a truncated
bitmap index (#9866)
new fa1178fa96 [core] Fix remaining byte[] identity comparisons in collect
and merge_map aggregators (#9870)
new eea8c4abf9 [cdc] Support per-table configuration for MySQL database
CDC (#9871)
new cdea2a1a9c [flink][cdc][iceberg] Include table identifier in
schema-change and CDC-commit exception messages (#9892)
new 754bb6fe44 [format] Close the codec probe stream in
HadoopCompressionUtils (#9902)
new 95311761aa [iceberg] Retry transient REST DNS failures (#9903)
new bbab900146 [format] Reject multi-character csv delimiter, quote and
escape (#9904)
new 5b80b55b70 [test] Wait for a commit past the aborted savepoint after
region failover (#9907)
new e5b1a676e2 [core] Use full-range anchor as row representative in DE
read-type pruning (#9899)
new 7d355b9527 [common] Read VECTOR fields from NestedRow (#9864)
new eecae6d22d [python] Search vector index shards concurrently (#9805)
new 54c00628fb [common] Fix snapshot and tag reads failing when the file
is deleted concurrently (#9851)
new 1fe52c9037 [python] Batch TorchCodec frame decoding for LeRobot
datasets (#9836)
new 4bcbcec5ef [core] Preserve quoted empty and null keys/values in
string-to-MAP cast (#9849)
new 7af936ce6a [core] Add compression extension to text format data file
names (#9853)
new f080135748 [codegen] Fix equaliser treating null array elements and
map values as unequal (#9860)
new b6d8a70499 [core] Fix the failing DataEvolutionUtilsTest (#9911)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/docs/cdc-ingestion/action-configuration.mdx | 14 +-
docs/docs/cdc-ingestion/mysql-cdc.mdx | 21 ++
docs/docs/concepts/spec/fileformat.md | 4 +-
docs/docs/multimodal-table/global-index/vector.mdx | 10 +
docs/generated/iceberg_configuration.html | 12 +
.../apache/paimon/codegen/ScalarOperatorGens.scala | 17 +-
.../paimon/codegen/EqualiserCodeGeneratorTest.java | 78 ++++++
.../apache/paimon/casting/StringToMapCastRule.java | 81 ++----
.../org/apache/paimon/casting/TokenSplitter.java | 62 ++++-
.../java/org/apache/paimon/data/NestedRow.java | 3 +-
.../paimon/fileindex/bitmap/BitmapFileIndex.java | 45 ++-
.../src/main/java/org/apache/paimon/fs/FileIO.java | 28 ++
.../java/org/apache/paimon/fs/PluginFileIO.java | 8 +
.../java/org/apache/paimon/fs/ResolvingFileIO.java | 8 +
.../org/apache/paimon/fs/cache/CachingFileIO.java | 9 +
.../apache/paimon/casting/CastExecutorTest.java | 93 ++++++-
.../java/org/apache/paimon/data/NestedRowTest.java | 38 +++
.../fileindex/bitmapindex/BitmapFileIndexTest.java | 69 +++++
.../test/java/org/apache/paimon/fs/FileIOTest.java | 104 +++++++
.../org/apache/paimon/fs/PluginFileIOTest.java | 27 ++
.../org/apache/paimon/fs/ResolvingFileIOTest.java | 20 ++
.../apache/paimon/fs/cache/CachingFileIOTest.java | 21 ++
.../java/org/apache/paimon/AbstractFileStore.java | 3 +-
.../paimon/iceberg/IcebergCommitCallback.java | 18 +-
.../org/apache/paimon/iceberg/IcebergOptions.java | 14 +
.../org/apache/paimon/io/DataFilePathFactory.java | 8 +-
.../mergetree/compact/aggregate/BinaryMapKeys.java | 19 +-
.../compact/aggregate/FieldCollectAgg.java | 18 +-
.../operation/DataEvolutionFileStoreScan.java | 45 ++-
.../paimon/operation/DataEvolutionSplitRead.java | 114 +++++++-
.../operation/DataEvolutionVectorReadPlanner.java | 183 +++++++++++++
.../apache/paimon/utils/DataEvolutionUtils.java | 2 +-
.../apache/paimon/append/VectorStoreTableTest.java | 303 ++++++++++++++++++++-
.../apache/paimon/io/DataFilePathFactoryTest.java | 22 ++
.../compact/aggregate/FieldAggregatorTest.java | 132 +++++++++
.../operation/DataEvolutionFileStoreScanTest.java | 79 +++++-
.../paimon/table/DataEvolutionFileIndexTest.java | 64 +++++
.../paimon/table/PrimaryKeySimpleTableTest.java | 51 ++++
.../paimon/utils/DataEvolutionUtilsTest.java | 2 +-
.../apache/paimon/utils/SnapshotManagerTest.java | 21 ++
.../org/apache/paimon/utils/TagManagerTest.java | 22 ++
.../flink/action/cdc/CdcActionCommonUtils.java | 38 +++
.../flink/action/cdc/SyncDatabaseActionBase.java | 1 +
.../action/cdc/SynchronizationActionBase.java | 23 +-
.../cdc/SynchronizationActionFactoryBase.java | 8 +
.../action/cdc/mysql/MySqlSyncDatabaseAction.java | 8 +-
.../cdc/mysql/MySqlSyncDatabaseActionFactory.java | 3 +-
.../cdc/CdcMultiplexRecordChannelComputer.java | 2 +-
.../sink/cdc/CdcRecordStoreMultiWriteOperator.java | 6 +-
.../flink/sink/cdc/NewTableSchemaBuilder.java | 14 +-
.../cdc/UpdatedDataFieldsProcessFunctionBase.java | 4 +-
.../flink/action/cdc/CdcActionCommonUtilsTest.java | 62 +++++
.../flink/action/cdc/CdcActionITCaseBase.java | 7 +
.../cdc/mysql/MySqlSyncDatabaseActionITCase.java | 39 +++
.../apache/paimon/flink/sink/StoreCommitter.java | 3 +-
.../AppendTableSavepointTagFailoverITCase.java | 33 ++-
.../org/apache/paimon/format/csv/CsvOptions.java | 23 +-
.../paimon/format/text/AbstractTextFileWriter.java | 3 +-
.../paimon/format/text/HadoopCompressionUtils.java | 10 +-
.../paimon/format/csv/CsvFileFormatTest.java | 26 ++
.../format/text/HadoopCompressionUtilsTest.java | 21 ++
.../iceberg/IcebergRestMetadataCommitter.java | 72 ++++-
.../iceberg/IcebergRestMetadataCommitterTest.java | 137 ++++++++++
.../pypaimon/multimodal/lerobot/dataset.py | 9 +-
paimon-python/pypaimon/multimodal/video.py | 40 ++-
.../pypaimon/table/source/vector_search_read.py | 97 ++++---
.../pypaimon/tests/multimodal_lerobot_test.py | 163 +++++++++++
.../pypaimon/tests/multimodal_video_test.py | 132 ++++++++-
.../tests/vector_index_parallel_search_test.py | 197 ++++++++++++++
69 files changed, 2878 insertions(+), 195 deletions(-)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionVectorReadPlanner.java
create mode 100644
paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/CdcActionCommonUtilsTest.java
create mode 100644
paimon-python/pypaimon/tests/vector_index_parallel_search_test.py
