This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a change to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
from 7152705145 [common] Stop answering value predicates from a truncated
BSI index (#9654)
new ed72e8ffc0 [common] Resolve the atomic rename under the Kerberos
FileSystem wrapper (#9652)
new 2d21a16da7 [common] Range-check the digit guard in DateTimeUtils
(#9649)
new 00ff568d09 [common] Never compare null literals when merging range
predicates (#9647)
new d9fda1ef09 [common] Version the memory-mode cache key in CachingFileIO
(#9642)
new 79b6f948ea [common] Skip null literals in btree global index IN
predicates (#9624)
new fc9658dbcd [common] Normalize row-id IN literals inside Range.toRanges
(#9630)
new 3ad516f276 [spark] Support source table fileio for blob descriptors on
spark (#9598)
new 38aea2cdcd [core] Introduce local cleanup for unreferenced managed
BLOB packs. (#9609)
new db45ed237a [core] Push partition and bucket filters into the uncached
manifest read path (#9739)
new 34db147b8c [mosaic] Size the writer's Arrow vectors by
write.batch-size (#9741)
new ab818a1b95 [mosaic] Prefetch row groups and read with one input stream
per concurrent read (#9740)
new e71c49410f [python] Add as-of joins for multimodal scans (#9536)
new 332d63ca5b [mosaic] Respect decoded byte budget when refilling prefetch
new f97a68eb86 [core][python] Add extra files to manifest metadata (#9746)
new f41cbcf860 [mosaic] Push row selection down to row-group reads (#9742)
new 8f8f3bb128 [mosaic] Fix a close race in the reader stream pool and add
pool and writer sizing tests (#9744)
new 7808aa7c3c [common] Order decimal z-values through the signed long
transform (#9626)
new 1f6c82cfd6 [api] Fall back to the HTTP status when the error body
omits code (#9661)
new 6111a4d437 [spark] Close the commit and release the cached RDD in
remove_unexisting_files (#9706)
new 08f3fa8bdf [core] Add interrupt coverage for snapshot existence
retries (#9745)
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/docs/concepts/spec/fileformat.md | 2 +
docs/docs/concepts/spec/manifest.md | 4 +
docs/docs/primary-key-table/blob-storage.md | 42 +-
docs/docs/pypaimon/multimodal-reading.md | 39 +
.../apache/paimon/rest/DefaultErrorHandler.java | 5 +-
.../paimon/rest/responses/ErrorResponse.java | 9 +-
.../paimon/rest/responses/ErrorResponseTest.java | 90 ++
.../org/apache/paimon/fs/hadoop/HadoopFileIO.java | 54 +-
.../paimon/fs/hadoop/HadoopSecuredFileSystem.java | 15 +
.../paimon/globalindex/btree/BTreeIndexReader.java | 5 +
.../java/org/apache/paimon/predicate/Between.java | 32 +-
.../paimon/predicate/RowIdPredicateVisitor.java | 11 +-
.../org/apache/paimon/sort/zorder/ZIndexer.java | 70 +-
.../paimon/utils}/BlobDescriptorResolvingRow.java | 171 +-
.../org/apache/paimon/utils/DateTimeUtils.java | 26 +-
.../java/org/apache/paimon/utils/FileType.java | 14 +-
.../main/java/org/apache/paimon/utils/Range.java | 17 +-
.../apache/paimon/fs/cache/CachingFileIOTest.java | 57 +
.../fs/hadoop/HadoopSecuredFileSystemTest.java | 127 ++
.../globalindex/btree/AbstractIndexReaderTest.java | 24 +
.../org/apache/paimon/predicate/BetweenTest.java | 39 +
.../predicate/RowIdPredicateVisitorTest.java | 48 +
.../paimon/sort/zorder/TestZOrderByteUtil.java | 124 ++
.../org/apache/paimon/utils/DateTimeUtilsTest.java | 26 +
.../java/org/apache/paimon/utils/FileTypeTest.java | 32 +
.../blob/ManagedBlobReachabilityCollector.java | 273 ++++
.../paimon/blob/ManagedBlobReferenceFile.java | 4 +
.../apache/paimon/manifest/ManifestEntryCache.java | 36 +
.../org/apache/paimon/manifest/ManifestFile.java | 13 +-
.../apache/paimon/manifest/ManifestFileMeta.java | 55 +-
.../manifest/ManifestFileMetaSerializer.java | 9 +-
.../apache/paimon/operation/FileDeletionBase.java | 3 +
.../LocalManagedBlobOrphanFilesClean.java | 415 +++++
.../operation/ManagedBlobOrphanFilesClean.java | 508 ++++++
.../apache/paimon/operation/OrphanFilesClean.java | 5 +
.../paimon/table/BlobDescriptorReadUtils.java | 67 +
.../java/org/apache/paimon/utils/ObjectsCache.java | 8 +-
.../blob/ManagedBlobReachabilityCollectorTest.java | 210 +++
.../manifest/ManifestFileMetaSerializerTest.java | 44 +
.../apache/paimon/manifest/ManifestFileTest.java | 58 +-
.../apache/paimon/manifest/ManifestListTest.java | 30 +-
.../paimon/operation/ExpireSnapshotsTest.java | 114 +-
.../operation/LocalOrphanFilesCleanTest.java | 55 +
.../operation/ManagedBlobOrphanFilesCleanTest.java | 1645 ++++++++++++++++++++
.../paimon/rest/DefaultErrorHandlerTest.java | 14 +
.../org/apache/paimon/rest/HttpClientTest.java | 24 +
.../paimon/table/DataEvolutionTableTest.java | 56 +
.../apache/paimon/utils/SnapshotManagerTest.java | 19 +
.../flink/sink/BlobDescriptorResolvingRow.java | 239 +--
.../paimon/format/mosaic/MosaicFileFormat.java | 30 +-
.../format/mosaic/MosaicInputFileAdapter.java | 160 +-
.../paimon/format/mosaic/MosaicReaderFactory.java | 17 +-
.../paimon/format/mosaic/MosaicRecordsReader.java | 406 ++++-
.../paimon/format/mosaic/MosaicRecordsWriter.java | 19 +
.../format/mosaic/MosaicInputFileAdapterTest.java | 263 ++++
.../format/mosaic/MosaicReaderWriterTest.java | 240 +++
.../format/mosaic/MosaicRecordsReaderTest.java | 488 ++++++
.../format/mosaic/MosaicRecordsWriterTest.java | 184 +++
.../pypaimon/manifest/manifest_list_manager.py | 2 +
.../pypaimon/manifest/schema/manifest_file_meta.py | 4 +-
paimon-python/pypaimon/multimodal/__init__.py | 6 +
paimon-python/pypaimon/multimodal/temporal.py | 1025 ++++++++++++
.../pypaimon/read/reader/format_pyarrow_reader.py | 90 +-
paimon-python/pypaimon/read/split_read.py | 4 +-
paimon-python/pypaimon/read/table_read.py | 2 +
.../tests/manifest/manifest_manager_test.py | 40 +
.../tests/manifest/manifest_schema_test.py | 10 +-
.../pypaimon/tests/multimodal_temporal_test.py | 1411 +++++++++++++++++
.../pypaimon/tests/parquet_metadata_cache_test.py | 6 +-
.../pypaimon/tests/parquet_row_range_test.py | 54 +-
.../org/apache/paimon/spark/PaimonBatch.scala | 13 +-
.../paimon/spark/PaimonPartitionReader.scala | 17 +-
.../spark/PaimonPartitionReaderFactory.scala | 14 +-
.../apache/paimon/spark/PostponeMergeOnRead.scala | 25 +-
.../spark/execution/PostponeMergeOnReadExec.scala | 37 +-
.../procedure/SparkRemoveUnexistingFiles.scala | 44 +-
.../org/apache/paimon/spark/read/BaseScan.scala | 14 +-
.../spark/sources/PaimonMicroBatchStream.scala | 15 +-
.../apache/paimon/spark/SparkInternalRowTest.java | 80 +
.../procedure/CreateGlobalIndexProcedureTest.scala | 28 +
.../spark/sources/PaimonMicroBatchStreamTest.scala | 47 +-
.../org/apache/paimon/spark/sql/BlobTestBase.scala | 35 +
82 files changed, 9364 insertions(+), 453 deletions(-)
create mode 100644
paimon-api/src/test/java/org/apache/paimon/rest/responses/ErrorResponseTest.java
copy
{paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink =>
paimon-common/src/main/java/org/apache/paimon/utils}/BlobDescriptorResolvingRow.java
(58%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/blob/ManagedBlobReachabilityCollector.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/operation/LocalManagedBlobOrphanFilesClean.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/operation/ManagedBlobOrphanFilesClean.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/table/BlobDescriptorReadUtils.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/blob/ManagedBlobReachabilityCollectorTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/operation/ManagedBlobOrphanFilesCleanTest.java
create mode 100644
paimon-mosaic/src/test/java/org/apache/paimon/format/mosaic/MosaicInputFileAdapterTest.java
create mode 100644 paimon-python/pypaimon/multimodal/temporal.py
create mode 100644 paimon-python/pypaimon/tests/multimodal_temporal_test.py
