This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a change to branch release-1.3
in repository https://gitbox.apache.org/repos/asf/paimon.git
from 0a5004c78d [python] Support ignore_if_exists param for database and
table (#6314)
new 501d1dbeeb [arrow] ArrowBatchConverter support reset the reused
VectorSchemaRoot (#6322)
new 388fc6853b [python] Modify package version to be compatible with
python version (#6331)
new f684400125 [core] Apply 'file-operation-thread-num' to commit (#6339)
new 86de753145 [spark] Support version-specific default configuration
values (#6334)
new 30efebed35 [python] Fix OSSParam to access DLF (#6332)
new 2df6a06b93 [spark] Improve error msg for creating a function on an
existing tmp function (#6343)
new 565b8aa3c2 [core] Fix comment of paged api (#6348)
new c4bf236874 [core] Support incremental clustering for append unaware
table (#6338)
new 5caa6f6fd7 [core] Add dv conflict detection during commit (#6303)
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../content/append-table/incremental-clustering.md | 134 ++++++++
.../shortcodes/generated/core_configuration.html | 18 +-
.../main/java/org/apache/paimon/CoreOptions.java | 22 +-
.../arrow/converter/ArrowBatchConverter.java | 3 +
.../arrow/converter/ArrowPerRowBatchConverter.java | 8 +
.../converter/ArrowVectorizedBatchConverter.java | 13 +
...hreadPool.java => FileOperationThreadPool.java} | 6 +-
.../java/org/apache/paimon/AbstractFileStore.java | 11 +-
.../append/cluster/IncrementalClusterManager.java | 242 +++++++++++++++
.../append/cluster/IncrementalClusterStrategy.java | 89 ++++++
.../java/org/apache/paimon/catalog/Catalog.java | 10 +-
.../java/org/apache/paimon/manifest/FileEntry.java | 6 +-
.../apache/paimon/manifest/SimpleFileEntry.java | 20 ++
.../paimon/manifest/SimpleFileEntryWithDV.java | 124 ++++++++
.../apache/paimon/operation/FileDeletionBase.java | 4 +-
.../paimon/operation/FileStoreCommitImpl.java | 154 +++++++---
.../paimon/operation/ListUnexistingFiles.java | 7 +-
.../paimon/operation/LocalOrphanFilesClean.java | 4 +-
.../org/apache/paimon/schema/SchemaValidation.java | 15 +
.../paimon/table/AbstractFileStoreTable.java | 3 +-
.../apache/paimon/table/sink/TableCommitImpl.java | 24 +-
.../apache/paimon/utils/ConflictDeletionUtils.java | 149 +++++++++
.../org/apache/paimon/TestAppendFileStore.java | 35 +++
.../cluster/IncrementalClusterManagerTest.java | 206 +++++++++++++
.../cluster/IncrementalClusterStrategyTest.java | 222 +++++++++++++
.../deletionvectors/BucketedDvMaintainerTest.java | 87 ++++--
.../apache/paimon/operation/FileDeletionTest.java | 4 +-
.../paimon/operation/FileStoreCommitTest.java | 26 +-
.../paimon/utils/ConflictDeletionUtilsTest.java | 342 +++++++++++++++++++++
.../apache/paimon/flink/action/CompactAction.java | 4 +
.../apache/paimon/flink/sink/AppendTableSink.java | 6 +-
LICENSE => paimon-python/LICENSE | 112 +------
.../{pypaimon/manifest/__init__.py => MANIFEST.in} | 3 +
paimon-python/README.md | 26 +-
.../dev/requirements-dev.txt | 15 +-
.../dev/requirements.txt | 48 ++-
paimon-python/pypaimon/__init__.py | 1 +
.../pypaimon/catalog/rest/rest_catalog.py | 22 +-
.../pypaimon/catalog/rest/rest_token_file_io.py | 8 +-
paimon-python/pypaimon/common/config.py | 1 -
paimon-python/pypaimon/common/file_io.py | 33 +-
.../pypaimon/tests/py36/ao_simple_test.py | 34 ++
paimon-python/setup.py | 43 ++-
.../paimon/spark/procedure/CompactProcedure.java | 131 +++++++-
.../commands/DeleteFromPaimonTableCommand.scala | 3 +-
.../spark/commands/MergeIntoPaimonTable.scala | 4 +-
.../paimon/spark/commands/PaimonSparkWriter.scala | 9 +-
.../spark/commands/UpdatePaimonTableCommand.scala | 3 +-
.../org/apache/paimon/spark/util/OptionUtils.scala | 23 +-
.../extensions/RewritePaimonFunctionCommands.scala | 45 ++-
.../spark/procedure/CompactProcedureTestBase.scala | 337 +++++++++++++++++++-
.../paimon/spark/sql/MergeIntoTableTestBase.scala | 71 +++--
.../spark/sql/PaimonV1FunctionTestBase.scala | 47 ++-
.../paimon/spark/sql/UpdateTableTestBase.scala | 39 +++
54 files changed, 2661 insertions(+), 395 deletions(-)
create mode 100644 docs/content/append-table/incremental-clustering.md
rename
paimon-common/src/main/java/org/apache/paimon/utils/{FileDeletionThreadPool.java
=> FileOperationThreadPool.java} (89%)
create mode 100644
paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterStrategy.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/manifest/SimpleFileEntryWithDV.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/utils/ConflictDeletionUtils.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterStrategyTest.java
create mode 100644
paimon-core/src/test/java/org/apache/paimon/utils/ConflictDeletionUtilsTest.java
copy LICENSE => paimon-python/LICENSE (60%)
copy paimon-python/{pypaimon/manifest/__init__.py => MANIFEST.in} (92%)
copy
paimon-benchmark/paimon-cluster-benchmark/src/main/resources/bin/shutdown_cluster.sh
=> paimon-python/dev/requirements-dev.txt (85%)
copy docs/setup_hugo.sh => paimon-python/dev/requirements.txt (51%)
mode change 100755 => 100644
