This is an automated email from the ASF dual-hosted git repository.
czweng pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
from bcafd296de [flink] All lookup writers should store active buckets in
state to make sure changelog can be produced (#5796)
add 359fb2150f [core][flink] Introduce write operator coordinator to
optimize write restore (#5787)
No new revisions were added by this update.
Summary of changes:
docs/content/maintenance/write-performance.md | 5 +-
.../generated/flink_connector_configuration.html | 12 ++
.../java/org/apache/paimon/KeyValueFileStore.java | 7 +-
.../deletionvectors/DeletionVectorsMaintainer.java | 40 ++----
.../append/BaseAppendDeleteFileMaintainer.java | 7 +-
...iner.java => DynamicBucketIndexMaintainer.java} | 40 ++----
.../org/apache/paimon/index/IndexFileHandler.java | 3 +
.../org/apache/paimon/index/IndexMaintainer.java | 40 ------
.../paimon/operation/AbstractFileStoreWrite.java | 146 ++++++++++----------
.../operation/BucketedAppendFileStoreWrite.java | 2 +-
.../apache/paimon/operation/FileStoreWrite.java | 8 +-
.../paimon/operation/FileSystemWriteRestore.java | 98 +++++++++++++
.../paimon/operation/KeyValueFileStoreWrite.java | 12 +-
.../paimon/operation/MemoryFileStoreWrite.java | 6 +-
.../org/apache/paimon/operation/RestoreFiles.java | 79 +++++++++++
.../org/apache/paimon/operation/WriteRestore.java | 55 ++++++++
.../paimon/table/AbstractFileStoreTable.java | 6 +
.../paimon/table/DelegatedFileStoreTable.java | 6 +
.../org/apache/paimon/table/FileStoreTable.java | 3 +
.../apache/paimon/table/sink/InnerTableWrite.java | 4 +
.../apache/paimon/table/sink/TableWriteImpl.java | 7 +
.../org/apache/paimon/TestAppendFileStore.java | 4 +-
.../DeletionVectorsMaintainerTest.java | 45 ++++--
....java => DynamicBucketIndexMaintainerTest.java} | 4 +-
.../paimon/table/DynamicBucketTableTest.java | 8 +-
.../flink/sink/cdc/CdcDynamicBucketSink.java | 10 +-
.../sink/cdc/CdcDynamicBucketWriteOperator.java | 27 ++++
.../paimon/flink/sink/cdc/CdcFixedBucketSink.java | 10 +-
.../sink/cdc/CdcRecordStoreWriteOperator.java | 27 ++++
.../apache/paimon/flink/FlinkConnectorOptions.java | 14 ++
.../flink/sink/DynamicBucketRowWriteOperator.java | 27 ++++
.../apache/paimon/flink/sink/FixedBucketSink.java | 12 +-
.../flink/sink/RowDataStoreWriteOperator.java | 35 +++++
.../paimon/flink/sink/RowDynamicBucketSink.java | 10 +-
.../apache/paimon/flink/sink/StoreSinkWrite.java | 3 +
.../paimon/flink/sink/StoreSinkWriteImpl.java | 6 +
.../paimon/flink/sink/TableWriteOperator.java | 77 ++++++++++-
.../sink/coordinator/CoordinatedWriteRestore.java | 90 ++++++++++++
.../sink/coordinator/LatestIdentifierRequest.java | 21 ++-
.../sink/coordinator/LatestIdentifierResponse.java | 21 ++-
.../sink/coordinator/ScanCoordinationRequest.java | 45 +++---
.../sink/coordinator/ScanCoordinationResponse.java | 126 +++++++++++++++++
.../sink/coordinator/TableWriteCoordinator.java | 129 ++++++++++++++++++
.../sink/coordinator/WriteOperatorCoordinator.java | 151 +++++++++++++++++++++
.../flink/sink/index/GlobalDynamicBucketSink.java | 9 +-
.../apache/paimon/flink/BatchFileStoreITCase.java | 38 ++++++
.../flink/PrimaryKeyFileStoreTableITCase.java | 4 +
.../flink/sink/StoreCompactOperatorTest.java | 4 +
.../coordinator/TableWriteCoordinatorTest.java | 79 +++++++++++
.../paimon/spark/sql/DeletionVectorTest.scala | 10 +-
50 files changed, 1370 insertions(+), 262 deletions(-)
rename
paimon-core/src/main/java/org/apache/paimon/index/{HashIndexMaintainer.java =>
DynamicBucketIndexMaintainer.java} (71%)
delete mode 100644
paimon-core/src/main/java/org/apache/paimon/index/IndexMaintainer.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/operation/FileSystemWriteRestore.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/operation/RestoreFiles.java
create mode 100644
paimon-core/src/main/java/org/apache/paimon/operation/WriteRestore.java
rename
paimon-core/src/test/java/org/apache/paimon/index/{HashIndexMaintainerTest.java
=> DynamicBucketIndexMaintainerTest.java} (97%)
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/CoordinatedWriteRestore.java
copy
paimon-common/src/main/java/org/apache/paimon/fs/hadoop/HadoopFileIOLoader.java
=>
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/LatestIdentifierRequest.java
(68%)
copy
paimon-common/src/main/java/org/apache/paimon/fs/local/LocalFileIOLoader.java
=>
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/LatestIdentifierResponse.java
(64%)
copy paimon-core/src/main/java/org/apache/paimon/table/sink/SinkRecord.java =>
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/ScanCoordinationRequest.java
(51%)
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/ScanCoordinationResponse.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinator.java
create mode 100644
paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/coordinator/WriteOperatorCoordinator.java
create mode 100644
paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/coordinator/TableWriteCoordinatorTest.java
