This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 6053cfafe [flink][cdc] Setting memory pool for multiple tables sink
(#1622)
6053cfafe is described below
commit 6053cfafe07f308d77bb47fe23f753256407d2a7
Author: yuzelin <[email protected]>
AuthorDate: Fri Jul 21 19:11:06 2023 +0800
[flink][cdc] Setting memory pool for multiple tables sink (#1622)
---
.../flink/sink/cdc/CdcRecordStoreMultiWriteOperator.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperator.java
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperator.java
index 6b9f922ae..8f607d607 100644
---
a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperator.java
+++
b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperator.java
@@ -26,6 +26,7 @@ import org.apache.paimon.flink.sink.PrepareCommitOperator;
import org.apache.paimon.flink.sink.StateUtils;
import org.apache.paimon.flink.sink.StoreSinkWrite;
import org.apache.paimon.flink.sink.StoreSinkWriteState;
+import org.apache.paimon.memory.HeapMemorySegmentPool;
import org.apache.paimon.options.Options;
import org.apache.paimon.table.BucketMode;
import org.apache.paimon.table.FileStoreTable;
@@ -104,8 +105,16 @@ public class CdcRecordStoreMultiWriteOperator
FileStoreTable table = getTable(tableId);
- // TODO memoryPool should not be null
// TODO set executor service to write
+
+ // avoid allocating memory pool for every sink write
+ // the options of all tables should be the same in CDC
+ if (memoryPool == null) {
+ memoryPool =
+ new HeapMemorySegmentPool(
+ table.coreOptions().writeBufferSize(),
table.coreOptions().pageSize());
+ }
+
StoreSinkWrite write =
writes.computeIfAbsent(
tableId,
