(hudi) branch master updated: [HUDI-9331] Fix Incorrect memory estimation for CDC block flushing can lead to OOM (#13193)

[danny0405]

This is an automated email from the ASF dual-hosted git repository.

danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new b9dbc6a6f86 [HUDI-9331] Fix Incorrect memory estimation for CDC block 
flushing can lead to OOM (#13193)
b9dbc6a6f86 is described below

commit b9dbc6a6f86f4d2e4e5ec5f7cf8ec9ec2d218256
Author: mrzhao <imrz...@qq.com>
AuthorDate: Tue Apr 22 11:08:43 2025 +0800

    [HUDI-9331] Fix Incorrect memory estimation for CDC block flushing can lead 
to OOM (#13193)
---
 .../src/main/java/org/apache/hudi/io/HoodieCDCLogger.java               | 2 ++
 1 file changed, 2 insertions(+)

diff --git 
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
 
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
index c30cfca9949..411de8c7305 100644
--- 
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
+++ 
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
@@ -177,6 +177,8 @@ public class HoodieCDCLogger implements Closeable {
     HoodieAvroPayload payload = new HoodieAvroPayload(Option.of(cdcRecord));
     if (cdcData.isEmpty()) {
       averageCDCRecordSize = sizeEstimator.sizeEstimate(payload);
+    } else if (numOfCDCRecordsInMemory.get() % 100 == 0) {
+      averageCDCRecordSize = (long) (averageCDCRecordSize * 0.8 + 
sizeEstimator.sizeEstimate(payload) * 0.2);
     }
     cdcData.put(recordKey, payload);
     numOfCDCRecordsInMemory.incrementAndGet();