This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20570-d5230b69e22721d621b09d4089457c39feecfc5e in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 46ccedc365cb46fb8d2e419601c747dc07487bdb Author: Danny Chan <[email protected]> AuthorDate: Wed Nov 11 02:03:00 2026 +0000 fix(flink): retain latest file size when merging COW mini-batches (#20570) --- .../java/org/apache/hudi/util/WriteStatMerger.java | 5 +- .../org/apache/hudi/util/TestWriteStatMerger.java | 81 ++++++++++++++++++++++ .../apache/hudi/util/TestWriteStatusMerger.java | 5 +- 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/WriteStatMerger.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/WriteStatMerger.java index 996a5c22b316..9f488c4058e9 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/WriteStatMerger.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/WriteStatMerger.java @@ -101,7 +101,10 @@ public class WriteStatMerger { merged.setTotalLogBlocks(stat2.getTotalLogBlocks() + stat1.getTotalLogBlocks()); merged.setTotalCorruptLogBlock(stat2.getTotalCorruptLogBlock() + stat1.getTotalCorruptLogBlock()); merged.setTotalRollbackBlocks(stat2.getTotalRollbackBlocks() + stat1.getTotalRollbackBlocks()); - merged.setFileSizeInBytes(stat2.getFileSizeInBytes() + stat1.getFileSizeInBytes()); + // COW mini-batches replace the previous file, while delta writes retain size aggregation. + merged.setFileSizeInBytes(stat1 instanceof HoodieDeltaWriteStat + ? stat1.getFileSizeInBytes() + stat2.getFileSizeInBytes() + : stat2.getFileSizeInBytes()); // event time merged.setMinEventTime(minLong(stat1.getMinEventTime(), stat2.getMinEventTime())); merged.setMaxEventTime(maxLong(stat1.getMaxEventTime(), stat2.getMaxEventTime())); diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestWriteStatMerger.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestWriteStatMerger.java new file mode 100644 index 000000000000..d58a6482fce4 --- /dev/null +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestWriteStatMerger.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.util; + +import org.apache.hudi.common.model.HoodieDeltaWriteStat; +import org.apache.hudi.common.model.HoodieWriteStat; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests for mini-batch write stat merging. + */ +class TestWriteStatMerger { + + @ParameterizedTest + @ValueSource(longs = {110, 90, 0}) + void testReplacementFileSize(long latestSize) { + HoodieWriteStat first = new HoodieWriteStat(); + first.setPath("partition/first.parquet"); + first.setFileSizeInBytes(100); + first.setTotalWriteBytes(100); + HoodieWriteStat second = new HoodieWriteStat(); + second.setPath("partition/second.parquet"); + second.setFileSizeInBytes(latestSize); + second.setTotalWriteBytes(latestSize); + + HoodieWriteStat merged = WriteStatMerger.merge(first, second); + assertEquals(second.getPath(), merged.getPath()); + assertEquals(latestSize, merged.getFileSizeInBytes()); + assertEquals(100 + latestSize, merged.getTotalWriteBytes()); + + HoodieWriteStat third = new HoodieWriteStat(); + third.setPath("partition/third.parquet"); + third.setFileSizeInBytes(80); + third.setTotalWriteBytes(80); + merged = WriteStatMerger.merge(merged, third); + assertEquals(third.getPath(), merged.getPath()); + assertEquals(80, merged.getFileSizeInBytes()); + assertEquals(180 + latestSize, merged.getTotalWriteBytes()); + } + + @Test + void testDeltaWriteSizeAggregationIsPreserved() { + HoodieDeltaWriteStat first = new HoodieDeltaWriteStat(); + first.setPath("partition/first.log"); + first.addLogFiles("first.log"); + first.setFileSizeInBytes(100); + first.setTotalWriteBytes(100); + HoodieDeltaWriteStat second = new HoodieDeltaWriteStat(); + second.setPath("partition/second.log"); + second.addLogFiles("second.log"); + second.setFileSizeInBytes(110); + second.setTotalWriteBytes(110); + + HoodieDeltaWriteStat merged = (HoodieDeltaWriteStat) WriteStatMerger.merge(first, second); + assertEquals(second.getPath(), merged.getPath()); + assertEquals(210, merged.getFileSizeInBytes()); + assertEquals(210, merged.getTotalWriteBytes()); + assertEquals(2, merged.getLogFiles().size()); + } +} diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestWriteStatusMerger.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestWriteStatusMerger.java index 22f1a03aca43..6df1d1c0b130 100644 --- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestWriteStatusMerger.java +++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/util/TestWriteStatusMerger.java @@ -67,6 +67,7 @@ public class TestWriteStatusMerger extends HoodieFlinkClientTestHarness { writeStat1.setPath("path1"); writeStat1.setNumWrites(10); writeStat1.setTotalWriteBytes(1000); + writeStat1.setFileSizeInBytes(1000); writeStatus1.setStat(writeStat1); writeStatus1.setTotalRecords(10); @@ -92,6 +93,7 @@ public class TestWriteStatusMerger extends HoodieFlinkClientTestHarness { writeStat2.setPath("path2"); writeStat2.setNumWrites(15); writeStat2.setTotalWriteBytes(1500); + writeStat2.setFileSizeInBytes(1500); writeStatus2.setStat(writeStat2); writeStatus2.setTotalRecords(15); @@ -126,6 +128,7 @@ public class TestWriteStatusMerger extends HoodieFlinkClientTestHarness { assertEquals("path2", mergedStat.getPath()); // path from the 2nd write stat assertEquals(25, mergedStat.getNumWrites()); // 10 + 15 assertEquals(2500, mergedStat.getTotalWriteBytes()); // 1000 + 1500 + assertEquals(1500, mergedStat.getFileSizeInBytes()); // Verify merged record delegates List<HoodieRecordDelegate> writtenRecordDelegates = mergedStatus.getIndexStats().getWrittenRecordDelegates(); @@ -328,4 +331,4 @@ public class TestWriteStatusMerger extends HoodieFlinkClientTestHarness { assertTrue(mergedStatus.hasGlobalError()); assertEquals(globalError2, mergedStatus.getGlobalError()); } -} \ No newline at end of file +}
