This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release-0.15.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit b350a26cabf8fe0f3e04def635709f3f5120869c Author: Y Ethan Guo <[email protected]> AuthorDate: Fri May 24 14:17:48 2024 -0700 [HUDI-7788] Fixing exception handling in AverageRecordSizeUtils (#11290) --- .../table/action/commit/AverageRecordSizeUtils.java | 5 ++--- .../action/commit/TestAverageRecordSizeUtils.java | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java index 9d9408e173b..b0cd1ca5f7e 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/AverageRecordSizeUtils.java @@ -29,7 +29,6 @@ import org.apache.hudi.storage.StoragePath; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.Iterator; import java.util.concurrent.atomic.AtomicLong; @@ -80,9 +79,9 @@ public class AverageRecordSizeUtils { break; } } - } catch (IOException ioe) { + } catch (Throwable t) { // make this fail safe. - LOG.error("Error trying to compute average bytes/record ", ioe); + LOG.error("Error trying to compute average bytes/record ", t); } } } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java index 5db8c978b65..2addb0296f6 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestAverageRecordSizeUtils.java @@ -29,6 +29,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieWriteConfig; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -44,6 +45,7 @@ import java.util.stream.Stream; import static org.apache.hudi.common.model.HoodieFileFormat.HOODIE_LOG; import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET; import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes; +import static org.apache.hudi.config.HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -91,6 +93,24 @@ public class TestAverageRecordSizeUtils { assertEquals(expectedSize, AverageRecordSizeUtils.averageBytesPerRecord(mockTimeline, writeConfig)); } + @Test + public void testErrorHandling() { + int recordSize = 10000; + HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder() + .withProps(Collections.singletonMap(COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), String.valueOf(recordSize))) + .build(false); + HoodieDefaultTimeline commitsTimeline = new HoodieDefaultTimeline(); + List<HoodieInstant> instants = Collections.singletonList( + new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "1")); + + when(mockTimeline.getInstants()).thenReturn(instants); + when(mockTimeline.getReverseOrderedInstants()).then(i -> instants.stream()); + // Simulate a case where the instant details are absent + commitsTimeline.setInstants(new ArrayList<>()); + + assertEquals(recordSize, AverageRecordSizeUtils.averageBytesPerRecord(mockTimeline, writeConfig)); + } + private static String getBaseFileName(String instantTime) { String fileName = UUID.randomUUID().toString(); return FSUtils.makeBaseFileName(instantTime, TEST_WRITE_TOKEN, fileName, PARQUET.getFileExtension());
