This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 9db33ba33022 test(core): add file format dispatch exhaustiveness
coverage (#19253)
9db33ba33022 is described below
commit 9db33ba3302245b5bd077a5be0e76c16858f1461
Author: Y Ethan Guo <[email protected]>
AuthorDate: Fri Aug 28 08:50:00 2026 -0700
test(core): add file format dispatch exhaustiveness coverage (#19253)
Per-HoodieFileFormat dispatch is duplicated across factory overloads and
utility switches, so a new format can be wired into one dispatch point and
missed in another. That shipped with Lance (the getLogBlockType arm landed
16 days after #17731, in #17768) and twice with Vortex (the StoragePathInfo
reader overload and getLogBlockType, both still pending in #19252). These
tests sweep HoodieFileFormat.values() instead of a hardcoded list so such
gaps fail mechanically.
- TestHoodieHadoopIOFactory:
testGetFileFormatUtilsEntryPointsCoverEveryBaseFileFormat
drives every format except HOODIE_LOG through both
HoodieIOFactory#getFileFormatUtils(StoragePath) (the extension if-chain)
and HoodieHadoopIOFactory#getFileFormatUtils(HoodieFileFormat) (the format
switch); a case missing from either surfaces as
UnsupportedOperationException, so a format wired into neither fails too.
The hardcoded testGetFileFormatUtils gains its missing VORTEX rows and the
class builds its factory from HoodieTestUtils.getDefaultStorage().
- TestCommonClientUtils: testGetLogBlockTypeMapsEveryBaseFileFormat pins the
expected log block type per base file format through a switch with a
failing default (HFILE -> HFILE_DATA_BLOCK, the rest -> AVRO_DATA_BLOCK).
VORTEX is excluded with a TODO(#19252) until its case lands; the expected
VORTEX arm is already in place, so #19252 only drops the exclusion.
Not covered: the HoodieFileReaderFactory overload sweep is owned by
TestHoodieFileReaderFactory in #19255; shouldWriteNativeLogs is
format-independent since #19283; HoodieInputFormatUtils (hudi-hadoop-mr)
remains the unswept dispatch family.
---------
Co-authored-by: voon <[email protected]>
---
.../apache/hudi/utils/TestCommonClientUtils.java | 35 ++++++++++++
.../hudi/io/hadoop/TestHoodieHadoopIOFactory.java | 66 +++++++++++++---------
2 files changed, 74 insertions(+), 27 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java
index 59429de2b0d1..572fa34f722e 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java
@@ -20,8 +20,11 @@
package org.apache.hudi.utils;
+import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableVersion;
+import org.apache.hudi.common.table.log.block.HoodieLogBlock;
+import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieNotSupportedException;
import org.apache.hudi.util.CommonClientUtils;
@@ -29,6 +32,7 @@ import org.apache.hudi.util.CommonClientUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.MethodSource;
import java.util.stream.Stream;
@@ -110,6 +114,37 @@ class TestCommonClientUtils {
);
}
+ /**
+ * Every base file format needs a case in the getLogBlockType switch; a
missing one only surfaces as a
+ * HoodieException on the inline log write path (write version below TEN),
which is how VORTEX shipped (#19252).
+ */
+ // TODO(#19252): drop the VORTEX exclusion once the VORTEX case lands in
getLogBlockType.
+ @ParameterizedTest
+ @EnumSource(value = HoodieFileFormat.class, mode = EnumSource.Mode.EXCLUDE,
names = {"HOODIE_LOG", "VORTEX"})
+ void testGetLogBlockTypeMapsEveryBaseFileFormat(HoodieFileFormat format) {
+ HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class);
+ HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(writeConfig.getLogDataBlockFormat()).thenReturn(Option.empty());
+ when(tableConfig.getBaseFileFormat()).thenReturn(format);
+
+ assertEquals(expectedLogBlockType(format),
CommonClientUtils.getLogBlockType(writeConfig, tableConfig));
+ }
+
+ private static HoodieLogBlock.HoodieLogBlockType
expectedLogBlockType(HoodieFileFormat format) {
+ switch (format) {
+ case PARQUET:
+ case ORC:
+ case LANCE:
+ case VORTEX:
+ return HoodieLogBlock.HoodieLogBlockType.AVRO_DATA_BLOCK;
+ case HFILE:
+ return HoodieLogBlock.HoodieLogBlockType.HFILE_DATA_BLOCK;
+ default:
+ throw new IllegalArgumentException("No expected log block type for " +
format
+ + "; add it here and to CommonClientUtils#getLogBlockType");
+ }
+ }
+
@ParameterizedTest(name = "Table version {0} with write version {1} should
be valid: {2}")
@MethodSource("provideValidTableVersionWriteVersionPairs")
void testValidTableVersionWriteVersionPairs(
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java
index 053d9c41e66a..31b9b74123d7 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java
@@ -20,50 +20,62 @@
package org.apache.hudi.io.hadoop;
import org.apache.hudi.common.model.HoodieFileFormat;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.util.FileFormatUtils;
import org.apache.hudi.common.util.HFileUtils;
import org.apache.hudi.common.util.LanceUtils;
import org.apache.hudi.common.util.OrcUtils;
import org.apache.hudi.common.util.ParquetUtils;
+import org.apache.hudi.common.util.VortexUtils;
import org.apache.hudi.core.io.storage.HoodieIOFactory;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.hadoop.HoodieHadoopIOFactory;
-import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
-import java.io.IOException;
-
-import static org.apache.hudi.common.testutils.HoodieTestUtils.DEFAULT_URI;
-import static
org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Tests {@link HoodieHadoopIOFactory}
*/
-public class TestHoodieHadoopIOFactory {
+class TestHoodieHadoopIOFactory {
+ private final HoodieIOFactory ioFactory = new
HoodieHadoopIOFactory(HoodieTestUtils.getDefaultStorage());
+
@Test
- public void testGetFileFormatUtils() throws IOException {
- try (HoodieStorage storage =
- new HoodieHadoopStorage(HadoopFSUtils.getFs(DEFAULT_URI,
getDefaultStorageConf()))) {
- HoodieIOFactory ioFactory = new HoodieHadoopIOFactory(storage);
- assertTrue(ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b.parquet")) instanceof ParquetUtils);
- assertTrue(ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b.orc")) instanceof OrcUtils);
- assertTrue(ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b.hfile")) instanceof HFileUtils);
- assertTrue(ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b.lance")) instanceof LanceUtils);
- assertThrows(
- UnsupportedOperationException.class,
- () -> ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b.log")));
+ void testGetFileFormatUtils() {
+ assertTrue(ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b.parquet")) instanceof ParquetUtils);
+ assertTrue(ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b.orc")) instanceof OrcUtils);
+ assertTrue(ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b.hfile")) instanceof HFileUtils);
+ assertTrue(ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b.lance")) instanceof LanceUtils);
+ assertTrue(ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b.vortex")) instanceof VortexUtils);
+ assertThrows(
+ UnsupportedOperationException.class,
+ () -> ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b.log")));
+
+ assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.PARQUET)
instanceof ParquetUtils);
+ assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.ORC) instanceof
OrcUtils);
+ assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.HFILE) instanceof
HFileUtils);
+ assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.LANCE) instanceof
LanceUtils);
+ assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.VORTEX)
instanceof VortexUtils);
+ assertThrows(
+ UnsupportedOperationException.class,
+ () -> ioFactory.getFileFormatUtils(HoodieFileFormat.HOODIE_LOG));
+ }
- assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.PARQUET)
instanceof ParquetUtils);
- assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.ORC) instanceof
OrcUtils);
- assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.HFILE)
instanceof HFileUtils);
- assertTrue(ioFactory.getFileFormatUtils(HoodieFileFormat.LANCE)
instanceof LanceUtils);
- assertThrows(
- UnsupportedOperationException.class,
- () -> ioFactory.getFileFormatUtils(HoodieFileFormat.HOODIE_LOG));
- }
+ /**
+ * The extension if-chain in {@link
HoodieIOFactory#getFileFormatUtils(StoragePath)} and the format switch in
+ * {@link HoodieHadoopIOFactory#getFileFormatUtils(HoodieFileFormat)} must
both cover every base file format;
+ * a case missing from either surfaces here as {@link
UnsupportedOperationException}.
+ */
+ @ParameterizedTest
+ @EnumSource(value = HoodieFileFormat.class, mode = EnumSource.Mode.EXCLUDE,
names = {"HOODIE_LOG"})
+ void
testGetFileFormatUtilsEntryPointsCoverEveryBaseFileFormat(HoodieFileFormat
format) {
+ FileFormatUtils byFormat = ioFactory.getFileFormatUtils(format);
+ FileFormatUtils byPath = ioFactory.getFileFormatUtils(new
StoragePath("file:///a/b" + format.getFileExtension()));
+ assertEquals(byFormat.getClass(), byPath.getClass());
}
}
