This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release-0.14.1-spark35-scala213 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 415c0660a25ac0951901579d8f2e64c1c44a8603 Author: Jon Vexler <[email protected]> AuthorDate: Wed Jan 17 16:17:19 2024 -0500 [HUDI-7296] Reduce CI Time by Minimizing Duplicate Code Coverage in Tests (#10492) * reduce combos of tests * build success --------- Co-authored-by: Jonathan Vexler <=> --- .../apache/hudi/functional/TestBootstrapRead.java | 30 +++++++----- ...estHoodieDeltaStreamerSchemaEvolutionQuick.java | 53 ++++++++++++++-------- 2 files changed, 53 insertions(+), 30 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrapRead.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrapRead.java index d926a3be5a4..1e36f491b3f 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrapRead.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrapRead.java @@ -40,23 +40,29 @@ import static org.apache.hudi.common.model.HoodieTableType.MERGE_ON_READ; @Tag("functional") public class TestBootstrapRead extends TestBootstrapReadBase { private static Stream<Arguments> testArgs() { + boolean fullTest = false; Stream.Builder<Arguments> b = Stream.builder(); - String[] bootstrapType = {"full", "metadata", "mixed"}; - Boolean[] dashPartitions = {true,false}; - HoodieTableType[] tableType = {COPY_ON_WRITE, MERGE_ON_READ}; - Integer[] nPartitions = {0, 1, 2}; - for (HoodieTableType tt : tableType) { - for (Boolean dash : dashPartitions) { - for (String bt : bootstrapType) { - for (Integer n : nPartitions) { - // can't be mixed bootstrap if it's nonpartitioned - // don't need to test slash partitions if it's nonpartitioned - if ((!bt.equals("mixed") && dash) || n > 0) { - b.add(Arguments.of(bt, dash, tt, n)); + if (fullTest) { + String[] bootstrapType = {"full", "metadata", "mixed"}; + Boolean[] dashPartitions = {true,false}; + HoodieTableType[] tableType = {COPY_ON_WRITE, MERGE_ON_READ}; + Integer[] nPartitions = {0, 1, 2}; + for (HoodieTableType tt : tableType) { + for (Boolean dash : dashPartitions) { + for (String bt : bootstrapType) { + for (Integer n : nPartitions) { + // can't be mixed bootstrap if it's nonpartitioned + // don't need to test slash partitions if it's nonpartitioned + if ((!bt.equals("mixed") && dash) || n > 0) { + b.add(Arguments.of(bt, dash, tt, n)); + } } } } } + } else { + b.add(Arguments.of("metadata", true, COPY_ON_WRITE, 0)); + b.add(Arguments.of("mixed", false, MERGE_ON_READ, 2)); } return b.build(); } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionQuick.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionQuick.java index de21b33fff4..81f27eec7fb 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionQuick.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionQuick.java @@ -58,25 +58,34 @@ public class TestHoodieDeltaStreamerSchemaEvolutionQuick extends TestHoodieDelta } protected static Stream<Arguments> testArgs() { + boolean fullTest = false; Stream.Builder<Arguments> b = Stream.builder(); - //only testing row-writer enabled for now - for (Boolean rowWriterEnable : new Boolean[] {true}) { - for (Boolean nullForDeletedCols : new Boolean[] {false, true}) { - for (Boolean useKafkaSource : new Boolean[] {false, true}) { - for (Boolean addFilegroups : new Boolean[] {false, true}) { - for (Boolean multiLogFiles : new Boolean[] {false, true}) { - for (Boolean shouldCluster : new Boolean[] {false, true}) { - for (String tableType : new String[] {"COPY_ON_WRITE", "MERGE_ON_READ"}) { - if (!multiLogFiles || tableType.equals("MERGE_ON_READ")) { - b.add(Arguments.of(tableType, shouldCluster, false, rowWriterEnable, addFilegroups, multiLogFiles, useKafkaSource, nullForDeletedCols)); + if (fullTest) { + //only testing row-writer enabled for now + for (Boolean rowWriterEnable : new Boolean[] {true}) { + for (Boolean nullForDeletedCols : new Boolean[] {false, true}) { + for (Boolean useKafkaSource : new Boolean[] {false, true}) { + for (Boolean addFilegroups : new Boolean[] {false, true}) { + for (Boolean multiLogFiles : new Boolean[] {false, true}) { + for (Boolean shouldCluster : new Boolean[] {false, true}) { + for (String tableType : new String[] {"COPY_ON_WRITE", "MERGE_ON_READ"}) { + if (!multiLogFiles || tableType.equals("MERGE_ON_READ")) { + b.add(Arguments.of(tableType, shouldCluster, false, rowWriterEnable, addFilegroups, multiLogFiles, useKafkaSource, nullForDeletedCols)); + } } } + b.add(Arguments.of("MERGE_ON_READ", false, true, rowWriterEnable, addFilegroups, multiLogFiles, useKafkaSource, nullForDeletedCols)); } - b.add(Arguments.of("MERGE_ON_READ", false, true, rowWriterEnable, addFilegroups, multiLogFiles, useKafkaSource, nullForDeletedCols)); } } } } + } else { + b.add(Arguments.of("COPY_ON_WRITE", true, false, true, false, false, true, false)); + b.add(Arguments.of("COPY_ON_WRITE", true, false, true, false, false, true, true)); + b.add(Arguments.of("MERGE_ON_READ", false, true, true, true, true, true, true)); + b.add(Arguments.of("MERGE_ON_READ", false, true, true, true, true, true, true)); + b.add(Arguments.of("MERGE_ON_READ", false, false, true, true, true, false, true)); } return b.build(); } @@ -96,19 +105,27 @@ public class TestHoodieDeltaStreamerSchemaEvolutionQuick extends TestHoodieDelta } protected static Stream<Arguments> testParamsWithSchemaTransformer() { + boolean fullTest = false; Stream.Builder<Arguments> b = Stream.builder(); - for (Boolean useTransformer : new Boolean[] {false, true}) { - for (Boolean setSchema : new Boolean[] {false, true}) { - for (Boolean rowWriterEnable : new Boolean[] {true}) { - for (Boolean nullForDeletedCols : new Boolean[] {false, true}) { - for (Boolean useKafkaSource : new Boolean[] {false, true}) { - for (String tableType : new String[] {"COPY_ON_WRITE", "MERGE_ON_READ"}) { - b.add(Arguments.of(tableType, rowWriterEnable, useKafkaSource, nullForDeletedCols, useTransformer, setSchema)); + if (fullTest) { + for (Boolean useTransformer : new Boolean[] {false, true}) { + for (Boolean setSchema : new Boolean[] {false, true}) { + for (Boolean rowWriterEnable : new Boolean[] {true}) { + for (Boolean nullForDeletedCols : new Boolean[] {false, true}) { + for (Boolean useKafkaSource : new Boolean[] {false, true}) { + for (String tableType : new String[] {"COPY_ON_WRITE", "MERGE_ON_READ"}) { + b.add(Arguments.of(tableType, rowWriterEnable, useKafkaSource, nullForDeletedCols, useTransformer, setSchema)); + } } } } } } + } else { + b.add(Arguments.of("COPY_ON_WRITE", true, true, true, true, true)); + b.add(Arguments.of("COPY_ON_WRITE", true, false, false, false, true)); + b.add(Arguments.of("MERGE_ON_READ", true, true, true, false, false)); + b.add(Arguments.of("MERGE_ON_READ", true, false, true, true, false)); } return b.build(); }
