This is an automated email from the ASF dual-hosted git repository.
vinoyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new df8f099 [HUDI-532] Add java doc for the test classes of hudi test
suite (#1901)
df8f099 is described below
commit df8f099c999379038bf189491ff71dfb958806dd
Author: Mathieu <[email protected]>
AuthorDate: Wed Aug 26 08:49:01 2020 +0800
[HUDI-532] Add java doc for the test classes of hudi test suite (#1901)
---
.../src/test/java/org/apache/hudi/integ/ITTestBase.java | 3 +++
.../integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java | 3 +++
.../hudi/integ/testsuite/TestFileDeltaInputWriter.java | 3 +++
.../integ/testsuite/configuration/TestWorkflowBuilder.java | 3 +++
.../hudi/integ/testsuite/converter/TestUpdateConverter.java | 9 +++++++++
.../apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java | 9 +++++++++
.../hudi/integ/testsuite/dag/HiveSyncDagGenerator.java | 3 +++
.../hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java | 4 ++++
.../org/apache/hudi/integ/testsuite/dag/TestDagUtils.java | 3 +++
.../generator/TestGenericRecordPayloadEstimator.java | 3 +++
.../generator/TestGenericRecordPayloadGenerator.java | 3 +++
.../hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java | 3 +++
.../integ/testsuite/reader/TestDFSAvroDeltaInputReader.java | 3 +++
.../testsuite/reader/TestDFSHoodieDatasetInputReader.java | 3 +++
.../org/apache/hudi/integ/testsuite/utils/TestUtils.java | 12 ++++++++++++
15 files changed, 67 insertions(+)
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java
index d0b32ee..eda776e 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java
@@ -49,6 +49,9 @@ import static org.awaitility.Awaitility.await;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+/**
+ * Base test class for IT Test helps to run command and generate data.
+ */
public abstract class ITTestBase {
public static final Logger LOG = LogManager.getLogger(ITTestBase.class);
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java
index f6d9073..ff41b44 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestDFSHoodieTestSuiteWriterAdapter.java
@@ -52,6 +52,9 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+/**
+ * Unit test against DeltaWriterAdapter, by testing writing DFS files.
+ */
public class TestDFSHoodieTestSuiteWriterAdapter extends UtilitiesTestBase {
private FilebasedSchemaProvider schemaProvider;
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java
index 7419667..8e175c5 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/TestFileDeltaInputWriter.java
@@ -48,6 +48,9 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for {@link DeltaInputWriter}.
+ */
public class TestFileDeltaInputWriter extends UtilitiesTestBase {
private FilebasedSchemaProvider schemaProvider;
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java
index f414c92..1e5ca68 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/configuration/TestWorkflowBuilder.java
@@ -30,6 +30,9 @@ import org.apache.hudi.integ.testsuite.dag.nodes.UpsertNode;
import org.apache.hudi.integ.testsuite.dag.WorkflowDag;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for the build process of {@link DagNode} and {@link WorkflowDag}.
+ */
public class TestWorkflowBuilder {
@Test
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java
index d1c2b76..c48d1b1 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/converter/TestUpdateConverter.java
@@ -23,6 +23,7 @@ import static junit.framework.TestCase.assertTrue;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+
import org.apache.avro.Schema.Field;
import org.apache.avro.generic.GenericRecord;
import org.apache.hudi.integ.testsuite.utils.TestUtils;
@@ -34,6 +35,9 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import scala.Tuple2;
+/**
+ * Test Cases for {@link UpdateConverter} APIs.
+ */
public class TestUpdateConverter {
private JavaSparkContext jsc;
@@ -49,11 +53,16 @@ public class TestUpdateConverter {
jsc.stop();
}
+ /**
+ * Test {@link UpdateConverter} by generates random updates from existing
records.
+ */
@Test
public void testGenerateUpdateRecordsFromInputRecords() throws Exception {
+ // 1. prepare input records
JavaRDD<GenericRecord> inputRDD = TestUtils.makeRDD(jsc, 10);
String schemaStr = inputRDD.take(1).get(0).getSchema().toString();
int minPayloadSize = 1000;
+
// 2. DFS converter reads existing records and generates random updates
for the same row keys
UpdateConverter updateConverter = new UpdateConverter(schemaStr,
minPayloadSize,
Arrays.asList("timestamp"), Arrays.asList("_row_key"));
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java
index a91ea01..44b0ccd 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/ComplexDagGenerator.java
@@ -29,15 +29,20 @@ import
org.apache.hudi.integ.testsuite.dag.nodes.ValidateNode;
import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
import org.apache.spark.api.java.JavaRDD;
+/**
+ * An implementation of {@link WorkflowDagGenerator}, that generates complex
workflowDag.
+ */
public class ComplexDagGenerator implements WorkflowDagGenerator {
@Override
public WorkflowDag build() {
+ // root node
DagNode root = new InsertNode(Config.newBuilder()
.withNumRecordsToInsert(1000)
.withNumInsertPartitions(3)
.withRecordSize(1000).build());
+ // child node1
DagNode child1 = new UpsertNode(Config.newBuilder()
.withNumRecordsToUpdate(999)
.withNumRecordsToInsert(1000)
@@ -46,6 +51,7 @@ public class ComplexDagGenerator implements
WorkflowDagGenerator {
.withNumInsertPartitions(1)
.withRecordSize(10000).build());
+ // function used to build ValidateNode
Function<List<DagNode<JavaRDD<WriteStatus>>>, Boolean> function =
(dagNodes) -> {
DagNode<JavaRDD<WriteStatus>> parent1 = dagNodes.get(0);
List<WriteStatus> statuses = parent1.getResult().collect();
@@ -63,8 +69,11 @@ public class ComplexDagGenerator implements
WorkflowDagGenerator {
* parent2.getConfig().getNumInsertPartitions() +
parent2.getConfig().getNumRecordsUpsert();
return b1 & b2 & b3;
};
+
+ // child node2
DagNode child2 = new ValidateNode(Config.newBuilder().build(), function);
+ // create relationship between nodes
root.addChildNode(child1);
// child1.addParentNode(root);
child1.addChildNode(child2);
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java
index 7309d43..e2f6ee5 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGenerator.java
@@ -28,6 +28,9 @@ import org.apache.hudi.integ.testsuite.dag.nodes.InsertNode;
import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
import org.apache.hudi.integ.testsuite.dag.nodes.HiveQueryNode;
+/**
+ * An implementation of {@link WorkflowDagGenerator}, helps to generate a
workflowDag with two hive nodes as child node.
+ */
public class HiveSyncDagGenerator implements WorkflowDagGenerator {
@Override
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java
index 26269fc..07dd467 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/HiveSyncDagGeneratorMOR.java
@@ -28,6 +28,10 @@ import org.apache.hudi.integ.testsuite.dag.nodes.InsertNode;
import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
import org.apache.hudi.integ.testsuite.dag.nodes.HiveQueryNode;
+/**
+ * An implementation of {@link WorkflowDagGenerator}, that generates
workflowDag with hive nodes as child node
+ * for MOR table.
+ */
public class HiveSyncDagGeneratorMOR implements WorkflowDagGenerator {
@Override
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java
index b5a0cd3..418e644 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/dag/TestDagUtils.java
@@ -29,6 +29,9 @@ import org.apache.hudi.utilities.testutils.UtilitiesTestBase;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+/**
+ * A utility class for DAG test.
+ */
public class TestDagUtils {
private static final String COW_DAG_DOCKER_DEMO_RELATIVE_PATH =
"/docker/demo/config/test-suite/complex-dag-cow.yaml";
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java
index 85d5358..c31a7d6 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadEstimator.java
@@ -25,6 +25,9 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.utilities.testutils.UtilitiesTestBase;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for {@link GenericRecordFullPayloadSizeEstimator}.
+ */
public class TestGenericRecordPayloadEstimator {
private static final String SOURCE_SCHEMA_DOCKER_DEMO_RELATIVE_PATH =
"/docker/demo/config/test-suite/source.avsc";
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java
index 886fb16..7524d4a 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/generator/TestGenericRecordPayloadGenerator.java
@@ -32,6 +32,9 @@ import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.utilities.testutils.UtilitiesTestBase;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for {@link GenericRecordFullPayloadGenerator} and {@link
GenericRecordPartialPayloadGenerator}.
+ */
public class TestGenericRecordPayloadGenerator {
private static final String SOURCE_SCHEMA_DOCKER_DEMO_RELATIVE_PATH =
"/docker/demo/config/test-suite/source.avsc";
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
index 4c71b60..5826019 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
@@ -49,6 +49,9 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
+/**
+ * Unit test against {@link HoodieTestSuiteJob}.
+ */
public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
private static final String TEST_NAME_WITH_PARAMS = "[{index}] Test with
useDeltaStreamer={0}, tableType={1}";
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java
index 6d0b994..fa8f4ac 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSAvroDeltaInputReader.java
@@ -36,6 +36,9 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for {@link DFSAvroDeltaInputReader} by issuing analyzeSingleFile
and read from it.
+ */
public class TestDFSAvroDeltaInputReader extends UtilitiesTestBase {
@BeforeAll
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java
index 911f168..941ef43 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/reader/TestDFSHoodieDatasetInputReader.java
@@ -40,6 +40,9 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+/**
+ * Unit test for {@link DFSHoodieDatasetInputReader}.
+ */
public class TestDFSHoodieDatasetInputReader extends UtilitiesTestBase {
@BeforeAll
diff --git
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java
index 4699324..372d674 100644
---
a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java
+++
b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/utils/TestUtils.java
@@ -28,6 +28,9 @@ import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;
+/**
+ * A utility class for testing purpose.
+ */
public class TestUtils {
/**
@@ -45,6 +48,15 @@ public class TestUtils {
return dataGenerator.generateGenericRecords(numRecords);
}
+ /**
+ * Method helps to create avro files and save it to file.
+ *
+ * @param jsc Java Spark Context jsc.
+ * @param sparkSession Spark Session, the entry point to programming
Spark with the Dataset and DataFrame API.
+ * @param basePath The basePath where files are written.
+ * @param numFiles The number of files to create.
+ * @param numRecordsPerFile The number of records per file.
+ */
public static void createAvroFiles(JavaSparkContext jsc, SparkSession
sparkSession, String basePath, int numFiles,
int numRecordsPerFile) {
Schema schema = HoodieTestDataGenerator.AVRO_SCHEMA;
