This is an automated email from the ASF dual-hosted git repository. vinoyang pushed a commit to branch hudi_test_suite in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
commit 4ac72f4f62800725a469b9a8bc8110acb0b18fb1 Author: yanghua <yanghua1...@gmail.com> AuthorDate: Tue Dec 3 19:44:15 2019 +0800 Rename module name from hudi-bench to hudi-end-to-end-tests --- docker/hoodie/hadoop/hive_base/Dockerfile | 2 +- docker/hoodie/hadoop/hive_base/pom.xml | 4 ++-- {hudi-bench => hudi-end-to-end-tests}/pom.xml | 2 +- .../prepare_integration_suite.sh | 4 ++-- .../apache/hudi/e2e}/DFSDeltaWriterAdapter.java | 6 +++--- .../apache/hudi/e2e}/DFSSparkAvroDeltaWriter.java | 8 ++++---- .../org/apache/hudi/e2e}/DeltaInputFormat.java | 2 +- .../java/org/apache/hudi/e2e}/DeltaOutputType.java | 2 +- .../org/apache/hudi/e2e}/DeltaWriterAdapter.java | 4 ++-- .../org/apache/hudi/e2e}/DeltaWriterFactory.java | 10 ++++----- .../hudi/e2e}/configuration/DFSDeltaConfig.java | 6 +++--- .../hudi/e2e}/configuration/DeltaConfig.java | 6 +++--- .../hudi/e2e}/converter/UpdateConverter.java | 6 +++--- .../java/org/apache/hudi/e2e}/dag/DagUtils.java | 9 ++++---- .../org/apache/hudi/e2e}/dag/ExecutionContext.java | 9 ++++---- .../java/org/apache/hudi/e2e}/dag/WorkflowDag.java | 4 ++-- .../apache/hudi/e2e}/dag/WorkflowDagGenerator.java | 12 +++++------ .../apache/hudi/e2e}/dag/nodes/BulkInsertNode.java | 6 +++--- .../org/apache/hudi/e2e}/dag/nodes/CleanNode.java | 4 ++-- .../apache/hudi/e2e}/dag/nodes/CompactNode.java | 6 +++--- .../org/apache/hudi/e2e}/dag/nodes/DagNode.java | 6 +++--- .../apache/hudi/e2e}/dag/nodes/HiveQueryNode.java | 8 ++++---- .../apache/hudi/e2e}/dag/nodes/HiveSyncNode.java | 8 ++++---- .../org/apache/hudi/e2e}/dag/nodes/InsertNode.java | 10 ++++----- .../apache/hudi/e2e}/dag/nodes/RollbackNode.java | 6 +++--- .../hudi/e2e}/dag/nodes/ScheduleCompactNode.java | 6 +++--- .../hudi/e2e}/dag/nodes/SparkSQLQueryNode.java | 8 ++++---- .../org/apache/hudi/e2e}/dag/nodes/UpsertNode.java | 8 ++++---- .../apache/hudi/e2e}/dag/nodes/ValidateNode.java | 6 +++--- .../hudi/e2e}/dag/scheduler/DagScheduler.java | 12 +++++------ .../apache/hudi/e2e}/generator/DeltaGenerator.java | 24 +++++++++++----------- .../FlexibleSchemaRecordGenerationIterator.java | 2 +- .../GenericRecordFullPayloadGenerator.java | 2 +- .../GenericRecordFullPayloadSizeEstimator.java | 2 +- .../GenericRecordPartialPayloadGenerator.java | 2 +- .../generator/LazyRecordGeneratorIterator.java | 2 +- .../e2e}/generator/UpdateGeneratorIterator.java | 2 +- .../e2e}/helpers/DFSTestSuitePathSelector.java | 2 +- .../hudi/e2e}/helpers/HiveServiceProvider.java | 6 +++--- .../hudi/e2e}/job/HoodieDeltaStreamerWrapper.java | 2 +- .../apache/hudi/e2e}/job/HoodieTestSuiteJob.java | 24 +++++++++++----------- .../hudi/e2e}/reader/DFSAvroDeltaInputReader.java | 8 ++++---- .../hudi/e2e}/reader/DFSDeltaInputReader.java | 2 +- .../e2e}/reader/DFSHoodieDatasetInputReader.java | 2 +- .../e2e}/reader/DFSParquetDeltaInputReader.java | 6 +++--- .../apache/hudi/e2e}/reader/DeltaInputReader.java | 2 +- .../apache/hudi/e2e}/reader/SparkBasedReader.java | 2 +- .../hudi/e2e}/writer/AvroDeltaInputWriter.java | 2 +- .../apache/hudi/e2e}/writer/DeltaInputWriter.java | 2 +- .../org/apache/hudi/e2e}/writer/DeltaWriter.java | 6 +++--- .../hudi/e2e}/writer/FileDeltaInputWriter.java | 2 +- .../e2e}/writer/SparkAvroDeltaInputWriter.java | 2 +- .../org/apache/hudi/e2e}/writer/WriteStats.java | 2 +- .../hudi/e2e}/TestDFSDeltaWriterAdapter.java | 16 +++++++-------- .../apache/hudi/e2e}/TestFileDeltaInputWriter.java | 14 ++++++------- .../e2e}/configuration/TestWorkflowBuilder.java | 12 +++++------ .../hudi/e2e}/converter/TestUpdateConverter.java | 4 ++-- .../org/apache/hudi/e2e}/dag/TestComplexDag.java | 14 +++++++------ .../org/apache/hudi/e2e}/dag/TestDagUtils.java | 13 +++++++----- .../org/apache/hudi/e2e}/dag/TestHiveSyncDag.java | 14 +++++++------ .../apache/hudi/e2e}/dag/TestInsertOnlyDag.java | 10 +++++---- .../apache/hudi/e2e}/dag/TestInsertUpsertDag.java | 12 ++++++----- .../TestGenericRecordPayloadEstimator.java | 7 ++++--- .../TestGenericRecordPayloadGenerator.java | 16 ++++++++------- .../hudi/e2e}/generator/TestWorkloadGenerator.java | 22 ++++++++++---------- .../hudi/e2e}/job/TestHoodieTestSuiteJob.java | 22 ++++++++++---------- .../e2e}/reader/TestDFSAvroDeltaInputReader.java | 4 ++-- .../reader/TestDFSHoodieDatasetInputReader.java | 5 +++-- .../java/org/apache/hudi/e2e}/utils/TestUtils.java | 2 +- .../apache/hudi/e2e}/writer/TestDeltaWriter.java | 7 ++++--- .../resources/hudi-bench-config/base.properties | 0 .../hudi-bench-config/complex-source.avsc | 0 .../complex-workflow-dag-cow.yaml | 0 .../test/resources/hudi-bench-config/source.avsc | 0 .../test/resources/hudi-bench-config/target.avsc | 0 .../test/resources/log4j-surefire-quiet.properties | 0 .../src/test/resources/log4j-surefire.properties | 0 .../pom.xml | 6 +++--- .../java/org/apache/hudi/e2e}/bundle/Main.java | 2 +- 79 files changed, 258 insertions(+), 242 deletions(-) diff --git a/docker/hoodie/hadoop/hive_base/Dockerfile b/docker/hoodie/hadoop/hive_base/Dockerfile index 1273c7e..d2b5650 100644 --- a/docker/hoodie/hadoop/hive_base/Dockerfile +++ b/docker/hoodie/hadoop/hive_base/Dockerfile @@ -56,7 +56,7 @@ ENV HUDI_HADOOP_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hood ENV HUDI_HIVE_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-hive-bundle.jar ENV HUDI_SPARK_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-spark-bundle.jar ENV HUDI_UTILITIES_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-utilities.jar -ENV HUDI_BENCH_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-bench-bundle.jar +ENV HUDI_END_TO_END_TESTS_BUNDLE=/var/hoodie/ws/docker/hoodie/hadoop/hive_base/target/hoodie-end-to-end-tests-bundle.jar COPY startup.sh /usr/local/bin/ RUN chmod +x /usr/local/bin/startup.sh diff --git a/docker/hoodie/hadoop/hive_base/pom.xml b/docker/hoodie/hadoop/hive_base/pom.xml index 8a13fb4..12608e4 100644 --- a/docker/hoodie/hadoop/hive_base/pom.xml +++ b/docker/hoodie/hadoop/hive_base/pom.xml @@ -62,8 +62,8 @@ file="${project.basedir}/../../../../packaging/hudi-utilities-bundle/target/hudi-utilities-bundle-${project.version}.jar" tofile="target/hoodie-utilities.jar"/> <copy - file="${project.basedir}/../../../../packaging/hudi-bench-bundle/target/hudi-bench-bundle-${project.version}.jar" - tofile="target/hoodie-bench-bundle.jar"/> + file="${project.basedir}/../../../../packaging/hudi-end-to-end-tests-bundle/target/hudi-end-to-end-tests-bundle-${project.version}.jar" + tofile="target/hoodie-end-to-end-tests-bundle.jar"/> </tasks> </configuration> <goals> diff --git a/hudi-bench/pom.xml b/hudi-end-to-end-tests/pom.xml similarity index 99% rename from hudi-bench/pom.xml rename to hudi-end-to-end-tests/pom.xml index 69cc78b..7cb1bd2 100644 --- a/hudi-bench/pom.xml +++ b/hudi-end-to-end-tests/pom.xml @@ -24,7 +24,7 @@ </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>hudi-bench</artifactId> + <artifactId>hudi-end-to-end-tests</artifactId> <packaging>jar</packaging> <properties> diff --git a/hudi-bench/prepare_integration_suite.sh b/hudi-end-to-end-tests/prepare_integration_suite.sh similarity index 95% rename from hudi-bench/prepare_integration_suite.sh rename to hudi-end-to-end-tests/prepare_integration_suite.sh index 69ecdd9..9e28988 100644 --- a/hudi-bench/prepare_integration_suite.sh +++ b/hudi-end-to-end-tests/prepare_integration_suite.sh @@ -24,8 +24,8 @@ echo "spark-submit --packages com.databricks:spark-avro_2.11:4.0.0 \ --master $0 \ --deploy-mode $1 \ --properties-file $2 \ ---class org.apache.hudi.bench.job.HoodieTestSuiteJob \ -`ls target/hudi-bench-*-SNAPSHOT.jar` \ +--class org.apache.hudi.e2e.job.HoodieTestSuiteJob \ +`ls target/hudi-end-to-end-tests-*-SNAPSHOT.jar` \ --source-class $3 \ --source-ordering-field $4 \ --input-base-path $5 \ diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DFSDeltaWriterAdapter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSDeltaWriterAdapter.java similarity index 94% rename from hudi-bench/src/main/java/org/apache/hudi/bench/DFSDeltaWriterAdapter.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSDeltaWriterAdapter.java index e3f1f7b..d743dc3 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/DFSDeltaWriterAdapter.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSDeltaWriterAdapter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench; +package org.apache.hudi.e2e; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; @@ -24,8 +24,8 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; import org.apache.avro.generic.GenericRecord; -import org.apache.hudi.bench.writer.FileDeltaInputWriter; -import org.apache.hudi.bench.writer.WriteStats; +import org.apache.hudi.e2e.writer.FileDeltaInputWriter; +import org.apache.hudi.e2e.writer.WriteStats; /** * {@link org.apache.hadoop.hdfs.DistributedFileSystem} (or {@link org.apache.hadoop.fs.LocalFileSystem}) based delta diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DFSSparkAvroDeltaWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSSparkAvroDeltaWriter.java similarity index 88% rename from hudi-bench/src/main/java/org/apache/hudi/bench/DFSSparkAvroDeltaWriter.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSSparkAvroDeltaWriter.java index 8c7c8b5..515d0da 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/DFSSparkAvroDeltaWriter.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DFSSparkAvroDeltaWriter.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package org.apache.hudi.bench; +package org.apache.hudi.e2e; import java.io.IOException; import org.apache.avro.generic.GenericRecord; -import org.apache.hudi.bench.DeltaWriterAdapter.SparkBasedDeltaWriter; -import org.apache.hudi.bench.writer.DeltaInputWriter; -import org.apache.hudi.bench.writer.WriteStats; +import org.apache.hudi.e2e.DeltaWriterAdapter.SparkBasedDeltaWriter; +import org.apache.hudi.e2e.writer.DeltaInputWriter; +import org.apache.hudi.e2e.writer.WriteStats; import org.apache.spark.api.java.JavaRDD; /** diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaInputFormat.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaInputFormat.java similarity index 96% rename from hudi-bench/src/main/java/org/apache/hudi/bench/DeltaInputFormat.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaInputFormat.java index 46d1425..6175b7c 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaInputFormat.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaInputFormat.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench; +package org.apache.hudi.e2e; /** * Supported delta input data formats diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaOutputType.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaOutputType.java similarity index 96% rename from hudi-bench/src/main/java/org/apache/hudi/bench/DeltaOutputType.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaOutputType.java index 77a1b0a..bdd9c29 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaOutputType.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaOutputType.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench; +package org.apache.hudi.e2e; /** * Supported output destination types for the generated delta workload diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterAdapter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterAdapter.java similarity index 93% rename from hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterAdapter.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterAdapter.java index da001db..06955b9 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterAdapter.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterAdapter.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.hudi.bench; +package org.apache.hudi.e2e; import java.io.IOException; import java.util.Iterator; import java.util.List; -import org.apache.hudi.bench.writer.WriteStats; +import org.apache.hudi.e2e.writer.WriteStats; import org.apache.spark.api.java.JavaRDD; public interface DeltaWriterAdapter<I> { diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterFactory.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterFactory.java similarity index 89% rename from hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterFactory.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterFactory.java index aeefa48..078d04d 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/DeltaWriterFactory.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/DeltaWriterFactory.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package org.apache.hudi.bench; +package org.apache.hudi.e2e; import java.io.IOException; import org.apache.avro.generic.GenericRecord; -import org.apache.hudi.bench.configuration.DFSDeltaConfig; -import org.apache.hudi.bench.configuration.DeltaConfig; -import org.apache.hudi.bench.writer.AvroDeltaInputWriter; -import org.apache.hudi.bench.writer.FileDeltaInputWriter; +import org.apache.hudi.e2e.configuration.DFSDeltaConfig; +import org.apache.hudi.e2e.configuration.DeltaConfig; +import org.apache.hudi.e2e.writer.AvroDeltaInputWriter; +import org.apache.hudi.e2e.writer.FileDeltaInputWriter; import org.apache.hudi.common.util.StringUtils; /** diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DFSDeltaConfig.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DFSDeltaConfig.java similarity index 94% rename from hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DFSDeltaConfig.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DFSDeltaConfig.java index 3781f3c..57d4a56 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DFSDeltaConfig.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DFSDeltaConfig.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.hudi.bench.configuration; +package org.apache.hudi.e2e.configuration; -import org.apache.hudi.bench.DeltaInputFormat; -import org.apache.hudi.bench.DeltaOutputType; +import org.apache.hudi.e2e.DeltaInputFormat; +import org.apache.hudi.e2e.DeltaOutputType; import org.apache.hudi.common.SerializableConfiguration; /** diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DeltaConfig.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DeltaConfig.java similarity index 98% rename from hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DeltaConfig.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DeltaConfig.java index 11aa072..6c0b351 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/configuration/DeltaConfig.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/configuration/DeltaConfig.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.configuration; +package org.apache.hudi.e2e.configuration; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; @@ -26,8 +26,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.hudi.bench.DeltaInputFormat; -import org.apache.hudi.bench.DeltaOutputType; +import org.apache.hudi.e2e.DeltaInputFormat; +import org.apache.hudi.e2e.DeltaOutputType; import org.apache.hudi.common.SerializableConfiguration; import org.apache.hudi.common.util.collection.Pair; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/converter/UpdateConverter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/converter/UpdateConverter.java similarity index 92% rename from hudi-bench/src/main/java/org/apache/hudi/bench/converter/UpdateConverter.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/converter/UpdateConverter.java index 7cf48c3..5d58bd3 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/converter/UpdateConverter.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/converter/UpdateConverter.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.hudi.bench.converter; +package org.apache.hudi.e2e.converter; import java.util.List; import org.apache.avro.generic.GenericRecord; -import org.apache.hudi.bench.generator.LazyRecordGeneratorIterator; -import org.apache.hudi.bench.generator.UpdateGeneratorIterator; +import org.apache.hudi.e2e.generator.LazyRecordGeneratorIterator; +import org.apache.hudi.e2e.generator.UpdateGeneratorIterator; import org.apache.hudi.utilities.converter.Converter; import org.apache.spark.api.java.JavaRDD; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/DagUtils.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/DagUtils.java similarity index 96% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/DagUtils.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/DagUtils.java index 93a04b3..2d93e7a 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/DagUtils.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/DagUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag; +package org.apache.hudi.e2e.dag; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.JsonNode; @@ -37,9 +37,8 @@ import java.util.Map.Entry; import java.util.stream.Collectors; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hudi.bench.configuration.DeltaConfig; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.nodes.DagNode; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.nodes.DagNode; import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.Pair; @@ -119,7 +118,7 @@ public class DagUtils { private static DagNode convertJsonToDagNode(JsonNode node, String type) { try { - DeltaConfig.Config config = Config.newBuilder().withConfigsMap(convertJsonNodeToMap(node)).build(); + Config config = Config.newBuilder().withConfigsMap(convertJsonNodeToMap(node)).build(); return (DagNode) ReflectionUtils.loadClass(generateFQN(type), config); } catch (ClassNotFoundException e) { throw new RuntimeException(e); diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/ExecutionContext.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/ExecutionContext.java similarity index 87% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/ExecutionContext.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/ExecutionContext.java index 03b585f..6b640f5 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/ExecutionContext.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/ExecutionContext.java @@ -16,16 +16,17 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag; +package org.apache.hudi.e2e.dag; import java.io.Serializable; -import org.apache.hudi.bench.generator.DeltaGenerator; -import org.apache.hudi.bench.writer.DeltaWriter; +import org.apache.hudi.e2e.generator.DeltaGenerator; +import org.apache.hudi.e2e.writer.DeltaWriter; +import org.apache.hudi.e2e.dag.nodes.DagNode; import org.apache.spark.api.java.JavaSparkContext; /** * This wraps the context needed for an execution of - * a {@link org.apache.hudi.bench.dag.nodes.DagNode#execute(ExecutionContext)} + * a {@link DagNode#execute(ExecutionContext)} */ public class ExecutionContext implements Serializable { diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDag.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDag.java similarity index 93% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDag.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDag.java index b88bd74..bcd45c2 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDag.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDag.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag; +package org.apache.hudi.e2e.dag; import java.util.List; -import org.apache.hudi.bench.dag.nodes.DagNode; +import org.apache.hudi.e2e.dag.nodes.DagNode; /** * Workflow dag that encapsulates all execute nodes diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDagGenerator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDagGenerator.java similarity index 88% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDagGenerator.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDagGenerator.java index a5602ce..171e27f 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/WorkflowDagGenerator.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/WorkflowDagGenerator.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag; +package org.apache.hudi.e2e.dag; import java.util.ArrayList; import java.util.List; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.nodes.DagNode; -import org.apache.hudi.bench.dag.nodes.HiveQueryNode; -import org.apache.hudi.bench.dag.nodes.InsertNode; -import org.apache.hudi.bench.dag.nodes.UpsertNode; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.nodes.DagNode; +import org.apache.hudi.e2e.dag.nodes.HiveQueryNode; +import org.apache.hudi.e2e.dag.nodes.InsertNode; +import org.apache.hudi.e2e.dag.nodes.UpsertNode; import org.apache.hudi.common.util.collection.Pair; /** diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/BulkInsertNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/BulkInsertNode.java similarity index 89% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/BulkInsertNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/BulkInsertNode.java index e855075..b4bf25d 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/BulkInsertNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/BulkInsertNode.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; import org.apache.hudi.WriteStatus; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.writer.DeltaWriter; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.writer.DeltaWriter; import org.apache.hudi.common.util.Option; import org.apache.spark.api.java.JavaRDD; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CleanNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CleanNode.java similarity index 92% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CleanNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CleanNode.java index 6e43259..861c338 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CleanNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CleanNode.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; -import org.apache.hudi.bench.dag.ExecutionContext; +import org.apache.hudi.e2e.dag.ExecutionContext; public class CleanNode extends DagNode<Boolean> { diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CompactNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CompactNode.java similarity index 92% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CompactNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CompactNode.java index 2c65252..bd62fb9 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/CompactNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/CompactNode.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; import org.apache.hudi.WriteStatus; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.ExecutionContext; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.ExecutionContext; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.Option; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/DagNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/DagNode.java similarity index 95% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/DagNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/DagNode.java index 3f4083a..321cee9 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/DagNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/DagNode.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Objects; import java.util.UUID; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.ExecutionContext; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.ExecutionContext; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveQueryNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveQueryNode.java similarity index 93% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveQueryNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveQueryNode.java index 8c5cac5..c58f00a 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveQueryNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveQueryNode.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; import java.sql.Connection; import java.sql.DriverManager; @@ -24,9 +24,9 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import org.apache.hudi.DataSourceUtils; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.ExecutionContext; -import org.apache.hudi.bench.helpers.HiveServiceProvider; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.ExecutionContext; +import org.apache.hudi.e2e.helpers.HiveServiceProvider; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.hive.HiveSyncConfig; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveSyncNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveSyncNode.java similarity index 88% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveSyncNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveSyncNode.java index 7ad06c6..c91a77a 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/HiveSyncNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/HiveSyncNode.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.ExecutionContext; -import org.apache.hudi.bench.helpers.HiveServiceProvider; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.ExecutionContext; +import org.apache.hudi.e2e.helpers.HiveServiceProvider; public class HiveSyncNode extends DagNode<Boolean> { diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/InsertNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/InsertNode.java similarity index 89% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/InsertNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/InsertNode.java index 73a1e22..8573a28 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/InsertNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/InsertNode.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; import org.apache.hudi.WriteStatus; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.ExecutionContext; -import org.apache.hudi.bench.generator.DeltaGenerator; -import org.apache.hudi.bench.writer.DeltaWriter; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.ExecutionContext; +import org.apache.hudi.e2e.generator.DeltaGenerator; +import org.apache.hudi.e2e.writer.DeltaWriter; import org.apache.hudi.common.util.Option; import org.apache.spark.api.java.JavaRDD; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/RollbackNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/RollbackNode.java similarity index 92% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/RollbackNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/RollbackNode.java index 1128368..16bcc1c 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/RollbackNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/RollbackNode.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.ExecutionContext; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.ExecutionContext; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.Option; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ScheduleCompactNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ScheduleCompactNode.java similarity index 94% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ScheduleCompactNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ScheduleCompactNode.java index d983af2..1ffda7b 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ScheduleCompactNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ScheduleCompactNode.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.ExecutionContext; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.ExecutionContext; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/SparkSQLQueryNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/SparkSQLQueryNode.java similarity index 91% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/SparkSQLQueryNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/SparkSQLQueryNode.java index d6d931e..14cb7d2 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/SparkSQLQueryNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/SparkSQLQueryNode.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.ExecutionContext; -import org.apache.hudi.bench.helpers.HiveServiceProvider; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.ExecutionContext; +import org.apache.hudi.e2e.helpers.HiveServiceProvider; import org.apache.hudi.common.util.collection.Pair; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/UpsertNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/UpsertNode.java similarity index 90% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/UpsertNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/UpsertNode.java index 60b013f..b9345df 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/UpsertNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/UpsertNode.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; import org.apache.hudi.WriteStatus; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.generator.DeltaGenerator; -import org.apache.hudi.bench.writer.DeltaWriter; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.generator.DeltaGenerator; +import org.apache.hudi.e2e.writer.DeltaWriter; import org.apache.hudi.common.util.Option; import org.apache.spark.api.java.JavaRDD; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ValidateNode.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ValidateNode.java similarity index 91% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ValidateNode.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ValidateNode.java index 3f28abc..ea51864 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/nodes/ValidateNode.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/nodes/ValidateNode.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.nodes; +package org.apache.hudi.e2e.dag.nodes; import java.util.List; import java.util.function.Function; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.ExecutionContext; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.ExecutionContext; public class ValidateNode<R> extends DagNode { diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/scheduler/DagScheduler.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/scheduler/DagScheduler.java similarity index 91% rename from hudi-bench/src/main/java/org/apache/hudi/bench/dag/scheduler/DagScheduler.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/scheduler/DagScheduler.java index 1dde398..db74f0c 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/dag/scheduler/DagScheduler.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/dag/scheduler/DagScheduler.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag.scheduler; +package org.apache.hudi.e2e.dag.scheduler; import java.util.ArrayList; import java.util.HashSet; @@ -28,11 +28,11 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.apache.hudi.bench.dag.ExecutionContext; -import org.apache.hudi.bench.dag.WorkflowDag; -import org.apache.hudi.bench.dag.nodes.DagNode; -import org.apache.hudi.bench.generator.DeltaGenerator; -import org.apache.hudi.bench.writer.DeltaWriter; +import org.apache.hudi.e2e.dag.ExecutionContext; +import org.apache.hudi.e2e.dag.WorkflowDag; +import org.apache.hudi.e2e.dag.nodes.DagNode; +import org.apache.hudi.e2e.generator.DeltaGenerator; +import org.apache.hudi.e2e.writer.DeltaWriter; import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/DeltaGenerator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/DeltaGenerator.java similarity index 94% rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/DeltaGenerator.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/DeltaGenerator.java index 335fecf..a7fe7bd 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/DeltaGenerator.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/DeltaGenerator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.generator; +package org.apache.hudi.e2e.generator; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; @@ -32,17 +32,17 @@ import java.util.Map; import java.util.stream.StreamSupport; import org.apache.avro.generic.GenericRecord; import org.apache.hudi.KeyGenerator; -import org.apache.hudi.bench.DeltaOutputType; -import org.apache.hudi.bench.DeltaWriterAdapter; -import org.apache.hudi.bench.DeltaWriterFactory; -import org.apache.hudi.bench.configuration.DFSDeltaConfig; -import org.apache.hudi.bench.configuration.DeltaConfig; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.converter.UpdateConverter; -import org.apache.hudi.bench.reader.DFSAvroDeltaInputReader; -import org.apache.hudi.bench.reader.DFSHoodieDatasetInputReader; -import org.apache.hudi.bench.reader.DeltaInputReader; -import org.apache.hudi.bench.writer.WriteStats; +import org.apache.hudi.e2e.DeltaOutputType; +import org.apache.hudi.e2e.DeltaWriterAdapter; +import org.apache.hudi.e2e.DeltaWriterFactory; +import org.apache.hudi.e2e.configuration.DFSDeltaConfig; +import org.apache.hudi.e2e.configuration.DeltaConfig; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.converter.UpdateConverter; +import org.apache.hudi.e2e.reader.DFSAvroDeltaInputReader; +import org.apache.hudi.e2e.reader.DFSHoodieDatasetInputReader; +import org.apache.hudi.e2e.reader.DeltaInputReader; +import org.apache.hudi.e2e.writer.WriteStats; import org.apache.hudi.common.util.Option; import org.apache.hudi.utilities.converter.Converter; import org.apache.log4j.LogManager; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/FlexibleSchemaRecordGenerationIterator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/FlexibleSchemaRecordGenerationIterator.java similarity index 98% rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/FlexibleSchemaRecordGenerationIterator.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/FlexibleSchemaRecordGenerationIterator.java index a982792..cf23460 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/FlexibleSchemaRecordGenerationIterator.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/FlexibleSchemaRecordGenerationIterator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.generator; +package org.apache.hudi.e2e.generator; import java.util.Iterator; import java.util.List; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadGenerator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadGenerator.java similarity index 99% rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadGenerator.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadGenerator.java index 2232cc5..09001e0 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadGenerator.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadGenerator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.generator; +package org.apache.hudi.e2e.generator; import com.google.common.annotations.VisibleForTesting; import java.io.Serializable; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadSizeEstimator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadSizeEstimator.java similarity index 99% rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadSizeEstimator.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadSizeEstimator.java index 1d515e9..ca32cf8 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordFullPayloadSizeEstimator.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordFullPayloadSizeEstimator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.generator; +package org.apache.hudi.e2e.generator; import java.io.Serializable; import java.util.List; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordPartialPayloadGenerator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordPartialPayloadGenerator.java similarity index 98% rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordPartialPayloadGenerator.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordPartialPayloadGenerator.java index 04174cd..091724f 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/GenericRecordPartialPayloadGenerator.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/GenericRecordPartialPayloadGenerator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.generator; +package org.apache.hudi.e2e.generator; import com.google.common.annotations.VisibleForTesting; import org.apache.avro.Schema; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/LazyRecordGeneratorIterator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/LazyRecordGeneratorIterator.java similarity index 97% rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/LazyRecordGeneratorIterator.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/LazyRecordGeneratorIterator.java index 902aaee..0878736 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/LazyRecordGeneratorIterator.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/LazyRecordGeneratorIterator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.generator; +package org.apache.hudi.e2e.generator; import java.util.Iterator; import org.apache.avro.generic.GenericRecord; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/UpdateGeneratorIterator.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/UpdateGeneratorIterator.java similarity index 97% rename from hudi-bench/src/main/java/org/apache/hudi/bench/generator/UpdateGeneratorIterator.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/UpdateGeneratorIterator.java index 235b090..bfe0526 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/generator/UpdateGeneratorIterator.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/generator/UpdateGeneratorIterator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.generator; +package org.apache.hudi.e2e.generator; import java.util.ArrayList; import java.util.Iterator; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/helpers/DFSTestSuitePathSelector.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/DFSTestSuitePathSelector.java similarity index 98% rename from hudi-bench/src/main/java/org/apache/hudi/bench/helpers/DFSTestSuitePathSelector.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/DFSTestSuitePathSelector.java index 3167407..182d5aa 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/helpers/DFSTestSuitePathSelector.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/DFSTestSuitePathSelector.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.helpers; +package org.apache.hudi.e2e.helpers; import java.io.IOException; import java.util.ArrayList; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/helpers/HiveServiceProvider.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/HiveServiceProvider.java similarity index 92% rename from hudi-bench/src/main/java/org/apache/hudi/bench/helpers/HiveServiceProvider.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/HiveServiceProvider.java index 5ae2ed1..6cee004 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/helpers/HiveServiceProvider.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/helpers/HiveServiceProvider.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package org.apache.hudi.bench.helpers; +package org.apache.hudi.e2e.helpers; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hive.service.server.HiveServer2; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.writer.DeltaWriter; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.writer.DeltaWriter; import org.apache.hudi.hive.util.HiveTestService; public class HiveServiceProvider { diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieDeltaStreamerWrapper.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieDeltaStreamerWrapper.java similarity index 98% rename from hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieDeltaStreamerWrapper.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieDeltaStreamerWrapper.java index e0f77df..96cf6e4 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieDeltaStreamerWrapper.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieDeltaStreamerWrapper.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.job; +package org.apache.hudi.e2e.job; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.hive.conf.HiveConf; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieTestSuiteJob.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieTestSuiteJob.java similarity index 91% rename from hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieTestSuiteJob.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieTestSuiteJob.java index b9e52d3..90bd9af 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/job/HoodieTestSuiteJob.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/job/HoodieTestSuiteJob.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.job; +package org.apache.hudi.e2e.job; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; @@ -27,15 +27,15 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hudi.DataSourceUtils; import org.apache.hudi.KeyGenerator; -import org.apache.hudi.bench.DeltaInputFormat; -import org.apache.hudi.bench.DeltaOutputType; -import org.apache.hudi.bench.configuration.DFSDeltaConfig; -import org.apache.hudi.bench.dag.DagUtils; -import org.apache.hudi.bench.dag.WorkflowDag; -import org.apache.hudi.bench.dag.WorkflowDagGenerator; -import org.apache.hudi.bench.dag.scheduler.DagScheduler; -import org.apache.hudi.bench.generator.DeltaGenerator; -import org.apache.hudi.bench.writer.DeltaWriter; +import org.apache.hudi.e2e.DeltaInputFormat; +import org.apache.hudi.e2e.DeltaOutputType; +import org.apache.hudi.e2e.configuration.DFSDeltaConfig; +import org.apache.hudi.e2e.dag.DagUtils; +import org.apache.hudi.e2e.dag.WorkflowDag; +import org.apache.hudi.e2e.dag.WorkflowDagGenerator; +import org.apache.hudi.e2e.dag.scheduler.DagScheduler; +import org.apache.hudi.e2e.generator.DeltaGenerator; +import org.apache.hudi.e2e.writer.DeltaWriter; import org.apache.hudi.common.SerializableConfiguration; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.FSUtils; @@ -164,11 +164,11 @@ public class HoodieTestSuiteJob { public String workloadYamlPath; @Parameter(names = {"--delta-output-type"}, description = "Subclass of " - + "org.apache.hudi.bench.workload.DeltaOutputType to readAvro data.") + + "org.apache.hudi.e2e.workload.DeltaOutputType to readAvro data.") public String outputTypeName = DeltaOutputType.DFS.name(); @Parameter(names = {"--delta-input-format"}, description = "Subclass of " - + "org.apache.hudi.bench.workload.DeltaOutputType to read avro data.") + + "org.apache.hudi.e2e.workload.DeltaOutputType to read avro data.") public String inputFormatName = DeltaInputFormat.AVRO.name(); @Parameter(names = {"--input-file-size"}, description = "The min/max size of the input files to generate", diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSAvroDeltaInputReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSAvroDeltaInputReader.java similarity index 94% rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSAvroDeltaInputReader.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSAvroDeltaInputReader.java index 7d06c77..622687b 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSAvroDeltaInputReader.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSAvroDeltaInputReader.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package org.apache.hudi.bench.reader; +package org.apache.hudi.e2e.reader; import java.io.IOException; import java.util.Arrays; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.PathFilter; -import org.apache.hudi.bench.DeltaInputFormat; -import org.apache.hudi.bench.DeltaOutputType; -import org.apache.hudi.bench.writer.AvroDeltaInputWriter; +import org.apache.hudi.e2e.DeltaInputFormat; +import org.apache.hudi.e2e.DeltaOutputType; +import org.apache.hudi.e2e.writer.AvroDeltaInputWriter; import org.apache.hudi.common.util.Option; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.SparkSession; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSDeltaInputReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSDeltaInputReader.java similarity index 99% rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSDeltaInputReader.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSDeltaInputReader.java index 9d3e5f2..bc2ffb3 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSDeltaInputReader.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSDeltaInputReader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.reader; +package org.apache.hudi.e2e.reader; import java.io.IOException; import java.util.ArrayList; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSHoodieDatasetInputReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSHoodieDatasetInputReader.java similarity index 99% rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSHoodieDatasetInputReader.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSHoodieDatasetInputReader.java index 462acc7..f96cdef 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSHoodieDatasetInputReader.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSHoodieDatasetInputReader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.reader; +package org.apache.hudi.e2e.reader; import static java.util.Map.Entry.comparingByValue; import static java.util.stream.Collectors.toMap; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSParquetDeltaInputReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSParquetDeltaInputReader.java similarity index 96% rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSParquetDeltaInputReader.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSParquetDeltaInputReader.java index c0016a2..776b16e 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DFSParquetDeltaInputReader.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DFSParquetDeltaInputReader.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package org.apache.hudi.bench.reader; +package org.apache.hudi.e2e.reader; import java.io.IOException; import java.util.Arrays; import java.util.List; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.PathFilter; -import org.apache.hudi.bench.DeltaInputFormat; -import org.apache.hudi.bench.DeltaOutputType; +import org.apache.hudi.e2e.DeltaInputFormat; +import org.apache.hudi.e2e.DeltaOutputType; import org.apache.hudi.common.util.Option; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.SparkSession; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DeltaInputReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DeltaInputReader.java similarity index 98% rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/DeltaInputReader.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DeltaInputReader.java index c322c88..e72b2c5 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/DeltaInputReader.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/DeltaInputReader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.reader; +package org.apache.hudi.e2e.reader; import java.io.IOException; import java.io.Serializable; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/SparkBasedReader.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/SparkBasedReader.java similarity index 98% rename from hudi-bench/src/main/java/org/apache/hudi/bench/reader/SparkBasedReader.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/SparkBasedReader.java index 6a28923..31bffc7 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/reader/SparkBasedReader.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/reader/SparkBasedReader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.reader; +package org.apache.hudi.e2e.reader; import java.util.List; import org.apache.avro.generic.GenericRecord; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/AvroDeltaInputWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/AvroDeltaInputWriter.java similarity index 99% rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/AvroDeltaInputWriter.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/AvroDeltaInputWriter.java index d53c39c..38f42e8 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/AvroDeltaInputWriter.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/AvroDeltaInputWriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.writer; +package org.apache.hudi.e2e.writer; import java.io.IOException; import java.io.OutputStream; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaInputWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaInputWriter.java similarity index 97% rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaInputWriter.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaInputWriter.java index e16e25d..41153b3 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaInputWriter.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaInputWriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.writer; +package org.apache.hudi.e2e.writer; import java.io.IOException; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaWriter.java similarity index 97% rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaWriter.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaWriter.java index 5c18ff2..13dedd9 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/DeltaWriter.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/DeltaWriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.writer; +package org.apache.hudi.e2e.writer; import java.util.HashMap; import java.util.Map; @@ -26,8 +26,8 @@ import org.apache.hudi.HoodieReadClient; import org.apache.hudi.HoodieWriteClient; import org.apache.hudi.WriteStatus; import org.apache.hudi.avro.model.HoodieCompactionPlan; -import org.apache.hudi.bench.job.HoodieDeltaStreamerWrapper; -import org.apache.hudi.bench.job.HoodieTestSuiteJob.HoodieTestSuiteConfig; +import org.apache.hudi.e2e.job.HoodieDeltaStreamerWrapper; +import org.apache.hudi.e2e.job.HoodieTestSuiteJob.HoodieTestSuiteConfig; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.util.Option; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/FileDeltaInputWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/FileDeltaInputWriter.java similarity index 97% rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/FileDeltaInputWriter.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/FileDeltaInputWriter.java index 4dc6bed..9210a27 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/FileDeltaInputWriter.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/FileDeltaInputWriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.writer; +package org.apache.hudi.e2e.writer; import java.io.IOException; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/SparkAvroDeltaInputWriter.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/SparkAvroDeltaInputWriter.java similarity index 98% rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/SparkAvroDeltaInputWriter.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/SparkAvroDeltaInputWriter.java index dcf744a..a9aee9c 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/SparkAvroDeltaInputWriter.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/SparkAvroDeltaInputWriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.writer; +package org.apache.hudi.e2e.writer; import java.io.IOException; import javax.ws.rs.NotSupportedException; diff --git a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/WriteStats.java b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/WriteStats.java similarity index 98% rename from hudi-bench/src/main/java/org/apache/hudi/bench/writer/WriteStats.java rename to hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/WriteStats.java index 1ba673e..6ded854 100644 --- a/hudi-bench/src/main/java/org/apache/hudi/bench/writer/WriteStats.java +++ b/hudi-end-to-end-tests/src/main/java/org/apache/hudi/e2e/writer/WriteStats.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.writer; +package org.apache.hudi.e2e.writer; import java.io.Serializable; import java.util.ArrayList; diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/TestDFSDeltaWriterAdapter.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestDFSDeltaWriterAdapter.java similarity index 91% rename from hudi-bench/src/test/java/org/apache/hudi/bench/TestDFSDeltaWriterAdapter.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestDFSDeltaWriterAdapter.java index e9e596a..b4cb13c 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/TestDFSDeltaWriterAdapter.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestDFSDeltaWriterAdapter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench; +package org.apache.hudi.e2e; import static org.mockito.Mockito.times; import static org.mockito.Mockito.when; @@ -27,12 +27,12 @@ import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hudi.bench.configuration.DFSDeltaConfig; -import org.apache.hudi.bench.configuration.DeltaConfig; -import org.apache.hudi.bench.generator.FlexibleSchemaRecordGenerationIterator; -import org.apache.hudi.bench.utils.TestUtils; -import org.apache.hudi.bench.writer.FileDeltaInputWriter; -import org.apache.hudi.bench.writer.WriteStats; +import org.apache.hudi.e2e.configuration.DFSDeltaConfig; +import org.apache.hudi.e2e.configuration.DeltaConfig; +import org.apache.hudi.e2e.generator.FlexibleSchemaRecordGenerationIterator; +import org.apache.hudi.e2e.utils.TestUtils; +import org.apache.hudi.e2e.writer.FileDeltaInputWriter; +import org.apache.hudi.e2e.writer.WriteStats; import org.apache.hudi.common.SerializableConfiguration; import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.utilities.UtilitiesTestBase; @@ -63,7 +63,7 @@ public class TestDFSDeltaWriterAdapter extends UtilitiesTestBase { @Before public void setup() throws Exception { super.setup(); - schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-bench-config/complex-source.avsc"), + schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-end-to-end-tests-config/complex-source.avsc"), jsc); } diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/TestFileDeltaInputWriter.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestFileDeltaInputWriter.java similarity index 93% rename from hudi-bench/src/test/java/org/apache/hudi/bench/TestFileDeltaInputWriter.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestFileDeltaInputWriter.java index 30e0190..78f8756 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/TestFileDeltaInputWriter.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/TestFileDeltaInputWriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench; +package org.apache.hudi.e2e; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -32,11 +32,11 @@ import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hudi.bench.generator.GenericRecordFullPayloadGenerator; -import org.apache.hudi.bench.reader.SparkBasedReader; -import org.apache.hudi.bench.writer.AvroDeltaInputWriter; -import org.apache.hudi.bench.writer.FileDeltaInputWriter; -import org.apache.hudi.bench.writer.WriteStats; +import org.apache.hudi.e2e.generator.GenericRecordFullPayloadGenerator; +import org.apache.hudi.e2e.reader.SparkBasedReader; +import org.apache.hudi.e2e.writer.AvroDeltaInputWriter; +import org.apache.hudi.e2e.writer.FileDeltaInputWriter; +import org.apache.hudi.e2e.writer.WriteStats; import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.utilities.UtilitiesTestBase; @@ -65,7 +65,7 @@ public class TestFileDeltaInputWriter extends UtilitiesTestBase { @Before public void setup() throws Exception { super.setup(); - schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-bench-config/complex-source.avsc"), + schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-end-to-end-tests-config/complex-source.avsc"), jsc); } diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/configuration/TestWorkflowBuilder.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/configuration/TestWorkflowBuilder.java similarity index 89% rename from hudi-bench/src/test/java/org/apache/hudi/bench/configuration/TestWorkflowBuilder.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/configuration/TestWorkflowBuilder.java index 77cc42c..8838518 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/configuration/TestWorkflowBuilder.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/configuration/TestWorkflowBuilder.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package org.apache.hudi.bench.configuration; +package org.apache.hudi.e2e.configuration; import static junit.framework.Assert.assertTrue; import static junit.framework.TestCase.assertEquals; import java.util.ArrayList; import java.util.List; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.WorkflowDag; -import org.apache.hudi.bench.dag.nodes.DagNode; -import org.apache.hudi.bench.dag.nodes.InsertNode; -import org.apache.hudi.bench.dag.nodes.UpsertNode; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.WorkflowDag; +import org.apache.hudi.e2e.dag.nodes.DagNode; +import org.apache.hudi.e2e.dag.nodes.InsertNode; +import org.apache.hudi.e2e.dag.nodes.UpsertNode; import org.junit.Test; public class TestWorkflowBuilder { diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/converter/TestUpdateConverter.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/converter/TestUpdateConverter.java similarity index 97% rename from hudi-bench/src/test/java/org/apache/hudi/bench/converter/TestUpdateConverter.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/converter/TestUpdateConverter.java index cb476d7..d8440f0 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/converter/TestUpdateConverter.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/converter/TestUpdateConverter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.converter; +package org.apache.hudi.e2e.converter; import static junit.framework.TestCase.assertTrue; @@ -25,7 +25,7 @@ import java.util.List; import java.util.Map; import org.apache.avro.Schema.Field; import org.apache.avro.generic.GenericRecord; -import org.apache.hudi.bench.utils.TestUtils; +import org.apache.hudi.e2e.utils.TestUtils; import org.apache.hudi.utilities.UtilHelpers; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestComplexDag.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestComplexDag.java similarity index 88% rename from hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestComplexDag.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestComplexDag.java index c9492b4..4bdb0dd 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestComplexDag.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestComplexDag.java @@ -16,17 +16,19 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag; +package org.apache.hudi.e2e.dag; import java.util.ArrayList; import java.util.List; import java.util.function.Function; import org.apache.hudi.WriteStatus; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.nodes.DagNode; -import org.apache.hudi.bench.dag.nodes.InsertNode; -import org.apache.hudi.bench.dag.nodes.UpsertNode; -import org.apache.hudi.bench.dag.nodes.ValidateNode; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.WorkflowDag; +import org.apache.hudi.e2e.dag.WorkflowDagGenerator; +import org.apache.hudi.e2e.dag.nodes.DagNode; +import org.apache.hudi.e2e.dag.nodes.InsertNode; +import org.apache.hudi.e2e.dag.nodes.UpsertNode; +import org.apache.hudi.e2e.dag.nodes.ValidateNode; import org.apache.spark.api.java.JavaRDD; public class TestComplexDag extends WorkflowDagGenerator { diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestDagUtils.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestDagUtils.java similarity index 88% rename from hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestDagUtils.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestDagUtils.java index afeaf6a..640c4fa 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestDagUtils.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestDagUtils.java @@ -16,13 +16,16 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag; +package org.apache.hudi.e2e.dag; import java.util.ArrayList; import java.util.List; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.nodes.DagNode; -import org.apache.hudi.bench.dag.nodes.InsertNode; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.DagUtils; +import org.apache.hudi.e2e.dag.WorkflowDag; +import org.apache.hudi.e2e.dag.WorkflowDagGenerator; +import org.apache.hudi.e2e.dag.nodes.DagNode; +import org.apache.hudi.e2e.dag.nodes.InsertNode; import org.junit.Assert; import org.junit.Test; @@ -38,7 +41,7 @@ public class TestDagUtils { @Test public void testConvertYamlToDag() throws Exception { WorkflowDag dag = DagUtils.convertYamlToDag( - DagUtils.toString(getClass().getResourceAsStream("/hudi-bench-config/complex-workflow-dag-cow.yaml"))); + DagUtils.toString(getClass().getResourceAsStream("/hudi-end-to-end-tests-config/complex-workflow-dag-cow.yaml"))); Assert.assertEquals(dag.getNodeList().size(), 1); Assert.assertEquals(((DagNode) dag.getNodeList().get(0)).getParentNodes().size(), 0); Assert.assertEquals(((DagNode) dag.getNodeList().get(0)).getChildNodes().size(), 1); diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestHiveSyncDag.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestHiveSyncDag.java similarity index 81% rename from hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestHiveSyncDag.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestHiveSyncDag.java index 17c9730..ee85e18 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestHiveSyncDag.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestHiveSyncDag.java @@ -16,16 +16,18 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag; +package org.apache.hudi.e2e.dag; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.nodes.DagNode; -import org.apache.hudi.bench.dag.nodes.HiveQueryNode; -import org.apache.hudi.bench.dag.nodes.HiveSyncNode; -import org.apache.hudi.bench.dag.nodes.InsertNode; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.WorkflowDag; +import org.apache.hudi.e2e.dag.WorkflowDagGenerator; +import org.apache.hudi.e2e.dag.nodes.DagNode; +import org.apache.hudi.e2e.dag.nodes.HiveQueryNode; +import org.apache.hudi.e2e.dag.nodes.HiveSyncNode; +import org.apache.hudi.e2e.dag.nodes.InsertNode; import org.apache.hudi.common.util.collection.Pair; public class TestHiveSyncDag extends WorkflowDagGenerator { diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertOnlyDag.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertOnlyDag.java similarity index 84% rename from hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertOnlyDag.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertOnlyDag.java index f50fe97..23dc61e 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertOnlyDag.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertOnlyDag.java @@ -16,13 +16,15 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag; +package org.apache.hudi.e2e.dag; import java.util.ArrayList; import java.util.List; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.nodes.DagNode; -import org.apache.hudi.bench.dag.nodes.InsertNode; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.WorkflowDag; +import org.apache.hudi.e2e.dag.WorkflowDagGenerator; +import org.apache.hudi.e2e.dag.nodes.DagNode; +import org.apache.hudi.e2e.dag.nodes.InsertNode; public class TestInsertOnlyDag extends WorkflowDagGenerator { diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertUpsertDag.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertUpsertDag.java similarity index 82% rename from hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertUpsertDag.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertUpsertDag.java index 63d898d..1b0d5a9 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/dag/TestInsertUpsertDag.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/dag/TestInsertUpsertDag.java @@ -16,14 +16,16 @@ * limitations under the License. */ -package org.apache.hudi.bench.dag; +package org.apache.hudi.e2e.dag; import java.util.ArrayList; import java.util.List; -import org.apache.hudi.bench.configuration.DeltaConfig.Config; -import org.apache.hudi.bench.dag.nodes.DagNode; -import org.apache.hudi.bench.dag.nodes.InsertNode; -import org.apache.hudi.bench.dag.nodes.UpsertNode; +import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +import org.apache.hudi.e2e.dag.WorkflowDag; +import org.apache.hudi.e2e.dag.WorkflowDagGenerator; +import org.apache.hudi.e2e.dag.nodes.DagNode; +import org.apache.hudi.e2e.dag.nodes.InsertNode; +import org.apache.hudi.e2e.dag.nodes.UpsertNode; public class TestInsertUpsertDag extends WorkflowDagGenerator { diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadEstimator.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadEstimator.java similarity index 88% rename from hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadEstimator.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadEstimator.java index e616813..a3fe808 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadEstimator.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadEstimator.java @@ -16,12 +16,13 @@ * limitations under the License. */ -package org.apache.hudi.bench.generator; +package org.apache.hudi.e2e.generator; import static junit.framework.TestCase.assertEquals; import org.apache.avro.Schema; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.e2e.generator.GenericRecordFullPayloadSizeEstimator; import org.junit.Test; public class TestGenericRecordPayloadEstimator { @@ -29,7 +30,7 @@ public class TestGenericRecordPayloadEstimator { @Test public void testSimpleSchemaSize() throws Exception { Schema schema = new Schema.Parser().parse(getClass().getClassLoader() - .getResourceAsStream("hudi-bench-config/source.avsc")); + .getResourceAsStream("hudi-end-to-end-tests-config/source.avsc")); GenericRecordFullPayloadSizeEstimator estimator = new GenericRecordFullPayloadSizeEstimator(schema); Pair<Integer, Integer> estimateAndNumComplexFields = estimator.typeEstimateAndNumComplexFields(); @@ -40,7 +41,7 @@ public class TestGenericRecordPayloadEstimator { @Test public void testComplexSchemaSize() throws Exception { Schema schema = new Schema.Parser().parse(getClass().getClassLoader() - .getResourceAsStream("hudi-bench-config/complex-source.avsc")); + .getResourceAsStream("hudi-end-to-end-tests-config/complex-source.avsc")); GenericRecordFullPayloadSizeEstimator estimator = new GenericRecordFullPayloadSizeEstimator(schema); Pair<Integer, Integer> estimateAndNumComplexFields = estimator.typeEstimateAndNumComplexFields(); diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadGenerator.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadGenerator.java similarity index 88% rename from hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadGenerator.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadGenerator.java index e2c3ae6..bb3e9b3 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestGenericRecordPayloadGenerator.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestGenericRecordPayloadGenerator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.generator; +package org.apache.hudi.e2e.generator; import static junit.framework.TestCase.assertTrue; @@ -28,6 +28,8 @@ import java.util.stream.IntStream; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.e2e.generator.GenericRecordFullPayloadGenerator; +import org.apache.hudi.e2e.generator.GenericRecordPartialPayloadGenerator; import org.junit.Assert; import org.junit.Test; @@ -36,7 +38,7 @@ public class TestGenericRecordPayloadGenerator { @Test public void testSimplePayload() throws Exception { Schema schema = new Schema.Parser().parse(getClass().getClassLoader() - .getResourceAsStream("hudi-bench-config/source.avsc")); + .getResourceAsStream("hudi-end-to-end-tests-config/source.avsc")); GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator(schema); GenericRecord record = payloadGenerator.getNewPayload(); // The generated payload should validate with the provided schema @@ -46,7 +48,7 @@ public class TestGenericRecordPayloadGenerator { @Test public void testComplexPayload() throws IOException { Schema schema = new Schema.Parser().parse(getClass().getClassLoader() - .getResourceAsStream("hudi-bench-config/complex-source.avsc")); + .getResourceAsStream("hudi-end-to-end-tests-config/complex-source.avsc")); GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator(schema); GenericRecord record = payloadGenerator.getNewPayload(); // The generated payload should validate with the provided schema @@ -56,7 +58,7 @@ public class TestGenericRecordPayloadGenerator { @Test public void testComplexPartialPayload() throws IOException { Schema schema = new Schema.Parser().parse(getClass().getClassLoader() - .getResourceAsStream("hudi-bench-config/complex-source.avsc")); + .getResourceAsStream("hudi-end-to-end-tests-config/complex-source.avsc")); GenericRecordPartialPayloadGenerator payloadGenerator = new GenericRecordPartialPayloadGenerator(schema); IntStream.range(0, 10).forEach(a -> { GenericRecord record = payloadGenerator.getNewPayload(); @@ -68,7 +70,7 @@ public class TestGenericRecordPayloadGenerator { @Test public void testUpdatePayloadGenerator() throws IOException { Schema schema = new Schema.Parser().parse(getClass().getClassLoader() - .getResourceAsStream("hudi-bench-config/source.avsc")); + .getResourceAsStream("hudi-end-to-end-tests-config/source.avsc")); GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator(schema); List<String> insertRowKeys = new ArrayList<>(); List<String> updateRowKeys = new ArrayList<>(); @@ -98,7 +100,7 @@ public class TestGenericRecordPayloadGenerator { @Test public void testSimplePayloadWithLargeMinSize() throws Exception { Schema schema = new Schema.Parser().parse(getClass().getClassLoader() - .getResourceAsStream("hudi-bench-config/source.avsc")); + .getResourceAsStream("hudi-end-to-end-tests-config/source.avsc")); int minPayloadSize = 1000; GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator(schema, minPayloadSize); @@ -110,7 +112,7 @@ public class TestGenericRecordPayloadGenerator { @Test public void testComplexPayloadWithLargeMinSize() throws Exception { Schema schema = new Schema.Parser().parse(getClass().getClassLoader() - .getResourceAsStream("hudi-bench-config/complex-source.avsc")); + .getResourceAsStream("hudi-end-to-end-tests-config/complex-source.avsc")); int minPayloadSize = 10000; GenericRecordFullPayloadGenerator payloadGenerator = new GenericRecordFullPayloadGenerator( schema, minPayloadSize); diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestWorkloadGenerator.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestWorkloadGenerator.java similarity index 96% rename from hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestWorkloadGenerator.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestWorkloadGenerator.java index 5f10005..1c1d26b 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/generator/TestWorkloadGenerator.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/generator/TestWorkloadGenerator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.generator;///* +package org.apache.hudi.e2e.generator;///* // * Copyright (c) 2019 Uber Technologies, Inc. (hoodie-dev-gr...@uber.com) // * // * Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,22 +34,22 @@ package org.apache.hudi.bench.generator;///* // * // */ // -//package org.apache.hudi.bench.generator; +//package org.apache.hudi.e2e.generator; // //import static junit.framework.TestCase.assertEquals; //import static junit.framework.TestCase.assertFalse; //import static org.mockito.Mockito.when; // -//import org.apache.hudi.bench.configuration.DeltaConfig.Config; -//import org.apache.hudi.bench.input.config.DFSDeltaConfig; -//import org.apache.hudi.bench.input.config.DeltaConfig; +//import org.apache.hudi.e2e.configuration.DeltaConfig.Config; +//import org.apache.hudi.e2e.input.config.DFSDeltaConfig; +//import org.apache.hudi.e2e.input.config.DeltaConfig; //import org.apache.hudi.common.SerializableConfiguration; //import org.apache.hudi.common.util.FSUtils; -//import org.apache.hudi.bench.job.operation.Operation; -//import org.apache.hudi.bench.input.DeltaInputFormat; -//import org.apache.hudi.bench.input.DeltaOutputType; -//import org.apache.hudi.bench.input.reader.DFSAvroDeltaInputReader; -//import org.apache.hudi.bench.utils.TestUtils; +//import org.apache.hudi.e2e.job.operation.Operation; +//import org.apache.hudi.e2e.input.DeltaInputFormat; +//import org.apache.hudi.e2e.input.DeltaOutputType; +//import org.apache.hudi.e2e.input.reader.DFSAvroDeltaInputReader; +//import org.apache.hudi.e2e.utils.TestUtils; //import org.apache.hudi.utilities.UtilitiesTestBase; //import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; //import java.io.IOException; @@ -95,7 +95,7 @@ package org.apache.hudi.bench.generator;///* // @Before // public void setup() throws Exception { // super.setup(); -// schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-bench-config/source.avsc"), +// schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-end-to-end-tests-config/source.avsc"), // jsc); // } // diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/job/TestHoodieTestSuiteJob.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/job/TestHoodieTestSuiteJob.java similarity index 92% rename from hudi-bench/src/test/java/org/apache/hudi/bench/job/TestHoodieTestSuiteJob.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/job/TestHoodieTestSuiteJob.java index d1acd5f..2918003 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/job/TestHoodieTestSuiteJob.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/job/TestHoodieTestSuiteJob.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.job; +package org.apache.hudi.e2e.job; import static org.junit.Assert.assertEquals; @@ -26,13 +26,13 @@ import java.util.UUID; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hudi.DataSourceWriteOptions; -import org.apache.hudi.bench.DeltaInputFormat; -import org.apache.hudi.bench.DeltaOutputType; -import org.apache.hudi.bench.dag.TestHiveSyncDag; -import org.apache.hudi.bench.dag.TestInsertOnlyDag; -import org.apache.hudi.bench.dag.TestInsertUpsertDag; -import org.apache.hudi.bench.dag.WorkflowDagGenerator; -import org.apache.hudi.bench.job.HoodieTestSuiteJob.HoodieTestSuiteConfig; +import org.apache.hudi.e2e.DeltaInputFormat; +import org.apache.hudi.e2e.DeltaOutputType; +import org.apache.hudi.e2e.dag.TestHiveSyncDag; +import org.apache.hudi.e2e.dag.TestInsertOnlyDag; +import org.apache.hudi.e2e.dag.TestInsertUpsertDag; +import org.apache.hudi.e2e.dag.WorkflowDagGenerator; +import org.apache.hudi.e2e.job.HoodieTestSuiteJob.HoodieTestSuiteConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.TypedProperties; import org.apache.hudi.utilities.UtilitiesTestBase; @@ -72,9 +72,9 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase { public static void initClass() throws Exception { UtilitiesTestBase.initClass(); // prepare the configs. - UtilitiesTestBase.Helpers.copyToDFS("hudi-bench-config/base.properties", dfs, dfsBasePath + "/base.properties"); - UtilitiesTestBase.Helpers.copyToDFS("hudi-bench-config/source.avsc", dfs, dfsBasePath + "/source.avsc"); - UtilitiesTestBase.Helpers.copyToDFS("hudi-bench-config/target.avsc", dfs, dfsBasePath + "/target.avsc"); + UtilitiesTestBase.Helpers.copyToDFS("hudi-end-to-end-tests-config/base.properties", dfs, dfsBasePath + "/base.properties"); + UtilitiesTestBase.Helpers.copyToDFS("hudi-end-to-end-tests-config/source.avsc", dfs, dfsBasePath + "/source.avsc"); + UtilitiesTestBase.Helpers.copyToDFS("hudi-end-to-end-tests-config/target.avsc", dfs, dfsBasePath + "/target.avsc"); TypedProperties props = new TypedProperties(); props.setProperty("hoodie.datasource.write.recordkey.field", "_row_key"); diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSAvroDeltaInputReader.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSAvroDeltaInputReader.java similarity index 96% rename from hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSAvroDeltaInputReader.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSAvroDeltaInputReader.java index 9cfb70f..1529cbc 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSAvroDeltaInputReader.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSAvroDeltaInputReader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.reader; +package org.apache.hudi.e2e.reader; import static junit.framework.Assert.assertTrue; import static junit.framework.TestCase.assertEquals; @@ -26,7 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hudi.bench.utils.TestUtils; +import org.apache.hudi.e2e.utils.TestUtils; import org.apache.hudi.common.util.FSUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.utilities.UtilitiesTestBase; diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSHoodieDatasetInputReader.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSHoodieDatasetInputReader.java similarity index 96% rename from hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSHoodieDatasetInputReader.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSHoodieDatasetInputReader.java index dba57b0..db817c6 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/reader/TestDFSHoodieDatasetInputReader.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/reader/TestDFSHoodieDatasetInputReader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.reader; +package org.apache.hudi.e2e.reader; import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertTrue; @@ -32,6 +32,7 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.util.HoodieAvroUtils; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.e2e.reader.DFSHoodieDatasetInputReader; import org.apache.hudi.utilities.UtilitiesTestBase; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; import org.apache.spark.api.java.JavaRDD; @@ -58,7 +59,7 @@ public class TestDFSHoodieDatasetInputReader extends UtilitiesTestBase { @Before public void setup() throws Exception { super.setup(); - schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-bench-config/complex-source.avsc"), + schemaProvider = new FilebasedSchemaProvider(Helpers.setupSchemaOnDFS("hudi-end-to-end-tests-config/complex-source.avsc"), jsc); HoodieTestUtils.init(jsc.hadoopConfiguration(), dfsBasePath); } diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/utils/TestUtils.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/utils/TestUtils.java similarity index 98% rename from hudi-bench/src/test/java/org/apache/hudi/bench/utils/TestUtils.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/utils/TestUtils.java index af2b575..742e3ef 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/utils/TestUtils.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/utils/TestUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.utils; +package org.apache.hudi.e2e.utils; import java.util.List; import org.apache.avro.Schema; diff --git a/hudi-bench/src/test/java/org/apache/hudi/bench/writer/TestDeltaWriter.java b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/writer/TestDeltaWriter.java similarity index 96% rename from hudi-bench/src/test/java/org/apache/hudi/bench/writer/TestDeltaWriter.java rename to hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/writer/TestDeltaWriter.java index 692f552..3316b4c 100644 --- a/hudi-bench/src/test/java/org/apache/hudi/bench/writer/TestDeltaWriter.java +++ b/hudi-end-to-end-tests/src/test/java/org/apache/hudi/e2e/writer/TestDeltaWriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.writer; +package org.apache.hudi.e2e.writer; import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertEquals; @@ -29,10 +29,11 @@ import static org.powermock.api.mockito.PowerMockito.when; import java.util.Properties; import org.apache.hudi.HoodieReadClient; import org.apache.hudi.HoodieWriteClient; -import org.apache.hudi.bench.job.HoodieDeltaStreamerWrapper; -import org.apache.hudi.bench.job.HoodieTestSuiteJob.HoodieTestSuiteConfig; +import org.apache.hudi.e2e.job.HoodieDeltaStreamerWrapper; +import org.apache.hudi.e2e.job.HoodieTestSuiteJob.HoodieTestSuiteConfig; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.e2e.writer.DeltaWriter; import org.apache.spark.api.java.JavaSparkContext; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/hudi-bench/src/test/resources/hudi-bench-config/base.properties b/hudi-end-to-end-tests/src/test/resources/hudi-bench-config/base.properties similarity index 100% rename from hudi-bench/src/test/resources/hudi-bench-config/base.properties rename to hudi-end-to-end-tests/src/test/resources/hudi-bench-config/base.properties diff --git a/hudi-bench/src/test/resources/hudi-bench-config/complex-source.avsc b/hudi-end-to-end-tests/src/test/resources/hudi-bench-config/complex-source.avsc similarity index 100% rename from hudi-bench/src/test/resources/hudi-bench-config/complex-source.avsc rename to hudi-end-to-end-tests/src/test/resources/hudi-bench-config/complex-source.avsc diff --git a/hudi-bench/src/test/resources/hudi-bench-config/complex-workflow-dag-cow.yaml b/hudi-end-to-end-tests/src/test/resources/hudi-bench-config/complex-workflow-dag-cow.yaml similarity index 100% rename from hudi-bench/src/test/resources/hudi-bench-config/complex-workflow-dag-cow.yaml rename to hudi-end-to-end-tests/src/test/resources/hudi-bench-config/complex-workflow-dag-cow.yaml diff --git a/hudi-bench/src/test/resources/hudi-bench-config/source.avsc b/hudi-end-to-end-tests/src/test/resources/hudi-bench-config/source.avsc similarity index 100% rename from hudi-bench/src/test/resources/hudi-bench-config/source.avsc rename to hudi-end-to-end-tests/src/test/resources/hudi-bench-config/source.avsc diff --git a/hudi-bench/src/test/resources/hudi-bench-config/target.avsc b/hudi-end-to-end-tests/src/test/resources/hudi-bench-config/target.avsc similarity index 100% rename from hudi-bench/src/test/resources/hudi-bench-config/target.avsc rename to hudi-end-to-end-tests/src/test/resources/hudi-bench-config/target.avsc diff --git a/hudi-bench/src/test/resources/log4j-surefire-quiet.properties b/hudi-end-to-end-tests/src/test/resources/log4j-surefire-quiet.properties similarity index 100% rename from hudi-bench/src/test/resources/log4j-surefire-quiet.properties rename to hudi-end-to-end-tests/src/test/resources/log4j-surefire-quiet.properties diff --git a/hudi-bench/src/test/resources/log4j-surefire.properties b/hudi-end-to-end-tests/src/test/resources/log4j-surefire.properties similarity index 100% rename from hudi-bench/src/test/resources/log4j-surefire.properties rename to hudi-end-to-end-tests/src/test/resources/log4j-surefire.properties diff --git a/packaging/hudi-bench-bundle/pom.xml b/packaging/hudi-end-to-end-tests-bundle/pom.xml similarity index 99% rename from packaging/hudi-bench-bundle/pom.xml rename to packaging/hudi-end-to-end-tests-bundle/pom.xml index 60c12bd..07dfb20 100644 --- a/packaging/hudi-bench-bundle/pom.xml +++ b/packaging/hudi-end-to-end-tests-bundle/pom.xml @@ -24,7 +24,7 @@ <relativePath>../../pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> - <artifactId>hudi-bench-bundle</artifactId> + <artifactId>hudi-end-to-end-tests-bundle</artifactId> <packaging>jar</packaging> <properties> @@ -72,7 +72,7 @@ <include>org.apache.hudi:hudi-hive</include> <include>org.apache.hudi:hudi-hadoop-mr</include> <include>org.apache.hudi:hudi-timeline-service</include> - <include>org.apache.hudi:hudi-bench</include> + <include>org.apache.hudi:hudi-end-to-end-tests</include> <include>com.beust:jcommander</include> <include>com.twitter:bijection-avro_2.11</include> <include>com.twitter:bijection-core_2.11</include> @@ -396,7 +396,7 @@ <dependency> <groupId>org.apache.hudi</groupId> - <artifactId>hudi-bench</artifactId> + <artifactId>hudi-end-to-end-tests</artifactId> <version>${project.version}</version> </dependency> diff --git a/packaging/hudi-bench-bundle/src/main/java/org/apache/hudi/bench/bundle/Main.java b/packaging/hudi-end-to-end-tests-bundle/src/main/java/org/apache/hudi/e2e/bundle/Main.java similarity index 97% rename from packaging/hudi-bench-bundle/src/main/java/org/apache/hudi/bench/bundle/Main.java rename to packaging/hudi-end-to-end-tests-bundle/src/main/java/org/apache/hudi/e2e/bundle/Main.java index 134141e..7e9e573 100644 --- a/packaging/hudi-bench-bundle/src/main/java/org/apache/hudi/bench/bundle/Main.java +++ b/packaging/hudi-end-to-end-tests-bundle/src/main/java/org/apache/hudi/e2e/bundle/Main.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.hudi.bench.bundle; +package org.apache.hudi.e2e.bundle; import org.apache.hudi.common.util.ReflectionUtils;