This is an automated email from the ASF dual-hosted git repository. vinoyang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new ca36c44 [HUDI-995] Move TestRawTripPayload and HoodieTestDataGenerator to hudi-common (#1873) ca36c44 is described below commit ca36c44cb3a081ce68282742f177f471fb7ed8c3 Author: Raymond Xu <2701446+xushi...@users.noreply.github.com> AuthorDate: Mon Jul 27 04:21:45 2020 -0700 [HUDI-995] Move TestRawTripPayload and HoodieTestDataGenerator to hudi-common (#1873) --- .../hudi/cli/commands/TestCleansCommand.java | 4 +- .../hudi/cli/commands/TestCommitsCommand.java | 4 +- .../hudi/cli/commands/TestRepairsCommand.java | 2 +- .../hudi/cli/commands/TestRollbacksCommand.java | 2 +- .../hudi/cli/commands/TestSavepointsCommand.java | 2 +- .../apache/hudi/cli/commands/TestStatsCommand.java | 2 +- .../apache/hudi/cli/commands/TestTableCommand.java | 8 +- .../hudi/cli/integ/ITTestCommitsCommand.java | 2 +- .../cli/integ/ITTestHDFSParquetImportCommand.java | 2 +- .../hudi/cli/integ/ITTestRepairsCommand.java | 2 +- .../hudi/cli/integ/ITTestSavepointsCommand.java | 2 +- .../HoodieTestCommitMetadataGenerator.java | 4 +- .../org/apache/hudi/client/TestClientRollback.java | 2 +- .../TestHoodieClientOnCopyOnWriteStorage.java | 20 ++--- .../java/org/apache/hudi/client/TestMultiFS.java | 2 +- .../hudi/client/TestTableSchemaEvolution.java | 20 ++--- .../hudi/client/TestUpdateSchemaEvolution.java | 12 +-- .../hudi/execution/TestBoundedInMemoryQueue.java | 2 +- .../TestSparkBoundedInMemoryExecutor.java | 2 +- .../org/apache/hudi/index/TestHoodieIndex.java | 31 ++++---- .../hudi/index/bloom/TestHoodieBloomIndex.java | 46 ++++++------ .../index/bloom/TestHoodieGlobalBloomIndex.java | 64 ++++++++-------- .../apache/hudi/index/hbase/TestHBaseIndex.java | 2 +- .../index/hbase/TestHBaseQPSResourceAllocator.java | 2 +- .../hudi/io/TestHoodieKeyLocationFetchHandle.java | 8 +- .../org/apache/hudi/io/TestHoodieMergeHandle.java | 2 +- .../hudi/io/TestHoodieTimelineArchiveLog.java | 2 +- .../io/storage/TestHoodieFileWriterFactory.java | 2 +- .../java/org/apache/hudi/table/TestCleaner.java | 2 +- .../hudi/table/TestHoodieMergeOnReadTable.java | 6 +- .../commit/TestCopyOnWriteActionExecutor.java | 26 +++---- .../table/action/commit/TestUpsertPartitioner.java | 2 +- .../table/action/compact/CompactionTestBase.java | 7 +- .../table/action/compact/TestHoodieCompactor.java | 2 +- .../rollback/HoodieClientRollbackTestBase.java | 7 +- .../TestCopyOnWriteRollbackActionExecutor.java | 9 ++- .../TestMergeOnReadRollbackActionExecutor.java | 9 ++- .../hudi/testutils/HoodieClientTestBase.java | 5 +- .../hudi/testutils/HoodieClientTestHarness.java | 6 +- .../hudi/testutils/HoodieMergeOnReadTestUtils.java | 2 + .../hudi/testutils/MetadataMergeWriteStatus.java | 85 ++++++++++++++++++++++ .../common/fs/inline}/TestParquetInLining.java | 22 +++--- .../common}/testutils/HoodieTestDataGenerator.java | 54 +++++++------- .../hudi/common/testutils/RawTripTestPayload.java | 84 ++++----------------- hudi-spark/src/test/java/HoodieJavaApp.java | 2 +- .../src/test/java/HoodieJavaStreamingApp.java | 2 +- .../apache/hudi/testutils/DataSourceTestUtils.java | 3 +- .../apache/hudi/functional/TestDataSource.scala | 3 +- .../functional/TestHDFSParquetImporter.java | 2 +- .../functional/TestHoodieDeltaStreamer.java | 2 +- .../TestHoodieMultiTableDeltaStreamer.java | 2 +- .../functional/TestHoodieSnapshotCopier.java | 2 +- .../functional/TestHoodieSnapshotExporter.java | 6 +- .../hudi/utilities/sources/TestKafkaSource.java | 2 +- .../utilities/testutils/UtilitiesTestBase.java | 6 +- .../testutils/sources/AbstractBaseTestSource.java | 6 +- .../sources/AbstractDFSSourceTestBase.java | 2 +- 57 files changed, 329 insertions(+), 294 deletions(-) diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java index c14cf0b..3da7189 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java @@ -33,10 +33,10 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.util.Option; import org.apache.hadoop.conf.Configuration; -import org.apache.hudi.common.util.Option; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java index 4761252..45c340d 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java @@ -18,7 +18,6 @@ package org.apache.hudi.cli.commands; -import org.apache.hadoop.fs.FileSystem; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.HoodieTableHeaderFields; @@ -30,14 +29,15 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.NumericUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTimelineArchiveLog; -import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hadoop.fs.FileSystem; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java index 9764b9a..afad53b 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java @@ -27,7 +27,7 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; -import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java index 245b76b..e7a4603 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java @@ -30,12 +30,12 @@ import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java index 1158ac6..949a764 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java @@ -25,7 +25,7 @@ import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; -import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java index 61c2fcb..72a86b5 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java @@ -26,9 +26,9 @@ import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import com.codahale.metrics.Histogram; import com.codahale.metrics.Snapshot; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java index 411ca4d..cdf9db3 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java @@ -18,16 +18,16 @@ package org.apache.hudi.cli.commands; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.common.fs.ConsistencyGuardConfig; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; - import org.apache.hudi.common.table.timeline.HoodieTimeline; -import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java index dc1b51e..aa87cdd 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java @@ -26,8 +26,8 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java index 8bc51fc..f68b59c 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java @@ -24,8 +24,8 @@ import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.HDFSParquetImporter; import org.apache.hudi.utilities.functional.TestHDFSParquetImporter; import org.apache.hudi.utilities.functional.TestHDFSParquetImporter.HoodieTripModel; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java index 2c1bb9a..7167965 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java @@ -31,9 +31,9 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.SchemaTestUtil; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.avro.Schema; import org.apache.hadoop.fs.FileStatus; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java index 166f891..e933239 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java @@ -26,7 +26,7 @@ import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; -import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java index 94904c5..b86e748 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java @@ -18,17 +18,16 @@ package org.apache.hudi.cli.testutils; -import java.util.UUID; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -41,6 +40,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; /** * Class to be used in tests to keep generating test inserts and updates against a corpus. diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java b/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java index b37a350..943a1ea 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestClientRollback.java @@ -25,6 +25,7 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; @@ -33,7 +34,6 @@ import org.apache.hudi.exception.HoodieRollbackException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestBase; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.spark.api.java.JavaRDD; import org.junit.jupiter.api.Test; diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java index 3d1d31d..cb3bdf9 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java @@ -31,7 +31,9 @@ import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ParquetUtils; @@ -50,8 +52,6 @@ import org.apache.hudi.table.MarkerFiles; import org.apache.hudi.table.action.commit.WriteHelper; import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; -import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.Path; @@ -78,13 +78,13 @@ import java.util.UUID; import java.util.stream.Collectors; import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_0; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.NULL_SCHEMA; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.apache.hudi.common.util.ParquetUtils.readRowKeysFromParquet; import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.NULL_SCHEMA; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -208,7 +208,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { String recordKey = UUID.randomUUID().toString(); HoodieKey keyOne = new HoodieKey(recordKey, "2018-01-01"); - HoodieRecord<TestRawTripPayload> recordOne = + HoodieRecord<RawTripTestPayload> recordOne = new HoodieRecord(keyOne, dataGen.generateRandomValue(keyOne, newCommitTime)); HoodieKey keyTwo = new HoodieKey(recordKey, "2018-02-01"); @@ -219,13 +219,13 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { HoodieRecord recordThree = new HoodieRecord(keyTwo, dataGen.generateRandomValue(keyTwo, newCommitTime)); - JavaRDD<HoodieRecord<TestRawTripPayload>> records = + JavaRDD<HoodieRecord<RawTripTestPayload>> records = jsc.parallelize(Arrays.asList(recordOne, recordTwo, recordThree), 1); // Global dedup should be done based on recordKey only HoodieIndex index = mock(HoodieIndex.class); when(index.isGlobal()).thenReturn(true); - List<HoodieRecord<TestRawTripPayload>> dedupedRecs = WriteHelper.deduplicateRecords(records, index, 1).collect(); + List<HoodieRecord<RawTripTestPayload>> dedupedRecs = WriteHelper.deduplicateRecords(records, index, 1).collect(); assertEquals(1, dedupedRecs.size()); assertNodupesWithinPartition(dedupedRecs); diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java b/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java index 6a78bc5..b4369d8 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestMultiFS.java @@ -25,13 +25,13 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.testutils.HoodieClientTestHarness; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.LogManager; diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java b/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java index 6a93e31..21bf601 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java @@ -25,6 +25,8 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; @@ -34,8 +36,6 @@ import org.apache.hudi.exception.HoodieUpsertException; import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; -import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; @@ -46,13 +46,13 @@ import java.util.List; import java.util.stream.Collectors; import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_1; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.EXTRA_TYPE_SCHEMA; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.FARE_NESTED_SCHEMA; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.MAP_TYPE_SCHEMA; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.TIP_NESTED_SCHEMA; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_SCHEMA_PREFIX; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_SCHEMA_SUFFIX; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.EXTRA_TYPE_SCHEMA; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.FARE_NESTED_SCHEMA; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.MAP_TYPE_SCHEMA; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TIP_NESTED_SCHEMA; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_SCHEMA_PREFIX; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_SCHEMA_SUFFIX; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -459,7 +459,7 @@ public class TestTableSchemaEvolution extends HoodieClientTestBase { try { payload = (GenericRecord)r.getData().getInsertValue(HoodieTestDataGenerator.AVRO_SCHEMA).get(); GenericRecord newPayload = HoodieAvroUtils.rewriteRecordWithOnlyNewSchemaFields(payload, newSchema); - return new HoodieRecord(key, new TestRawTripPayload(newPayload.toString(), key.getRecordKey(), key.getPartitionPath(), schemaStr)); + return new HoodieRecord(key, new RawTripTestPayload(newPayload.toString(), key.getRecordKey(), key.getPartitionPath(), schemaStr)); } catch (IOException e) { throw new RuntimeException("Conversion to new schema failed"); } diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java b/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java index 2c985f3..db35069 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestUpdateSchemaEvolution.java @@ -18,13 +18,13 @@ package org.apache.hudi.client; -import java.io.IOException; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.ParquetUtils; import org.apache.hudi.config.HoodieWriteConfig; @@ -32,7 +32,6 @@ import org.apache.hudi.io.HoodieCreateHandle; import org.apache.hudi.io.HoodieMergeHandle; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestHarness; -import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; @@ -42,6 +41,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -78,13 +78,13 @@ public class TestUpdateSchemaEvolution extends HoodieClientTestHarness { String recordStr3 = "{\"_row_key\":\"8eb5b87c-1fej-4edd-87b4-6ec96dc405a0\"," + "\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":15}"; List<HoodieRecord> insertRecords = new ArrayList<>(); - TestRawTripPayload rowChange1 = new TestRawTripPayload(recordStr1); + RawTripTestPayload rowChange1 = new RawTripTestPayload(recordStr1); insertRecords .add(new HoodieRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1)); - TestRawTripPayload rowChange2 = new TestRawTripPayload(recordStr2); + RawTripTestPayload rowChange2 = new RawTripTestPayload(recordStr2); insertRecords .add(new HoodieRecord(new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()), rowChange2)); - TestRawTripPayload rowChange3 = new TestRawTripPayload(recordStr3); + RawTripTestPayload rowChange3 = new RawTripTestPayload(recordStr3); insertRecords .add(new HoodieRecord(new HoodieKey(rowChange3.getRowKey(), rowChange3.getPartitionPath()), rowChange3)); @@ -109,7 +109,7 @@ public class TestUpdateSchemaEvolution extends HoodieClientTestHarness { String recordStr1 = "{\"_row_key\":\"8eb5b87a-1feh-4edd-87b4-6ec96dc405a0\"," + "\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12,\"added_field\":1}"; List<HoodieRecord> updateRecords = new ArrayList<>(); - TestRawTripPayload rowChange1 = new TestRawTripPayload(recordStr1); + RawTripTestPayload rowChange1 = new RawTripTestPayload(recordStr1); HoodieRecord record1 = new HoodieRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1); record1.unseal(); diff --git a/hudi-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java b/hudi-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java index 4b52926..5520ba8 100644 --- a/hudi-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java +++ b/hudi-client/src/test/java/org/apache/hudi/execution/TestBoundedInMemoryQueue.java @@ -20,6 +20,7 @@ package org.apache.hudi.execution; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.DefaultSizeEstimator; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; @@ -31,7 +32,6 @@ import org.apache.hudi.common.util.queue.IteratorBasedQueueProducer; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.execution.LazyInsertIterable.HoodieInsertValueGenResult; import org.apache.hudi.testutils.HoodieClientTestHarness; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.avro.generic.IndexedRecord; import org.junit.jupiter.api.AfterEach; diff --git a/hudi-client/src/test/java/org/apache/hudi/execution/TestSparkBoundedInMemoryExecutor.java b/hudi-client/src/test/java/org/apache/hudi/execution/TestSparkBoundedInMemoryExecutor.java index c55f275..487dd22 100644 --- a/hudi-client/src/test/java/org/apache/hudi/execution/TestSparkBoundedInMemoryExecutor.java +++ b/hudi-client/src/test/java/org/apache/hudi/execution/TestSparkBoundedInMemoryExecutor.java @@ -20,12 +20,12 @@ package org.apache.hudi.execution; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.execution.LazyInsertIterable.HoodieInsertValueGenResult; import org.apache.hudi.testutils.HoodieClientTestHarness; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.avro.generic.IndexedRecord; import org.junit.jupiter.api.AfterEach; diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java index ab3e888..8ac53dd 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java @@ -29,6 +29,8 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.table.view.FileSystemViewStorageType; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; @@ -40,8 +42,7 @@ import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestHarness; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; -import org.apache.hudi.testutils.TestRawTripPayload; +import org.apache.hudi.testutils.MetadataMergeWriteStatus; import org.apache.avro.Schema; import org.apache.hadoop.fs.Path; @@ -258,16 +259,16 @@ public class TestHoodieIndex extends HoodieClientTestHarness { String recordStr3 = "{\"_row_key\":\"" + rowKey3 + "\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":15}"; // place same row key under a different partition. String recordStr4 = "{\"_row_key\":\"" + rowKey1 + "\",\"time\":\"2015-01-31T03:16:41.415Z\",\"number\":32}"; - TestRawTripPayload rowChange1 = new TestRawTripPayload(recordStr1); + RawTripTestPayload rowChange1 = new RawTripTestPayload(recordStr1); HoodieRecord record1 = new HoodieRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1); - TestRawTripPayload rowChange2 = new TestRawTripPayload(recordStr2); + RawTripTestPayload rowChange2 = new RawTripTestPayload(recordStr2); HoodieRecord record2 = new HoodieRecord(new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()), rowChange2); - TestRawTripPayload rowChange3 = new TestRawTripPayload(recordStr3); + RawTripTestPayload rowChange3 = new RawTripTestPayload(recordStr3); HoodieRecord record3 = new HoodieRecord(new HoodieKey(rowChange3.getRowKey(), rowChange3.getPartitionPath()), rowChange3); - TestRawTripPayload rowChange4 = new TestRawTripPayload(recordStr4); + RawTripTestPayload rowChange4 = new RawTripTestPayload(recordStr4); HoodieRecord record4 = new HoodieRecord(new HoodieKey(rowChange4.getRowKey(), rowChange4.getPartitionPath()), rowChange4); JavaRDD<HoodieRecord> recordRDD = jsc.parallelize(Arrays.asList(record1, record2, record3, record4)); @@ -348,8 +349,8 @@ public class TestHoodieIndex extends HoodieClientTestHarness { new File(basePath + "/2016/01/31/" + HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE).createNewFile(); // this record will be saved in table and will be tagged to an empty record - TestRawTripPayload originalPayload = - new TestRawTripPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload originalPayload = + new RawTripTestPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord originalRecord = new HoodieRecord(new HoodieKey(originalPayload.getRowKey(), originalPayload.getPartitionPath()), originalPayload); @@ -361,8 +362,8 @@ public class TestHoodieIndex extends HoodieClientTestHarness { - tag the original partition of the originalRecord to an empty record for deletion, and - tag the new partition of the incomingRecord */ - TestRawTripPayload incomingPayload = - new TestRawTripPayload("{\"_row_key\":\"000\",\"time\":\"2016-02-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload incomingPayload = + new RawTripTestPayload("{\"_row_key\":\"000\",\"time\":\"2016-02-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord incomingRecord = new HoodieRecord(new HoodieKey(incomingPayload.getRowKey(), incomingPayload.getPartitionPath()), incomingPayload); @@ -371,8 +372,8 @@ public class TestHoodieIndex extends HoodieClientTestHarness { Though GLOBAL_BLOOM_INDEX_SHOULD_UPDATE_PARTITION_PATH = true, globalBloomIndex should just tag the original partition */ - TestRawTripPayload incomingPayloadSamePartition = - new TestRawTripPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T04:16:41.415Z\",\"number\":15}"); + RawTripTestPayload incomingPayloadSamePartition = + new RawTripTestPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T04:16:41.415Z\",\"number\":15}"); HoodieRecord incomingRecordSamePartition = new HoodieRecord( new HoodieKey(incomingPayloadSamePartition.getRowKey(), incomingPayloadSamePartition.getPartitionPath()), @@ -404,7 +405,7 @@ public class TestHoodieIndex extends HoodieClientTestHarness { break; case "2016/02/31": assertEquals("000", record.getRecordKey()); - assertEquals(incomingPayload.getJsonData(), ((TestRawTripPayload) record.getData()).getJsonData()); + assertEquals(incomingPayload.getJsonData(), ((RawTripTestPayload) record.getData()).getJsonData()); break; default: assertFalse(true, String.format("Should not get partition path: %s", record.getPartitionPath())); @@ -420,7 +421,7 @@ public class TestHoodieIndex extends HoodieClientTestHarness { HoodieRecord record = taggedRecordRDDSamePartition.first(); assertEquals("000", record.getRecordKey()); assertEquals("2016/01/31", record.getPartitionPath()); - assertEquals(incomingPayloadSamePartition.getJsonData(), ((TestRawTripPayload) record.getData()).getJsonData()); + assertEquals(incomingPayloadSamePartition.getJsonData(), ((RawTripTestPayload) record.getData()).getJsonData()); } /** @@ -444,7 +445,7 @@ public class TestHoodieIndex extends HoodieClientTestHarness { private HoodieWriteConfig.Builder getConfigBuilder(String schemaStr, IndexType indexType) { return HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(schemaStr) .withParallelism(2, 2).withBulkInsertParallelism(2).withFinalizeWriteParallelism(2) - .withWriteStatusClass(TestRawTripPayload.MetadataMergeWriteStatus.class) + .withWriteStatusClass(MetadataMergeWriteStatus.class) .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true).build()) .withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(1024 * 1024).build()) .withStorageConfig(HoodieStorageConfig.newBuilder().limitFileSize(1024 * 1024).build()) diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java index 719f042..f0a116f 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java @@ -26,6 +26,7 @@ import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; @@ -35,7 +36,6 @@ import org.apache.hudi.io.HoodieKeyLookupHandle; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestHarness; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.avro.Schema; import org.apache.hadoop.fs.Path; @@ -123,20 +123,20 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness { Files.createDirectories(Paths.get(basePath, "2016", "04", "01")); Files.createDirectories(Paths.get(basePath, "2015", "03", "12")); - TestRawTripPayload rowChange1 = - new TestRawTripPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange1 = + new RawTripTestPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record1 = new HoodieRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1); - TestRawTripPayload rowChange2 = - new TestRawTripPayload("{\"_row_key\":\"001\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange2 = + new RawTripTestPayload("{\"_row_key\":\"001\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record2 = new HoodieRecord(new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()), rowChange2); - TestRawTripPayload rowChange3 = - new TestRawTripPayload("{\"_row_key\":\"002\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange3 = + new RawTripTestPayload("{\"_row_key\":\"002\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record3 = new HoodieRecord(new HoodieKey(rowChange3.getRowKey(), rowChange3.getPartitionPath()), rowChange3); - TestRawTripPayload rowChange4 = - new TestRawTripPayload("{\"_row_key\":\"003\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange4 = + new RawTripTestPayload("{\"_row_key\":\"003\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record4 = new HoodieRecord(new HoodieKey(rowChange4.getRowKey(), rowChange4.getPartitionPath()), rowChange4); @@ -227,16 +227,16 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness { + "\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":15}"; String recordStr4 = "{\"_row_key\":\"4eb5b87c-1fej-4edd-87b4-6ec96dc405a0\"," + "\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":32}"; - TestRawTripPayload rowChange1 = new TestRawTripPayload(recordStr1); + RawTripTestPayload rowChange1 = new RawTripTestPayload(recordStr1); HoodieRecord record1 = new HoodieRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1); - TestRawTripPayload rowChange2 = new TestRawTripPayload(recordStr2); + RawTripTestPayload rowChange2 = new RawTripTestPayload(recordStr2); HoodieRecord record2 = new HoodieRecord(new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()), rowChange2); - TestRawTripPayload rowChange3 = new TestRawTripPayload(recordStr3); + RawTripTestPayload rowChange3 = new RawTripTestPayload(recordStr3); HoodieRecord record3 = new HoodieRecord(new HoodieKey(rowChange3.getRowKey(), rowChange3.getPartitionPath()), rowChange3); - TestRawTripPayload rowChange4 = new TestRawTripPayload(recordStr4); + RawTripTestPayload rowChange4 = new RawTripTestPayload(recordStr4); HoodieRecord record4 = new HoodieRecord(new HoodieKey(rowChange4.getRowKey(), rowChange4.getPartitionPath()), rowChange4); @@ -303,16 +303,16 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness { String recordStr3 = "{\"_row_key\":\"" + rowKey3 + "\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":15}"; // place same row key under a different partition. String recordStr4 = "{\"_row_key\":\"" + rowKey1 + "\",\"time\":\"2015-01-31T03:16:41.415Z\",\"number\":32}"; - TestRawTripPayload rowChange1 = new TestRawTripPayload(recordStr1); + RawTripTestPayload rowChange1 = new RawTripTestPayload(recordStr1); HoodieRecord record1 = new HoodieRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1); - TestRawTripPayload rowChange2 = new TestRawTripPayload(recordStr2); + RawTripTestPayload rowChange2 = new RawTripTestPayload(recordStr2); HoodieRecord record2 = new HoodieRecord(new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()), rowChange2); - TestRawTripPayload rowChange3 = new TestRawTripPayload(recordStr3); + RawTripTestPayload rowChange3 = new RawTripTestPayload(recordStr3); HoodieRecord record3 = new HoodieRecord(new HoodieKey(rowChange3.getRowKey(), rowChange3.getPartitionPath()), rowChange3); - TestRawTripPayload rowChange4 = new TestRawTripPayload(recordStr4); + RawTripTestPayload rowChange4 = new RawTripTestPayload(recordStr4); HoodieRecord record4 = new HoodieRecord(new HoodieKey(rowChange4.getRowKey(), rowChange4.getPartitionPath()), rowChange4); JavaRDD<HoodieRecord> recordRDD = jsc.parallelize(Arrays.asList(record1, record2, record3, record4)); @@ -375,15 +375,15 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness { // record key same as recordStr2 String recordStr4 = "{\"_row_key\":\"2eb5b87b-1feu-4edd-87b4-6ec96dc405a0\"," + "\"time\":\"2015-01-31T03:16:41.415Z\",\"number\":32}"; - TestRawTripPayload rowChange1 = new TestRawTripPayload(recordStr1); + RawTripTestPayload rowChange1 = new RawTripTestPayload(recordStr1); HoodieKey key1 = new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()); HoodieRecord record1 = new HoodieRecord(key1, rowChange1); - TestRawTripPayload rowChange2 = new TestRawTripPayload(recordStr2); + RawTripTestPayload rowChange2 = new RawTripTestPayload(recordStr2); HoodieKey key2 = new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()); HoodieRecord record2 = new HoodieRecord(key2, rowChange2); - TestRawTripPayload rowChange3 = new TestRawTripPayload(recordStr3); + RawTripTestPayload rowChange3 = new RawTripTestPayload(recordStr3); HoodieKey key3 = new HoodieKey(rowChange3.getRowKey(), rowChange3.getPartitionPath()); - TestRawTripPayload rowChange4 = new TestRawTripPayload(recordStr4); + RawTripTestPayload rowChange4 = new RawTripTestPayload(recordStr4); HoodieKey key4 = new HoodieKey(rowChange4.getRowKey(), rowChange4.getPartitionPath()); HoodieRecord record4 = new HoodieRecord(key4, rowChange4); JavaRDD<HoodieKey> keysRDD = jsc.parallelize(Arrays.asList(key1, key2, key3, key4)); @@ -444,10 +444,10 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness { + "\"time\":\"2016-01-31T03:20:41.415Z\",\"number\":100}"; // We write record1 to a parquet file, using a bloom filter having both records - TestRawTripPayload rowChange1 = new TestRawTripPayload(recordStr1); + RawTripTestPayload rowChange1 = new RawTripTestPayload(recordStr1); HoodieRecord record1 = new HoodieRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1); - TestRawTripPayload rowChange2 = new TestRawTripPayload(recordStr2); + RawTripTestPayload rowChange2 = new RawTripTestPayload(recordStr2); HoodieRecord record2 = new HoodieRecord(new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()), rowChange2); diff --git a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java index bb227ae..c5f3045 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java @@ -25,13 +25,13 @@ import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodiePartitionMetadata; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestHarness; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.avro.Schema; import org.apache.spark.api.java.JavaPairRDD; @@ -101,20 +101,20 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { Path dir3 = Files.createDirectories(Paths.get(basePath, "2015", "03", "12")); Files.createFile(dir3.resolve(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE)); - TestRawTripPayload rowChange1 = - new TestRawTripPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange1 = + new RawTripTestPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record1 = new HoodieRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1); - TestRawTripPayload rowChange2 = - new TestRawTripPayload("{\"_row_key\":\"001\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange2 = + new RawTripTestPayload("{\"_row_key\":\"001\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record2 = new HoodieRecord(new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()), rowChange2); - TestRawTripPayload rowChange3 = - new TestRawTripPayload("{\"_row_key\":\"002\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange3 = + new RawTripTestPayload("{\"_row_key\":\"002\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record3 = new HoodieRecord(new HoodieKey(rowChange3.getRowKey(), rowChange3.getPartitionPath()), rowChange3); - TestRawTripPayload rowChange4 = - new TestRawTripPayload("{\"_row_key\":\"003\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange4 = + new RawTripTestPayload("{\"_row_key\":\"003\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record4 = new HoodieRecord(new HoodieKey(rowChange4.getRowKey(), rowChange4.getPartitionPath()), rowChange4); @@ -221,29 +221,29 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { Path dir3 = Files.createDirectories(Paths.get(basePath, "2015", "03", "12")); Files.createFile(dir3.resolve(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE)); - TestRawTripPayload rowChange1 = - new TestRawTripPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange1 = + new RawTripTestPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record1 = new HoodieRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1); - TestRawTripPayload rowChange2 = - new TestRawTripPayload("{\"_row_key\":\"001\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange2 = + new RawTripTestPayload("{\"_row_key\":\"001\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record2 = new HoodieRecord(new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()), rowChange2); - TestRawTripPayload rowChange3 = - new TestRawTripPayload("{\"_row_key\":\"002\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange3 = + new RawTripTestPayload("{\"_row_key\":\"002\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record3 = new HoodieRecord(new HoodieKey(rowChange3.getRowKey(), rowChange3.getPartitionPath()), rowChange3); // this record will be saved in table and will be tagged to the incoming record5 - TestRawTripPayload rowChange4 = - new TestRawTripPayload("{\"_row_key\":\"003\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange4 = + new RawTripTestPayload("{\"_row_key\":\"003\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record4 = new HoodieRecord(new HoodieKey(rowChange4.getRowKey(), rowChange4.getPartitionPath()), rowChange4); // this has the same record key as record4 but different time so different partition, but globalbloomIndex should // tag the original partition of the saved record4 - TestRawTripPayload rowChange5 = - new TestRawTripPayload("{\"_row_key\":\"003\",\"time\":\"2016-02-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload rowChange5 = + new RawTripTestPayload("{\"_row_key\":\"003\",\"time\":\"2016-02-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord record5 = new HoodieRecord(new HoodieKey(rowChange5.getRowKey(), rowChange5.getPartitionPath()), rowChange5); @@ -272,23 +272,23 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { switch (record.getRecordKey()) { case "000": assertEquals(record.getCurrentLocation().getFileId(), FSUtils.getFileId(filename0)); - assertEquals(((TestRawTripPayload) record.getData()).getJsonData(), rowChange1.getJsonData()); + assertEquals(((RawTripTestPayload) record.getData()).getJsonData(), rowChange1.getJsonData()); break; case "001": assertEquals(record.getCurrentLocation().getFileId(), FSUtils.getFileId(filename2)); - assertEquals(((TestRawTripPayload) record.getData()).getJsonData(), rowChange2.getJsonData()); + assertEquals(((RawTripTestPayload) record.getData()).getJsonData(), rowChange2.getJsonData()); break; case "002": assertFalse(record.isCurrentLocationKnown()); - assertEquals(((TestRawTripPayload) record.getData()).getJsonData(), rowChange3.getJsonData()); + assertEquals(((RawTripTestPayload) record.getData()).getJsonData(), rowChange3.getJsonData()); break; case "003": assertEquals(record.getCurrentLocation().getFileId(), FSUtils.getFileId(filename3)); - assertEquals(((TestRawTripPayload) record.getData()).getJsonData(), rowChange5.getJsonData()); + assertEquals(((RawTripTestPayload) record.getData()).getJsonData(), rowChange5.getJsonData()); break; case "004": assertEquals(record.getCurrentLocation().getFileId(), FSUtils.getFileId(filename3)); - assertEquals(((TestRawTripPayload) record.getData()).getJsonData(), rowChange4.getJsonData()); + assertEquals(((RawTripTestPayload) record.getData()).getJsonData(), rowChange4.getJsonData()); break; default: throw new IllegalArgumentException("Unknown Key: " + record.getRecordKey()); @@ -310,8 +310,8 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { Files.createFile(dir.resolve(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE)); // this record will be saved in table and will be tagged to an empty record - TestRawTripPayload originalPayload = - new TestRawTripPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload originalPayload = + new RawTripTestPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord originalRecord = new HoodieRecord(new HoodieKey(originalPayload.getRowKey(), originalPayload.getPartitionPath()), originalPayload); @@ -323,8 +323,8 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { - tag the original partition of the originalRecord to an empty record for deletion, and - tag the new partition of the incomingRecord */ - TestRawTripPayload incomingPayload = - new TestRawTripPayload("{\"_row_key\":\"000\",\"time\":\"2016-02-31T03:16:41.415Z\",\"number\":12}"); + RawTripTestPayload incomingPayload = + new RawTripTestPayload("{\"_row_key\":\"000\",\"time\":\"2016-02-31T03:16:41.415Z\",\"number\":12}"); HoodieRecord incomingRecord = new HoodieRecord(new HoodieKey(incomingPayload.getRowKey(), incomingPayload.getPartitionPath()), incomingPayload); @@ -334,8 +334,8 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { Though GLOBAL_BLOOM_INDEX_SHOULD_UPDATE_PARTITION_PATH = true, globalBloomIndex should just tag the original partition */ - TestRawTripPayload incomingPayloadSamePartition = - new TestRawTripPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T04:16:41.415Z\",\"number\":15}"); + RawTripTestPayload incomingPayloadSamePartition = + new RawTripTestPayload("{\"_row_key\":\"000\",\"time\":\"2016-01-31T04:16:41.415Z\",\"number\":15}"); HoodieRecord incomingRecordSamePartition = new HoodieRecord( new HoodieKey(incomingPayloadSamePartition.getRowKey(), incomingPayloadSamePartition.getPartitionPath()), @@ -363,7 +363,7 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { break; case "2016/02/31": assertEquals("000", record.getRecordKey()); - assertEquals(incomingPayload.getJsonData(), ((TestRawTripPayload) record.getData()).getJsonData()); + assertEquals(incomingPayload.getJsonData(), ((RawTripTestPayload) record.getData()).getJsonData()); break; default: fail(String.format("Should not get partition path: %s", record.getPartitionPath())); @@ -379,7 +379,7 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness { HoodieRecord record = taggedRecordRDDSamePartition.first(); assertEquals("000", record.getRecordKey()); assertEquals("2016/01/31", record.getPartitionPath()); - assertEquals(incomingPayloadSamePartition.getJsonData(), ((TestRawTripPayload) record.getData()).getJsonData()); + assertEquals(incomingPayloadSamePartition.getJsonData(), ((RawTripTestPayload) record.getData()).getJsonData()); } // convert list to map to avoid sorting order dependencies diff --git a/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java index b3d2f5a..7339855 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java @@ -23,6 +23,7 @@ import org.apache.hudi.client.WriteStatus; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieHBaseIndexConfig; @@ -32,7 +33,6 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.FunctionalTestHarness; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; diff --git a/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java index cb976e7..d7591e6 100644 --- a/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java @@ -18,6 +18,7 @@ package org.apache.hudi.index.hbase; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieHBaseIndexConfig; @@ -25,7 +26,6 @@ import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.junit.jupiter.api.Test; diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java index 5c3c5ad..8228f03 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieKeyLocationFetchHandle.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.table.view.FileSystemViewStorageType; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieCompactionConfig; @@ -37,8 +38,7 @@ import org.apache.hudi.index.HoodieIndexUtils; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestHarness; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; -import org.apache.hudi.testutils.TestRawTripPayload; +import org.apache.hudi.testutils.MetadataMergeWriteStatus; import org.apache.spark.api.java.JavaSparkContext; import org.junit.jupiter.api.AfterEach; @@ -58,7 +58,7 @@ import java.util.UUID; import scala.Tuple2; import static java.util.stream.Collectors.toList; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.AVRO_SCHEMA_WITH_METADATA_FIELDS; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.AVRO_SCHEMA_WITH_METADATA_FIELDS; import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -185,7 +185,7 @@ public class TestHoodieKeyLocationFetchHandle extends HoodieClientTestHarness { private HoodieWriteConfig.Builder getConfigBuilder(String schemaStr) { return HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(schemaStr) .withParallelism(2, 2).withBulkInsertParallelism(2).withFinalizeWriteParallelism(2) - .withWriteStatusClass(TestRawTripPayload.MetadataMergeWriteStatus.class) + .withWriteStatusClass(MetadataMergeWriteStatus.class) .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true).build()) .withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(1024 * 1024).build()) .withStorageConfig(HoodieStorageConfig.newBuilder().limitFileSize(1024 * 1024).build()) diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java index 18744ef..b45eae0 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java @@ -26,6 +26,7 @@ import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieStorageConfig; @@ -33,7 +34,6 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.testutils.HoodieClientTestHarness; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.fs.FileSystem; import org.apache.spark.api.java.JavaRDD; diff --git a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiveLog.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiveLog.java index 7b2114f..484caf7 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiveLog.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieTimelineArchiveLog.java @@ -26,12 +26,12 @@ import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTimelineArchiveLog; import org.apache.hudi.testutils.HoodieClientTestHarness; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; diff --git a/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieFileWriterFactory.java b/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieFileWriterFactory.java index 2584643..ed740ea 100644 --- a/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieFileWriterFactory.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/storage/TestHoodieFileWriterFactory.java @@ -19,10 +19,10 @@ package org.apache.hudi.io.storage; import org.apache.hudi.client.SparkTaskContextSupplier; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestBase; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.Path; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java index dc36005..0376ec3 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java @@ -44,6 +44,7 @@ import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.common.table.timeline.versioning.clean.CleanMetadataMigrator; import org.apache.hudi.common.table.view.TableFileSystemView; import org.apache.hudi.common.testutils.FileSystemTestUtils; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.CleanerUtils; import org.apache.hudi.common.util.CollectionUtils; @@ -56,7 +57,6 @@ import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java index 1551df1..d4ac533 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java @@ -40,6 +40,7 @@ import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.table.view.SyncableFileSystemView; import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; import org.apache.hudi.common.table.view.TableFileSystemView.SliceView; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; @@ -58,8 +59,7 @@ import org.apache.hudi.table.action.deltacommit.DeltaCommitActionExecutor; import org.apache.hudi.testutils.HoodieClientTestHarness; import org.apache.hudi.testutils.HoodieClientTestUtils; import org.apache.hudi.testutils.HoodieMergeOnReadTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; -import org.apache.hudi.testutils.TestRawTripPayload.MetadataMergeWriteStatus; +import org.apache.hudi.testutils.MetadataMergeWriteStatus; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.FileStatus; @@ -86,8 +86,8 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java index c95a917..4d16e57 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java @@ -27,20 +27,20 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.ParquetUtils; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.config.HoodieStorageConfig; import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.hadoop.utils.HoodieHiveUtils; import org.apache.hudi.hadoop.HoodieParquetInputFormat; +import org.apache.hudi.hadoop.utils.HoodieHiveUtils; import org.apache.hudi.io.HoodieCreateHandle; import org.apache.hudi.table.HoodieCopyOnWriteTable; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.TestRawTripPayload; -import org.apache.hudi.testutils.TestRawTripPayload.MetadataMergeWriteStatus; +import org.apache.hudi.testutils.MetadataMergeWriteStatus; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.FileStatus; @@ -128,11 +128,11 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestBase { + "\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":51}"; List<HoodieRecord> records = new ArrayList<>(); - TestRawTripPayload rowChange1 = new TestRawTripPayload(recordStr1); + RawTripTestPayload rowChange1 = new RawTripTestPayload(recordStr1); records.add(new HoodieRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1)); - TestRawTripPayload rowChange2 = new TestRawTripPayload(recordStr2); + RawTripTestPayload rowChange2 = new RawTripTestPayload(recordStr2); records.add(new HoodieRecord(new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()), rowChange2)); - TestRawTripPayload rowChange3 = new TestRawTripPayload(recordStr3); + RawTripTestPayload rowChange3 = new RawTripTestPayload(recordStr3); records.add(new HoodieRecord(new HoodieKey(rowChange3.getRowKey(), rowChange3.getPartitionPath()), rowChange3)); // Insert new records @@ -162,11 +162,11 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestBase { // We update the 1st record & add a new record String updateRecordStr1 = "{\"_row_key\":\"8eb5b87a-1feh-4edd-87b4-6ec96dc405a0\"," + "\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":15}"; - TestRawTripPayload updateRowChanges1 = new TestRawTripPayload(updateRecordStr1); + RawTripTestPayload updateRowChanges1 = new RawTripTestPayload(updateRecordStr1); HoodieRecord updatedRecord1 = new HoodieRecord( new HoodieKey(updateRowChanges1.getRowKey(), updateRowChanges1.getPartitionPath()), updateRowChanges1); - TestRawTripPayload rowChange4 = new TestRawTripPayload(recordStr4); + RawTripTestPayload rowChange4 = new RawTripTestPayload(recordStr4); HoodieRecord insertedRecord1 = new HoodieRecord(new HoodieKey(rowChange4.getRowKey(), rowChange4.getPartitionPath()), rowChange4); @@ -242,7 +242,7 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestBase { for (int i = 0; i < n; i++) { String recordStr = String.format("{\"_row_key\":\"%s\",\"time\":\"%s\",\"number\":%d}", UUID.randomUUID().toString(), time, i); - TestRawTripPayload rowChange = new TestRawTripPayload(recordStr); + RawTripTestPayload rowChange = new RawTripTestPayload(recordStr); records.add(new HoodieRecord(new HoodieKey(rowChange.getRowKey(), rowChange.getPartitionPath()), rowChange)); } return records; @@ -268,11 +268,11 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestBase { + "\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":15}"; List<HoodieRecord> records = new ArrayList<>(); - TestRawTripPayload rowChange1 = new TestRawTripPayload(recordStr1); + RawTripTestPayload rowChange1 = new RawTripTestPayload(recordStr1); records.add(new HoodieRecord(new HoodieKey(rowChange1.getRowKey(), rowChange1.getPartitionPath()), rowChange1)); - TestRawTripPayload rowChange2 = new TestRawTripPayload(recordStr2); + RawTripTestPayload rowChange2 = new RawTripTestPayload(recordStr2); records.add(new HoodieRecord(new HoodieKey(rowChange2.getRowKey(), rowChange2.getPartitionPath()), rowChange2)); - TestRawTripPayload rowChange3 = new TestRawTripPayload(recordStr3); + RawTripTestPayload rowChange3 = new RawTripTestPayload(recordStr3); records.add(new HoodieRecord(new HoodieKey(rowChange3.getRowKey(), rowChange3.getPartitionPath()), rowChange3)); // Insert new records @@ -357,7 +357,7 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestBase { for (int i = 0; i < 2000; i++) { String recordStr = "{\"_row_key\":\"" + UUID.randomUUID().toString() + "\",\"time\":\"2016-01-31T03:16:41.415Z\",\"number\":" + i + "}"; - TestRawTripPayload rowChange = new TestRawTripPayload(recordStr); + RawTripTestPayload rowChange = new RawTripTestPayload(recordStr); records.add(new HoodieRecord(new HoodieKey(rowChange.getRowKey(), rowChange.getPartitionPath()), rowChange)); } diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java index c526ad1..afde632 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java @@ -25,6 +25,7 @@ import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; @@ -35,7 +36,6 @@ import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.WorkloadProfile; import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java index c171255..697febe 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java @@ -18,7 +18,6 @@ package org.apache.hudi.table.action.compact; -import org.apache.hadoop.fs.FileStatus; import org.apache.hudi.avro.model.HoodieCompactionOperation; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.client.HoodieReadClient; @@ -35,6 +34,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.table.view.FileSystemViewStorageType; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.CompactionUtils; import org.apache.hudi.common.util.Option; @@ -47,7 +47,8 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestBase; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; + +import org.apache.hadoop.fs.FileStatus; import org.apache.spark.api.java.JavaRDD; import java.io.IOException; @@ -56,8 +57,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java index 6819cc5..1529d79 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestHoodieCompactor.java @@ -30,6 +30,7 @@ import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; @@ -42,7 +43,6 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.bloom.HoodieBloomIndex; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.HoodieClientTestHarness; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hadoop.conf.Configuration; import org.apache.spark.api.java.JavaRDD; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/HoodieClientRollbackTestBase.java b/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/HoodieClientRollbackTestBase.java index 24aa869..ed97f43 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/HoodieClientRollbackTestBase.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/HoodieClientRollbackTestBase.java @@ -25,19 +25,20 @@ import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.view.SyncableFileSystemView; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.testutils.Assertions; import org.apache.hudi.testutils.HoodieClientTestBase; -import org.apache.hudi.testutils.HoodieTestDataGenerator; + import org.apache.spark.api.java.JavaRDD; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; import static org.junit.jupiter.api.Assertions.assertEquals; public class HoodieClientRollbackTestBase extends HoodieClientTestBase { diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestCopyOnWriteRollbackActionExecutor.java b/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestCopyOnWriteRollbackActionExecutor.java index 1c5a85a..26a85e1 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestCopyOnWriteRollbackActionExecutor.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestCopyOnWriteRollbackActionExecutor.java @@ -19,17 +19,18 @@ package org.apache.hudi.table.action.rollback; import org.apache.hudi.avro.model.HoodieRollbackPartitionMetadata; +import org.apache.hudi.common.HoodieRollbackStat; import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; -import org.apache.hudi.common.HoodieRollbackStat; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.MarkerFiles; -import org.apache.hudi.testutils.HoodieTestDataGenerator; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -44,8 +45,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestMergeOnReadRollbackActionExecutor.java b/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestMergeOnReadRollbackActionExecutor.java index 759e7f6..1edebcd 100644 --- a/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestMergeOnReadRollbackActionExecutor.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestMergeOnReadRollbackActionExecutor.java @@ -25,10 +25,11 @@ import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.MarkerFiles; -import org.apache.hudi.testutils.HoodieTestDataGenerator; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -42,11 +43,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; -import static org.apache.hudi.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; +import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestMergeOnReadRollbackActionExecutor extends HoodieClientRollbackTestBase { @Override diff --git a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java index 1f5aa54..667181c 100644 --- a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java @@ -33,6 +33,8 @@ import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; import org.apache.hudi.common.table.view.FileSystemViewStorageType; import org.apache.hudi.common.table.view.SyncableFileSystemView; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; @@ -41,7 +43,6 @@ import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.hudi.table.HoodieTable; -import org.apache.hudi.testutils.TestRawTripPayload.MetadataMergeWriteStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -192,7 +193,7 @@ public class HoodieClientTestBase extends HoodieClientTestHarness { * * @param records List of Hoodie records */ - public void assertNodupesWithinPartition(List<HoodieRecord<TestRawTripPayload>> records) { + public void assertNodupesWithinPartition(List<HoodieRecord<RawTripTestPayload>> records) { Map<String, Set<String>> partitionToKeys = new HashMap<>(); for (HoodieRecord r : records) { String key = r.getRecordKey(); diff --git a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java index 71be8f3..17b9c35 100644 --- a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java @@ -25,8 +25,11 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.view.HoodieTableFileSystemView; import org.apache.hudi.common.testutils.HoodieCommonTestHarness; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.testutils.minicluster.HdfsTestService; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -35,9 +38,6 @@ import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hudi.config.HoodieWriteConfig; -import org.apache.hudi.index.HoodieIndex; - import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; import org.junit.jupiter.api.BeforeEach; diff --git a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java index b88cfff..7ee3dc9 100644 --- a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java @@ -20,8 +20,10 @@ package org.apache.hudi.testutils; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils; + import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecordBuilder; diff --git a/hudi-client/src/test/java/org/apache/hudi/testutils/MetadataMergeWriteStatus.java b/hudi-client/src/test/java/org/apache/hudi/testutils/MetadataMergeWriteStatus.java new file mode 100644 index 0000000..2f2240f --- /dev/null +++ b/hudi-client/src/test/java/org/apache/hudi/testutils/MetadataMergeWriteStatus.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.testutils; + +import org.apache.hudi.client.WriteStatus; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.Option; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +/** + * A custom {@link WriteStatus} that merges passed metadata key value map to {@code WriteStatus.markSuccess()} and + * {@code WriteStatus.markFailure()}. + */ +public class MetadataMergeWriteStatus extends WriteStatus { + + private Map<String, String> mergedMetadataMap = new HashMap<>(); + + public MetadataMergeWriteStatus(Boolean trackSuccessRecords, Double failureFraction) { + super(trackSuccessRecords, failureFraction); + } + + public static Map<String, String> mergeMetadataForWriteStatuses(List<WriteStatus> writeStatuses) { + Map<String, String> allWriteStatusMergedMetadataMap = new HashMap<>(); + for (WriteStatus writeStatus : writeStatuses) { + MetadataMergeWriteStatus.mergeMetadataMaps(((MetadataMergeWriteStatus) writeStatus).getMergedMetadataMap(), + allWriteStatusMergedMetadataMap); + } + return allWriteStatusMergedMetadataMap; + } + + private static void mergeMetadataMaps(Map<String, String> mergeFromMap, Map<String, String> mergeToMap) { + for (Entry<String, String> entry : mergeFromMap.entrySet()) { + String key = entry.getKey(); + if (!mergeToMap.containsKey(key)) { + mergeToMap.put(key, "0"); + } + mergeToMap.put(key, addStrsAsInt(entry.getValue(), mergeToMap.get(key))); + } + } + + private static String addStrsAsInt(String a, String b) { + return String.valueOf(Integer.parseInt(a) + Integer.parseInt(b)); + } + + @Override + public void markSuccess(HoodieRecord record, Option<Map<String, String>> recordMetadata) { + super.markSuccess(record, recordMetadata); + if (recordMetadata.isPresent()) { + mergeMetadataMaps(recordMetadata.get(), mergedMetadataMap); + } + } + + @Override + public void markFailure(HoodieRecord record, Throwable t, Option<Map<String, String>> recordMetadata) { + super.markFailure(record, t, recordMetadata); + if (recordMetadata.isPresent()) { + mergeMetadataMaps(recordMetadata.get(), mergedMetadataMap); + } + } + + private Map<String, String> getMergedMetadataMap() { + return mergedMetadataMap; + } +} diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestParquetInLining.java similarity index 87% rename from hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java rename to hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestParquetInLining.java index ce21dd3..c4e728d 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs/TestParquetInLining.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/fs/inline/TestParquetInLining.java @@ -7,22 +7,21 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package org.apache.hudi.utilities.inline.fs; +package org.apache.hudi.common.fs.inline; -import org.apache.hudi.common.fs.inline.InLineFileSystem; -import org.apache.hudi.common.fs.inline.InMemoryFileSystem; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.testutils.FileSystemTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; @@ -48,8 +47,7 @@ import static org.apache.hudi.common.testutils.FileSystemTestUtils.getRandomOute import static org.junit.jupiter.api.Assertions.assertArrayEquals; /** - * Tests {@link InLineFileSystem} with Parquet writer and reader. hudi-common can't access HoodieTestDataGenerator. - * Hence keeping this test in hudi-utilities. + * Tests for {@link InLineFileSystem} with Parquet writer and reader. */ public class TestParquetInLining { diff --git a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieTestDataGenerator.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java similarity index 96% rename from hudi-client/src/test/java/org/apache/hudi/testutils/HoodieTestDataGenerator.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java index 4d08e8f..e23d885 100644 --- a/hudi-client/src/test/java/org/apache/hudi/testutils/HoodieTestDataGenerator.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java @@ -7,20 +7,18 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package org.apache.hudi.testutils; +package org.apache.hudi.common.testutils; -import org.apache.avro.Conversions; -import org.apache.avro.LogicalTypes; -import org.apache.avro.generic.GenericFixed; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.common.fs.FSUtils; @@ -36,9 +34,12 @@ import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.exception.HoodieIOException; +import org.apache.avro.Conversions; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.generic.GenericArray; import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericFixed; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -164,7 +165,7 @@ public class HoodieTestDataGenerator { return numOfRecords * BYTES_PER_RECORD + BLOOM_FILTER_BYTES; } - public TestRawTripPayload generateRandomValueAsPerSchema(String schemaStr, HoodieKey key, String commitTime, boolean isFlattened) throws IOException { + public RawTripTestPayload generateRandomValueAsPerSchema(String schemaStr, HoodieKey key, String commitTime, boolean isFlattened) throws IOException { if (TRIP_EXAMPLE_SCHEMA.equals(schemaStr)) { return generateRandomValue(key, commitTime, isFlattened); } else if (TRIP_SCHEMA.equals(schemaStr)) { @@ -180,12 +181,11 @@ public class HoodieTestDataGenerator { * Generates a new avro record of the above nested schema format, * retaining the key if optionally provided. * - * @param key Hoodie key. - * @param instantTime Instant time to use. - * @return Raw paylaod of a test record. - * @throws IOException + * @param key Hoodie key. + * @param instantTime Instant time to use. + * @return Raw paylaod of a test record. */ - public static TestRawTripPayload generateRandomValue(HoodieKey key, String instantTime) throws IOException { + public static RawTripTestPayload generateRandomValue(HoodieKey key, String instantTime) throws IOException { return generateRandomValue(key, instantTime, false); } @@ -196,37 +196,37 @@ public class HoodieTestDataGenerator { * @param key Hoodie key. * @param instantTime Commit time to use. * @param isFlattened whether the schema of the record should be flattened. - * @return Raw paylaod of a test record. + * @return Raw paylaod of a test record. * @throws IOException */ - public static TestRawTripPayload generateRandomValue( + public static RawTripTestPayload generateRandomValue( HoodieKey key, String instantTime, boolean isFlattened) throws IOException { GenericRecord rec = generateGenericRecord( key.getRecordKey(), "rider-" + instantTime, "driver-" + instantTime, 0.0, false, isFlattened); - return new TestRawTripPayload(rec.toString(), key.getRecordKey(), key.getPartitionPath(), TRIP_EXAMPLE_SCHEMA); + return new RawTripTestPayload(rec.toString(), key.getRecordKey(), key.getPartitionPath(), TRIP_EXAMPLE_SCHEMA); } /** * Generates a new avro record with TRIP_SCHEMA, retaining the key if optionally provided. */ - public TestRawTripPayload generatePayloadForTripSchema(HoodieKey key, String commitTime) throws IOException { + public RawTripTestPayload generatePayloadForTripSchema(HoodieKey key, String commitTime) throws IOException { GenericRecord rec = generateRecordForTripSchema(key.getRecordKey(), "rider-" + commitTime, "driver-" + commitTime, 0.0); - return new TestRawTripPayload(rec.toString(), key.getRecordKey(), key.getPartitionPath(), TRIP_SCHEMA); + return new RawTripTestPayload(rec.toString(), key.getRecordKey(), key.getPartitionPath(), TRIP_SCHEMA); } - public TestRawTripPayload generatePayloadForShortTripSchema(HoodieKey key, String commitTime) throws IOException { + public RawTripTestPayload generatePayloadForShortTripSchema(HoodieKey key, String commitTime) throws IOException { GenericRecord rec = generateRecordForShortTripSchema(key.getRecordKey(), "rider-" + commitTime, "driver-" + commitTime, 0.0); - return new TestRawTripPayload(rec.toString(), key.getRecordKey(), key.getPartitionPath(), SHORT_TRIP_SCHEMA); + return new RawTripTestPayload(rec.toString(), key.getRecordKey(), key.getPartitionPath(), SHORT_TRIP_SCHEMA); } /** * Generates a new avro record of the above schema format for a delete. */ - public static TestRawTripPayload generateRandomDeleteValue(HoodieKey key, String instantTime) throws IOException { + public static RawTripTestPayload generateRandomDeleteValue(HoodieKey key, String instantTime) throws IOException { GenericRecord rec = generateGenericRecord(key.getRecordKey(), "rider-" + instantTime, "driver-" + instantTime, 0.0, true, false); - return new TestRawTripPayload(Option.of(rec.toString()), key.getRecordKey(), key.getPartitionPath(), TRIP_EXAMPLE_SCHEMA, true); + return new RawTripTestPayload(Option.of(rec.toString()), key.getRecordKey(), key.getPartitionPath(), TRIP_EXAMPLE_SCHEMA, true); } /** @@ -545,8 +545,8 @@ public class HoodieTestDataGenerator { } public HoodieRecord generateDeleteRecord(HoodieKey key) throws IOException { - TestRawTripPayload payload = - new TestRawTripPayload(Option.empty(), key.getRecordKey(), key.getPartitionPath(), null, true); + RawTripTestPayload payload = + new RawTripTestPayload(Option.empty(), key.getRecordKey(), key.getPartitionPath(), null, true); return new HoodieRecord(key, payload); } diff --git a/hudi-client/src/test/java/org/apache/hudi/testutils/TestRawTripPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/RawTripTestPayload.java similarity index 58% rename from hudi-client/src/test/java/org/apache/hudi/testutils/TestRawTripPayload.java rename to hudi-common/src/test/java/org/apache/hudi/common/testutils/RawTripTestPayload.java index d34c7ce..54525cb 100644 --- a/hudi-client/src/test/java/org/apache/hudi/testutils/TestRawTripPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/RawTripTestPayload.java @@ -7,20 +7,19 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package org.apache.hudi.testutils; +package org.apache.hudi.common.testutils; import org.apache.hudi.avro.MercifulJsonConverter; -import org.apache.hudi.client.WriteStatus; -import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hudi.common.util.Option; @@ -33,9 +32,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; @@ -44,7 +41,7 @@ import java.util.zip.InflaterInputStream; * Example row change event based on some example data used by testcases. The data avro schema is * src/test/resources/schema1. */ -public class TestRawTripPayload implements HoodieRecordPayload<TestRawTripPayload> { +public class RawTripTestPayload implements HoodieRecordPayload<RawTripTestPayload> { private static final transient ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private String partitionPath; @@ -53,7 +50,7 @@ public class TestRawTripPayload implements HoodieRecordPayload<TestRawTripPayloa private int dataSize; private boolean isDeleted; - public TestRawTripPayload(Option<String> jsonData, String rowKey, String partitionPath, String schemaStr, + public RawTripTestPayload(Option<String> jsonData, String rowKey, String partitionPath, String schemaStr, Boolean isDeleted) throws IOException { if (jsonData.isPresent()) { this.jsonDataCompressed = compressData(jsonData.get()); @@ -64,11 +61,11 @@ public class TestRawTripPayload implements HoodieRecordPayload<TestRawTripPayloa this.isDeleted = isDeleted; } - public TestRawTripPayload(String jsonData, String rowKey, String partitionPath, String schemaStr) throws IOException { + public RawTripTestPayload(String jsonData, String rowKey, String partitionPath, String schemaStr) throws IOException { this(Option.of(jsonData), rowKey, partitionPath, schemaStr, false); } - public TestRawTripPayload(String jsonData) throws IOException { + public RawTripTestPayload(String jsonData) throws IOException { this.jsonDataCompressed = compressData(jsonData); this.dataSize = jsonData.length(); Map<String, Object> jsonRecordMap = OBJECT_MAPPER.readValue(jsonData, Map.class); @@ -82,7 +79,7 @@ public class TestRawTripPayload implements HoodieRecordPayload<TestRawTripPayloa } @Override - public TestRawTripPayload preCombine(TestRawTripPayload another) { + public RawTripTestPayload preCombine(RawTripTestPayload another) { return another; } @@ -141,59 +138,4 @@ public class TestRawTripPayload implements HoodieRecordPayload<TestRawTripPayloa } } - /** - * A custom {@link WriteStatus} that merges passed metadata key value map to {@code WriteStatus.markSuccess()} and - * {@code WriteStatus.markFailure()}. - */ - public static class MetadataMergeWriteStatus extends WriteStatus { - - private Map<String, String> mergedMetadataMap = new HashMap<>(); - - public MetadataMergeWriteStatus(Boolean trackSuccessRecords, Double failureFraction) { - super(trackSuccessRecords, failureFraction); - } - - public static Map<String, String> mergeMetadataForWriteStatuses(List<WriteStatus> writeStatuses) { - Map<String, String> allWriteStatusMergedMetadataMap = new HashMap<>(); - for (WriteStatus writeStatus : writeStatuses) { - MetadataMergeWriteStatus.mergeMetadataMaps(((MetadataMergeWriteStatus) writeStatus).getMergedMetadataMap(), - allWriteStatusMergedMetadataMap); - } - return allWriteStatusMergedMetadataMap; - } - - private static void mergeMetadataMaps(Map<String, String> mergeFromMap, Map<String, String> mergeToMap) { - for (Entry<String, String> entry : mergeFromMap.entrySet()) { - String key = entry.getKey(); - if (!mergeToMap.containsKey(key)) { - mergeToMap.put(key, "0"); - } - mergeToMap.put(key, addStrsAsInt(entry.getValue(), mergeToMap.get(key))); - } - } - - private static String addStrsAsInt(String a, String b) { - return String.valueOf(Integer.parseInt(a) + Integer.parseInt(b)); - } - - @Override - public void markSuccess(HoodieRecord record, Option<Map<String, String>> recordMetadata) { - super.markSuccess(record, recordMetadata); - if (recordMetadata.isPresent()) { - mergeMetadataMaps(recordMetadata.get(), mergedMetadataMap); - } - } - - @Override - public void markFailure(HoodieRecord record, Throwable t, Option<Map<String, String>> recordMetadata) { - super.markFailure(record, t, recordMetadata); - if (recordMetadata.isPresent()) { - mergeMetadataMaps(recordMetadata.get(), mergedMetadataMap); - } - } - - private Map<String, String> getMergedMetadataMap() { - return mergedMetadataMap; - } - } } diff --git a/hudi-spark/src/test/java/HoodieJavaApp.java b/hudi-spark/src/test/java/HoodieJavaApp.java index 93493a6..90e133e 100644 --- a/hudi-spark/src/test/java/HoodieJavaApp.java +++ b/hudi-spark/src/test/java/HoodieJavaApp.java @@ -21,6 +21,7 @@ import org.apache.hudi.DataSourceWriteOptions; import org.apache.hudi.HoodieDataSourceHelpers; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.hive.MultiPartKeysValueExtractor; @@ -29,7 +30,6 @@ import org.apache.hudi.keygen.NonpartitionedKeyGenerator; import org.apache.hudi.keygen.SimpleKeyGenerator; import org.apache.hudi.testutils.DataSourceTestUtils; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; diff --git a/hudi-spark/src/test/java/HoodieJavaStreamingApp.java b/hudi-spark/src/test/java/HoodieJavaStreamingApp.java index 70ec3fa..b88eefb 100644 --- a/hudi-spark/src/test/java/HoodieJavaStreamingApp.java +++ b/hudi-spark/src/test/java/HoodieJavaStreamingApp.java @@ -20,10 +20,10 @@ import org.apache.hudi.DataSourceReadOptions; import org.apache.hudi.DataSourceWriteOptions; import org.apache.hudi.HoodieDataSourceHelpers; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.hive.MultiPartKeysValueExtractor; import org.apache.hudi.testutils.DataSourceTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; diff --git a/hudi-spark/src/test/java/org/apache/hudi/testutils/DataSourceTestUtils.java b/hudi-spark/src/test/java/org/apache/hudi/testutils/DataSourceTestUtils.java index 130a6e4..14e66a5 100644 --- a/hudi-spark/src/test/java/org/apache/hudi/testutils/DataSourceTestUtils.java +++ b/hudi-spark/src/test/java/org/apache/hudi/testutils/DataSourceTestUtils.java @@ -21,6 +21,7 @@ package org.apache.hudi.testutils; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.Option; import org.apache.hudi.table.UserDefinedBulkInsertPartitioner; @@ -37,7 +38,7 @@ public class DataSourceTestUtils { public static Option<String> convertToString(HoodieRecord record) { try { - String str = ((TestRawTripPayload) record.getData()).getJsonData(); + String str = ((RawTripTestPayload) record.getData()).getJsonData(); str = "{" + str.substring(str.indexOf("\"timestamp\":")); // Remove the last } bracket str = str.substring(0, str.length() - 1); diff --git a/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSource.scala b/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSource.scala index d2ca589..7dfff55 100644 --- a/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSource.scala +++ b/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDataSource.scala @@ -19,8 +19,9 @@ package org.apache.hudi.functional import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hudi.common.fs.FSUtils +import org.apache.hudi.common.testutils.HoodieTestDataGenerator import org.apache.hudi.config.HoodieWriteConfig -import org.apache.hudi.testutils.{DataSourceTestUtils, HoodieTestDataGenerator} +import org.apache.hudi.testutils.DataSourceTestUtils import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.spark.sql._ import org.apache.spark.sql.functions.col diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java index 90fa31d..414cfb8 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java @@ -20,10 +20,10 @@ package org.apache.hudi.utilities.functional; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.testutils.FunctionalTestHarness; import org.apache.hudi.testutils.HoodieClientTestUtils; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.HDFSParquetImporter; import org.apache.avro.generic.GenericRecord; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java index 079f88d..f99b738 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java @@ -29,6 +29,7 @@ import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.exception.HoodieException; @@ -37,7 +38,6 @@ import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.hive.HoodieHiveClient; import org.apache.hudi.hive.MultiPartKeysValueExtractor; import org.apache.hudi.keygen.SimpleKeyGenerator; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer; import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.Operation; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java index 130f7a8..a6f4edf 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java @@ -20,8 +20,8 @@ package org.apache.hudi.utilities.functional; import org.apache.hudi.DataSourceWriteOptions; import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.deltastreamer.HoodieMultiTableDeltaStreamer; import org.apache.hudi.utilities.deltastreamer.TableExecutionContext; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java index 7c47ce9..95af888 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotCopier.java @@ -19,9 +19,9 @@ package org.apache.hudi.utilities.functional; import org.apache.hudi.common.fs.FSUtils; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.testutils.FunctionalTestHarness; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.HoodieSnapshotCopier; import org.apache.hadoop.conf.Configuration; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java index 20c7811..618af51 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieSnapshotExporter.java @@ -18,18 +18,16 @@ package org.apache.hudi.utilities.functional; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.RemoteIterator; import org.apache.hudi.client.HoodieWriteClient; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.hudi.testutils.FunctionalTestHarness; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.HoodieSnapshotExporter; import org.apache.hudi.utilities.HoodieSnapshotExporter.Config; import org.apache.hudi.utilities.HoodieSnapshotExporter.OutputFormatValidator; @@ -37,7 +35,9 @@ import org.apache.hudi.utilities.HoodieSnapshotExporter.Partitioner; import org.apache.hudi.utilities.exception.HoodieSnapshotExporterException; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java index 76a494c..ea77e26 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java @@ -20,8 +20,8 @@ package org.apache.hudi.utilities.sources; import org.apache.hudi.AvroConversionUtils; import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.deltastreamer.SourceFormatAdapter; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; import org.apache.hudi.utilities.sources.helpers.KafkaOffsetGen.CheckpointUtils; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java index 44aaefd..e8a20d8 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java @@ -22,7 +22,9 @@ import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.testutils.minicluster.HdfsTestService; import org.apache.hudi.common.util.CollectionUtils; import org.apache.hudi.common.util.Option; @@ -30,8 +32,6 @@ import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.hive.HoodieHiveClient; import org.apache.hudi.hive.testutils.HiveTestService; -import org.apache.hudi.testutils.HoodieTestDataGenerator; -import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.hudi.utilities.UtilHelpers; import org.apache.hudi.utilities.sources.TestDataSource; @@ -287,7 +287,7 @@ public class UtilitiesTestBase { public static String toJsonString(HoodieRecord hr) { try { - return ((TestRawTripPayload) hr.getData()).getJsonData(); + return ((RawTripTestPayload) hr.getData()).getJsonData(); } catch (IOException ioe) { return null; } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java index 8370282..8441693 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractBaseTestSource.java @@ -20,10 +20,10 @@ package org.apache.hudi.utilities.testutils.sources; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; +import org.apache.hudi.common.testutils.RawTripTestPayload; import org.apache.hudi.common.util.collection.RocksDBBasedMap; import org.apache.hudi.exception.HoodieIOException; -import org.apache.hudi.testutils.HoodieTestDataGenerator; -import org.apache.hudi.testutils.TestRawTripPayload; import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.hudi.utilities.sources.AvroSource; import org.apache.hudi.utilities.testutils.sources.config.SourceConfigs; @@ -131,7 +131,7 @@ public abstract class AbstractBaseTestSource extends AvroSource { private static GenericRecord toGenericRecord(HoodieRecord hoodieRecord) { try { - TestRawTripPayload payload = (TestRawTripPayload) hoodieRecord.getData(); + RawTripTestPayload payload = (RawTripTestPayload) hoodieRecord.getData(); return (GenericRecord) payload.getRecordToInsert(HoodieTestDataGenerator.AVRO_SCHEMA); } catch (IOException e) { return null; diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractDFSSourceTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractDFSSourceTestBase.java index e30a183..f63f3e9 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractDFSSourceTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/sources/AbstractDFSSourceTestBase.java @@ -20,8 +20,8 @@ package org.apache.hudi.utilities.testutils.sources; import org.apache.hudi.AvroConversionUtils; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.testutils.HoodieTestDataGenerator; import org.apache.hudi.common.util.Option; -import org.apache.hudi.testutils.HoodieTestDataGenerator; import org.apache.hudi.utilities.deltastreamer.SourceFormatAdapter; import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; import org.apache.hudi.utilities.sources.InputBatch;