This is an automated email from the ASF dual-hosted git repository. vinoyang pushed a change to branch hudi_test_suite_refactor in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git.
omit dcfbab1 [HUDI-441] Rename WorkflowDagGenerator and some class names in test package omit 9151ccf [HUDI-442] Fix TestComplexKeyGenerator#testSingleValueKeyGenerator and testMultipleValueKeyGenerator NPE (#1118) omit ae5bd06 [HUDI-391] Rename module name from hudi-bench to hudi-test-suite and fix some checkstyle issues (#1102) omit eaaf3f6 [HUDI-394] Provide a basic implementation of test suite add f324057 [MINOR] Unify Lists import (#1103) add 8963a68 [HUDI-398]Add spark env set/get for spark launcher (#1096) add 9a1f698 [HUDI-308] Avoid Renames for tracking state transitions of all actions on dataset add 7498ca7 [MINOR] Add slack invite icon in README (#1108) add 14881e9 [HUDI-106] Adding support for DynamicBloomFilter (#976) add 36b3b6f [HUDI-415] Get commit time when Spark start (#1113) add b284091 [HUDI-386] Refactor hudi scala checkstyle rules (#1099) add 313fab5 [HUDI-444] Refactor the codes based on scala codestyle ReturnChecker rule (#1121) add 350b0ec [HUDI-311] : Support for AWS Database Migration Service in DeltaStreamer add 9b55d37 [HUDI-394] Provide a basic implementation of test suite add 1d2ecbc [HUDI-391] Rename module name from hudi-bench to hudi-test-suite and fix some checkstyle issues (#1102) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (dcfbab1) \ N -- N -- N refs/heads/hudi_test_suite_refactor (1d2ecbc) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: LICENSE | 28 +++ README.md | 1 + .../main/java/org/apache/hudi/cli/HoodieCLI.java | 15 +- .../hudi/cli/commands/CompactionCommand.java | 30 +-- .../apache/hudi/cli/commands/DatasetsCommand.java | 10 +- .../apache/hudi/cli/commands/SparkEnvCommand.java | 68 ++++++ .../java/org/apache/hudi/cli/utils/SparkUtil.java | 6 +- .../scala/org/apache/hudi/cli/DedupeSparkJob.scala | 4 +- .../scala/org/apache/hudi/cli/SparkHelpers.scala | 10 +- .../org/apache/hudi/CompactionAdminClient.java | 8 +- .../java/org/apache/hudi/HoodieCleanClient.java | 26 ++- .../java/org/apache/hudi/HoodieWriteClient.java | 125 ++++++----- .../org/apache/hudi/client/utils/ClientUtils.java | 4 +- .../org/apache/hudi/config/HoodieIndexConfig.java | 11 +- .../org/apache/hudi/config/HoodieWriteConfig.java | 29 ++- .../org/apache/hudi/io/HoodieCommitArchiveLog.java | 42 +++- .../org/apache/hudi/io/HoodieKeyLookupHandle.java | 8 +- .../io/storage/HoodieStorageWriterFactory.java | 8 +- .../apache/hudi/table/HoodieCopyOnWriteTable.java | 128 ++++++----- .../apache/hudi/table/HoodieMergeOnReadTable.java | 42 ++-- .../java/org/apache/hudi/table/HoodieTable.java | 10 +- .../java/org/apache/hudi/TestAsyncCompaction.java | 10 +- .../src/test/java/org/apache/hudi/TestCleaner.java | 31 ++- .../java/org/apache/hudi/TestClientRollback.java | 6 +- .../hudi/TestHoodieClientOnCopyOnWriteStorage.java | 47 +++- .../apache/hudi/common/HoodieClientTestUtils.java | 18 +- .../hudi/common/HoodieTestDataGenerator.java | 37 +-- .../hudi/func/TestBoundedInMemoryExecutor.java | 2 +- .../apache/hudi/func/TestBoundedInMemoryQueue.java | 2 +- .../java/org/apache/hudi/index/TestHbaseIndex.java | 11 +- .../hudi/index/bloom/TestHoodieBloomIndex.java | 9 +- .../apache/hudi/io/TestHoodieCommitArchiveLog.java | 34 +-- .../org/apache/hudi/io/TestHoodieCompactor.java | 6 +- .../apache/hudi/table/TestCopyOnWriteTable.java | 2 +- .../apache/hudi/table/TestMergeOnReadTable.java | 12 +- hudi-common/pom.xml | 1 + .../src/main/avro/HoodieArchivedMetaEntry.avsc | 22 ++ .../apache/hudi/avro/HoodieAvroWriteSupport.java | 7 +- .../hudi/common/bloom/filter/BloomFilter.java | 35 +-- .../common/bloom/filter/BloomFilterFactory.java | 63 ++++++ .../common/bloom/filter/BloomFilterTypeCode.java | 10 +- .../filter/BloomFilterUtils.java} | 34 +-- .../filter/HoodieDynamicBoundedBloomFilter.java | 109 +++++++++ .../bloom/filter/InternalDynamicBloomFilter.java | 249 +++++++++++++++++++++ .../hudi/common/bloom/filter/InternalFilter.java | 178 +++++++++++++++ .../filter/SimpleBloomFilter.java} | 66 ++++-- .../hudi/common/model/TimelineLayoutVersion.java | 79 +++++++ .../hudi/common/table/HoodieTableConfig.java | 14 ++ .../hudi/common/table/HoodieTableMetaClient.java | 89 +++++--- .../apache/hudi/common/table/HoodieTimeline.java | 25 ++- .../apache/hudi/common/table/TimelineLayout.java | 79 +++++++ .../table/timeline/HoodieActiveTimeline.java | 208 ++++++++++++----- .../table/timeline/HoodieDefaultTimeline.java | 8 +- .../hudi/common/table/timeline/HoodieInstant.java | 33 ++- .../IncrementalTimelineSyncFileSystemView.java | 4 +- .../org/apache/hudi/common/util/CleanerUtils.java | 33 ++- .../apache/hudi/common/util/CompactionUtils.java | 5 +- .../common/util/DFSPropertiesConfiguration.java | 15 +- .../org/apache/hudi/common/util/ParquetUtils.java | 32 ++- .../org/apache/hudi/common/TestBloomFilter.java | 49 ---- .../hudi/common/bloom/filter/TestBloomFilter.java | 110 +++++++++ .../filter/TestInternalDynamicBloomFilter.java | 60 +++++ .../apache/hudi/common/model/HoodieTestUtils.java | 50 ++++- .../common/table/TestHoodieTableMetaClient.java | 4 +- .../hudi/common/table/TestTimelineLayout.java | 77 +++++++ .../table/string/TestHoodieActiveTimeline.java | 35 ++- .../table/view/TestHoodieTableFileSystemView.java | 60 ++--- .../table/view/TestIncrementalFSViewSync.java | 25 ++- .../hudi/common/util/CompactionTestUtils.java | 7 +- .../apache/hudi/common/util/SchemaTestUtil.java | 2 +- .../apache/hudi/common/util/TestParquetUtils.java | 38 +++- .../common/util/collection/TestDiskBasedMap.java | 2 +- .../util/collection/TestExternalSpillableMap.java | 6 +- .../apache/hudi/hadoop/InputFormatTestUtil.java | 2 +- .../SlashEncodedDayPartitionValueExtractor.java | 5 +- .../test/java/org/apache/hudi/hive/TestUtil.java | 15 +- .../java/org/apache/hudi/ComplexKeyGenerator.java | 5 + .../org/apache/hudi/payload/AWSDmsAvroPayload.java | 68 ++++++ .../org/apache/hudi/HoodieSparkSqlWriter.scala | 11 +- .../testsuite/dag/SimpleWorkflowDagGenerator.java | 76 ------- .../hudi/testsuite/dag/WorkflowDagGenerator.java | 56 ++++- ...omplexDagGenerator.java => TestComplexDag.java} | 2 +- .../apache/hudi/testsuite/dag/TestDagUtils.java | 4 +- ...eSyncDagGenerator.java => TestHiveSyncDag.java} | 2 +- ...nlyDagGenerator.java => TestInsertOnlyDag.java} | 2 +- ...tDagGenerator.java => TestInsertUpsertDag.java} | 2 +- .../hudi/testsuite/job/TestHoodieTestSuiteJob.java | 12 +- .../org/apache/hudi/utilities/UtilHelpers.java | 18 +- .../deltastreamer/SourceFormatAdapter.java | 15 -- .../utilities/schema/RowBasedSchemaProvider.java | 6 + .../hudi/utilities/sources/ParquetDFSSource.java | 20 +- .../hudi/utilities/sources/ParquetSource.java | 35 --- .../org/apache/hudi/utilities/sources/Source.java | 2 +- ...tityTransformer.java => AWSDmsTransformer.java} | 19 +- .../TestAWSDatabaseMigrationServiceSource.java | 107 +++++++++ .../apache/hudi/utilities/UtilitiesTestBase.java | 1 - .../hudi/utilities/sources/TestDFSSource.java | 2 +- pom.xml | 2 +- style/checkstyle.xml | 2 +- style/scalastyle-config.xml | 114 ---------- style/scalastyle.xml | 134 +++++++++++ 101 files changed, 2648 insertions(+), 854 deletions(-) create mode 100644 hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkEnvCommand.java copy hudi-test-suite/src/main/java/org/apache/hudi/testsuite/writer/DeltaInputWriter.java => hudi-common/src/main/java/org/apache/hudi/common/bloom/filter/BloomFilter.java (57%) create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/bloom/filter/BloomFilterFactory.java copy hudi-test-suite/src/main/java/org/apache/hudi/testsuite/DeltaInputFormat.java => hudi-common/src/main/java/org/apache/hudi/common/bloom/filter/BloomFilterTypeCode.java (81%) copy hudi-common/src/main/java/org/apache/hudi/common/{util/Functions.java => bloom/filter/BloomFilterUtils.java} (55%) create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/bloom/filter/HoodieDynamicBoundedBloomFilter.java create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/bloom/filter/InternalDynamicBloomFilter.java create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/bloom/filter/InternalFilter.java rename hudi-common/src/main/java/org/apache/hudi/common/{BloomFilter.java => bloom/filter/SimpleBloomFilter.java} (62%) create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/TimelineLayoutVersion.java create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/table/TimelineLayout.java delete mode 100644 hudi-common/src/test/java/org/apache/hudi/common/TestBloomFilter.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/bloom/filter/TestBloomFilter.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/bloom/filter/TestInternalDynamicBloomFilter.java create mode 100644 hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineLayout.java create mode 100644 hudi-spark/src/main/java/org/apache/hudi/payload/AWSDmsAvroPayload.java delete mode 100644 hudi-test-suite/src/main/java/org/apache/hudi/testsuite/dag/SimpleWorkflowDagGenerator.java rename hudi-test-suite/src/test/java/org/apache/hudi/testsuite/dag/{ComplexDagGenerator.java => TestComplexDag.java} (97%) rename hudi-test-suite/src/test/java/org/apache/hudi/testsuite/dag/{HiveSyncDagGenerator.java => TestHiveSyncDag.java} (96%) rename hudi-test-suite/src/test/java/org/apache/hudi/testsuite/dag/{InsertOnlyDagGenerator.java => TestInsertOnlyDag.java} (96%) rename hudi-test-suite/src/test/java/org/apache/hudi/testsuite/dag/{InsertUpsertDagGenerator.java => TestInsertUpsertDag.java} (96%) delete mode 100644 hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/ParquetSource.java copy hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/{IdentityTransformer.java => AWSDmsTransformer.java} (65%) create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/TestAWSDatabaseMigrationServiceSource.java delete mode 100644 style/scalastyle-config.xml create mode 100644 style/scalastyle.xml