This is an automated email from the ASF dual-hosted git repository.
nagarwal pushed a change to branch hudi_test_suite_refactor
in repository https://gitbox.apache.org/repos/asf/hudi.git.
discard f4ff5d6 [HUDI-394] Provide a basic implementation of test suite
add 1aae437 [HUDI-1102] Add common useful Spark related and Table path
detection utilities (#1841)
add b71f25f [HUDI-92] Provide reasonable names for Spark DAG stages in
HUDI. (#1289)
add 1ec89e9 [HUDI-839] Introducing support for rollbacks using marker
files (#1756)
add 5e7ab11 [HUDI-994] Move TestHoodieIndex test cases to unit tests
(#1850)
add 743ef32 [HUDI-871] Add support for Tencent Cloud Object Storage(COS)
(#1855)
add 12ef8c9 [HUDI-708] Add temps show and unit test for TempViewCommand
(#1770)
add 5b6026b [HUDI-802] Fixing deletes for inserts in same batch in write
path (#1792)
add 9bd37ef [MINOR] Fix flaky testUpsertsUpdatePartitionPath* tests
(#1863)
add a8bd76c [HUDI-1029] In inline compaction mode, previously failed
compactions needs to be retried before new compactions (#1857)
add 3dd189e [MINOR] Fix checkstyle issue on
TestHoodieClientOnCopyOnWriteStorage (#1865)
add c39778c [HUDI-1113] Add user define metrics reporter (#1851)
add f61cd10 [HUDI-985] Introduce rerun ci bot (#1693)
add da10680 [HUDI-1037] Introduce a write committed callback hook and
given a default http callback implementation (#1842)
add c3279cd [HUDI-1082] Fix minor bug in deciding the insert buckets
(#1838)
add 467d097 [MINOR] Add Databricks File System to StorageSchemes (#1877)
add 0cb24e4 [MINOR] Use HoodieActiveTimeline.COMMIT_FORMATTER (#1874)
add ca36c44 [HUDI-995] Move TestRawTripPayload and
HoodieTestDataGenerator to hudi-common (#1873)
add fa41921 [HUDI-703] Add test for HoodieSyncCommand (#1774)
add 5e7931b [MINOR] Fix master compilation failure (#1881)
add b2763f4 [MINOR] Fixing default index parallelism for simple index
(#1882)
add d5b593b [MINOR] change log.info to log.debug (#1883)
add d2b5125 [HUDI-394] Provide a basic implementation of test suite
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f4ff5d6)
\
N -- N -- N refs/heads/hudi_test_suite_refactor (d2b5125)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/actions/bot/package.json | 36 +++
.github/actions/bot/src/action.js | 100 +++++++++
.github/workflows/bot.yml | 43 ++++
.gitignore | 2 +
README.md | 12 +
.../{base.properties => hoodie-incr.properties} | 14 +-
docker/demo/config/hoodie-schema.avsc | 145 ++++++++++++
...n_commit_time_mor.sh => sync-validate.commands} | 7 +-
hudi-cli/pom.xml | 17 ++
.../main/java/org/apache/hudi/cli/HoodieCLI.java | 12 +
.../hudi/cli/commands/HoodieSyncCommand.java | 4 +-
.../apache/hudi/cli/commands/TempViewCommand.java | 45 ++--
.../hudi/cli/utils/SparkTempViewProvider.java | 18 ++
.../apache/hudi/cli/utils/TempViewProvider.java | 7 +-
.../cli/commands/TestArchivedCommitsCommand.java | 4 +-
.../hudi/cli/commands/TestCleansCommand.java | 4 +-
.../hudi/cli/commands/TestCommitsCommand.java | 8 +-
.../hudi/cli/commands/TestRepairsCommand.java | 2 +-
.../hudi/cli/commands/TestRollbacksCommand.java | 2 +-
.../hudi/cli/commands/TestSavepointsCommand.java | 2 +-
.../apache/hudi/cli/commands/TestStatsCommand.java | 2 +-
.../apache/hudi/cli/commands/TestTableCommand.java | 8 +-
.../hudi/cli/commands/TestTempViewCommand.java | 84 +++++++
.../hudi/cli/integ/ITTestCommitsCommand.java | 2 +-
.../cli/integ/ITTestHDFSParquetImportCommand.java | 2 +-
.../hudi/cli/integ/ITTestRepairsCommand.java | 2 +-
.../hudi/cli/integ/ITTestSavepointsCommand.java | 2 +-
....java => AbstractShellBaseIntegrationTest.java} | 6 +-
.../testutils/AbstractShellIntegrationTest.java | 38 +---
.../HoodieTestCommitMetadataGenerator.java | 4 +-
.../hudi/callback/HoodieWriteCommitCallback.java | 35 +++
.../http/HoodieWriteCommitHttpCallbackClient.java | 108 +++++++++
.../common/HoodieWriteCommitCallbackMessage.java | 76 +++++++
.../impl/HoodieWriteCommitHttpCallback.java | 58 +++++
.../callback/util/HoodieCommitCallbackFactory.java | 47 ++++
.../hudi/client/AbstractHoodieWriteClient.java | 18 +-
.../apache/hudi/client/CompactionAdminClient.java | 3 +
.../org/apache/hudi/client/HoodieWriteClient.java | 23 +-
.../org/apache/hudi/config/HoodieIndexConfig.java | 4 +-
.../apache/hudi/config/HoodieMetricsConfig.java | 12 +
.../config/HoodieWriteCommitCallbackConfig.java | 106 +++++++++
.../org/apache/hudi/config/HoodieWriteConfig.java | 37 ++++
.../exception/HoodieCommitCallbackException.java | 16 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 1 +
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 1 +
.../org/apache/hudi/io/HoodieAppendHandle.java | 18 +-
.../org/apache/hudi/io/HoodieCreateHandle.java | 12 +-
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 15 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 30 +--
.../src/main/java/org/apache/hudi/io/IOType.java | 10 +-
.../hudi/metrics/MetricsReporterFactory.java | 18 ++
.../AbstractUserDefinedMetricsReporter.java | 33 ++-
.../java/org/apache/hudi/table/HoodieTable.java | 106 ++++-----
.../hudi/table/HoodieTimelineArchiveLog.java | 91 ++++----
.../java/org/apache/hudi/table/MarkerFiles.java | 153 +++++++++++++
.../table/action/clean/CleanActionExecutor.java | 3 +
.../table/action/commit/UpsertPartitioner.java | 12 +-
.../compact/HoodieMergeOnReadTableCompactor.java | 2 +
.../compact/ScheduleCompactionActionExecutor.java | 2 +-
.../restore/CopyOnWriteRestoreActionExecutor.java | 3 +-
.../restore/MergeOnReadRestoreActionExecutor.java | 3 +-
.../rollback/BaseRollbackActionExecutor.java | 103 ++++++---
.../CopyOnWriteRollbackActionExecutor.java | 43 ++--
...Helper.java => ListingBasedRollbackHelper.java} | 70 +++---
...quest.java => ListingBasedRollbackRequest.java} | 56 ++---
.../rollback/MarkerBasedRollbackStrategy.java | 161 ++++++++++++++
.../MergeOnReadRollbackActionExecutor.java | 69 +++---
.../hudi/table/action/rollback/RollbackUtils.java | 66 ++++++
.../action/savepoint/SavepointActionExecutor.java | 1 +
.../http/TestCallbackHttpClient.java} | 73 ++----
.../org/apache/hudi/client/TestClientRollback.java | 2 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 97 +++++---
.../java/org/apache/hudi/client/TestMultiFS.java | 2 +-
.../hudi/client/TestTableSchemaEvolution.java | 20 +-
.../hudi/client/TestUpdateSchemaEvolution.java | 12 +-
.../hudi/execution/TestBoundedInMemoryQueue.java | 2 +-
.../TestSparkBoundedInMemoryExecutor.java | 2 +-
.../org/apache/hudi/index/TestHoodieIndex.java | 186 ++--------------
.../apache/hudi/index/TestHoodieIndexConfigs.java | 181 +++++++++++++++
.../hudi/index/bloom/TestHoodieBloomIndex.java | 48 ++--
.../index/bloom/TestHoodieGlobalBloomIndex.java | 66 +++---
.../apache/hudi/index/hbase/TestHBaseIndex.java | 2 +-
.../index/hbase/TestHBaseQPSResourceAllocator.java | 2 +-
.../hudi/io/TestHoodieKeyLocationFetchHandle.java | 8 +-
.../org/apache/hudi/io/TestHoodieMergeHandle.java | 4 +-
...eLog.java => TestHoodieTimelineArchiveLog.java} | 49 ++---
.../io/storage/TestHoodieFileWriterFactory.java | 2 +-
.../hudi/metrics/TestMetricsReporterFactory.java | 56 +++++
.../java/org/apache/hudi/table/TestCleaner.java | 16 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 213 ++++++++++++------
.../org/apache/hudi/table/TestMarkerFiles.java | 148 +++++++++++++
.../commit/TestCopyOnWriteActionExecutor.java | 26 +--
.../table/action/commit/TestUpsertPartitioner.java | 61 +++++-
.../table/action/compact/CompactionTestBase.java | 244 +++++++++++++++++++++
.../table/action/compact/TestAsyncCompaction.java | 213 +-----------------
.../table/action/compact/TestHoodieCompactor.java | 4 +-
.../table/action/compact/TestInlineCompaction.java | 116 ++++++++++
.../rollback/HoodieClientRollbackTestBase.java | 99 +++++++++
.../TestCopyOnWriteRollbackActionExecutor.java | 199 +++++++++++++++++
.../rollback/TestMarkerBasedRollbackStrategy.java | 135 ++++++++++++
.../TestMergeOnReadRollbackActionExecutor.java | 158 +++++++++++++
.../table/action/rollback/TestRollbackUtils.java | 120 ++++++++++
.../hudi/testutils/HoodieClientTestBase.java | 5 +-
.../hudi/testutils/HoodieClientTestHarness.java | 30 ++-
.../hudi/testutils/HoodieClientTestUtils.java | 70 +++++-
.../hudi/testutils/HoodieMergeOnReadTestUtils.java | 4 +-
.../hudi/testutils/MetadataMergeWriteStatus.java | 85 +++++++
.../org/apache/hudi/common/HoodieRollbackStat.java | 19 ++
.../java/org/apache/hudi/common/fs/FSUtils.java | 65 +-----
.../org/apache/hudi/common/fs/StorageSchemes.java | 6 +-
.../model/OverwriteWithLatestAvroPayload.java | 26 ++-
.../hudi/common/table/HoodieTableMetaClient.java | 1 +
.../apache/hudi/common/util/TablePathUtils.java | 110 ++++++++++
.../org/apache/hudi/common/fs/TestFSUtils.java | 10 +-
.../apache/hudi/common/fs/TestStorageSchemes.java | 2 +
.../common/fs/inline}/TestParquetInLining.java | 22 +-
.../hudi/common/model/TestHoodieWriteStat.java | 4 +-
.../model/TestOverwriteWithLatestAvroPayload.java | 103 +++++++++
.../hudi/common/testutils/FileSystemTestUtils.java | 15 ++
.../common/testutils/HoodieCommonTestHarness.java | 12 +
.../common}/testutils/HoodieTestDataGenerator.java | 76 ++++---
.../hudi/common/testutils/HoodieTestUtils.java | 36 +--
.../hudi/common/testutils/RawTripTestPayload.java | 89 ++------
.../hudi/common/util/TestTablePathUtils.java | 126 +++++++++++
.../realtime/AbstractRealtimeRecordReader.java | 2 +-
.../org/apache/hudi/integ/HoodieTestHiveBase.java | 121 ++++++++++
.../java/org/apache/hudi/integ/ITTestBase.java | 4 +-
.../org/apache/hudi/integ/ITTestHoodieSanity.java | 6 +-
.../integ/command/ITTestHoodieSyncCommand.java | 75 +++++++
.../src/test/resources/hoodie-docker.properties | 3 +-
hudi-spark/run_hoodie_app.sh | 4 +-
.../main/java/org/apache/hudi/DataSourceUtils.java | 23 ++
.../scala/org/apache/hudi/HudiSparkUtils.scala | 50 +++++
hudi-spark/src/test/java/HoodieJavaApp.java | 2 +-
...odieJavaApp.java => HoodieJavaGenerateApp.java} | 211 ++++++------------
.../src/test/java/HoodieJavaStreamingApp.java | 2 +-
.../apache/hudi/testutils/DataSourceTestUtils.java | 3 +-
.../scala/org/apache/hudi/TestHudiSparkUtils.scala | 105 +++++++++
.../apache/hudi/functional/TestDataSource.scala | 3 +-
.../apache/hudi/utilities/HDFSParquetImporter.java | 1 +
.../hudi/utilities/HoodieSnapshotCopier.java | 1 +
.../hudi/utilities/HoodieSnapshotExporter.java | 2 +
.../hudi/utilities/HoodieWithTimelineServer.java | 1 +
.../hudi/utilities/perf/TimelineServerPerf.java | 1 +
.../hudi/utilities/sources/AvroDFSSource.java | 1 +
.../hudi/utilities/sources/HiveIncrPullSource.java | 1 +
.../functional/TestHDFSParquetImporter.java | 2 +-
.../functional/TestHoodieDeltaStreamer.java | 2 +-
.../TestHoodieMultiTableDeltaStreamer.java | 2 +-
.../functional/TestHoodieSnapshotCopier.java | 2 +-
.../functional/TestHoodieSnapshotExporter.java | 3 +-
.../hudi/utilities/sources/TestKafkaSource.java | 2 +-
.../utilities/testutils/UtilitiesTestBase.java | 6 +-
.../testutils/sources/AbstractBaseTestSource.java | 17 +-
.../sources/AbstractDFSSourceTestBase.java | 2 +-
packaging/hudi-spark-bundle/pom.xml | 1 +
packaging/hudi-utilities-bundle/pom.xml | 1 +
157 files changed, 4916 insertions(+), 1487 deletions(-)
create mode 100644 .github/actions/bot/package.json
create mode 100644 .github/actions/bot/src/action.js
create mode 100644 .github/workflows/bot.yml
copy docker/demo/config/{base.properties => hoodie-incr.properties} (55%)
create mode 100644 docker/demo/config/hoodie-schema.avsc
copy docker/demo/{get_min_commit_time_mor.sh => sync-validate.commands} (77%)
mode change 100755 => 100644
create mode 100644
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTempViewCommand.java
copy
hudi-cli/src/test/java/org/apache/hudi/cli/testutils/{AbstractShellIntegrationTest.java
=> AbstractShellBaseIntegrationTest.java} (93%)
create mode 100644
hudi-client/src/main/java/org/apache/hudi/callback/HoodieWriteCommitCallback.java
create mode 100644
hudi-client/src/main/java/org/apache/hudi/callback/client/http/HoodieWriteCommitHttpCallbackClient.java
create mode 100644
hudi-client/src/main/java/org/apache/hudi/callback/common/HoodieWriteCommitCallbackMessage.java
create mode 100644
hudi-client/src/main/java/org/apache/hudi/callback/impl/HoodieWriteCommitHttpCallback.java
create mode 100644
hudi-client/src/main/java/org/apache/hudi/callback/util/HoodieCommitCallbackFactory.java
create mode 100644
hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java
copy
hudi-common/src/main/java/org/apache/hudi/exception/MetadataNotFoundException.java
=>
hudi-client/src/main/java/org/apache/hudi/exception/HoodieCommitCallbackException.java
(71%)
copy
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DeltaInputType.java
=> hudi-client/src/main/java/org/apache/hudi/io/IOType.java (85%)
copy
hudi-common/src/main/java/org/apache/hudi/common/config/DefaultHoodieConfig.java
=>
hudi-client/src/main/java/org/apache/hudi/metrics/userdefined/AbstractUserDefinedMetricsReporter.java
(58%)
create mode 100644
hudi-client/src/main/java/org/apache/hudi/table/MarkerFiles.java
rename
hudi-client/src/main/java/org/apache/hudi/table/action/rollback/{RollbackHelper.java
=> ListingBasedRollbackHelper.java} (71%)
rename
hudi-client/src/main/java/org/apache/hudi/table/action/rollback/{RollbackRequest.java
=> ListingBasedRollbackRequest.java} (51%)
create mode 100644
hudi-client/src/main/java/org/apache/hudi/table/action/rollback/MarkerBasedRollbackStrategy.java
create mode 100644
hudi-client/src/main/java/org/apache/hudi/table/action/rollback/RollbackUtils.java
copy
hudi-client/src/test/java/org/apache/hudi/{metrics/datadog/TestDatadogHttpClient.java
=> callback/http/TestCallbackHttpClient.java} (59%)
create mode 100644
hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndexConfigs.java
rename
hudi-client/src/test/java/org/apache/hudi/io/{TestHoodieCommitArchiveLog.java
=> TestHoodieTimelineArchiveLog.java} (95%)
create mode 100644
hudi-client/src/test/java/org/apache/hudi/table/TestMarkerFiles.java
create mode 100644
hudi-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java
create mode 100644
hudi-client/src/test/java/org/apache/hudi/table/action/compact/TestInlineCompaction.java
create mode 100644
hudi-client/src/test/java/org/apache/hudi/table/action/rollback/HoodieClientRollbackTestBase.java
create mode 100644
hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestCopyOnWriteRollbackActionExecutor.java
create mode 100644
hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestMarkerBasedRollbackStrategy.java
create mode 100644
hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestMergeOnReadRollbackActionExecutor.java
create mode 100644
hudi-client/src/test/java/org/apache/hudi/table/action/rollback/TestRollbackUtils.java
create mode 100644
hudi-client/src/test/java/org/apache/hudi/testutils/MetadataMergeWriteStatus.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/util/TablePathUtils.java
rename {hudi-utilities/src/test/java/org/apache/hudi/utilities/inline/fs =>
hudi-common/src/test/java/org/apache/hudi/common/fs/inline}/TestParquetInLining.java
(87%)
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/model/TestOverwriteWithLatestAvroPayload.java
rename {hudi-client/src/test/java/org/apache/hudi =>
hudi-common/src/test/java/org/apache/hudi/common}/testutils/HoodieTestDataGenerator.java
(93%)
rename
hudi-client/src/test/java/org/apache/hudi/testutils/TestRawTripPayload.java =>
hudi-common/src/test/java/org/apache/hudi/common/testutils/RawTripTestPayload.java
(56%)
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/util/TestTablePathUtils.java
create mode 100644
hudi-integ-test/src/test/java/org/apache/hudi/integ/HoodieTestHiveBase.java
create mode 100644
hudi-integ-test/src/test/java/org/apache/hudi/integ/command/ITTestHoodieSyncCommand.java
copy hudi-cli/src/test/resources/clean.properties =>
hudi-integ-test/src/test/resources/hoodie-docker.properties (91%)
create mode 100644
hudi-spark/src/main/scala/org/apache/hudi/HudiSparkUtils.scala
copy hudi-spark/src/test/java/{HoodieJavaApp.java =>
HoodieJavaGenerateApp.java} (55%)
create mode 100644
hudi-spark/src/test/scala/org/apache/hudi/TestHudiSparkUtils.scala
