This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new d8e675d035a [HUDI-5167] Reducing total test run time: reducing tests
for virtual keys (#7153)
d8e675d035a is described below
commit d8e675d035aa7e77c7e476d2c132c56979948e38
Author: Sivabalan Narayanan <[email protected]>
AuthorDate: Thu Mar 7 06:54:43 2024 -0800
[HUDI-5167] Reducing total test run time: reducing tests for virtual keys
(#7153)
---
.../TestHoodieClientOnCopyOnWriteStorage.java | 111 +++++++++------------
1 file changed, 49 insertions(+), 62 deletions(-)
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index 44105a41983..61f73207aeb 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -204,7 +204,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
private static Stream<Arguments> rollbackFailedCommitsParams() {
return Stream.of(
Arguments.of(HoodieFailedWritesCleaningPolicy.LAZY, true),
- Arguments.of(HoodieFailedWritesCleaningPolicy.LAZY, false),
Arguments.of(HoodieFailedWritesCleaningPolicy.NEVER, true),
Arguments.of(HoodieFailedWritesCleaningPolicy.NEVER, false)
);
@@ -240,10 +239,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
/**
* Test Auto Commit behavior for HoodieWriteClient insertPrepped API.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testAutoCommitOnInsertPrepped(boolean populateMetaFields) throws
Exception {
- testAutoCommit(SparkRDDWriteClient::insertPreppedRecords, true,
populateMetaFields);
+ @Test
+ public void testAutoCommitOnInsertPrepped() throws Exception {
+ testAutoCommit(SparkRDDWriteClient::insertPreppedRecords, true, true);
}
/**
@@ -276,11 +274,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
/**
* Test Auto Commit behavior for HoodieWriteClient bulk-insert prepped API.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testAutoCommitOnBulkInsertPrepped(boolean populateMetaFields)
throws Exception {
+ @Test
+ public void testAutoCommitOnBulkInsertPrepped() throws Exception {
testAutoCommit((writeClient, recordRDD, instantTime) ->
writeClient.bulkInsertPreppedRecords(recordRDD, instantTime,
- Option.empty()), true, populateMetaFields);
+ Option.empty()), true, true);
}
/**
@@ -440,10 +437,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
/**
* Test De-duplication behavior for HoodieWriteClient upsert API.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testDeduplicationOnUpsert(boolean populateMetaFields) throws
Exception {
- testDeduplication(SparkRDDWriteClient::upsert, populateMetaFields);
+ @Test
+ public void testDeduplicationOnUpsert() throws Exception {
+ testDeduplication(SparkRDDWriteClient::upsert, true);
}
/**
@@ -598,11 +594,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
/**
* Test UpsertPrepped API.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testUpsertsPrepped(boolean populateMetaFields) throws Exception {
+ @Test
+ public void testUpsertsPrepped() throws Exception {
HoodieWriteConfig.Builder cfgBuilder =
getConfigBuilder().withRollbackUsingMarkers(true);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
testUpsertsInternal(cfgBuilder.build(),
SparkRDDWriteClient::upsertPreppedRecords, true);
}
@@ -837,11 +832,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
/**
* Test InsertPrepped API for HoodieConcatHandle.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testInsertsPreppedWithHoodieConcatHandle(boolean
populateMetaFields) throws Exception {
+ @Test
+ public void testInsertsPreppedWithHoodieConcatHandle() throws Exception {
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder();
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
testHoodieConcatHandle(cfgBuilder.build(), true);
}
@@ -994,11 +988,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
/**
* Tests deletion of records.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testDeletes(boolean populateMetaFields) throws Exception {
+ @Test
+ public void testDeletes() throws Exception {
HoodieWriteConfig.Builder cfgBuilder =
getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());
/**
* Write 1 (inserts and deletes) Write actual 200 insert records and
ignore 100 delete records
@@ -1019,7 +1012,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
writeBatch(client, newCommitTime, initCommitTime, Option.empty(),
initCommitTime,
// unused as genFn uses hard-coded number of inserts/updates/deletes
-1, recordGenFunction, SparkRDDWriteClient::upsert, true, 200, 200, 1,
false,
- populateMetaFields);
+ true);
/**
* Write 2 (deletes+writes).
@@ -1037,7 +1030,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
};
writeBatch(client, newCommitTime, prevCommitTime, Option.empty(),
initCommitTime, 100, recordGenFunction,
SparkRDDWriteClient::upsert, true, 50, 150, 2, false,
- populateMetaFields);
+ true);
}
/**
@@ -1046,11 +1039,10 @@ public class TestHoodieClientOnCopyOnWriteStorage
extends HoodieClientTestBase {
*
* @throws Exception
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testDeletesForInsertsInSameBatch(boolean populateMetaFields)
throws Exception {
+ @Test
+ public void testDeletesForInsertsInSameBatch() throws Exception {
HoodieWriteConfig.Builder cfgBuilder =
getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build());
/**
* Write 200 inserts and issue deletes to a subset(50) of inserts.
@@ -1071,7 +1063,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
writeBatch(client, newCommitTime, initCommitTime, Option.empty(),
initCommitTime,
-1, recordGenFunction, SparkRDDWriteClient::upsert, true, 150, 150, 1,
false,
- populateMetaFields);
+ true);
}
private void assertPartitionPathRecordKeys(List<Pair<String, String>>
expectedPartitionPathRecKeyPairs, String[] fullPartitionPaths) {
@@ -1898,19 +1890,17 @@ public class TestHoodieClientOnCopyOnWriteStorage
extends HoodieClientTestBase {
/**
* Test scenario of writing fewer file groups than existing number of file
groups in partition.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testInsertOverwritePartitionHandlingWithFewerRecords(boolean
populateMetaFields) throws Exception {
- verifyInsertOverwritePartitionHandling(3000, 1000, populateMetaFields);
+ @Test
+ public void testInsertOverwritePartitionHandlingWithFewerRecords() throws
Exception {
+ verifyInsertOverwritePartitionHandling(3000, 1000, true);
}
/**
* Test scenario of writing similar number file groups in partition.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void
testInsertOverwritePartitionHandlingWithSimilarNumberOfRecords(boolean
populateMetaFields) throws Exception {
- verifyInsertOverwritePartitionHandling(3000, 3000, populateMetaFields);
+ @Test
+ public void testInsertOverwritePartitionHandlingWithSimilarNumberOfRecords()
throws Exception {
+ verifyInsertOverwritePartitionHandling(3000, 3000, true);
}
/**
@@ -1963,19 +1953,17 @@ public class TestHoodieClientOnCopyOnWriteStorage
extends HoodieClientTestBase {
/**
* Test scenario of writing similar number file groups in partition.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void verifyDeletePartitionsHandlingWithSimilarNumberOfRecords(boolean
populateMetaFields) throws Exception {
- verifyDeletePartitionsHandling(3000, 3000, 3000, populateMetaFields);
+ @Test
+ public void verifyDeletePartitionsHandlingWithSimilarNumberOfRecords()
throws Exception {
+ verifyDeletePartitionsHandling(3000, 3000, 3000, true);
}
/**
* Test scenario of writing more file groups for first partition than second
and third partition.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void
verifyDeletePartitionsHandlingHandlingWithFewerRecordsSecondThirdPartition(boolean
populateMetaFields) throws Exception {
- verifyDeletePartitionsHandling(3000, 1000, 1000, populateMetaFields);
+ @Test
+ public void
verifyDeletePartitionsHandlingHandlingWithFewerRecordsSecondThirdPartition()
throws Exception {
+ verifyDeletePartitionsHandling(3000, 1000, 1000, true);
}
private Set<String> insertPartitionRecordsWithCommit(SparkRDDWriteClient
client, int recordsCount, String commitTime1, String partitionPath) throws
IOException {
@@ -2217,11 +2205,11 @@ public class TestHoodieClientOnCopyOnWriteStorage
extends HoodieClientTestBase {
/**
* Test to ensure commit metadata points to valid files.
*/
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testCommitWritesRelativePaths(boolean populateMetaFields) throws
Exception {
+ @Test
+ public void testCommitWritesRelativePaths() throws Exception {
+
HoodieWriteConfig.Builder cfgBuilder =
getConfigBuilder().withAutoCommit(false);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
try (SparkRDDWriteClient client =
getHoodieWriteClient(cfgBuilder.build())) {
HoodieTableMetaClient metaClient =
HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
HoodieSparkTable table = HoodieSparkTable.create(cfgBuilder.build(),
context, metaClient);
@@ -2393,9 +2381,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
}
@ParameterizedTest
- @MethodSource("rollbackAfterConsistencyCheckFailureParams")
- public void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean
enableOptimisticConsistencyGuard, boolean populateMetCols) throws Exception {
- testRollbackAfterConsistencyCheckFailureUsingFileList(false,
enableOptimisticConsistencyGuard, populateMetCols);
+ @ValueSource(booleans = {true, false})
+ public void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean
enableOptimisticConsistencyGuard) throws Exception {
+ testRollbackAfterConsistencyCheckFailureUsingFileList(false,
enableOptimisticConsistencyGuard, true);
}
@ParameterizedTest
@@ -2486,9 +2474,9 @@ public class TestHoodieClientOnCopyOnWriteStorage extends
HoodieClientTestBase {
}
}
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testRollbackFailedCommitsToggleCleaningPolicy(boolean
populateMetaFields) throws Exception {
+ @Test
+ public void testRollbackFailedCommitsToggleCleaningPolicy() throws Exception
{
+ boolean populateMetaFields = true;
HoodieTestUtils.init(hadoopConf, basePath);
HoodieFailedWritesCleaningPolicy cleaningPolicy = EAGER;
SparkRDDWriteClient client = new SparkRDDWriteClient(context,
getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
@@ -2655,12 +2643,11 @@ public class TestHoodieClientOnCopyOnWriteStorage
extends HoodieClientTestBase {
return Pair.of(markerFilePath.get(), result);
}
- @ParameterizedTest
- @MethodSource("populateMetaFieldsParams")
- public void testMultiOperationsPerCommit(boolean populateMetaFields) throws
IOException {
+ @Test
+ public void testMultiOperationsPerCommit() throws IOException {
HoodieWriteConfig.Builder cfgBuilder =
getConfigBuilder().withAutoCommit(false)
.withAllowMultiWriteOnSameInstant(true);
- addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields);
+ addConfigsForPopulateMetaFields(cfgBuilder, true);
HoodieWriteConfig cfg = cfgBuilder.build();
SparkRDDWriteClient client = getHoodieWriteClient(cfg);
String firstInstantTime = "0000";
