This is an automated email from the ASF dual-hosted git repository. pwason pushed a commit to branch release-0.14.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit a7690eca670f7c69884fa36770f931663cbb34fc Author: Sivabalan Narayanan <[email protected]> AuthorDate: Fri Aug 25 09:54:06 2023 -0400 [HUDI-6445] Triage ci flakiness and some test fies (#9534) Fixed metrics in tests. (disabled metrics). Fixed Java tests to use local FS instead of hdfs. Removed some of parametrized tests for java. --------- Co-authored-by: Sagar Sumit <[email protected]> --- .../hudi/client/TestJavaHoodieBackedMetadata.java | 16 +- .../TestHoodieJavaClientOnCopyOnWriteStorage.java | 185 +++++++++------------ .../testutils/HoodieJavaClientTestHarness.java | 140 ++++++++-------- .../hudi/testutils/TestHoodieMetadataBase.java | 2 +- .../functional/TestHoodieBackedMetadata.java | 18 +- .../client/functional/TestHoodieMetadataBase.java | 2 +- .../realtime/TestHoodieRealtimeRecordReader.java | 7 +- .../apache/hudi/functional/TestBootstrapRead.java | 2 +- .../functional/TestNewHoodieParquetFileFormat.java | 4 +- 9 files changed, 174 insertions(+), 202 deletions(-) diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java index 7226563feaa..b22fa76788d 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java @@ -185,14 +185,10 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase { public static List<Arguments> tableOperationsTestArgs() { return asList( - Arguments.of(COPY_ON_WRITE, true, true), - Arguments.of(COPY_ON_WRITE, true, false), - Arguments.of(COPY_ON_WRITE, false, true), - Arguments.of(COPY_ON_WRITE, false, false), - Arguments.of(MERGE_ON_READ, true, true), - Arguments.of(MERGE_ON_READ, true, false), - Arguments.of(MERGE_ON_READ, false, true), - Arguments.of(MERGE_ON_READ, false, false) + Arguments.of(COPY_ON_WRITE, true), + Arguments.of(COPY_ON_WRITE, false), + Arguments.of(MERGE_ON_READ, true), + Arguments.of(MERGE_ON_READ, false) ); } @@ -284,14 +280,14 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase { */ @ParameterizedTest @MethodSource("tableOperationsTestArgs") - public void testTableOperations(HoodieTableType tableType, boolean enableFullScan, boolean enableMetrics) throws Exception { + public void testTableOperations(HoodieTableType tableType, boolean enableFullScan) throws Exception { List<Long> commitTimeList = new ArrayList<>(); commitTimeList.add(Long.parseLong(HoodieActiveTimeline.createNewInstantTime())); for (int i = 0; i < 8; i++) { long nextCommitTime = getNextCommitTime(commitTimeList.get(commitTimeList.size() - 1)); commitTimeList.add(nextCommitTime); } - init(tableType, true, enableFullScan, enableMetrics, false); + init(tableType, true, enableFullScan, false, false); doWriteInsertAndUpsert(testTable, commitTimeList.get(0).toString(), commitTimeList.get(1).toString(), false); // trigger an upsert diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java index a3a0b726619..211dc0129e6 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java @@ -150,16 +150,10 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe private static final String CLUSTERING_FAILURE = "CLUSTERING FAILURE"; - private static Stream<Arguments> populateMetaFieldsParams() { - return Arrays.stream(new Boolean[][] {{true}, {false}}).map(Arguments::of); - } - private static Stream<Arguments> rollbackAfterConsistencyCheckFailureParams() { return Stream.of( - Arguments.of(true, true), - Arguments.of(true, false), - Arguments.of(false, true), - Arguments.of(false, false) + Arguments.of(true), + Arguments.of(false) ); } @@ -173,56 +167,50 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe /** * Test Auto Commit behavior for HoodieWriteClient insert API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testAutoCommitOnInsert(boolean populateMetaFields) throws Exception { - testAutoCommit(HoodieJavaWriteClient::insert, false, populateMetaFields); + @Test + public void testAutoCommitOnInsert() throws Exception { + testAutoCommit(HoodieJavaWriteClient::insert, false, true); } /** * Test Auto Commit behavior for HoodieWriteClient insertPrepped API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testAutoCommitOnInsertPrepped(boolean populateMetaFields) throws Exception { - testAutoCommit(HoodieJavaWriteClient::insertPreppedRecords, true, populateMetaFields); + @Test + public void testAutoCommitOnInsertPrepped() throws Exception { + testAutoCommit(HoodieJavaWriteClient::insertPreppedRecords, true, true); } /** * Test Auto Commit behavior for HoodieWriteClient upsert API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testAutoCommitOnUpsert(boolean populateMetaFields) throws Exception { - testAutoCommit(HoodieJavaWriteClient::upsert, false, populateMetaFields); + @Test + public void testAutoCommitOnUpsert() throws Exception { + testAutoCommit(HoodieJavaWriteClient::upsert, false, true); } /** * Test Auto Commit behavior for HoodieWriteClient upsert Prepped API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testAutoCommitOnUpsertPrepped(boolean populateMetaFields) throws Exception { - testAutoCommit(HoodieJavaWriteClient::upsertPreppedRecords, true, populateMetaFields); + @Test + public void testAutoCommitOnUpsertPrepped() throws Exception { + testAutoCommit(HoodieJavaWriteClient::upsertPreppedRecords, true, true); } /** * Test Auto Commit behavior for HoodieWriteClient bulk-insert API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testAutoCommitOnBulkInsert(boolean populateMetaFields) throws Exception { - testAutoCommit(HoodieJavaWriteClient::bulkInsert, false, populateMetaFields); + @Test + public void testAutoCommitOnBulkInsert() throws Exception { + testAutoCommit(HoodieJavaWriteClient::bulkInsert, false, true); } /** * Test Auto Commit behavior for HoodieWriteClient bulk-insert prepped API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testAutoCommitOnBulkInsertPrepped(boolean populateMetaFields) throws Exception { + @Test + public void testAutoCommitOnBulkInsertPrepped() throws Exception { testAutoCommit((writeClient, recordRDD, instantTime) -> writeClient.bulkInsertPreppedRecords(recordRDD, instantTime, - Option.empty()), true, populateMetaFields); + Option.empty()), true, true); } /** @@ -264,37 +252,33 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe /** * Test De-duplication behavior for HoodieWriteClient insert API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testDeduplicationOnInsert(boolean populateMetaFields) throws Exception { - testDeduplication(HoodieJavaWriteClient::insert, populateMetaFields); + @Test + public void testDeduplicationOnInsert() throws Exception { + testDeduplication(HoodieJavaWriteClient::insert, true); } /** * Test De-duplication behavior for HoodieWriteClient insert API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testDeduplicationKeepOperationFieldOnInsert(boolean populateMetaFields) throws Exception { - testDeduplicationKeepOperation(HoodieJavaWriteClient::insert, populateMetaFields); + @Test + public void testDeduplicationKeepOperationFieldOnInsert() throws Exception { + testDeduplicationKeepOperation(HoodieJavaWriteClient::insert, true); } /** * Test De-duplication behavior for HoodieWriteClient bulk-insert API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testDeduplicationOnBulkInsert(boolean populateMetaFields) throws Exception { - testDeduplication(HoodieJavaWriteClient::bulkInsert, populateMetaFields); + @Test + public void testDeduplicationOnBulkInsert() throws Exception { + testDeduplication(HoodieJavaWriteClient::bulkInsert, true); } /** * Test De-duplication behavior for HoodieWriteClient upsert API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testDeduplicationOnUpsert(boolean populateMetaFields) throws Exception { - testDeduplication(HoodieJavaWriteClient::upsert, populateMetaFields); + @Test + public void testDeduplicationOnUpsert() throws Exception { + testDeduplication(HoodieJavaWriteClient::upsert, true); } /** @@ -436,22 +420,20 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe /** * Test Upsert API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testUpserts(boolean populateMetaFields) throws Exception { + @Test + public void testUpserts() throws Exception { HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withRollbackUsingMarkers(true); - addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); + addConfigsForPopulateMetaFields(cfgBuilder, true); testUpsertsInternal(cfgBuilder.build(), HoodieJavaWriteClient::upsert, false); } /** * Test UpsertPrepped API. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testUpsertsPrepped(boolean populateMetaFields) throws Exception { + @Test + public void testUpsertsPrepped() throws Exception { HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withRollbackUsingMarkers(true); - addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); + addConfigsForPopulateMetaFields(cfgBuilder, true); testUpsertsInternal(cfgBuilder.build(), HoodieJavaWriteClient::upsertPreppedRecords, true); } @@ -602,22 +584,19 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe /** * Test Insert API for HoodieConcatHandle. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testInsertsWithHoodieConcatHandle(boolean populateMetaFields) throws Exception { + @Test + public void testInsertsWithHoodieConcatHandle() throws Exception { HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(); - addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); + addConfigsForPopulateMetaFields(cfgBuilder, true); testHoodieConcatHandle(cfgBuilder.build(), false); } /** * Test InsertPrepped API for HoodieConcatHandle. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testInsertsPreppedWithHoodieConcatHandle(boolean populateMetaFields) throws Exception { + public void testInsertsPreppedWithHoodieConcatHandle() throws Exception { HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(); - addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); + addConfigsForPopulateMetaFields(cfgBuilder, true); testHoodieConcatHandle(cfgBuilder.build(), true); } @@ -711,11 +690,9 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe /** * Tests deletion of records. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testDeletes(boolean populateMetaFields) throws Exception { + public void testDeletes() throws Exception { HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY); - addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); + addConfigsForPopulateMetaFields(cfgBuilder, true); HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build()); /** * Write 1 (inserts and deletes) Write actual 200 insert records and ignore 100 delete records @@ -736,7 +713,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime, // unused as genFn uses hard-coded number of inserts/updates/deletes -1, recordGenFunction, HoodieJavaWriteClient::upsert, true, 200, 200, 1, false, - populateMetaFields); + true); /** * Write 2 (deletes+writes). @@ -753,7 +730,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe }; writeBatch(client, newCommitTime, prevCommitTime, Option.empty(), initCommitTime, 75, recordGenFunction, HoodieJavaWriteClient::upsert, true, 25, 175, 2, false, - populateMetaFields); + true); } /** @@ -762,11 +739,10 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe * * @throws Exception */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testDeletesForInsertsInSameBatch(boolean populateMetaFields) throws Exception { + @Test + public void testDeletesForInsertsInSameBatch() throws Exception { HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder(HoodieFailedWritesCleaningPolicy.LAZY); - addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); + addConfigsForPopulateMetaFields(cfgBuilder, true); HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build()); /** * Write 200 inserts and issue deletes to a subset(50) of inserts. @@ -787,7 +763,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe writeBatch(client, newCommitTime, initCommitTime, Option.empty(), initCommitTime, -1, recordGenFunction, HoodieJavaWriteClient::upsert, true, 150, 150, 1, false, - populateMetaFields); + true); } @Test @@ -958,11 +934,11 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe } private HoodieWriteMetadata<List<WriteStatus>> performClustering(HoodieClusteringConfig clusteringConfig, - boolean populateMetaFields, - boolean completeClustering, - String validatorClasses, - String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation, - Pair<List<HoodieRecord>, List<String>> allRecords) throws IOException { + boolean populateMetaFields, + boolean completeClustering, + String validatorClasses, + String sqlQueryForEqualityValidation, String sqlQueryForSingleResultValidation, + Pair<List<HoodieRecord>, List<String>> allRecords) throws IOException { HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder() .withPreCommitValidator(StringUtils.nullToEmpty(validatorClasses)) .withPrecommitValidatorEqualitySqlQueries(sqlQueryForEqualityValidation) @@ -1101,14 +1077,13 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe /** * Test delete with delete api. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testDeletesWithoutInserts(boolean populateMetaFields) { + @Test + public void testDeletesWithoutInserts() { final String testPartitionPath = "2016/09/26"; final int insertSplitLimit = 100; // setup the small file handling params HoodieWriteConfig config = getSmallInsertWriteConfig(insertSplitLimit, - TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), populateMetaFields, populateMetaFields + TRIP_EXAMPLE_SCHEMA, dataGen.getEstimatedFileSizeInBytes(150), true, true ? new Properties() : getPropertiesForKeyGen()); dataGen = new HoodieTestDataGenerator(new String[] {testPartitionPath}); HoodieJavaWriteClient client = getHoodieWriteClient(config); @@ -1125,12 +1100,11 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe /** * Test to ensure commit metadata points to valid files. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testCommitWritesRelativePaths(boolean populateMetaFields) throws Exception { + @Test + public void testCommitWritesRelativePaths() throws Exception { HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false); - addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); + addConfigsForPopulateMetaFields(cfgBuilder, true); try (HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build());) { HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build(); HoodieJavaTable table = HoodieJavaTable.create(cfgBuilder.build(), context, metaClient); @@ -1171,11 +1145,10 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe /** * Test to ensure commit metadata points to valid files.10. */ - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testMetadataStatsOnCommit(boolean populateMetaFields) throws Exception { + @Test + public void testMetadataStatsOnCommit() throws Exception { HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false); - addConfigsForPopulateMetaFields(cfgBuilder, populateMetaFields); + addConfigsForPopulateMetaFields(cfgBuilder, true); HoodieWriteConfig cfg = cfgBuilder.build(); HoodieJavaWriteClient client = getHoodieWriteClient(cfg); @@ -1304,18 +1277,16 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe @ParameterizedTest @MethodSource("rollbackAfterConsistencyCheckFailureParams") - public void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean enableOptimisticConsistencyGuard, boolean populateMetCols) throws Exception { - testRollbackAfterConsistencyCheckFailureUsingFileList(false, enableOptimisticConsistencyGuard, populateMetCols); + public void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean enableOptimisticConsistencyGuard) throws Exception { + testRollbackAfterConsistencyCheckFailureUsingFileList(false, enableOptimisticConsistencyGuard, true); } @ParameterizedTest @MethodSource("rollbackAfterConsistencyCheckFailureParams") - public void testRollbackAfterConsistencyCheckFailureUsingMarkers(boolean enableOptimisticConsistencyGuard, boolean populateMetCols) throws Exception { - testRollbackAfterConsistencyCheckFailureUsingFileList(true, enableOptimisticConsistencyGuard, populateMetCols); + public void testRollbackAfterConsistencyCheckFailureUsingMarkers(boolean enableOptimisticConsistencyGuard) throws Exception { + testRollbackAfterConsistencyCheckFailureUsingFileList(true, enableOptimisticConsistencyGuard, true); } - //@ParameterizedTest - //@MethodSource("rollbackFailedCommitsParams") @Test public void testRollbackFailedCommits() throws Exception { // HoodieFailedWritesCleaningPolicy cleaningPolicy, boolean populateMetaFields @@ -1395,12 +1366,11 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe } } - @ParameterizedTest - @MethodSource("populateMetaFieldsParams") - public void testRollbackFailedCommitsToggleCleaningPolicy(boolean populateMetaFields) throws Exception { + @Test + public void testRollbackFailedCommitsToggleCleaningPolicy() throws Exception { HoodieTestUtils.init(hadoopConf, basePath); HoodieFailedWritesCleaningPolicy cleaningPolicy = EAGER; - HoodieJavaWriteClient client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)); + HoodieJavaWriteClient client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)); // Perform 1 successful writes to table writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100", 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300, @@ -1414,12 +1384,12 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe // Toggle cleaning policy to LAZY cleaningPolicy = HoodieFailedWritesCleaningPolicy.LAZY; // Perform 2 failed writes to table - client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)); + client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)); writeBatch(client, "300", "200", Option.of(Arrays.asList("300")), "300", 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300, 0, false); client.close(); - client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)); + client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)); writeBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400", 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300, 0, false); @@ -1435,25 +1405,26 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe assertTrue(timeline.getTimelineOfActions( CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 3); // Perform 2 failed commits - client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)); + client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)); writeBatch(client, "500", "400", Option.of(Arrays.asList("300")), "300", 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300, 0, false); client.close(); - client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)); + client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)); writeBatch(client, "600", "500", Option.of(Arrays.asList("400")), "400", 100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300, 0, false); client.close(); // Toggle cleaning policy to EAGER cleaningPolicy = EAGER; - client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields)); + client = new HoodieJavaWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)); client.startCommit(); timeline = metaClient.getActiveTimeline().reload(); // since OCC is enabled, hudi auto flips the cleaningPolicy to Lazy. assertTrue(timeline.getTimelineOfActions( CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 3); assertTrue(timeline.getCommitsTimeline().filterCompletedInstants().countInstants() == 1); + client.close(); } @Test diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java index aaf072e7b98..68b7ed18a7f 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java @@ -86,6 +86,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileScanner; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.slf4j.Logger; @@ -131,9 +132,14 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest protected HoodieTableFileSystemView tableView; protected HoodieJavaWriteClient writeClient; + @AfterAll + public static void tearDownAll() throws IOException { + FileSystem.closeAll(); + } + @BeforeEach protected void initResources() throws IOException { - basePath = tempDir.resolve("java_client_tests" + System.currentTimeMillis()).toUri().getPath(); + basePath = tempDir.resolve("java_client_tests" + System.currentTimeMillis()).toAbsolutePath().toUri().getPath(); hadoopConf = new Configuration(); taskContextSupplier = new TestJavaTaskContextSupplier(); context = new HoodieJavaEngineContext(hadoopConf, taskContextSupplier); @@ -142,6 +148,14 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest initMetaClient(); } + @AfterEach + protected void cleanupResources() throws IOException { + cleanupClients(); + cleanupTestDataGenerator(); + cleanupFileSystem(); + cleanupExecutorService(); + } + public class TestJavaTaskContextSupplier extends TaskContextSupplier { int partitionId = 0; int stageId = 0; @@ -172,14 +186,6 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest } } - @AfterEach - protected void cleanupResources() throws IOException { - cleanupClients(); - cleanupTestDataGenerator(); - cleanupFileSystem(); - cleanupExecutorService(); - } - protected void initFileSystem(String basePath, Configuration hadoopConf) { if (basePath == null) { throw new IllegalStateException("The base path has not been initialized."); @@ -423,9 +429,9 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest } public List<WriteStatus> insertFirstBatch(HoodieWriteConfig writeConfig, HoodieJavaWriteClient client, String newCommitTime, - String initCommitTime, int numRecordsInThisCommit, - Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPreppedAPI, - boolean assertForCommit, int expRecordsInThisCommit) throws Exception { + String initCommitTime, int numRecordsInThisCommit, + Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPreppedAPI, + boolean assertForCommit, int expRecordsInThisCommit) throws Exception { return insertFirstBatch(writeConfig, client, newCommitTime, initCommitTime, numRecordsInThisCommit, writeFn, isPreppedAPI, assertForCommit, expRecordsInThisCommit, true); } @@ -445,9 +451,9 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest * @throws Exception in case of error */ public List<WriteStatus> insertFirstBatch(HoodieWriteConfig writeConfig, HoodieJavaWriteClient client, String newCommitTime, - String initCommitTime, int numRecordsInThisCommit, - Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPreppedAPI, - boolean assertForCommit, int expRecordsInThisCommit, boolean filterForCommitTimeWithAssert) throws Exception { + String initCommitTime, int numRecordsInThisCommit, + Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPreppedAPI, + boolean assertForCommit, int expRecordsInThisCommit, boolean filterForCommitTimeWithAssert) throws Exception { final Function2<List<HoodieRecord>, String, Integer> recordGenFunction = generateWrapRecordsFn(isPreppedAPI, writeConfig, dataGen::generateInserts); @@ -473,9 +479,9 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest * @throws Exception in case of error */ public List<WriteStatus> insertBatch(HoodieWriteConfig writeConfig, HoodieJavaWriteClient client, String newCommitTime, - String initCommitTime, int numRecordsInThisCommit, - Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPreppedAPI, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, Option<String> partition) throws Exception { + String initCommitTime, int numRecordsInThisCommit, + Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPreppedAPI, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, Option<String> partition) throws Exception { if (partition.isPresent()) { final Function3<List<HoodieRecord>, String, Integer, String> recordGenFunction = @@ -494,10 +500,10 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest } public List<WriteStatus> updateBatch(HoodieWriteConfig writeConfig, HoodieJavaWriteClient client, String newCommitTime, - String prevCommitTime, Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, - int numRecordsInThisCommit, - Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPreppedAPI, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits) throws Exception { + String prevCommitTime, Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, + int numRecordsInThisCommit, + Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPreppedAPI, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits) throws Exception { return updateBatch(writeConfig, client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, writeFn, isPreppedAPI, assertForCommit, expRecordsInThisCommit, expTotalRecords, expTotalCommits, true); } @@ -522,11 +528,11 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest * @throws Exception in case of error */ public List<WriteStatus> updateBatch(HoodieWriteConfig writeConfig, HoodieJavaWriteClient client, String newCommitTime, - String prevCommitTime, Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, - int numRecordsInThisCommit, - Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPreppedAPI, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, - boolean filterForCommitTimeWithAssert) throws Exception { + String prevCommitTime, Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, + int numRecordsInThisCommit, + Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, boolean isPreppedAPI, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, + boolean filterForCommitTimeWithAssert) throws Exception { final Function2<List<HoodieRecord>, String, Integer> recordGenFunction = generateWrapRecordsFn(isPreppedAPI, writeConfig, dataGen::generateUniqueUpdates); @@ -536,8 +542,8 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest } public List<WriteStatus> deleteBatch(HoodieWriteConfig writeConfig, HoodieJavaWriteClient client, String newCommitTime, String prevCommitTime, - String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI, boolean assertForCommit, - int expRecordsInThisCommit, int expTotalRecords) throws Exception { + String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI, boolean assertForCommit, + int expRecordsInThisCommit, int expTotalRecords) throws Exception { return deleteBatch(writeConfig, client, newCommitTime, prevCommitTime, initCommitTime, numRecordsInThisCommit, isPreppedAPI, assertForCommit, expRecordsInThisCommit, expTotalRecords, true); } @@ -559,8 +565,8 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest * @throws Exception in case of error */ public List<WriteStatus> deleteBatch(HoodieWriteConfig writeConfig, HoodieJavaWriteClient client, String newCommitTime, - String prevCommitTime, String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, boolean filterForCommitTimeWithAssert) throws Exception { + String prevCommitTime, String initCommitTime, int numRecordsInThisCommit, boolean isPreppedAPI, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, boolean filterForCommitTimeWithAssert) throws Exception { if (isPreppedAPI) { final Function2<List<HoodieRecord>, String, Integer> recordGenFunction = @@ -592,20 +598,20 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest } public List<WriteStatus> writeBatch(HoodieJavaWriteClient client, String newCommitTime, String prevCommitTime, - Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit, - Function2<List<HoodieRecord>, String, Integer> recordGenFunction, - Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit) throws Exception { + Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit, + Function2<List<HoodieRecord>, String, Integer> recordGenFunction, + Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit) throws Exception { return writeBatch(client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction, writeFn, assertForCommit, expRecordsInThisCommit, expTotalRecords, expTotalCommits, doCommit, true); } public List<WriteStatus> writeBatch(HoodieJavaWriteClient client, String newCommitTime, String prevCommitTime, - Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit, - Function3<List<HoodieRecord>, String, Integer, String> recordGenFunction, - Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, - boolean doCommit, String partition) throws Exception { + Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit, + Function3<List<HoodieRecord>, String, Integer, String> recordGenFunction, + Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, + boolean doCommit, String partition) throws Exception { return writeBatch(client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, numRecordsInThisCommit, recordGenFunction, writeFn, assertForCommit, expRecordsInThisCommit, expTotalRecords, expTotalCommits, doCommit, true, partition); } @@ -629,11 +635,11 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest * @throws Exception in case of error */ public List<WriteStatus> writeBatch(HoodieJavaWriteClient client, String newCommitTime, String prevCommitTime, - Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit, - Function2<List<HoodieRecord>, String, Integer> recordGenFunction, - Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit, - boolean filterForCommitTimeWithAssert) throws Exception { + Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit, + Function2<List<HoodieRecord>, String, Integer> recordGenFunction, + Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit, + boolean filterForCommitTimeWithAssert) throws Exception { List<HoodieRecord> records = recordGenFunction.apply(newCommitTime, numRecordsInThisCommit); return writeBatchHelper(client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, @@ -642,12 +648,12 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest } public List<WriteStatus> writeBatch(HoodieJavaWriteClient client, String newCommitTime, String prevCommitTime, - Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit, - Function3<List<HoodieRecord>, String, Integer, String> recordGenFunction, - Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit, - boolean filterForCommitTimeWithAssert, - String partition) throws Exception { + Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit, + Function3<List<HoodieRecord>, String, Integer, String> recordGenFunction, + Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean doCommit, + boolean filterForCommitTimeWithAssert, + String partition) throws Exception { List<HoodieRecord> records = recordGenFunction.apply(newCommitTime, numRecordsInThisCommit, partition); return writeBatchHelper(client, newCommitTime, prevCommitTime, commitTimesBetweenPrevAndNew, initCommitTime, @@ -656,11 +662,11 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest } private List<WriteStatus> writeBatchHelper(HoodieJavaWriteClient client, String newCommitTime, String prevCommitTime, - Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, - int numRecordsInThisCommit, List<HoodieRecord> records, - Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, - boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, - int expTotalCommits, boolean doCommit, boolean filterForCommitTimeWithAssert) throws IOException { + Option<List<String>> commitTimesBetweenPrevAndNew, String initCommitTime, + int numRecordsInThisCommit, List<HoodieRecord> records, + Function3<List<WriteStatus>, HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn, + boolean assertForCommit, int expRecordsInThisCommit, int expTotalRecords, + int expTotalCommits, boolean doCommit, boolean filterForCommitTimeWithAssert) throws IOException { // Write 1 (only inserts) client.startCommitWithTime(newCommitTime); @@ -716,8 +722,8 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest * Generate wrapper for record generation function for testing Prepped APIs. * * @param isPreppedAPI Flag to indicate if this is for testing prepped-version of APIs - * @param writeConfig Hoodie Write Config - * @param wrapped Actual Records Generation function + * @param writeConfig Hoodie Write Config + * @param wrapped Actual Records Generation function * @return Wrapped Function */ public Function2<List<HoodieRecord>, String, Integer> generateWrapRecordsFn(boolean isPreppedAPI, @@ -734,8 +740,8 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest * Generate wrapper for record generation function for testing Prepped APIs. * * @param isPreppedAPI Flag to indicate if this is for testing prepped-version of APIs - * @param writeConfig Hoodie Write Config - * @param wrapped Actual Records Generation function (for partition) + * @param writeConfig Hoodie Write Config + * @param wrapped Actual Records Generation function (for partition) * @return Wrapped Function */ public Function3<List<HoodieRecord>, String, Integer, String> generateWrapRecordsForPartitionFn(boolean isPreppedAPI, @@ -752,7 +758,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest * to be already de-duped and have location set. This wrapper takes care of record-location setting. Uniqueness is * guaranteed by record-generation function itself. * - * @param writeConfig Hoodie Write Config + * @param writeConfig Hoodie Write Config * @param recordsGenFunction Records Generation function * @return Wrapped function */ @@ -776,7 +782,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest * to be already de-duped and have location set. This wrapper takes care of record-location setting. Uniqueness is * guaranteed by record-generation function itself. * - * @param writeConfig Hoodie Write Config + * @param writeConfig Hoodie Write Config * @param recordsGenFunction Records Generation function (for partition) * @return Wrapped function */ @@ -799,8 +805,8 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest * Generate wrapper for delete key generation function for testing Prepped APIs. * * @param isPreppedAPI Flag to indicate if this is for testing prepped-version of APIs - * @param writeConfig Hoodie Write Config - * @param wrapped Actual Records Generation function + * @param writeConfig Hoodie Write Config + * @param wrapped Actual Records Generation function * @return Wrapped Function */ public Function<Integer, List<HoodieKey>> generateWrapDeleteKeysFn(boolean isPreppedAPI, @@ -845,7 +851,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest } private List<WriteStatus> getWriteStatusAndVerifyDeleteOperation(String newCommitTime, String prevCommitTime, String initCommitTime, boolean assertForCommit, int expRecordsInThisCommit, - int expTotalRecords, boolean filerForCommitTimeWithAssert, List<WriteStatus> result) { + int expTotalRecords, boolean filerForCommitTimeWithAssert, List<WriteStatus> result) { assertNoWriteErrors(result); // verify that there is a commit @@ -882,7 +888,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest } public long numRowsInCommit(String basePath, HoodieTimeline commitTimeline, - String instantTime, boolean filterByCommitTime) { + String instantTime, boolean filterByCommitTime) { HoodieInstant commitInstant = new HoodieInstant(false, HoodieTimeline.COMMIT_ACTION, instantTime); if (!commitTimeline.containsInstant(commitInstant)) { throw new HoodieException("No commit exists at " + instantTime); @@ -891,7 +897,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest HashMap<String, String> paths = getLatestFileIDsToFullPath(basePath, commitTimeline, Arrays.asList(commitInstant)); return paths.values().stream().flatMap(path -> - BaseFileUtils.getInstance(path).readAvroRecords(context.getHadoopConf().get(), new Path(path)).stream()) + BaseFileUtils.getInstance(path).readAvroRecords(context.getHadoopConf().get(), new Path(path)).stream()) .filter(record -> { if (filterByCommitTime) { Object commitTime = record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD); diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java index f556bc18541..e7f13991add 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java @@ -303,7 +303,7 @@ public class TestHoodieMetadataBase extends HoodieJavaClientTestHarness { .ignoreSpuriousDeletes(validateMetadataPayloadConsistency) .build()) .withMetricsConfig(HoodieMetricsConfig.newBuilder().on(enableMetrics) - .withExecutorMetrics(true).build()) + .withExecutorMetrics(enableMetrics).build()) .withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig.newBuilder() .usePrefix("unit-test").build()) .withRollbackUsingMarkers(useRollbackUsingMarkers) diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java index 464d47b2a27..26dc41f73a3 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java @@ -203,14 +203,10 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase { public static List<Arguments> tableOperationsTestArgs() { return asList( - Arguments.of(COPY_ON_WRITE, true, true), - Arguments.of(COPY_ON_WRITE, true, false), - Arguments.of(COPY_ON_WRITE, false, true), - Arguments.of(COPY_ON_WRITE, false, false), - Arguments.of(MERGE_ON_READ, true, true), - Arguments.of(MERGE_ON_READ, true, false), - Arguments.of(MERGE_ON_READ, false, true), - Arguments.of(MERGE_ON_READ, false, false) + Arguments.of(COPY_ON_WRITE, true), + Arguments.of(COPY_ON_WRITE, false), + Arguments.of(MERGE_ON_READ, true), + Arguments.of(MERGE_ON_READ, false) ); } @@ -479,14 +475,14 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase { */ @ParameterizedTest @MethodSource("tableOperationsTestArgs") - public void testTableOperations(HoodieTableType tableType, boolean enableFullScan, boolean enableMetrics) throws Exception { + public void testTableOperations(HoodieTableType tableType, boolean enableFullScan) throws Exception { List<Long> commitTimeList = new ArrayList<>(); commitTimeList.add(Long.parseLong(HoodieActiveTimeline.createNewInstantTime())); for (int i = 0; i < 8; i++) { long nextCommitTime = getNextCommitTime(commitTimeList.get(commitTimeList.size() - 1)); commitTimeList.add(nextCommitTime); } - init(tableType, true, enableFullScan, enableMetrics, false); + init(tableType, true, enableFullScan, false, false); doWriteInsertAndUpsert(testTable, commitTimeList.get(0).toString(), commitTimeList.get(1).toString(), false); // trigger an upsert @@ -2726,7 +2722,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase { public void testbootstrapWithEmptyCommit() throws Exception { init(HoodieTableType.COPY_ON_WRITE); - HoodieWriteConfig writeConfig = getWriteConfigBuilder(true, true, true).build(); + HoodieWriteConfig writeConfig = getWriteConfigBuilder(true, true, false).build(); initWriteConfigAndMetatableWriter(writeConfig, true); testTable.doWriteOperation(HoodieActiveTimeline.createNewInstantTime(), INSERT, Collections.EMPTY_LIST, 0); syncTableMetadata(writeConfig); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java index 62148acbf5b..e0a00c24e92 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java @@ -344,7 +344,7 @@ public class TestHoodieMetadataBase extends HoodieSparkClientTestHarness { .ignoreSpuriousDeletes(validateMetadataPayloadConsistency) .build()) .withMetricsConfig(HoodieMetricsConfig.newBuilder().on(enableMetrics) - .withExecutorMetrics(true).build()) + .withExecutorMetrics(enableMetrics).build()) .withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig.newBuilder() .usePrefix("unit-test").build()) .withRollbackUsingMarkers(useRollbackUsingMarkers) diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java index 6c530833d55..9fca206ac26 100644 --- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java @@ -44,12 +44,12 @@ import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.hadoop.RealtimeFileStatus; import org.apache.hudi.hadoop.config.HoodieRealtimeConfig; -import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils; import org.apache.hudi.hadoop.testutils.InputFormatTestUtil; +import org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils; -import org.apache.avro.generic.GenericRecord; import org.apache.avro.Schema; import org.apache.avro.Schema.Field; +import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -71,8 +71,8 @@ import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; - import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -101,6 +101,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; +@Disabled("HUDI-6755") public class TestHoodieRealtimeRecordReader { private static final String PARTITION_COLUMN = "datestr"; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrapRead.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrapRead.java index f57be60461a..d926a3be5a4 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrapRead.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrapRead.java @@ -63,7 +63,7 @@ public class TestBootstrapRead extends TestBootstrapReadBase { @ParameterizedTest @MethodSource("testArgs") - public void runTests(String bootstrapType, Boolean dashPartitions, HoodieTableType tableType, Integer nPartitions) { + public void testBootstrapFunctional(String bootstrapType, Boolean dashPartitions, HoodieTableType tableType, Integer nPartitions) { this.bootstrapType = bootstrapType; this.dashPartitions = dashPartitions; this.tableType = tableType; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestNewHoodieParquetFileFormat.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestNewHoodieParquetFileFormat.java index ef6814f21c5..ec719414dc8 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestNewHoodieParquetFileFormat.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestNewHoodieParquetFileFormat.java @@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieTableType; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SaveMode; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -38,6 +39,7 @@ import static org.apache.hudi.common.model.HoodieTableType.MERGE_ON_READ; import static org.junit.jupiter.api.Assertions.assertEquals; @Tag("functional") +@Disabled("HUDI-6756") public class TestNewHoodieParquetFileFormat extends TestBootstrapReadBase { private static Stream<Arguments> testArgs() { @@ -54,7 +56,7 @@ public class TestNewHoodieParquetFileFormat extends TestBootstrapReadBase { @ParameterizedTest @MethodSource("testArgs") - public void runTests(HoodieTableType tableType, Integer nPartitions) { + public void testNewParquetFileFormat(HoodieTableType tableType, Integer nPartitions) { this.bootstrapType = nPartitions == 0 ? "metadata" : "mixed"; this.dashPartitions = true; this.tableType = tableType;
