nsivabalan commented on code in PR #13229:
URL: https://github.com/apache/hudi/pull/13229#discussion_r2103615529
##########
hudi-utilities/src/test/java/org/apache/hudi/utilities/offlinejob/TestHoodieCompactorJob.java:
##########
@@ -59,7 +59,6 @@ public void testHoodieCompactorWithOptionalClean(boolean
skipClean) throws Excep
.withSchema(TRIP_EXAMPLE_SCHEMA)
.withParallelism(2, 2)
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build())
- .withAutoCommit(false)
Review Comment:
https://issues.apache.org/jira/browse/HUDI-9440
##########
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/clean/TestCleanerInsertAndCleanByVersions.java:
##########
@@ -123,7 +125,7 @@ private void testInsertAndCleanByVersions(
Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient,
JavaRDD<HoodieRecord>, String> upsertFn, boolean isPreppedAPI)
throws Exception {
int maxVersions = 2; // keep upto 2 versions for each file
- HoodieWriteConfig cfg = getConfigBuilder(true)
+ HoodieWriteConfig cfg = getConfigBuilder(false)
Review Comment:
https://issues.apache.org/jira/browse/HUDI-9440
##########
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java:
##########
@@ -331,7 +331,6 @@ protected HoodieWriteConfig.Builder
getWriteConfigBuilder(HoodieFailedWritesClea
Properties properties = new Properties();
return
HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(TRIP_EXAMPLE_SCHEMA)
.withParallelism(2,
2).withDeleteParallelism(2).withRollbackParallelism(2).withFinalizeWriteParallelism(2)
- .withAutoCommit(autoCommit)
Review Comment:
https://issues.apache.org/jira/browse/HUDI-9440
##########
hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java:
##########
@@ -672,49 +689,61 @@ public List<WriteStatus> writeBatch(HoodieJavaWriteClient
client, String newComm
* @param expRecordsInThisCommit Expected number of records in this
commit
* @param expTotalRecords Expected number of records when
scanned
* @param expTotalCommits Expected number of commits (including
this commit)
- * @param doCommit
* @throws Exception in case of error
*/
public List<WriteStatus> writeBatch(HoodieJavaWriteClient client, String
newCommitTime, String prevCommitTime,
Option<List<String>>
commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
Function2<List<HoodieRecord>, String,
Integer> recordGenFunction,
Function3<List<WriteStatus>,
HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn,
- boolean assertForCommit, int
expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean
doCommit,
- boolean filterForCommitTimeWithAssert,
InstantGenerator instantGenerator) throws Exception {
+ boolean assertForCommit, int
expRecordsInThisCommit, int expTotalRecords, int expTotalCommits,
+ boolean filterForCommitTimeWithAssert,
InstantGenerator instantGenerator,
+ boolean skipCommit) throws Exception {
List<HoodieRecord> records = recordGenFunction.apply(newCommitTime,
numRecordsInThisCommit);
return writeBatchHelper(client, newCommitTime, prevCommitTime,
commitTimesBetweenPrevAndNew, initCommitTime,
numRecordsInThisCommit, records, writeFn, assertForCommit,
expRecordsInThisCommit, expTotalRecords,
- expTotalCommits, doCommit, filterForCommitTimeWithAssert,
instantGenerator);
+ expTotalCommits, filterForCommitTimeWithAssert, instantGenerator,
skipCommit);
}
public List<WriteStatus> writeBatch(HoodieJavaWriteClient client, String
newCommitTime, String prevCommitTime,
Option<List<String>>
commitTimesBetweenPrevAndNew, String initCommitTime, int numRecordsInThisCommit,
Function3<List<HoodieRecord>, String,
Integer, String> recordGenFunction,
Function3<List<WriteStatus>,
HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn,
- boolean assertForCommit, int
expRecordsInThisCommit, int expTotalRecords, int expTotalCommits, boolean
doCommit,
+ boolean assertForCommit, int
expRecordsInThisCommit, int expTotalRecords, int expTotalCommits,
boolean filterForCommitTimeWithAssert,
String partition, InstantGenerator
instantGenerator) throws Exception {
List<HoodieRecord> records = recordGenFunction.apply(newCommitTime,
numRecordsInThisCommit, partition);
return writeBatchHelper(client, newCommitTime, prevCommitTime,
commitTimesBetweenPrevAndNew, initCommitTime,
numRecordsInThisCommit, records, writeFn, assertForCommit,
expRecordsInThisCommit, expTotalRecords,
- expTotalCommits, doCommit, filterForCommitTimeWithAssert,
instantGenerator);
+ expTotalCommits, filterForCommitTimeWithAssert, instantGenerator);
+ }
+
+ private List<WriteStatus> writeBatchHelper(HoodieJavaWriteClient client,
String newCommitTime, String prevCommitTime,
+ Option<List<String>>
commitTimesBetweenPrevAndNew, String initCommitTime,
+ int numRecordsInThisCommit,
List<HoodieRecord> records,
+ Function3<List<WriteStatus>,
HoodieJavaWriteClient, List<HoodieRecord>, String> writeFn,
+ boolean assertForCommit, int
expRecordsInThisCommit, int expTotalRecords,
+ int expTotalCommits, boolean
filterForCommitTimeWithAssert, InstantGenerator instantGenerator) throws
IOException {
+ return writeBatchHelper(client, newCommitTime, prevCommitTime,
commitTimesBetweenPrevAndNew, initCommitTime,
+ numRecordsInThisCommit, records, writeFn, assertForCommit,
expRecordsInThisCommit, expTotalRecords, expTotalCommits,
+ filterForCommitTimeWithAssert, instantGenerator, false);
Review Comment:
https://issues.apache.org/jira/browse/HUDI-9440
##########
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/functional/TestMetadataUtilRLIandSIRecordGeneration.java:
##########
@@ -203,22 +208,23 @@ public void testRecordGenerationAPIsForMOR() throws
IOException {
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
HoodieWriteConfig writeConfig =
getConfigBuilder(HoodieFailedWritesCleaningPolicy.EAGER)
-
.withCompactionConfig(HoodieCompactionConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(2)
- .withInlineCompaction(true)
+
.withCompactionConfig(HoodieCompactionConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(3)
Review Comment:
https://issues.apache.org/jira/browse/HUDI-9440
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
