satishkotha commented on a change in pull request #2502:
URL: https://github.com/apache/hudi/pull/2502#discussion_r566378842
##########
File path:
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala
##########
@@ -243,17 +243,24 @@ class TestStructuredStreaming extends
HoodieClientTestBase {
val f2 = Future {
inputDF1.coalesce(1).write.mode(SaveMode.Append).json(sourcePath)
// wait for spark streaming to process one microbatch
- val currNumCommits = waitTillAtleastNCommits(fs, destPath, 1, 120, 5)
+ var currNumCommits = waitTillAtleastNCommits(fs, destPath, 1, 120, 5)
assertTrue(HoodieDataSourceHelpers.hasNewCommits(fs, destPath, "000"))
inputDF2.coalesce(1).write.mode(SaveMode.Append).json(sourcePath)
// wait for spark streaming to process second microbatch
- waitTillAtleastNCommits(fs, destPath, currNumCommits + 1, 120, 5)
- assertEquals(2, HoodieDataSourceHelpers.listCommitsSince(fs, destPath,
"000").size())
-
- // check have more than one file group
- this.metaClient = new HoodieTableMetaClient(fs.getConf, destPath, true)
- assertTrue(getLatestFileGroupsFileId(partitionOfRecords).size > 1)
+ currNumCommits = waitTillAtleastNCommits(fs, destPath, currNumCommits +
1, 120, 5)
+ // for inline clustering, clustering may be complete along with 2nd
commit
+ if (HoodieDataSourceHelpers.allCompletedCommitsCompactions(fs,
destPath).getCompletedReplaceTimeline().countInstants() > 0) {
Review comment:
@lw309637554 It seems like this test has a race condition. PTAL and
confirm my fix is reasonable.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
