leesf commented on a change in pull request #2379:
URL: https://github.com/apache/hudi/pull/2379#discussion_r548949532
##########
File path:
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala
##########
@@ -177,4 +188,112 @@ class TestStructuredStreaming extends
HoodieClientTestBase {
if (!success) throw new IllegalStateException("Timed-out waiting for " +
numCommits + " commits to appear in " + tablePath)
numInstants
}
+
+ def getInlineClusteringOpts( isInlineClustering: String,
clusteringNumCommit: String, fileMaxRecordNum: Int):Map[String, String] = {
+ commonOpts + (HoodieClusteringConfig.INLINE_CLUSTERING_PROP ->
isInlineClustering,
+ HoodieClusteringConfig.INLINE_CLUSTERING_MAX_COMMIT_PROP ->
clusteringNumCommit,
+ HoodieStorageConfig.PARQUET_FILE_MAX_BYTES ->
dataGen.getEstimatedFileSizeInBytes(fileMaxRecordNum).toString
+ )
+ }
+
+ @Test
+ def testStructuredStreamingWithInlineClustering(): Unit = {
+ val (sourcePath, destPath) = initStreamingSourceAndDestPath("source",
"dest")
+
+ def checkClusteringResult(destPath: String):Unit = {
+ // check have schedule clustering and clustering file group to one
+ waitTillHasCompletedReplaceInstant(destPath, 120, 5)
+ metaClient.reloadActiveTimeline()
+ assertEquals(1, getLatestFileGroupsFileId.size)
+ }
+ structuredStreamingForTestClusteringRunner(sourcePath, destPath, true,
checkClusteringResult)
+ }
+
+ @Test
+ def testStructuredStreamingWithoutInlineClustering(): Unit = {
+ val (sourcePath, destPath) = initStreamingSourceAndDestPath("source",
"dest")
+
+ def checkClusteringResult(destPath: String):Unit = {
+ val msg = "Should have replace commit completed"
+ assertThrows(classOf[IllegalStateException], new Executable {
+ override def execute(): Unit = {
+ waitTillHasCompletedReplaceInstant(destPath, 120, 5)
+ }
+ }
+ , "Should have replace commit completed")
+ println(msg)
+ }
+ structuredStreamingForTestClusteringRunner(sourcePath, destPath, false,
checkClusteringResult)
+ }
+
+ def structuredStreamingForTestClusteringRunner(sourcePath: String, destPath:
String,
+ isInlineClustering: Boolean,
checkClusteringResult: String => Unit): Unit = {
+ // First insert of data
+ val records1 = recordsToStrings(dataGen.generateInsertsForPartition("000",
100, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)).toList
+ val inputDF1 = spark.read.json(spark.sparkContext.parallelize(records1, 2))
+
+ // Second insert of data
+ val records2 = recordsToStrings(dataGen.generateInsertsForPartition("001",
100, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)).toList
+ val inputDF2 = spark.read.json(spark.sparkContext.parallelize(records2, 2))
+
+ val hudiOptions = getInlineClusteringOpts(isInlineClustering.toString,
"2", 100)
+ val f1 = initStreamingWriteFuture(inputDF1.schema, sourcePath, destPath,
hudiOptions)
+
+ val f2 = Future {
+ inputDF1.coalesce(1).write.mode(SaveMode.Append).json(sourcePath)
+ // wait for spark streaming to process one microbatch
+ val currNumCommits = waitTillAtleastNCommits(fs, destPath, 1, 120, 5)
+ assertTrue(HoodieDataSourceHelpers.hasNewCommits(fs, destPath, "000"))
+
+ inputDF2.coalesce(1).write.mode(SaveMode.Append).json(sourcePath)
+ // wait for spark streaming to process second microbatch
+ waitTillAtleastNCommits(fs, destPath, currNumCommits + 1, 120, 5)
+ assertEquals(2, HoodieDataSourceHelpers.listCommitsSince(fs, destPath,
"000").size())
+
+ // check have more than one file group
+ this.metaClient = new HoodieTableMetaClient(fs.getConf, destPath, true)
+ assertTrue(getLatestFileGroupsFileId().size > 1)
+
+ // check clustering result
+ checkClusteringResult(destPath)
+
+ // check data correct after clustering
+ val hoodieROViewDF2 = spark.read.format("org.apache.hudi")
+ .load(destPath + "/*/*/*/*")
+ assertEquals(200, hoodieROViewDF2.count())
+ }
+ Await.result(Future.sequence(Seq(f1, f2)), Duration.Inf)
+ }
+
+ private def getLatestFileGroupsFileId():Array[String] = {
+ getHoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline,
+ HoodieTestTable.of(metaClient).listAllBaseFiles())
+
tableView.getLatestFileSlices(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)
+ .toArray().map(slice =>
slice.asInstanceOf[FileSlice].getFileGroupId.getFileId)
+ }
+
+ @throws[InterruptedException]
+ private def waitTillHasCompletedReplaceInstant(tablePath: String,
+ timeoutSecs: Int,
sleepSecsAfterEachRun: Int) = {
+ val beginTime = System.currentTimeMillis
+ var currTime = beginTime
+ val timeoutMsecs = timeoutSecs * 1000
+ var success = false
+ while ({!success && (currTime - beginTime) < timeoutMsecs}) try {
+ this.metaClient.reloadActiveTimeline()
+ val completeReplaceSize =
this.metaClient.getActiveTimeline.getCompletedReplaceTimeline().getInstants.toArray.size
+ println("completeReplaceSize:" + completeReplaceSize)
+ if(completeReplaceSize > 0) {
Review comment:
format
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
