yihua commented on code in PR #12146:
URL: https://github.com/apache/hudi/pull/12146#discussion_r1811763818
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala:
##########
@@ -25,11 +25,11 @@ import
org.apache.hudi.client.common.HoodieSparkEngineContext
import
org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictResolutionStrategy
import org.apache.hudi.client.transaction.lock.InProcessLockProvider
import org.apache.hudi.common.config.HoodieMetadataConfig
-import org.apache.hudi.common.model.{FileSlice, HoodieBaseFile,
HoodieCommitMetadata, HoodieFailedWritesCleaningPolicy, HoodieTableType,
WriteConcurrencyMode, WriteOperationType}
+import org.apache.hudi.common.model._
Review Comment:
Avoid using `_` unless there are more than 20 entries
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala:
##########
@@ -165,17 +166,27 @@ class TestPartitionStatsIndex extends
PartitionStatsIndexTestBase {
HoodieLockConfig.WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME.key() ->
classOf[SimpleConcurrentFileWritesConflictResolutionStrategy].getName
)
- doWriteAndValidateDataAndPartitionStats(hudiOpts,
+ val firstBatch: mutable.Buffer[String] =
Review Comment:
```suggestion
val insertRecords: mutable.Buffer[String] =
```
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala:
##########
@@ -25,11 +25,11 @@ import
org.apache.hudi.client.common.HoodieSparkEngineContext
import
org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictResolutionStrategy
import org.apache.hudi.client.transaction.lock.InProcessLockProvider
import org.apache.hudi.common.config.HoodieMetadataConfig
-import org.apache.hudi.common.model.{FileSlice, HoodieBaseFile,
HoodieCommitMetadata, HoodieFailedWritesCleaningPolicy, HoodieTableType,
WriteConcurrencyMode, WriteOperationType}
+import org.apache.hudi.common.model._
import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.HoodieInstant
import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings
-import org.apache.hudi.config.{HoodieCleanConfig, HoodieClusteringConfig,
HoodieCompactionConfig, HoodieLockConfig, HoodieWriteConfig}
+import org.apache.hudi.config._
Review Comment:
Same here
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala:
##########
@@ -165,17 +166,27 @@ class TestPartitionStatsIndex extends
PartitionStatsIndexTestBase {
HoodieLockConfig.WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME.key() ->
classOf[SimpleConcurrentFileWritesConflictResolutionStrategy].getName
)
- doWriteAndValidateDataAndPartitionStats(hudiOpts,
+ val firstBatch: mutable.Buffer[String] =
+ recordsToStrings(dataGen.generateInserts(getInstantTime, 20)).asScala
+ doWriteAndValidateDataAndPartitionStats(
+ firstBatch,
+ hudiOpts,
operation = DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL,
saveMode = SaveMode.Overwrite,
validate = false)
+ val latestBatch1: mutable.Buffer[String] = firstBatch
+ val latestBatch2: mutable.Buffer[String] =
Review Comment:
```suggestion
val writer2Records: mutable.Buffer[String] =
```
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala:
##########
@@ -165,17 +166,27 @@ class TestPartitionStatsIndex extends
PartitionStatsIndexTestBase {
HoodieLockConfig.WRITE_CONFLICT_RESOLUTION_STRATEGY_CLASS_NAME.key() ->
classOf[SimpleConcurrentFileWritesConflictResolutionStrategy].getName
)
- doWriteAndValidateDataAndPartitionStats(hudiOpts,
+ val firstBatch: mutable.Buffer[String] =
+ recordsToStrings(dataGen.generateInserts(getInstantTime, 20)).asScala
+ doWriteAndValidateDataAndPartitionStats(
+ firstBatch,
+ hudiOpts,
operation = DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL,
saveMode = SaveMode.Overwrite,
validate = false)
+ val latestBatch1: mutable.Buffer[String] = firstBatch
Review Comment:
```suggestion
val writer1Records: mutable.Buffer[String] = firstBatch
```
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/PartitionStatsIndexTestBase.scala:
##########
@@ -200,6 +200,29 @@ class PartitionStatsIndexTestBase extends
HoodieSparkClientTestBase {
latestBatchDf
}
+ protected def doWriteAndValidateDataAndPartitionStats(records:
mutable.Buffer[String],
Review Comment:
Can this be reused by the method above to avoid code duplication?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
