minihippo commented on a change in pull request #3173:
URL: https://github.com/apache/hudi/pull/3173#discussion_r745639798
##########
File path:
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieMergeOnReadRDD.scala
##########
@@ -58,32 +64,50 @@ class HoodieMergeOnReadRDD(@transient sc: SparkContext,
}
override def compute(split: Partition, context: TaskContext):
Iterator[InternalRow] = {
val mergeOnReadPartition = split.asInstanceOf[HoodieMergeOnReadPartition]
- mergeOnReadPartition.split match {
- case dataFileOnlySplit if dataFileOnlySplit.logPaths.isEmpty =>
- read(dataFileOnlySplit.dataFile.get, requiredSchemaFileReader)
- case logFileOnlySplit if logFileOnlySplit.dataFile.isEmpty =>
- logFileIterator(logFileOnlySplit, getConfig)
- case skipMergeSplit if skipMergeSplit.mergeType
- .equals(DataSourceReadOptions.REALTIME_SKIP_MERGE_OPT_VAL) =>
- skipMergeFileIterator(
- skipMergeSplit,
- read(skipMergeSplit.dataFile.get, requiredSchemaFileReader),
- getConfig
- )
- case payloadCombineSplit if payloadCombineSplit.mergeType
- .equals(DataSourceReadOptions.REALTIME_PAYLOAD_COMBINE_OPT_VAL) =>
- payloadCombineFileIterator(
- payloadCombineSplit,
- read(payloadCombineSplit.dataFile.get, fullSchemaFileReader),
- getConfig
- )
- case _ => throw new HoodieException(s"Unable to select an Iterator to
read the Hoodie MOR File Split for " +
- s"file path: ${mergeOnReadPartition.split.dataFile.get.filePath}" +
- s"log paths: ${mergeOnReadPartition.split.logPaths.toString}" +
- s"hoodie table path: ${mergeOnReadPartition.split.tablePath}" +
- s"spark partition Index: ${mergeOnReadPartition.index}" +
- s"merge type: ${mergeOnReadPartition.split.mergeType}")
- }
+ mergeOnReadPartition.split.iterator.flatMap { sp =>
Review comment:
Did u mean that read optimization for hudi table with bucket index needs
to have another pr?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
