linliu-code commented on code in PR #12935:
URL: https://github.com/apache/hudi/pull/12935#discussion_r1996116285
##########
hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/SparkFileFormatInternalRowReaderContext.scala:
##########
@@ -59,17 +60,16 @@ import scala.collection.mutable
* @param filters spark filters that might be pushed down into the
reader
* @param requiredFilters filters that are required and should always be
used, even in merging situations
*/
-class SparkFileFormatInternalRowReaderContext(parquetFileReader:
SparkParquetReader,
- filters: Seq[Filter],
- requiredFilters: Seq[Filter])
extends BaseSparkInternalRowReaderContext {
+class SparkFileFormatInternalRowReaderContext(parquetFileReader:
SparkParquetReader, filters: Seq[Filter],
+ requiredFilters: Seq[Filter],
tableVersion: HoodieTableVersion) extends BaseSparkInternalRowReaderContext {
lazy val sparkAdapter: SparkAdapter = SparkAdapterSupport.sparkAdapter
private lazy val bootstrapSafeFilters: Seq[Filter] =
filters.filter(filterIsSafeForBootstrap) ++ requiredFilters
private val deserializerMap: mutable.Map[Schema, HoodieAvroDeserializer] =
mutable.Map()
private val serializerMap: mutable.Map[Schema, HoodieAvroSerializer] =
mutable.Map()
private lazy val allFilters = filters ++ requiredFilters
override def supportsParquetRowIndex: Boolean = {
- HoodieSparkUtils.gteqSpark3_5
+ HoodieSparkUtils.gteqSpark3_5 &&
tableVersion.greaterThanOrEquals(HoodieTableVersion.EIGHT)
Review Comment:
> Can you please check this as well? The issue here was the schema with
positional merging had dropped precombine field and that lead to errors with
ordering value.
This should be fixed by https://github.com/apache/hudi/pull/12979.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
