yihua commented on code in PR #13711:
URL: https://github.com/apache/hudi/pull/13711#discussion_r2359756944
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala:
##########
@@ -445,6 +448,28 @@ object ColumnStatsIndexSupport {
@inline def composeColumnStatStructType(col: String, statName: String,
dataType: DataType) =
StructField(formatColName(col, statName), dataType, nullable = true,
Metadata.empty)
+ def extractColStatsValue(valueWrapper: AnyRef, dataType: DataType,
valueMetadata: ValueMetadata, useJava8api: Boolean): Any = {
+ valueMetadata.getValueType match {
+ case ValueType.V1 => extractWrapperValueV1(valueWrapper, dataType)
+ case _ => extractColStatsValueV2(valueWrapper, dataType, valueMetadata,
useJava8api)
+ }
+ }
+
+ private def extractColStatsValueV2(valueWrapper: AnyRef, dataType: DataType,
valueMetadata: ValueMetadata, useJava8api: Boolean): Any = {
+ val colStatsValue =
SparkValueMetadata.convertJavaTypeToSparkType(SparkValueMetadata.getValueMetadata(dataType,
HoodieIndexVersion.V2)
+ .standardizeJavaTypeAndPromote(valueMetadata.unwrapValue(valueWrapper)),
useJava8api)
+ // TODO: should this be done here? Should we handle this with adding more
value types?
+ // TODO: should this logic be in convertJavaTypeToSparkType?
+ dataType match {
+ case ShortType => colStatsValue.asInstanceOf[Int].toShort
+ case ByteType => colStatsValue.asInstanceOf[Int].toByte
Review Comment:
We discussed that this is needed for expression index.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
