chenhao-db commented on code in PR #48851:
URL: https://github.com/apache/spark/pull/48851#discussion_r1878752368
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/SparkShreddingUtils.scala:
##########
@@ -19,12 +19,32 @@ package org.apache.spark.sql.execution.datasources.parquet
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.util.GenericArrayData
+import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.types._
import org.apache.spark.types.variant._
import org.apache.spark.unsafe.types._
+case class SparkShreddedRow(row: SpecializedGetters) extends
ShreddingUtils.ShreddedRow {
+ override def isNullAt(ordinal: Int): Boolean = row.isNullAt(ordinal)
+ override def getBoolean(ordinal: Int): Boolean = row.getBoolean(ordinal)
+ override def getByte(ordinal: Int): Byte = row.getByte(ordinal)
+ override def getShort(ordinal: Int): Short = row.getShort(ordinal)
+ override def getInt(ordinal: Int): Int = row.getInt(ordinal)
+ override def getLong(ordinal: Int): Long = row.getLong(ordinal)
+ override def getFloat(ordinal: Int): Float = row.getFloat(ordinal)
+ override def getDouble(ordinal: Int): Double = row.getDouble(ordinal)
+ override def getDecimal(ordinal: Int, precision: Int, scale: Int):
java.math.BigDecimal =
+ row.getDecimal(ordinal, precision, scale).toJavaBigDecimal
+ override def getString(ordinal: Int): String =
row.getUTF8String(ordinal).toString
Review Comment:
The returned string is processed by the variant library, like
https://github.com/apache/spark/pull/48851/files#diff-0561edc5f2b44a87fc788bceba8d485847e1cf1db3eea4b3f4b3620c1e4ecad5R76.
I think the library currently doesn't have access to `UTF8String`, and it will
require extra dependency to make it have access.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
