yaooqinn commented on a change in pull request #28197: [SPARK-31431][SQL] Add
CalendarInterval encoder support
URL: https://github.com/apache/spark/pull/28197#discussion_r409978333
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
##########
@@ -235,6 +235,7 @@ object ScalaReflection extends ScalaReflection {
case t if isSubtype(t, localTypeOf[java.lang.String]) =>
createDeserializerForString(path, returnNullable = false)
+ case t if isSubtype(t, localTypeOf[CalendarInterval]) => path
Review comment:
They don't work
```scala
scala> Seq(Map(new org.apache.spark.unsafe.types.CalendarInterval(1,2,3) ->
new org.apache.spark.unsafe.types.CalendarInterval(1,2,3))).toDF
java.lang.UnsupportedOperationException: Schema for type
org.apache.spark.unsafe.types.CalendarInterval is not supported
at
org.apache.spark.sql.catalyst.ScalaReflection$.$anonfun$schemaFor$1(ScalaReflection.scala:735)
at
scala.reflect.internal.tpe.TypeConstraints$UndoLog.undo(TypeConstraints.scala:69)
at
org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects(ScalaReflection.scala:877)
at
org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects$(ScalaReflection.scala:876)
at
org.apache.spark.sql.catalyst.ScalaReflection$.cleanUpReflectionObjects(ScalaReflection.scala:47)
at
org.apache.spark.sql.catalyst.ScalaReflection$.schemaFor(ScalaReflection.scala:663)
at
org.apache.spark.sql.catalyst.ScalaReflection$.$anonfun$schemaFor$1(ScalaReflection.scala:689)
at
scala.reflect.internal.tpe.TypeConstraints$UndoLog.undo(TypeConstraints.scala:69)
at
org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects(ScalaReflection.scala:877)
at
org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects$(ScalaReflection.scala:876)
at
org.apache.spark.sql.catalyst.ScalaReflection$.cleanUpReflectionObjects(ScalaReflection.scala:47)
at
org.apache.spark.sql.catalyst.ScalaReflection$.schemaFor(ScalaReflection.scala:663)
at
org.apache.spark.sql.catalyst.ScalaReflection$.deserializerForType(ScalaReflection.scala:164)
at
org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:55)
at org.apache.spark.sql.SQLImplicits.newMapEncoder(SQLImplicits.scala:175)
... 49 elided
scala> Seq(Seq(new
org.apache.spark.unsafe.types.CalendarInterval(1,2,3)).toDF
| ;
<console>:2: error: ')' expected but ';' found.
;
^
scala> Seq(Seq(new
org.apache.spark.unsafe.types.CalendarInterval(1,2,3))).toDF
java.lang.UnsupportedOperationException: Schema for type
org.apache.spark.unsafe.types.CalendarInterval is not supported
at
org.apache.spark.sql.catalyst.ScalaReflection$.$anonfun$schemaFor$1(ScalaReflection.scala:735)
at
scala.reflect.internal.tpe.TypeConstraints$UndoLog.undo(TypeConstraints.scala:69)
at
org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects(ScalaReflection.scala:877)
at
org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects$(ScalaReflection.scala:876)
at
org.apache.spark.sql.catalyst.ScalaReflection$.cleanUpReflectionObjects(ScalaReflection.scala:47)
at
org.apache.spark.sql.catalyst.ScalaReflection$.schemaFor(ScalaReflection.scala:663)
at
org.apache.spark.sql.catalyst.ScalaReflection$.toCatalystArray$1(ScalaReflection.scala:430)
at
org.apache.spark.sql.catalyst.ScalaReflection$.$anonfun$serializerFor$1(ScalaReflection.scala:448)
at
scala.reflect.internal.tpe.TypeConstraints$UndoLog.undo(TypeConstraints.scala:69)
at
org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects(ScalaReflection.scala:877)
at
org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects$(ScalaReflection.scala:876)
at
org.apache.spark.sql.catalyst.ScalaReflection$.cleanUpReflectionObjects(ScalaReflection.scala:47)
at
org.apache.spark.sql.catalyst.ScalaReflection$.serializerFor(ScalaReflection.scala:410)
at
org.apache.spark.sql.catalyst.ScalaReflection$.$anonfun$serializerForType$1(ScalaReflection.scala:399)
at
scala.reflect.internal.tpe.TypeConstraints$UndoLog.undo(TypeConstraints.scala:69)
at
org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects(ScalaReflection.scala:877)
at
org.apache.spark.sql.catalyst.ScalaReflection.cleanUpReflectionObjects$(ScalaReflection.scala:876)
at
org.apache.spark.sql.catalyst.ScalaReflection$.cleanUpReflectionObjects(ScalaReflection.scala:47)
at
org.apache.spark.sql.catalyst.ScalaReflection$.serializerForType(ScalaReflection.scala:391)
at
org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:54)
at
org.apache.spark.sql.SQLImplicits.newSequenceEncoder(SQLImplicits.scala:171)
... 49 elided
scala> Seq(Array(new
org.apache.spark.unsafe.types.CalendarInterval(1,2,3))).toDF
<console>:28: error: value toDF is not a member of
Seq[Array[org.apache.spark.unsafe.types.CalendarInterval]]
Seq(Array(new
org.apache.spark.unsafe.types.CalendarInterval(1,2,3))).toDF
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
