HeartSaVioR commented on a change in pull request #24014: [SPARK-27092][SQL]
Apply refactored pattern in ScalaReflection and JavaTypeInference to RowEncoder
URL: https://github.com/apache/spark/pull/24014#discussion_r264112201
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SerializerBuildHelper.scala
##########
@@ -188,11 +240,45 @@ object SerializerBuildHelper {
expressions.If(IsNull(inputObject), nullOutput, nonNullOutput)
}
+ def createSerializerForExternalRow(
+ inputObject: Expression,
+ fields: Array[StructField],
+ fnSerializerFor: (Expression, DataType) => Expression): Expression = {
+ val nonNullOutput = CreateNamedStruct(fields.zipWithIndex.flatMap { case
(field, index) =>
+ val fieldValue = fnSerializerFor(
+ ValidateExternalType(
+ GetExternalRowField(inputObject, index, field.name),
+ field.dataType),
+ field.dataType)
+ val convertedField = if (field.nullable) {
+ If(
+ Invoke(inputObject, "isNullAt", BooleanType, Literal(index) :: Nil),
+ // Because we strip UDTs, `field.dataType` can be different from
`fieldValue.dataType`.
+ // We should use `fieldValue.dataType` here.
+ Literal.create(null, fieldValue.dataType),
+ fieldValue
+ )
+ } else {
+ fieldValue
+ }
+ Literal(field.name) :: convertedField :: Nil
+ })
+
+ if (inputObject.nullable) {
+ expressionForNullableExpr(inputObject, nonNullOutput)
+ } else {
+ nonNullOutput
+ }
+ }
+
def createSerializerForUserDefinedType(
inputObject: Expression,
udt: UserDefinedType[_],
- udtClass: Class[_]): Expression = {
- val obj = NewInstance(udtClass, Nil, dataType = ObjectType(udtClass))
- Invoke(obj, "serialize", udt, inputObject :: Nil)
+ udtClass: Class[_],
+ propagateNull: Boolean = true,
Review comment:
That's because of
[SPARK-26991](https://issues.apache.org/jira/browse/SPARK-26991) - the method
doesn't have argument if the value of `nullable` from both of Scala and Java
side are same, have argument otherwise.
If we want to be consistent, we can add `nullable` to all methods
consistently: ideal situation would be like we investigate SPARK-26991
successfully, and remove that argument and use one way consistently between
Scala and Java side. I just failed to investigate SPARK-26991 so can't do that.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
