cloud-fan commented on code in PR #39186:
URL: https://github.com/apache/spark/pull/39186#discussion_r1056263501
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala:
##########
@@ -432,224 +305,96 @@ object ScalaReflection extends ScalaReflection {
* * the element type of [[Array]] or [[Seq]]: `array element class:
"abc.xyz.MyClass"`
* * the field of [[Product]]: `field (class: "abc.xyz.MyClass", name:
"myField")`
*/
- def serializerForType(tpe: `Type`): Expression =
ScalaReflection.cleanUpReflectionObjects {
- val clsName = getClassNameFromType(tpe)
- val walkedTypePath = new WalkedTypePath().recordRoot(clsName)
-
- // The input object to `ExpressionEncoder` is located at first column of
an row.
- val isPrimitive = tpe.typeSymbol.asClass.isPrimitive
- val inputObject = BoundReference(0, dataTypeFor(tpe), nullable =
!isPrimitive)
-
- serializerFor(inputObject, tpe, walkedTypePath)
- }
-
- /**
- * Returns an expression for serializing the value of an input expression
into Spark SQL
- * internal representation.
- */
- private def serializerFor(
- inputObject: Expression,
- tpe: `Type`,
- walkedTypePath: WalkedTypePath,
- seenTypeSet: Set[`Type`] = Set.empty): Expression =
cleanUpReflectionObjects {
-
- def toCatalystArray(input: Expression, elementType: `Type`): Expression = {
- dataTypeFor(elementType) match {
- case dt: ObjectType =>
- val clsName = getClassNameFromType(elementType)
- val newPath = walkedTypePath.recordArray(clsName)
- createSerializerForMapObjects(input, dt,
- serializerFor(_, elementType, newPath, seenTypeSet))
-
- case dt @ (BooleanType | ByteType | ShortType | IntegerType | LongType
|
- FloatType | DoubleType) =>
- val cls = input.dataType.asInstanceOf[ObjectType].cls
- if (cls.isArray && cls.getComponentType.isPrimitive) {
- createSerializerForPrimitiveArray(input, dt)
- } else {
- createSerializerForGenericArray(input, dt, nullable =
schemaFor(elementType).nullable)
- }
-
- case dt =>
- createSerializerForGenericArray(input, dt, nullable =
schemaFor(elementType).nullable)
- }
- }
- baseType(tpe) match {
- case _ if !inputObject.dataType.isInstanceOf[ObjectType] => inputObject
-
- case t if isSubtype(t, localTypeOf[Option[_]]) =>
- val TypeRef(_, _, Seq(optType)) = t
- val className = getClassNameFromType(optType)
- val newPath = walkedTypePath.recordOption(className)
- val unwrapped = UnwrapOption(dataTypeFor(optType), inputObject)
- serializerFor(unwrapped, optType, newPath, seenTypeSet)
-
- // Since List[_] also belongs to localTypeOf[Product], we put this case
before
- // "case t if definedByConstructorParams(t)" to make sure it will match
to the
- // case "localTypeOf[Seq[_]]"
- case t if isSubtype(t, localTypeOf[scala.collection.Seq[_]]) =>
- val TypeRef(_, _, Seq(elementType)) = t
- toCatalystArray(inputObject, elementType)
- case t if isSubtype(t, localTypeOf[Array[_]]) =>
- val TypeRef(_, _, Seq(elementType)) = t
- toCatalystArray(inputObject, elementType)
-
- case t if isSubtype(t, localTypeOf[Map[_, _]]) =>
- val TypeRef(_, _, Seq(keyType, valueType)) = t
- val keyClsName = getClassNameFromType(keyType)
- val valueClsName = getClassNameFromType(valueType)
- val keyPath = walkedTypePath.recordKeyForMap(keyClsName)
- val valuePath = walkedTypePath.recordValueForMap(valueClsName)
-
- createSerializerForMap(
- inputObject,
- MapElementInformation(
- dataTypeFor(keyType),
- nullable = !keyType.typeSymbol.asClass.isPrimitive,
- serializerFor(_, keyType, keyPath, seenTypeSet)),
- MapElementInformation(
- dataTypeFor(valueType),
- nullable = !valueType.typeSymbol.asClass.isPrimitive,
- serializerFor(_, valueType, valuePath, seenTypeSet))
- )
-
- case t if isSubtype(t, localTypeOf[scala.collection.Set[_]]) =>
- val TypeRef(_, _, Seq(elementType)) = t
-
+ // TODO doc
+ def serializerFor(enc: AgnosticEncoder[_]): Expression = {
+ val input = BoundReference(0, dataTypeFor(enc), nullable = enc.nullable)
+ serializerFor(enc, input)
+ }
+
+ private def serializerFor(enc: AgnosticEncoder[_], input: Expression):
Expression = enc match {
+ case _ if isNativeEncoder(enc) => input
+ case BooleanEncoder => createSerializerForBoolean(input)
Review Comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
