Github user marmbrus commented on a diff in the pull request:
https://github.com/apache/spark/pull/9802#discussion_r45235789
--- Diff: sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala ---
@@ -43,28 +43,48 @@ trait Encoder[T] extends Serializable {
*/
object Encoders {
- /**
- * (Scala-specific) Creates an encoder that serializes objects of type T
using Kryo.
- * This encoder maps T into a single byte array (binary) field.
- */
- def kryo[T: ClassTag]: Encoder[T] = {
- val ser = SerializeWithKryo(BoundReference(0,
ObjectType(classOf[AnyRef]), nullable = true))
- val deser = DeserializeWithKryo[T](BoundReference(0, BinaryType,
nullable = true), classTag[T])
+ private def genericSerializer[T: ClassTag](useKryo: Boolean): Encoder[T]
= {
ExpressionEncoder[T](
schema = new StructType().add("value", BinaryType),
flat = true,
- toRowExpressions = Seq(ser),
- fromRowExpression = deser,
+ toRowExpressions = Seq(
+ EncodeUsingSerializer(
+ BoundReference(0, ObjectType(classOf[AnyRef]), nullable = true),
kryo = useKryo)),
+ fromRowExpression =
+ DecodeUsingSerializer[T](
+ BoundReference(0, BinaryType, nullable = true), classTag[T],
kryo = useKryo),
clsTag = classTag[T]
)
}
/**
+ * (Scala-specific) Creates an encoder that serializes objects of type T
using Kryo.
+ * This encoder maps T into a single byte array (binary) field.
+ */
+ def kryo[T: ClassTag]: Encoder[T] = genericSerializer(useKryo = true)
+
+ /**
* Creates an encoder that serializes objects of type T using Kryo.
* This encoder maps T into a single byte array (binary) field.
*/
def kryo[T](clazz: Class[T]): Encoder[T] = kryo(ClassTag[T](clazz))
+ /**
+ * (Scala-specific) Creates an encoder that serializes objects of type T
using generic Java
+ * serialization. This encoder maps T into a single byte array (binary)
field.
+ *
+ * Note that this is extremely inefficient and should only be used as
the last resort.
+ */
+ def genericJava[T: ClassTag]: Encoder[T] = genericSerializer(useKryo =
false)
--- End diff --
`javaSerializer` `javaSerialization`? `generic` doesn't make it clear to
me what this is doing.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
