Github user zsxwing commented on a diff in the pull request:
https://github.com/apache/spark/pull/10609#discussion_r49122800
--- Diff:
streaming/src/test/scala/org/apache/spark/streaming/StateMapSuite.scala ---
@@ -321,4 +336,59 @@ class StateMapSuite extends SparkFunSuite {
}
}
}
+
+ test("OpenHashMapBasedStateMap - serializing and deserializing with
KryoSerializable states") {
+ val map = new OpenHashMapBasedStateMap[KryoState, KryoState]()
+ map.put(new KryoState("a"), new KryoState("b"), 1)
+ testSerialization(
+ new KryoSerializer(conf), map, "error deserializing and serialized
KryoSerializable states")
+ }
+
+ test("EmptyStateMap - serializing and deserializing") {
+ val map = StateMap.empty[KryoState, KryoState]
+ // Since EmptyStateMap doesn't contains any date, KryoState won't
break JavaSerializer.
+ assert(serializeAndDeserialize(new JavaSerializer(conf), map).
+ isInstanceOf[EmptyStateMap[KryoState, KryoState]])
+ assert(serializeAndDeserialize(new KryoSerializer(conf), map).
+ isInstanceOf[EmptyStateMap[KryoState, KryoState]])
+ }
+
+ test("MapWithStateRDDRecord - serializing and deserializing with
KryoSerializable states") {
+ val map = new OpenHashMapBasedStateMap[KryoState, KryoState]()
+ map.put(new KryoState("a"), new KryoState("b"), 1)
+
+ val record =
+ MapWithStateRDDRecord[KryoState, KryoState, KryoState](map, Seq(new
KryoState("c")))
+ val deserRecord = serializeAndDeserialize(new KryoSerializer(conf),
record)
+ assert(!(record eq deserRecord))
+ assert(record.stateMap.getAll().toSeq ===
deserRecord.stateMap.getAll().toSeq)
+ assert(record.mappedData === deserRecord.mappedData)
+ }
+
+ private def serializeAndDeserialize[T: ClassTag](serializer: Serializer,
t: T): T = {
--- End diff --
Yes. `deserialize` needs the ClassTag
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
