srowen commented on a change in pull request #25470: [SPARK-28751][Core][WIP]
Improve java serializer deserialization performance
URL: https://github.com/apache/spark/pull/25470#discussion_r319106976
##########
File path:
core/src/test/scala/org/apache/spark/serializer/JavaSerializerSuite.scala
##########
@@ -18,26 +18,114 @@
package org.apache.spark.serializer
import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.internal.config.JAVA_SERIALIZER_CACHE_RESOLVED_CLASSES
class JavaSerializerSuite extends SparkFunSuite {
- test("JavaSerializer instances are serializable") {
- val serializer = new JavaSerializer(new SparkConf())
+ test("JavaSerializer(without resolve class cache) instances are
serializable") {
+ val serializer = getSerializer(false)
val instance = serializer.newInstance()
val obj =
instance.deserialize[JavaSerializer](instance.serialize(serializer))
// enforce class cast
obj.getClass
}
- test("Deserialize object containing a primitive Class as attribute") {
- val serializer = new JavaSerializer(new SparkConf())
+ test("JavaSerializer(with resolve class cache) instances are serializable") {
+ val serializer = getSerializer(true)
+ val instance = serializer.newInstance()
+ val obj =
instance.deserialize[JavaSerializer](instance.serialize(serializer))
+ // enforce class cast
+ obj.getClass
+ }
+
+ test("Deserialize(without resolve class cache) object containing a primitive
" +
+ "Class as attribute") {
+ val serializer = getSerializer(false)
+ val instance = serializer.newInstance()
+ val obj = instance.deserialize[ContainsPrimitiveClass](instance.serialize(
+ new ContainsPrimitiveClass()))
+ // enforce class cast
+ obj.getClass
+ }
+
+ test("Deserialize(with resolve class cache) object containing a primitive " +
+ "Class as attribute") {
+ val serializer = getSerializer(true)
val instance = serializer.newInstance()
val obj = instance.deserialize[ContainsPrimitiveClass](instance.serialize(
new ContainsPrimitiveClass()))
// enforce class cast
obj.getClass
}
+
+ test ("Deserialize(without resolve class cache) object with provider
ClassLoader") {
+ val serializer = getSerializer(false)
+ val instance = serializer.newInstance()
+ deserializeWithClassLoader(instance)
+ deserializeWithClassLoader(instance)
+ }
+
+ test ("Deserialize(with resolve class cache) object with provider
ClassLoader") {
+ val serializer = getSerializer(true)
+ val instance = serializer.newInstance()
+ deserializeWithClassLoader(instance)
+ deserializeWithClassLoader(instance)
+ }
+
+ private def deserializeWithClassLoader(instance: SerializerInstance): Unit =
{
+ val myClass1Instance1 = new MyClass1
+ val myClass2Instance1 = new MyClass2
+ val bytes1 = instance.serialize(myClass1Instance1)
+ val bytes2 = instance.serialize(myClass2Instance1)
+ val deserialized1 = instance.deserialize[MyClass1](bytes1, loader1)
+ val deserialized2 = instance.deserialize[MyClass2](bytes2, loader2)
+ deserialized1.getClass
+ deserialized2.getClass
+
+ bytes1.rewind()
+ bytes2.rewind()
+
+ val thrown1 = intercept[ClassNotFoundException] {
+ instance.deserialize(bytes1, loader2)
+ }
+ assert(thrown1.getMessage.contains("ClassLoader2 can't load class"))
+
+ val thrown2 = intercept[ClassNotFoundException] {
+ instance.deserialize(bytes2, loader1)
+ }
+ assert(thrown2.getMessage.contains("ClassLoader1 can't load class"))
+ }
+
+ def getSerializer(cache: Boolean): JavaSerializer = {
+ val conf = new SparkConf()
+ conf.set(JAVA_SERIALIZER_CACHE_RESOLVED_CLASSES, cache)
+
+ new JavaSerializer(conf)
+ }
+
+ lazy val loader1 = new ClassLoader(null) {
+ override def loadClass(name: String): Class[_] = {
+ if (classOf[MyClass1].getName == name) {
+ classOf[MyClass1]
+ } else {
+ throw new ClassNotFoundException(s"ClassLoader1 can't load class
:${name}")
+ }
+ }
+ }
+
+ lazy val loader2 = new ClassLoader(null) {
Review comment:
Do these need to be lazy, to defer instantiation?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
