ConeyLiu commented on a change in pull request #25470: [SPARK-28751][Core]
Improve java serializer deserialization performance
URL: https://github.com/apache/spark/pull/25470#discussion_r315054629
##########
File path: core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala
##########
@@ -58,19 +59,32 @@ private[spark] class JavaSerializationStream(
def close() { objOut.close() }
}
-private[spark] class JavaDeserializationStream(in: InputStream, loader:
ClassLoader)
- extends DeserializationStream {
+private[spark] class JavaDeserializationStream(
+ in: InputStream,
+ loader: ClassLoader,
+ useCache: Boolean,
+ serializerInstance: JavaSerializerInstance) extends DeserializationStream {
private val objIn = new ObjectInputStream(in) {
- override def resolveClass(desc: ObjectStreamClass): Class[_] =
+ override def resolveClass(desc: ObjectStreamClass): Class[_] = {
+ if (useCache) {
+ serializerInstance.resolvedClassesCache.computeIfAbsent(
+ (desc.getName, loader), pair => normalResolve(pair._1))
+ } else {
+ normalResolve(desc.getName)
+ }
+ }
+
+ private def normalResolve(name: String): Class[_] = {
try {
// scalastyle:off classforname
- Class.forName(desc.getName, false, loader)
+ Class.forName(name, false, loader)
Review comment:
Class.forName seems is not free. I have done a simple test in local.
```scala
object ResolveClass {
def main(args: Array[String]): Unit = {
var i = 0
while (i < 10) {
Class.forName(Person.getClass.getName)
i += 1
}
val start = System.currentTimeMillis()
i = 0
while (i < 100000) {
Class.forName(Person.getClass.getName)
i += 1
}
println(System.currentTimeMillis() - start)
}
}
```
it needs about 196 ms. The map just need the less than half time.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
