cloud-fan commented on a change in pull request #25470: [SPARK-28751][Core]
Imporve java serializer deserialization performance
URL: https://github.com/apache/spark/pull/25470#discussion_r314566092
##########
File path: core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala
##########
@@ -58,19 +59,32 @@ private[spark] class JavaSerializationStream(
def close() { objOut.close() }
}
-private[spark] class JavaDeserializationStream(in: InputStream, loader:
ClassLoader)
- extends DeserializationStream {
+private[spark] class JavaDeserializationStream(
+ in: InputStream,
+ loader: ClassLoader,
+ useCache: Boolean,
+ serializerInstance: JavaSerializerInstance) extends DeserializationStream {
private val objIn = new ObjectInputStream(in) {
- override def resolveClass(desc: ObjectStreamClass): Class[_] =
+ override def resolveClass(desc: ObjectStreamClass): Class[_] = {
+ if (useCache) {
+ serializerInstance.resolvedClassesCache.computeIfAbsent(
+ (desc.getName, loader), pair => normalResolve(pair._1))
+ } else {
+ normalResolve(desc.getName)
+ }
+ }
+
+ private def normalResolve(name: String): Class[_] = {
try {
// scalastyle:off classforname
- Class.forName(desc.getName, false, loader)
+ Class.forName(name, false, loader)
Review comment:
@rednaxelafx does JVM classloader have cache?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
