This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 408b5834e53 [SPARK-42071][CORE] Register `scala.math.Ordering$Reverse`
to KyroSerializer
408b5834e53 is described below
commit 408b5834e533bad0a620583498500a2526aaa529
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sun Jan 15 01:09:54 2023 -0800
[SPARK-42071][CORE] Register `scala.math.Ordering$Reverse` to KyroSerializer
### What changes were proposed in this pull request?
This PR aims to register `scala.math.Ordering$Reverse` to KyroSerializer.
### Why are the changes needed?
Scala 2.12.12 added a new class 'Reverse' via
https://github.com/scala/scala/pull/8965. This affects Apache Spark 3.2.0+.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs with newly added test case.
Closes #39578 from dongjoon-hyun/SPARK-42071.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit e3c0fbeadfe5242fa6265cb0646d72d3b5f6ef35)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../scala/org/apache/spark/serializer/KryoSerializer.scala | 1 +
.../org/apache/spark/serializer/KryoSerializerSuite.scala | 14 +++++++++++++-
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index ce45762d20e..6653bf654c4 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -219,6 +219,7 @@ class KryoSerializer(conf: SparkConf)
kryo.register(Nil.getClass)
kryo.register(Utils.classForName("scala.collection.immutable.$colon$colon"))
kryo.register(Utils.classForName("scala.collection.immutable.Map$EmptyMap$"))
+ kryo.register(Utils.classForName("scala.math.Ordering$Reverse"))
kryo.register(classOf[ArrayBuffer[Any]])
// We can't load those class directly in order to avoid unnecessary jar
dependencies.
diff --git
a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index dd2340a8672..6ab491545de 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -38,7 +38,7 @@ import
org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage
import org.apache.spark.scheduler.HighlyCompressedMapStatus
import org.apache.spark.serializer.KryoTest._
import org.apache.spark.storage.BlockManagerId
-import org.apache.spark.util.ThreadUtils
+import org.apache.spark.util.{ThreadUtils, Utils}
import org.apache.spark.util.collection.OpenHashMap
class KryoSerializerSuite extends SparkFunSuite with SharedSparkContext {
@@ -172,6 +172,18 @@ class KryoSerializerSuite extends SparkFunSuite with
SharedSparkContext {
mutable.HashMap(1 -> "one", 2 -> "two", 3 -> "three")))
}
+ test("SPARK-42071: Register scala.math.Ordering$Reverse") {
+ val conf = new SparkConf(false)
+ conf.set(KRYO_REGISTRATION_REQUIRED, true)
+
+ val ser = new KryoSerializer(conf).newInstance()
+ def check[T: ClassTag](t: T): Unit = {
+ assert(ser.deserialize[T](ser.serialize(t)) === t)
+ }
+ // Scala 2.12.12 added a new class 'Reverse' via
https://github.com/scala/scala/pull/8965
+ check(Utils.classForName("scala.math.Ordering$Reverse"))
+ }
+
test("Bug: SPARK-10251") {
val ser = new KryoSerializer(conf.clone.set(KRYO_REGISTRATION_REQUIRED,
true))
.newInstance()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
