[spark] branch master updated: [SPARK-42063][CORE] Register `byte[][]` to KyroSerializer

[yumwang]

This is an automated email from the ASF dual-hosted git repository.

yumwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new e6c01cedbec [SPARK-42063][CORE] Register `byte[][]` to KyroSerializer
e6c01cedbec is described below

commit e6c01cedbecebdc40863e552862beee1bd959710
Author: Dongjoon Hyun <dongj...@apache.org>
AuthorDate: Sat Jan 14 21:45:43 2023 +0800

    [SPARK-42063][CORE] Register `byte[][]` to KyroSerializer
    
    ### What changes were proposed in this pull request?
    
    This PR aims to register `byte[][]` to `KryoSerializer`.
    
    ### Why are the changes needed?
    
    Spark uses `byte[][]` internally and it causes failure when we use 
`spark.kryo.registrationRequired=true`.
    ```
    java.lang.IllegalArgumentException: Class is not registered: byte[][]
    Note: To register this class use: kryo.register(byte[][].class);
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added test case.
    
    Closes #39569 from dongjoon-hyun/SPARK-42063.
    
    Authored-by: Dongjoon Hyun <dongj...@apache.org>
    Signed-off-by: Yuming Wang <yumw...@ebay.com>
---
 core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala    | 1 +
 .../test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala    | 2 ++
 2 files changed, 3 insertions(+)

diff --git 
a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala 
b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index ce45762d20e..0f40cd809d2 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -220,6 +220,7 @@ class KryoSerializer(conf: SparkConf)
     
kryo.register(Utils.classForName("scala.collection.immutable.$colon$colon"))
     
kryo.register(Utils.classForName("scala.collection.immutable.Map$EmptyMap$"))
     kryo.register(classOf[ArrayBuffer[Any]])
+    kryo.register(classOf[Array[Array[Byte]]])
 
     // We can't load those class directly in order to avoid unnecessary jar 
dependencies.
     // We load them safely, ignore it if the class not found.
diff --git 
a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala 
b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
index dd2340a8672..8c4ba583070 100644
--- a/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/serializer/KryoSerializerSuite.scala
@@ -104,6 +104,7 @@ class KryoSerializerSuite extends SparkFunSuite with 
SharedSparkContext {
     check(java.lang.Long.MAX_VALUE)
     check(java.lang.Long.MIN_VALUE)
     check[String](null)
+    check(Array(1.toByte))
     check(Array(1, 2, 3))
     check(Array(1L, 2L, 3L))
     check(Array(1.0, 2.0, 3.0))
@@ -114,6 +115,7 @@ class KryoSerializerSuite extends SparkFunSuite with 
SharedSparkContext {
     check(Array('a', 'b', 'c'))
     check(Array.empty[Int])
     check(Array(Array("1", "2"), Array("1", "2", "3", "4")))
+    check(Array(Array(1.toByte)))
   }
 
   test("pairs") {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org