Repository: spark Updated Branches: refs/heads/branch-1.1 822a0b42f -> d6b8d2c03
Revert "[SPARK-4882] Register PythonBroadcast with Kryo so that PySpark works with KryoSerializer" This reverts commit 822a0b42f79acea2771d0b298e803c11c37aab81. This fix does not apply to branch-1.1 or branch-1.0, since PythonBroadcast is new in 1.2. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d6b8d2c0 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d6b8d2c0 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d6b8d2c0 Branch: refs/heads/branch-1.1 Commit: d6b8d2c0307b5cc2bacd0c07139003821055ca69 Parents: 822a0b4 Author: Josh Rosen <[email protected]> Authored: Tue Dec 30 09:33:01 2014 -0800 Committer: Josh Rosen <[email protected]> Committed: Tue Dec 30 09:33:07 2014 -0800 ---------------------------------------------------------------------- .../spark/serializer/KryoSerializer.scala | 2 - .../spark/api/python/PythonBroadcastSuite.scala | 60 -------------------- 2 files changed, 62 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/d6b8d2c0/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala index 0c8bd62..87ef9bb 100644 --- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala +++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala @@ -26,7 +26,6 @@ import com.esotericsoftware.kryo.serializers.{JavaSerializer => KryoJavaSerializ import com.twitter.chill.{AllScalaRegistrar, EmptyScalaKryoInstantiator} import org.apache.spark._ -import org.apache.spark.api.python.PythonBroadcast import org.apache.spark.broadcast.HttpBroadcast import org.apache.spark.scheduler.MapStatus import org.apache.spark.storage._ @@ -80,7 +79,6 @@ class KryoSerializer(conf: SparkConf) // Allow sending SerializableWritable kryo.register(classOf[SerializableWritable[_]], new KryoJavaSerializer()) kryo.register(classOf[HttpBroadcast[_]], new KryoJavaSerializer()) - kryo.register(classOf[PythonBroadcast], new KryoJavaSerializer()) // Allow the user to register their own classes by setting spark.kryo.registrator for (regCls <- registrator) { http://git-wip-us.apache.org/repos/asf/spark/blob/d6b8d2c0/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala b/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala deleted file mode 100644 index 8959a84..0000000 --- a/core/src/test/scala/org/apache/spark/api/python/PythonBroadcastSuite.scala +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.api.python - -import scala.io.Source - -import java.io.{PrintWriter, File} - -import org.scalatest.{Matchers, FunSuite} - -import org.apache.spark.{SharedSparkContext, SparkConf} -import org.apache.spark.serializer.KryoSerializer -import org.apache.spark.util.Utils - -// This test suite uses SharedSparkContext because we need a SparkEnv in order to deserialize -// a PythonBroadcast: -class PythonBroadcastSuite extends FunSuite with Matchers with SharedSparkContext { - test("PythonBroadcast can be serialized with Kryo (SPARK-4882)") { - val tempDir = Utils.createTempDir() - val broadcastedString = "Hello, world!" - def assertBroadcastIsValid(broadcast: PythonBroadcast): Unit = { - val source = Source.fromFile(broadcast.path) - val contents = source.mkString - source.close() - contents should be (broadcastedString) - } - try { - val broadcastDataFile: File = { - val file = new File(tempDir, "broadcastData") - val printWriter = new PrintWriter(file) - printWriter.write(broadcastedString) - printWriter.close() - file - } - val broadcast = new PythonBroadcast(broadcastDataFile.getAbsolutePath) - assertBroadcastIsValid(broadcast) - val conf = new SparkConf().set("spark.kryo.registrationRequired", "true") - val deserializedBroadcast = - Utils.clone[PythonBroadcast](broadcast, new KryoSerializer(conf).newInstance()) - assertBroadcastIsValid(deserializedBroadcast) - } finally { - Utils.deleteRecursively(tempDir) - } - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
