Updated Branches: refs/heads/master 3efc01956 -> 58d4f6c8a
Call Kryo setReferences before calling user specified Kryo registrator. Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/7c5f70d8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/7c5f70d8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/7c5f70d8 Branch: refs/heads/master Commit: 7c5f70d8739126d08f48fc6219421e4aea60cacd Parents: 3efc019 Author: Reynold Xin <[email protected]> Authored: Sat Nov 9 22:43:36 2013 -0800 Committer: Reynold Xin <[email protected]> Committed: Sat Nov 9 22:43:36 2013 -0800 ---------------------------------------------------------------------- .../apache/spark/serializer/KryoSerializer.scala | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/7c5f70d8/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala index 55b25f1..4f60f0b 100644 --- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala +++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala @@ -30,10 +30,14 @@ import org.apache.spark.broadcast.HttpBroadcast import org.apache.spark.storage.{GetBlock,GotBlock, PutBlock, StorageLevel, TestBlockId} /** - * A Spark serializer that uses the [[http://code.google.com/p/kryo/wiki/V1Documentation Kryo 1.x library]]. + * A Spark serializer that uses the + * [[http://code.google.com/p/kryo/wiki/V1Documentation Kryo 1.x library]]. */ class KryoSerializer extends org.apache.spark.serializer.Serializer with Logging { - private val bufferSize = System.getProperty("spark.kryoserializer.buffer.mb", "2").toInt * 1024 * 1024 + + private val bufferSize = { + System.getProperty("spark.kryoserializer.buffer.mb", "2").toInt * 1024 * 1024 + } def newKryoOutput() = new KryoOutput(bufferSize) @@ -42,6 +46,10 @@ class KryoSerializer extends org.apache.spark.serializer.Serializer with Logging val kryo = instantiator.newKryo() val classLoader = Thread.currentThread.getContextClassLoader + // Allow disabling Kryo reference tracking if user knows their object graphs don't have loops. + // Do this before we invoke the user registrator so the user registrator can override this. + kryo.setReferences(System.getProperty("spark.kryo.referenceTracking", "true").toBoolean) + val blockId = TestBlockId("1") // Register some commonly used classes val toRegister: Seq[AnyRef] = Seq( @@ -78,10 +86,6 @@ class KryoSerializer extends org.apache.spark.serializer.Serializer with Logging new AllScalaRegistrar().apply(kryo) kryo.setClassLoader(classLoader) - - // Allow disabling Kryo reference tracking if user knows their object graphs don't have loops - kryo.setReferences(System.getProperty("spark.kryo.referenceTracking", "true").toBoolean) - kryo }
