Repository: spark Updated Branches: refs/heads/branch-1.1 e77fa81a2 -> 5081b0da0
[SPARK-2856] Decrease initial buffer size for Kryo to 64KB. Author: Reynold Xin <[email protected]> Closes #1780 from rxin/kryo-init-size and squashes the following commits: 551b935 [Reynold Xin] [SPARK-2856] Decrease initial buffer size for Kryo to 64KB. (cherry picked from commit 184048f80b6fa160c89d5bb47b937a0a89534a95) Signed-off-by: Reynold Xin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5081b0da Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5081b0da Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5081b0da Branch: refs/heads/branch-1.1 Commit: 5081b0da06232f2221c1e9d600f95a66f06ea357 Parents: e77fa81 Author: Reynold Xin <[email protected]> Authored: Tue Aug 5 01:30:46 2014 -0700 Committer: Reynold Xin <[email protected]> Committed: Tue Aug 5 16:24:04 2014 -0700 ---------------------------------------------------------------------- .../main/scala/org/apache/spark/serializer/KryoSerializer.scala | 4 +++- docs/configuration.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/5081b0da/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala index e60b802..407cb9d 100644 --- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala +++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala @@ -47,7 +47,9 @@ class KryoSerializer(conf: SparkConf) with Logging with Serializable { - private val bufferSize = conf.getInt("spark.kryoserializer.buffer.mb", 2) * 1024 * 1024 + private val bufferSize = + (conf.getDouble("spark.kryoserializer.buffer.mb", 0.064) * 1024 * 1024).toInt + private val maxBufferSize = conf.getInt("spark.kryoserializer.buffer.max.mb", 64) * 1024 * 1024 private val referenceTracking = conf.getBoolean("spark.kryo.referenceTracking", true) private val registrationRequired = conf.getBoolean("spark.kryo.registrationRequired", false) http://git-wip-us.apache.org/repos/asf/spark/blob/5081b0da/docs/configuration.md ---------------------------------------------------------------------- diff --git a/docs/configuration.md b/docs/configuration.md index 6ae453d..5e7556c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -420,7 +420,7 @@ Apart from these, the following properties are also available, and may be useful </tr> <tr> <td><code>spark.kryoserializer.buffer.mb</code></td> - <td>2</td> + <td>0.064</td> <td> Initial size of Kryo's serialization buffer, in megabytes. Note that there will be one buffer <i>per core</i> on each worker. This buffer will grow up to --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
