This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
commit dd31736c0ca55dfe3842a6ad920d85f4301c563f Author: beliefer <[email protected]> AuthorDate: Mon Mar 2 15:14:47 2020 +0900 [SPARK-30908][CORE][DOC] Add version information to the configuration of Kryo ### What changes were proposed in this pull request? 1.Add version information to the configuration of `Kryo`. 2.Update the docs of `Kryo`. I sorted out some information show below. Item name | Since version | JIRA ID | Commit ID | Note -- | -- | -- | -- | -- spark.kryo.registrationRequired | 1.1.0 | SPARK-2102 | efdaeb111917dd0314f1d00ee8524bed1e2e21ca#diff-1f81c62dad0e2dfc387a974bb08c497c | spark.kryo.registrator | 0.5.0 | None | 91c07a33d90ab0357e8713507134ecef5c14e28a#diff-792ed56b3398163fa14e8578549d0d98 | This is not a release version, do we need to record it? spark.kryo.classesToRegister | 1.2.0 | SPARK-1813 | 6bb56faea8d238ea22c2de33db93b1b39f492b3a#diff-529fc5c06b9731c1fbda6f3db60b16aa | spark.kryo.unsafe | 2.1.0 | SPARK-928 | bc167a2a53f5a795d089e8a884569b1b3e2cd439#diff-1f81c62dad0e2dfc387a974bb08c497c | spark.kryo.pool | 3.0.0 | SPARK-26466 | 38f030725c561979ca98b2a6cc7ca6c02a1f80ed#diff-a3c6b992784f9abeb9f3047d3dcf3ed9 | spark.kryo.referenceTracking | 0.8.0 | None | 0a8cc309211c62f8824d76618705c817edcf2424#diff-1f81c62dad0e2dfc387a974bb08c497c | spark.kryoserializer.buffer | 1.4.0 | SPARK-5932 | 2d222fb39dd978e5a33cde6ceb59307cbdf7b171#diff-1f81c62dad0e2dfc387a974bb08c497c | spark.kryoserializer.buffer.max | 1.4.0 | SPARK-5932 | 2d222fb39dd978e5a33cde6ceb59307cbdf7b171#diff-1f81c62dad0e2dfc387a974bb08c497c | ### Why are the changes needed? Supplemental configuration version information. ### Does this PR introduce any user-facing change? No ### How was this patch tested? Exists UT Closes #27734 from beliefer/add-version-to-kryo-config. Authored-by: beliefer <[email protected]> Signed-off-by: HyukjinKwon <[email protected]> --- core/src/main/scala/org/apache/spark/internal/config/Kryo.scala | 8 ++++++++ docs/configuration.md | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala b/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala index 717a099..646d855 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/Kryo.scala @@ -22,35 +22,43 @@ import org.apache.spark.network.util.ByteUnit private[spark] object Kryo { val KRYO_REGISTRATION_REQUIRED = ConfigBuilder("spark.kryo.registrationRequired") + .version("1.1.0") .booleanConf .createWithDefault(false) val KRYO_USER_REGISTRATORS = ConfigBuilder("spark.kryo.registrator") + .version("0.5.0") .stringConf .createOptional val KRYO_CLASSES_TO_REGISTER = ConfigBuilder("spark.kryo.classesToRegister") + .version("1.2.0") .stringConf .toSequence .createWithDefault(Nil) val KRYO_USE_UNSAFE = ConfigBuilder("spark.kryo.unsafe") + .version("2.1.0") .booleanConf .createWithDefault(false) val KRYO_USE_POOL = ConfigBuilder("spark.kryo.pool") + .version("3.0.0") .booleanConf .createWithDefault(true) val KRYO_REFERENCE_TRACKING = ConfigBuilder("spark.kryo.referenceTracking") + .version("0.8.0") .booleanConf .createWithDefault(true) val KRYO_SERIALIZER_BUFFER_SIZE = ConfigBuilder("spark.kryoserializer.buffer") + .version("1.4.0") .bytesConf(ByteUnit.KiB) .createWithDefaultString("64k") val KRYO_SERIALIZER_MAX_BUFFER_SIZE = ConfigBuilder("spark.kryoserializer.buffer.max") + .version("1.4.0") .bytesConf(ByteUnit.MiB) .createWithDefaultString("64m") diff --git a/docs/configuration.md b/docs/configuration.md index 295cb3d..c586ecf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1204,7 +1204,7 @@ Apart from these, the following properties are also available, and may be useful ### Compression and Serialization <table class="table"> -<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr> +<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr> <tr> <td><code>spark.broadcast.compress</code></td> <td>true</td> @@ -1278,6 +1278,7 @@ Apart from these, the following properties are also available, and may be useful with Kryo. See the <a href="tuning.html#data-serialization">tuning guide</a> for more details. </td> + <th>1.2.0</th> </tr> <tr> <td><code>spark.kryo.referenceTracking</code></td> @@ -1288,6 +1289,7 @@ Apart from these, the following properties are also available, and may be useful copies of the same object. Can be disabled to improve performance if you know this is not the case. </td> + <th>0.8.0</th> </tr> <tr> <td><code>spark.kryo.registrationRequired</code></td> @@ -1299,6 +1301,7 @@ Apart from these, the following properties are also available, and may be useful significant performance overhead, so enabling this option can enforce strictly that a user has not omitted classes from registration. </td> + <th>1.1.0</th> </tr> <tr> <td><code>spark.kryo.registrator</code></td> @@ -1312,6 +1315,7 @@ Apart from these, the following properties are also available, and may be useful <code>KryoRegistrator</code></a>. See the <a href="tuning.html#data-serialization">tuning guide</a> for more details. </td> + <th>0.5.0</th> </tr> <tr> <td><code>spark.kryo.unsafe</code></td> @@ -1320,6 +1324,7 @@ Apart from these, the following properties are also available, and may be useful Whether to use unsafe based Kryo serializer. Can be substantially faster by using Unsafe Based IO. </td> + <th>2.1.0</th> </tr> <tr> <td><code>spark.kryoserializer.buffer.max</code></td> @@ -1329,6 +1334,7 @@ Apart from these, the following properties are also available, and may be useful This must be larger than any object you attempt to serialize and must be less than 2048m. Increase this if you get a "buffer limit exceeded" exception inside Kryo. </td> + <th>1.4.0</th> </tr> <tr> <td><code>spark.kryoserializer.buffer</code></td> @@ -1338,6 +1344,7 @@ Apart from these, the following properties are also available, and may be useful Note that there will be one buffer <i>per core</i> on each worker. This buffer will grow up to <code>spark.kryoserializer.buffer.max</code> if needed. </td> + <th>1.4.0</th> </tr> <tr> <td><code>spark.rdd.compress</code></td> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
