This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
commit 5017bd6e42ef8b0d4dcf7eabf4293da1eecef16e Author: beliefer <[email protected]> AuthorDate: Thu Mar 5 11:58:21 2020 +0900 [SPARK-30889][SPARK-30913][CORE][DOC] Add version information to the configuration of Tests.scala and Worker 1.Add version information to the configuration of `Tests` and `Worker`. 2.Update the docs of `Worker`. I sorted out some information of `Tests` show below. Item name | Since version | JIRA ID | Commit ID | Note -- | -- | -- | -- | -- spark.testing.memory | 1.6.0 | SPARK-10983 | b3ffac5178795f2d8e7908b3e77e8e89f50b5f6f#diff-395d07dcd46359cca610ce74357f0bb4 | spark.testing.dynamicAllocation.scheduleInterval | 2.3.0 | SPARK-22864 | 4e9e6aee44bb2ddb41b567d659358b22fd824222#diff-b096353602813e47074ace09a3890d56 | spark.testing | 1.0.1 | SPARK-1606 | ce57624b8232159fe3ec6db228afc622133df591#diff-d239aee594001f8391676e1047a0381e | spark.test.noStageRetry | 1.2.0 | SPARK-3796 | f55218aeb1e9d638df6229b36a59a15ce5363482#diff-6a9ff7fb74fd490a50462d45db2d5e11 | spark.testing.reservedMemory | 1.6.0 | SPARK-12081 | 84c44b500b5c90dffbe1a6b0aa86f01699b09b96#diff-395d07dcd46359cca610ce74357f0bb4 | spark.testing.nHosts | 3.0.0 | SPARK-26491 | 1a641525e60039cc6b10816e946cb6f44b3e2696#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 | spark.testing.nExecutorsPerHost | 3.0.0 | SPARK-26491 | 1a641525e60039cc6b10816e946cb6f44b3e2696#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 | spark.testing.nCoresPerExecutor | 3.0.0 | SPARK-26491 | 1a641525e60039cc6b10816e946cb6f44b3e2696#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 | spark.resources.warnings.testing | 3.1.0 | SPARK-29148 | 496f6ac86001d284cbfb7488a63dd3a168919c0f#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 | spark.testing.resourceProfileManager | 3.1.0 | SPARK-29148 | 496f6ac86001d284cbfb7488a63dd3a168919c0f#diff-8b4ea8f3b0cc1e7ce7e943de1abbb165 | I sorted out some information of `Worker` show below. Item name | Since version | JIRA ID | Commit ID | Note -- | -- | -- | -- | -- spark.worker.resourcesFile | 3.0.0 | SPARK-27369 | 7cbe01e8efc3f6cd3a0cac4bcfadea8fcc74a955#diff-b2fc8d6ab7ac5735085e2d6cfacb95da | spark.worker.timeout | 0.6.2 | None | e395aa295aeec6767df798bf1002b1f30983c1cd#diff-776a630ac2b2ec5fe85c07ca20a58fc0 | spark.worker.driverTerminateTimeout | 2.1.2 | SPARK-20843 | ebd72f453aa0b4f68760d28b3e93e6dd33856659#diff-829a8674171f92acd61007bedb1bfa4f | spark.worker.cleanup.enabled | 1.0.0 | SPARK-1154 | 1440154c27ca48b5a75103eccc9057286d3f6ca8#diff-916ca56b663f178f302c265b7ef38499 | spark.worker.cleanup.interval | 1.0.0 | SPARK-1154 | 1440154c27ca48b5a75103eccc9057286d3f6ca8#diff-916ca56b663f178f302c265b7ef38499 | spark.worker.cleanup.appDataTtl | 1.0.0 | SPARK-1154 | 1440154c27ca48b5a75103eccc9057286d3f6ca8#diff-916ca56b663f178f302c265b7ef38499 | spark.worker.preferConfiguredMasterAddress | 2.2.1 | SPARK-20529 | 75e5ea294c15ecfb7366ae15dce196aa92c87ca4#diff-916ca56b663f178f302c265b7ef38499 | spark.worker.ui.port | 1.1.0 | SPARK-2857 | 12f99cf5f88faf94d9dbfe85cb72d0010a3a25ac#diff-48ca297b6536cb92362bec1487581f05 | spark.worker.ui.retainedExecutors | 1.5.0 | SPARK-9202 | c0686668ae6a92b6bb4801a55c3b78aedbee816a#diff-916ca56b663f178f302c265b7ef38499 | spark.worker.ui.retainedDrivers | 1.5.0 | SPARK-9202 | c0686668ae6a92b6bb4801a55c3b78aedbee816a#diff-916ca56b663f178f302c265b7ef38499 | spark.worker.ui.compressedLogFileLengthCacheSize | 2.0.2 | SPARK-17711 | 26e978a93f029e1a1b5c7524d0b52c8141b70997#diff-d239aee594001f8391676e1047a0381e | spark.worker.decommission.enabled | 3.1.0 | SPARK-20628 | d273a2bb0fac452a97f5670edd69d3e452e3e57e#diff-b2fc8d6ab7ac5735085e2d6cfacb95da | Supplemental configuration version information. No Exists UT Closes #27783 from beliefer/add-version-to-tests-config. Authored-by: beliefer <[email protected]> Signed-off-by: HyukjinKwon <[email protected]> --- .../scala/org/apache/spark/internal/config/Tests.scala | 8 ++++++++ .../scala/org/apache/spark/internal/config/Worker.scala | 15 +++++++++++++-- docs/configuration.md | 4 +++- docs/spark-standalone.md | 10 ++++++++-- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala index 21660ab..232264d6 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/Tests.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/Tests.scala @@ -22,35 +22,43 @@ private[spark] object Tests { val TEST_USE_COMPRESSED_OOPS_KEY = "spark.test.useCompressedOops" val TEST_MEMORY = ConfigBuilder("spark.testing.memory") + .version("1.6.0") .longConf .createWithDefault(Runtime.getRuntime.maxMemory) val TEST_SCHEDULE_INTERVAL = ConfigBuilder("spark.testing.dynamicAllocation.scheduleInterval") + .version("2.3.0") .longConf .createWithDefault(100) val IS_TESTING = ConfigBuilder("spark.testing") + .version("1.0.1") .booleanConf .createOptional val TEST_NO_STAGE_RETRY = ConfigBuilder("spark.test.noStageRetry") + .version("1.2.0") .booleanConf .createWithDefault(false) val TEST_RESERVED_MEMORY = ConfigBuilder("spark.testing.reservedMemory") + .version("1.6.0") .longConf .createOptional val TEST_N_HOSTS = ConfigBuilder("spark.testing.nHosts") + .version("3.0.0") .intConf .createWithDefault(5) val TEST_N_EXECUTORS_HOST = ConfigBuilder("spark.testing.nExecutorsPerHost") + .version("3.0.0") .intConf .createWithDefault(4) val TEST_N_CORES_EXECUTOR = ConfigBuilder("spark.testing.nCoresPerExecutor") + .version("3.0.0") .intConf .createWithDefault(2) } diff --git a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala index f1eaae2..a807271 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/Worker.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/Worker.scala @@ -28,47 +28,58 @@ private[spark] object Worker { .doc("Path to a file containing the resources allocated to the worker. " + "The file should be formatted as a JSON array of ResourceAllocation objects. " + "Only used internally in standalone mode.") + .version("3.0.0") .stringConf .createOptional val WORKER_TIMEOUT = ConfigBuilder("spark.worker.timeout") + .version("0.6.2") .longConf .createWithDefault(60) val WORKER_DRIVER_TERMINATE_TIMEOUT = ConfigBuilder("spark.worker.driverTerminateTimeout") + .version("2.1.2") .timeConf(TimeUnit.MILLISECONDS) .createWithDefaultString("10s") val WORKER_CLEANUP_ENABLED = ConfigBuilder("spark.worker.cleanup.enabled") + .version("1.0.0") .booleanConf .createWithDefault(false) val WORKER_CLEANUP_INTERVAL = ConfigBuilder("spark.worker.cleanup.interval") + .version("1.0.0") .longConf .createWithDefault(60 * 30) val APP_DATA_RETENTION = ConfigBuilder("spark.worker.cleanup.appDataTtl") + .version("1.0.0") .longConf .createWithDefault(7 * 24 * 3600) val PREFER_CONFIGURED_MASTER_ADDRESS = ConfigBuilder("spark.worker.preferConfiguredMasterAddress") + .version("2.2.1") .booleanConf .createWithDefault(false) val WORKER_UI_PORT = ConfigBuilder("spark.worker.ui.port") + .version("1.1.0") .intConf .createOptional val WORKER_UI_RETAINED_EXECUTORS = ConfigBuilder("spark.worker.ui.retainedExecutors") + .version("1.5.0") .intConf .createWithDefault(1000) val WORKER_UI_RETAINED_DRIVERS = ConfigBuilder("spark.worker.ui.retainedDrivers") + .version("1.5.0") .intConf .createWithDefault(1000) val UNCOMPRESSED_LOG_FILE_LENGTH_CACHE_SIZE_CONF = ConfigBuilder("spark.worker.ui.compressedLogFileLengthCacheSize") - .intConf - .createWithDefault(100) + .version("2.0.2") + .intConf + .createWithDefault(100) } diff --git a/docs/configuration.md b/docs/configuration.md index 9cbe341..b336289 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -929,7 +929,7 @@ Apart from these, the following properties are also available, and may be useful ### Spark UI <table class="table"> -<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr> +<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr> <tr> <td><code>spark.eventLog.logBlockUpdates.enabled</code></td> <td>false</td> @@ -1153,6 +1153,7 @@ Apart from these, the following properties are also available, and may be useful <td> How many finished executors the Spark UI and status APIs remember before garbage collecting. </td> + <td>1.5.0</td> </tr> <tr> <td><code>spark.worker.ui.retainedDrivers</code></td> @@ -1160,6 +1161,7 @@ Apart from these, the following properties are also available, and may be useful <td> How many finished drivers the Spark UI and status APIs remember before garbage collecting. </td> + <td>1.5.0</td> </tr> <tr> <td><code>spark.sql.ui.retainedExecutions</code></td> diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md index 17b6772..4d4b85e 100644 --- a/docs/spark-standalone.md +++ b/docs/spark-standalone.md @@ -185,7 +185,7 @@ You can optionally configure the cluster further by setting environment variable SPARK_MASTER_OPTS supports the following system properties: <table class="table"> -<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr> +<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr> <tr> <td><code>spark.deploy.retainedApplications</code></td> <td>200</td> @@ -242,6 +242,7 @@ SPARK_MASTER_OPTS supports the following system properties: Number of seconds after which the standalone deploy master considers a worker lost if it receives no heartbeats. </td> + <td>0.6.2</td> </tr> <tr> <td><code>spark.worker.resource.{resourceName}.amount</code></td> @@ -269,13 +270,14 @@ SPARK_MASTER_OPTS supports the following system properties: find that resource. If the discovery script also does not find the resources, the worker will fail to start up. </td> + <td>3.0.0</td> </tr> </table> SPARK_WORKER_OPTS supports the following system properties: <table class="table"> -<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr> +<tr><th>Property Name</th><th>Default</th><th>Meaning</th><th>Since Version</th></tr> <tr> <td><code>spark.worker.cleanup.enabled</code></td> <td>false</td> @@ -284,6 +286,7 @@ SPARK_WORKER_OPTS supports the following system properties: mode, as YARN works differently. Only the directories of stopped applications are cleaned up. This should be enabled if spark.shuffle.service.db.enabled is "true" </td> + <td>1.0.0</td> </tr> <tr> <td><code>spark.worker.cleanup.interval</code></td> @@ -292,6 +295,7 @@ SPARK_WORKER_OPTS supports the following system properties: Controls the interval, in seconds, at which the worker cleans up old application work dirs on the local machine. </td> + <td>1.0.0</td> </tr> <tr> <td><code>spark.worker.cleanup.appDataTtl</code></td> @@ -302,6 +306,7 @@ SPARK_WORKER_OPTS supports the following system properties: downloaded to each application work dir. Over time, the work dirs can quickly fill up disk space, especially if you run jobs very frequently. </td> + <td>1.0.0</td> </tr> <tr> <td><code>spark.shuffle.service.db.enabled</code></td> @@ -333,6 +338,7 @@ SPARK_WORKER_OPTS supports the following system properties: Spark caches the uncompressed file size of compressed log files. This property controls the cache size. </td> + <td>2.0.2</td> </tr> </table> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
