This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3363c2af3f6a [MINOR][CORE] Validate spark.deploy.defaultCores value
during config setting
3363c2af3f6a is described below
commit 3363c2af3f6a59363135451d251f25e328a4fddf
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Sat Nov 4 00:23:33 2023 -0700
[MINOR][CORE] Validate spark.deploy.defaultCores value during config setting
### What changes were proposed in this pull request?
This aims to move `spark.deploy.defaultCores` validation logic to config
setting.
### Why are the changes needed?
In order to ensure the value range by early checking.
### Does this PR introduce _any_ user-facing change?
No. `Spark Master` will fail to start in both cases, *before* and *after*.
### How was this patch tested?
Manual review.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #43655 from dongjoon-hyun/spark.deploy.defaultCores.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
core/src/main/scala/org/apache/spark/deploy/master/Master.scala | 5 +----
core/src/main/scala/org/apache/spark/internal/config/Deploy.scala | 1 +
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 63d981c5fde8..e63d72ebb40d 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -24,7 +24,7 @@ import java.util.concurrent.{ScheduledFuture, TimeUnit}
import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
import scala.util.Random
-import org.apache.spark.{SecurityManager, SparkConf, SparkException}
+import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.{ApplicationDescription, DriverDescription,
ExecutorState}
import org.apache.spark.deploy.DeployMessages._
import org.apache.spark.deploy.master.DriverState.DriverState
@@ -119,9 +119,6 @@ private[deploy] class Master(
// Default maxCores for applications that don't specify it (i.e. pass
Int.MaxValue)
private val defaultCores = conf.get(DEFAULT_CORES)
val reverseProxy = conf.get(UI_REVERSE_PROXY)
- if (defaultCores < 1) {
- throw new SparkException(s"${DEFAULT_CORES.key} must be positive")
- }
val historyServerUrl = conf.get(MASTER_UI_HISTORY_SERVER_URL)
// Alternative application submission gateway that is stable across Spark
versions
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
b/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
index 906ec0fc9973..7b35e92022ae 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
@@ -73,6 +73,7 @@ private[spark] object Deploy {
val DEFAULT_CORES = ConfigBuilder("spark.deploy.defaultCores")
.version("0.9.0")
.intConf
+ .checkValue(_ > 0, "spark.deploy.defaultCores must be positive.")
.createWithDefault(Int.MaxValue)
val MAX_DRIVERS = ConfigBuilder("spark.deploy.maxDrivers")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
