This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a566099133ff [SPARK-45756][CORE] Support
`spark.master.useAppNameAsAppId.enabled`
a566099133ff is described below
commit a566099133ff38cd1b2cd2fe64879bf0ba75fa9b
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Thu Nov 9 18:34:43 2023 -0800
[SPARK-45756][CORE] Support `spark.master.useAppNameAsAppId.enabled`
### What changes were proposed in this pull request?
This PR aims to support `spark.master.useAppNameAsAppId.enabled` as an
experimental feature in Spark Standalone cluster.
### Why are the changes needed?
This allows the users to control the appID completely.
<img width="359" alt="Screenshot 2023-11-09 at 5 33 45 PM"
src="https://github.com/apache/spark/assets/9700541/ad2b89ce-9d7d-4144-bd52-f29b94051103";>
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Manual tests with the following procedure.
```
$ SPARK_MASTER_OPTS="-Dspark.master.useAppNameAsAppId.enabled=true"
sbin/start-master.sh
$ bin/spark-shell --master spark://max.local:7077
```
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #43743 from dongjoon-hyun/SPARK-45756.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../scala/org/apache/spark/deploy/master/Master.scala | 7 ++++++-
.../scala/org/apache/spark/internal/config/package.scala | 8 ++++++++
.../org/apache/spark/deploy/master/MasterSuite.scala | 16 ++++++++++++++++
3 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index dbb647252c5f..b3fbec1830e4 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -120,6 +120,7 @@ private[deploy] class Master(
private val defaultCores = conf.get(DEFAULT_CORES)
val reverseProxy = conf.get(UI_REVERSE_PROXY)
val historyServerUrl = conf.get(MASTER_UI_HISTORY_SERVER_URL)
+ val useAppNameAsAppId = conf.get(MASTER_USE_APP_NAME_AS_APP_ID)
// Alternative application submission gateway that is stable across Spark
versions
private val restServerEnabled = conf.get(MASTER_REST_SERVER_ENABLED)
@@ -1041,7 +1042,11 @@ private[deploy] class Master(
ApplicationInfo = {
val now = System.currentTimeMillis()
val date = new Date(now)
- val appId = newApplicationId(date)
+ val appId = if (useAppNameAsAppId) {
+ desc.name.toLowerCase().replaceAll("\\s+", "")
+ } else {
+ newApplicationId(date)
+ }
new ApplicationInfo(now, appId, desc, date, driver, defaultCores)
}
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala
b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index bbadf91fc41c..b2bf30863a91 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -1846,6 +1846,14 @@ package object config {
.stringConf
.createOptional
+ private[spark] val MASTER_USE_APP_NAME_AS_APP_ID =
+ ConfigBuilder("spark.master.useAppNameAsAppId.enabled")
+ .internal()
+ .doc("(Experimental) If true, Spark master uses the user-provided
appName for appId.")
+ .version("4.0.0")
+ .booleanConf
+ .createWithDefault(false)
+
private[spark] val IO_COMPRESSION_SNAPPY_BLOCKSIZE =
ConfigBuilder("spark.io.compression.snappy.blockSize")
.doc("Block size in bytes used in Snappy compression, in the case when "
+
diff --git
a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
index 4f8457f930e4..2e54673649c7 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
@@ -804,6 +804,7 @@ class MasterSuite extends SparkFunSuite
private val _state = PrivateMethod[RecoveryState.Value](Symbol("state"))
private val _newDriverId = PrivateMethod[String](Symbol("newDriverId"))
private val _newApplicationId =
PrivateMethod[String](Symbol("newApplicationId"))
+ private val _createApplication =
PrivateMethod[ApplicationInfo](Symbol("createApplication"))
private val workerInfo = makeWorkerInfo(4096, 10)
private val workerInfos = Array(workerInfo, workerInfo, workerInfo)
@@ -1275,6 +1276,21 @@ class MasterSuite extends SparkFunSuite
assert(master.invokePrivate(_newApplicationId(submitDate)) === s"${i %
1000}")
}
}
+
+ test("SPARK-45756: Use appName for appId") {
+ val conf = new SparkConf()
+ .set(MASTER_USE_APP_NAME_AS_APP_ID, true)
+ val master = makeMaster(conf)
+ val desc = new ApplicationDescription(
+ name = " spark - 45756 ",
+ maxCores = None,
+ command = null,
+ appUiUrl = "",
+ defaultProfile = DeployTestUtils.defaultResourceProfile,
+ eventLogDir = None,
+ eventLogCodec = None)
+ assert(master.invokePrivate(_createApplication(desc, null)).id ===
"spark-45756")
+ }
}
private class FakeRecoveryModeFactory(conf: SparkConf, ser:
serializer.Serializer)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
