This is an automated email from the ASF dual-hosted git repository.
gyogal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/livy.git
The following commit(s) were added to refs/heads/master by this push:
new 9e18a865 [LIVY-1061] Add livy.server.app-name-prefix for interactive
session YARN app names
9e18a865 is described below
commit 9e18a86524cdc7724946ba169f1769997153d3b8
Author: samuhasavak5 <[email protected]>
AuthorDate: Tue Jul 28 21:06:28 2026 +0530
[LIVY-1061] Add livy.server.app-name-prefix for interactive session YARN
app names
## What changes were proposed in this pull request?
This PR adds a new Livy server configuration property,
livy.server.app-name-prefix, to allow cluster administrators to customize the
default YARN application name for interactive Spark sessions.
Problem: When a client creates an interactive session via POST /sessions
without specifying a name, Livy sets spark.app.name to the hardcoded value
livy-session-<id>. In HA or multi-cluster environments, this makes it difficult
to distinguish Livy-managed applications in the YARN Resource Manager UI.
Solution:
Add LivyConf.SERVER_APP_NAME_PREFIX (livy.server.app-name-prefix, default:
livy-session)
Use the configured prefix when building the default spark.app.name in
InteractiveSession.create()
Document the new property in conf/livy.conf.template
Add unit tests for default and custom prefix behavior
Scope: Interactive sessions only. Batch sessions already support
per-request naming via the REST API name field and are unchanged.
Backward compatibility: When the property is unset, behavior remains
livy-session-<id>.
Example:
**livy.conf**
```
livy.server.app-name-prefix = my-cluster
```
Creating a session without a name yields YARN application name: my-cluster-3
JIRA: [LIVY-1061](https://issues.apache.org/jira/browse/LIVY-1061)
## How was this patch tested?
- Unit tests: Added tests in InteractiveSessionSpec for:
-Default prefix (livy-session) when config is not set
-Custom prefix (e.g. my-cluster) when livy.server.app-name-prefix is
configured
- Build: mvn package -Pspark3 -Pscala-2.12 — livy-server and dependent
modules built successfully
- Manual verification (recommended): Set livy.server.app-name-prefix in
livy.conf, create an interactive session without name, and confirm the YARN RM
UI shows <prefix>-<session-id>
No UI changes in this PR.
## Was this patch authored or co-authored using generative AI tooling?
Yes, this was co-authored using Cursor to help generate the new test cases.
---
conf/livy.conf.template | 4 ++++
.../src/main/scala/org/apache/livy/LivyConf.scala | 1 +
.../server/interactive/InteractiveSession.scala | 8 ++++++-
.../interactive/InteractiveSessionSpec.scala | 27 ++++++++++++++++++++++
4 files changed, 39 insertions(+), 1 deletion(-)
diff --git a/conf/livy.conf.template b/conf/livy.conf.template
index 30a3a1f4..f489d76f 100644
--- a/conf/livy.conf.template
+++ b/conf/livy.conf.template
@@ -37,6 +37,10 @@
# What port to start the server on.
# livy.server.port = 8998
+# Prefix for YARN application names of interactive sessions when no name is
provided
+# in the session creation request. The application name becomes
<prefix>-<session-id>.
+# livy.server.app-name-prefix = livy-session
+
# What base path ui should work on. By default UI is mounted on "/".
# E.g.: livy.ui.basePath = /my_livy - result in mounting UI on /my_livy/
# livy.ui.basePath = ""
diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala
b/server/src/main/scala/org/apache/livy/LivyConf.scala
index 46c851f3..03bdf0fc 100644
--- a/server/src/main/scala/org/apache/livy/LivyConf.scala
+++ b/server/src/main/scala/org/apache/livy/LivyConf.scala
@@ -62,6 +62,7 @@ object LivyConf {
val SERVER_HOST = Entry("livy.server.host", "0.0.0.0")
val SERVER_PORT = Entry("livy.server.port", 8998)
val SERVER_BASE_PATH = Entry("livy.ui.basePath", "")
+ val SERVER_APP_NAME_PREFIX = Entry("livy.server.app-name-prefix",
"livy-session")
val UI_ENABLED = Entry("livy.ui.enabled", true)
diff --git
a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
index e578326c..6af80133 100644
---
a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
+++
b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSession.scala
@@ -78,6 +78,12 @@ object InteractiveSession extends Logging {
val RECOVERY_SESSION_TYPE = "interactive"
+ private[interactive] def defaultSparkAppName(sessionId: Int, livyConf:
LivyConf): String = {
+ val prefix =
Option(livyConf.get(LivyConf.SERVER_APP_NAME_PREFIX)).filter(_.nonEmpty)
+ .getOrElse("livy-session")
+ s"$prefix-$sessionId"
+ }
+
def create(
id: Int,
name: Option[String],
@@ -114,7 +120,7 @@ object InteractiveSession extends Logging {
opt.foreach { value => builderProperties.put(key, value) }
}
- builderProperties.getOrElseUpdate("spark.app.name", s"livy-session-$id")
+ builderProperties.getOrElseUpdate("spark.app.name",
defaultSparkAppName(id, livyConf))
info(s"Creating Interactive session $id: [owner: $owner, request:
$request]")
val builder = new LivyClientBuilder()
diff --git
a/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
index 72a2a31d..c19e671b 100644
---
a/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
+++
b/server/src/test/scala/org/apache/livy/server/interactive/InteractiveSessionSpec.scala
@@ -323,6 +323,33 @@ class InteractiveSessionSpec extends FunSpec
}
}
+ describe("default spark application name") {
+ it("should use livy-session prefix by default") {
+ val conf = new LivyConf(false)
+ InteractiveSession.defaultSparkAppName(42, conf) should
be("livy-session-42")
+ }
+
+ it("should use configured app name prefix") {
+ val conf = new LivyConf(false)
+ .set(LivyConf.SERVER_APP_NAME_PREFIX, "my-cluster")
+ InteractiveSession.defaultSparkAppName(3, conf) should be("my-cluster-3")
+ }
+
+ it("should fall back to livy-session when prefix is empty") {
+ val conf = new LivyConf(false)
+ .set(LivyConf.SERVER_APP_NAME_PREFIX, "")
+ InteractiveSession.defaultSparkAppName(5, conf) should
be("livy-session-5")
+ }
+
+ it("should not override user-provided spark.app.name") {
+ val conf = new LivyConf(false)
+ .set(LivyConf.SERVER_APP_NAME_PREFIX, "my-cluster")
+ val props = scala.collection.mutable.Map("spark.app.name" ->
"user-provided-name")
+ props.getOrElseUpdate("spark.app.name",
InteractiveSession.defaultSparkAppName(1, conf))
+ props("spark.app.name") should be("user-provided-name")
+ }
+ }
+
describe("InteractiveSession") {
it("should inherit the default YARN queue from LivyConf when request queue
is empty") {
val testLivyConf = new LivyConf()
