This is an automated email from the ASF dual-hosted git repository.
ggal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-livy.git
The following commit(s) were added to refs/heads/master by this push:
new cb484ea6 [LIVY-975][SERVER] Fix compatibility issue related to a
change on the Spark side (#403)
cb484ea6 is described below
commit cb484ea66d80381ba28d6b3bdc84d1828084c84d
Author: Asif Khatri <[email protected]>
AuthorDate: Tue May 9 13:11:28 2023 +0530
[LIVY-975][SERVER] Fix compatibility issue related to a change on the Spark
side (#403)
## What changes were proposed in this pull request?
Recent change in spark required us to fix compatibility issue related to a
change on the Spark side.
## How was this patch tested?
Verified manually by creating interactive / batch sessions via REST API
call in a local Yarn cluster. Also, we have updated the unit tests.
---
conf/livy.conf.template | 4 ++++
.../src/main/scala/org/apache/livy/LivyConf.scala | 2 ++
.../scala/org/apache/livy/sessions/Session.scala | 9 ++++++-
.../org/apache/livy/sessions/SessionSpec.scala | 28 ++++++++++++++++++++++
4 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/conf/livy.conf.template b/conf/livy.conf.template
index edec95a0..aedc6329 100644
--- a/conf/livy.conf.template
+++ b/conf/livy.conf.template
@@ -188,3 +188,7 @@
# livy.server.auth.<custom>.class = <class of custom auth filter>
# livy.server.auth.<custom>.param.<foo1> = <bar1>
# livy.server.auth.<custom>.param.<foo2> = <bar2>
+
+# Enable to allow custom classpath by proxy user in cluster mode
+# The below configuration parameter is disabled by default.
+# livy.server.session.allow-custom-classpath = true
diff --git a/server/src/main/scala/org/apache/livy/LivyConf.scala
b/server/src/main/scala/org/apache/livy/LivyConf.scala
index 5808ba46..51179e13 100644
--- a/server/src/main/scala/org/apache/livy/LivyConf.scala
+++ b/server/src/main/scala/org/apache/livy/LivyConf.scala
@@ -264,6 +264,8 @@ object LivyConf {
// Max creating session in livyServer
val SESSION_MAX_CREATION = Entry("livy.server.session.max-creation", 100)
+ val SESSION_ALLOW_CUSTOM_CLASSPATH =
Entry("livy.server.session.allow-custom-classpath", false)
+
val SPARK_MASTER = "spark.master"
val SPARK_DEPLOY_MODE = "spark.submit.deployMode"
val SPARK_JARS = "spark.jars"
diff --git a/server/src/main/scala/org/apache/livy/sessions/Session.scala
b/server/src/main/scala/org/apache/livy/sessions/Session.scala
index 197e22f9..6b01c4f1 100644
--- a/server/src/main/scala/org/apache/livy/sessions/Session.scala
+++ b/server/src/main/scala/org/apache/livy/sessions/Session.scala
@@ -34,6 +34,9 @@ import org.apache.livy.utils.AppInfo
object Session {
trait RecoveryMetadata { val id: Int }
+ val BLACKLIST_CUSTOM_CLASSPATH: Set[String] = Set("spark.submit.deployMode",
+ "spark.submit.proxyUser.allowCustomClasspathInClusterMode")
+
lazy val configBlackList: Set[String] = {
val url = getClass.getResource("/spark-blacklist.conf")
if (url != null) Utils.loadProperties(url).keySet else Set()
@@ -58,7 +61,11 @@ object Session {
return Map()
}
- val errors = conf.keySet.filter(configBlackList.contains)
+ val errors = if
(livyConf.getBoolean(LivyConf.SESSION_ALLOW_CUSTOM_CLASSPATH)) {
+ conf.keySet.filter(configBlackList.contains)
+ } else {
+ conf.keySet.filter((configBlackList ++
BLACKLIST_CUSTOM_CLASSPATH).contains)
+ }
if (errors.nonEmpty) {
throw new IllegalArgumentException(
"Blacklisted configuration values in session config: " +
errors.mkString(", "))
diff --git a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
index 05b41bb0..aa5d4563 100644
--- a/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
+++ b/server/src/test/scala/org/apache/livy/sessions/SessionSpec.scala
@@ -65,6 +65,18 @@ class SessionSpec extends FunSuite with
LivyBaseUnitTestSuite {
intercept[IllegalArgumentException] {
Session.prepareConf(Map("spark.do_not_set" -> "1"), Nil, Nil, Nil, Nil,
conf)
}
+
+ // Test for "spark.submit.deployMode".
+ intercept[IllegalArgumentException] {
+ Session.prepareConf(Map("spark.submit.deployMode" -> "standalone"), Nil,
Nil, Nil, Nil, conf)
+ }
+
+ // Test for "spark.submit.proxyUser.allowCustomClasspathInClusterMode".
+ intercept[IllegalArgumentException] {
+
Session.prepareConf(Map("spark.submit.proxyUser.allowCustomClasspathInClusterMode"
+ -> "false"), Nil, Nil, Nil, Nil, conf)
+ }
+
conf.sparkFileLists.foreach { key =>
intercept[IllegalArgumentException] {
Session.prepareConf(Map(key -> "file:/not_allowed"), Nil, Nil, Nil,
Nil, conf)
@@ -95,4 +107,20 @@ class SessionSpec extends FunSuite with
LivyBaseUnitTestSuite {
userLists.foreach { key => assert(result.get(key) === expected) }
}
+
+ test("conf validation to allow custom classpath") {
+ val conf = new LivyConf(false)
+ conf.set(LivyConf.SESSION_ALLOW_CUSTOM_CLASSPATH, true)
+
+ // Test for "spark.submit.deployMode".
+ assert(Session.prepareConf(Map("spark.submit.deployMode" -> "standalone"),
Nil, Nil, Nil,
+ Nil, conf) === Map("spark.submit.deployMode" -> "standalone",
"spark.master" -> "local"))
+
+ // Test for "spark.submit.proxyUser.allowCustomClasspathInClusterMode".
+
assert(Session.prepareConf(Map("spark.submit.proxyUser.allowCustomClasspathInClusterMode"
->
+ "false"), Nil, Nil, Nil, Nil, conf) === Map("spark.master" -> "local",
+ "spark.submit.proxyUser.allowCustomClasspathInClusterMode" -> "false"))
+
+ }
+
}
