Github user ifilonenko commented on a diff in the pull request:
https://github.com/apache/spark/pull/21092#discussion_r194117385
--- Diff:
resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala
---
@@ -88,15 +90,42 @@ class KubernetesConfSuite extends SparkFunSuite {
APP_NAME,
RESOURCE_NAME_PREFIX,
APP_ID,
- None,
+ mainAppResource = None,
MAIN_CLASS,
- APP_ARGS)
+ APP_ARGS,
+ maybePyFiles = None)
assert(kubernetesConfWithoutMainJar.sparkConf.get("spark.jars").split(",")
=== Array("local:///opt/spark/jar1.jar"))
+
assert(kubernetesConfWithoutMainJar.sparkConf.get(MEMORY_OVERHEAD_FACTOR) ===
0.1)
}
- test("Resolve driver labels, annotations, secret mount paths, and
envs.") {
+ test("Creating driver conf with a python primary file") {
+ val mainResourceFile = "local:///opt/spark/main.py"
+ val inputPyFiles = Array("local:///opt/spark/example2.py",
"local:///example3.py")
val sparkConf = new SparkConf(false)
+ .setJars(Seq("local:///opt/spark/jar1.jar"))
+ .set("spark.files", "local:///opt/spark/example4.py")
+ val mainAppResource = Some(PythonMainAppResource(mainResourceFile))
+ val kubernetesConfWithMainResource = KubernetesConf.createDriverConf(
+ sparkConf,
+ APP_NAME,
+ RESOURCE_NAME_PREFIX,
+ APP_ID,
+ mainAppResource,
+ MAIN_CLASS,
+ APP_ARGS,
+ Some(inputPyFiles.mkString(",")))
+
assert(kubernetesConfWithMainResource.sparkConf.get("spark.jars").split(",")
+ === Array("local:///opt/spark/jar1.jar"))
+
assert(kubernetesConfWithMainResource.sparkConf.get(MEMORY_OVERHEAD_FACTOR) ===
0.4)
--- End diff --
Done
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
