Github user jerryshao commented on a diff in the pull request:
https://github.com/apache/spark/pull/18901#discussion_r132443619
--- Diff:
core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala ---
@@ -250,6 +250,72 @@ class SparkSubmitSuite
sysProps("SPARK_SUBMIT") should be ("true")
}
+ test("handles YARN cluster mode with remote user jar") {
+ val hadoopConf = new Configuration()
+ // Set hdfs implementation to local file system for testing.
+ hadoopConf.set("fs.hdfs.impl",
"org.apache.spark.deploy.TestFileSystem")
+ // Disable file system impl cache to make sure the test file system is
picked up.
+ hadoopConf.set("fs.hdfs.impl.disable.cache", "true")
+ val jarFile = File.createTempFile("thejar", ".jar")
+ jarFile.deleteOnExit()
+ val clArgs = Seq(
+ "--deploy-mode", "cluster",
+ "--master", "yarn",
+ "--executor-memory", "5g",
+ "--class", "org.SomeClass",
+ "--jars", "one.jar,two.jar,three.jar",
+ "--driver-memory", "4g",
+ s"hdfs://${jarFile.getAbsolutePath}")
+ val appArgs = new SparkSubmitArguments(clArgs)
+ val (childArgs, classpath, sysProps, mainClass) =
prepareSubmitEnvironment(appArgs, hadoopConf)
+ val childArgsStr = childArgs.mkString(" ")
+ childArgsStr should include ("--class org.SomeClass")
+ childArgsStr should include regex ("--jar .*thejar.*.jar")
+ mainClass should be ("org.apache.spark.deploy.yarn.Client")
+
+ // In yarn cluster mode, also adding remote jars to classpath
+ classpath(0) should endWith regex ("thejar.*.jar")
+ classpath(1) should endWith ("one.jar")
+ classpath(2) should endWith ("two.jar")
+ classpath(3) should endWith ("three.jar")
+
+ sysProps("spark.executor.memory") should be ("5g")
+ sysProps("spark.driver.memory") should be ("4g")
--- End diff --
I don't think it is necessary to test other unrelated things, they're
already covered in other tests.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
