igozali commented on a change in pull request #25910: [SPARK-28762][CORE] Read
JAR main class if JAR is not located in local file system
URL: https://github.com/apache/spark/pull/25910#discussion_r335811062
##########
File path: core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
##########
@@ -453,6 +453,86 @@ class SparkSubmitSuite
conf.get("spark.kubernetes.driver.container.image") should be ("bar")
}
+ test("automatically sets mainClass if primary resource is S3 JAR in client
mode") {
+ withTempDir { tempDir =>
+ val excFile = TestUtils.createCompiledClass("SomeMainClass", tempDir,
"", null, Seq.empty)
+ val jarFile = new File(tempDir,
"s3-mainClass-test-%s.jar".format(System.currentTimeMillis()))
+ val jarUrl = TestUtils.createJar(
+ Seq(excFile),
+ jarFile,
+ directoryPrefix = Some(tempDir.toString),
+ mainClass = Some("SomeMainClass"))
+
+ val hadoopConf = new Configuration()
+ updateConfWithFakeS3Fs(hadoopConf)
+
+ val clArgs = Seq(
+ "--name", "testApp",
+ "--master", "yarn",
+ "--conf",
"spark.hadoop.fs.s3a.impl=org.apache.spark.deploy.TestFileSystem",
+ "--conf", "spark.hadoop.fs.s3a.impl.disable.cache=true",
+ s"s3a://${jarUrl.getPath}",
+ "arg1", "arg2")
+
+ val appArgs = new SparkSubmitArguments(clArgs)
+ val (childArgs, classpaths, _, mainClass_) =
submit.prepareSubmitEnvironment(
+ appArgs, conf = Some(hadoopConf))
+
+ mainClass_ should be ("SomeMainClass")
+ classpaths should have length 1
+ classpaths.head should endWith (jarFile.getName)
+ childArgs.mkString(" ") should be ("arg1 arg2")
+ }
+ }
+
+ test("automatically sets mainClass if primary resource is S3 JAR in cluster
mode") {
Review comment:
The assertions for both tests are slightly different, though. I will try
deduplicating the common logic in both tests.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
