Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/19130#discussion_r138981630
--- Diff:
core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala ---
@@ -897,6 +897,76 @@ class SparkSubmitSuite
sysProps("spark.submit.pyFiles") should (startWith("/"))
}
+ test("download remote resource if it is not supported by yarn service") {
+ testRemoteResources(isHttpSchemeBlacklisted = false, supportMockHttpFs
= false)
+ }
+
+ test("avoid downloading remote resource if it is supported by yarn
service") {
+ testRemoteResources(isHttpSchemeBlacklisted = false, supportMockHttpFs
= true)
+ }
+
+ test("force downloading remote resource if it's scheme is configured") {
+ testRemoteResources(isHttpSchemeBlacklisted = true, supportMockHttpFs
= true)
+ }
+
+ private def testRemoteResources(
+ isHttpSchemeBlacklisted: Boolean, supportMockHttpFs: Boolean): Unit
= {
+ val hadoopConf = new Configuration()
+ updateConfWithFakeS3Fs(hadoopConf)
+ if (supportMockHttpFs) {
+ hadoopConf.set("fs.http.impl",
classOf[TestFileSystem].getCanonicalName)
+ hadoopConf.set("fs.http.impl.disable.cache", "true")
+ }
+
+ val tmpDir = Utils.createTempDir()
+ val mainResource = File.createTempFile("tmpPy", ".py", tmpDir)
+ val tmpS3Jar = TestUtils.createJarWithFiles(Map("test.resource" ->
"USER"), tmpDir)
+ val tmpS3JarPath = s"s3a://${new File(tmpS3Jar.toURI).getAbsolutePath}"
+ val tmpHttpJar = TestUtils.createJarWithFiles(Map("test.resource" ->
"USER"), tmpDir)
+ val tmpHttpJarPath = s"http://${new
File(tmpHttpJar.toURI).getAbsolutePath}"
+
+ val args = Seq(
+ "--class", UserClasspathFirstTest.getClass.getName.stripPrefix("$"),
+ "--name", "testApp",
+ "--master", "yarn",
+ "--deploy-mode", "client",
+ "--jars", s"$tmpS3JarPath,$tmpHttpJarPath",
+ s"s3a://$mainResource"
+ ) ++ (
+ if (isHttpSchemeBlacklisted) {
+ Seq("--conf", "spark.yarn.dist.forceDownloadSchemes=http,https")
+ } else {
+ Nil
+ }
+ )
+
+ sys.props.put("spark.testing", "1")
--- End diff --
This should already be set by the build scripts, was it not working for you?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
