HeartSaVioR commented on a change in pull request #24465:
[SPARK-27575][CORE][YARN] Yarn file-related confs should merge new value with
existing value
URL: https://github.com/apache/spark/pull/24465#discussion_r279178447
##########
File path: core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
##########
@@ -897,6 +897,74 @@ class SparkSubmitSuite
}
}
+ test("SPARK-27575: yarn confs should merge new value with existing value") {
+ val tmpJarDir = Utils.createTempDir()
+ val jar1 = TestUtils.createJarWithFiles(Map("test.resource" -> "1"),
tmpJarDir)
+ val jar2 = TestUtils.createJarWithFiles(Map("test.resource" -> "USER"),
tmpJarDir)
+
+ val tmpJarDirYarnOpt = Utils.createTempDir()
+ val jar1YarnOpt = TestUtils.createJarWithFiles(Map("test.resource" ->
"2"), tmpJarDirYarnOpt)
+ val jar2YarnOpt = TestUtils.createJarWithFiles(Map("test.resource" ->
"USER2"),
+ tmpJarDirYarnOpt)
+
+ val tmpFileDir = Utils.createTempDir()
+ val file1 = File.createTempFile("tmpFile1", "", tmpFileDir)
+ val file2 = File.createTempFile("tmpFile2", "", tmpFileDir)
+
+ val tmpFileDirYarnOpt = Utils.createTempDir()
+ val file1YarnOpt = File.createTempFile("tmpPy1YarnOpt", ".py",
tmpFileDirYarnOpt)
+ val file2YarnOpt = File.createTempFile("tmpPy2YarnOpt", ".egg",
tmpFileDirYarnOpt)
+
+ val tmpPyFileDir = Utils.createTempDir()
+ val pyFile1 = File.createTempFile("tmpPy1", ".py", tmpPyFileDir)
+ val pyFile2 = File.createTempFile("tmpPy2", ".egg", tmpPyFileDir)
+
+ val tmpPyFileDirYarnOpt = Utils.createTempDir()
+ val pyFile1YarnOpt = File.createTempFile("tmpPy1YarnOpt", ".py",
tmpPyFileDirYarnOpt)
+ val pyFile2YarnOpt = File.createTempFile("tmpPy2YarnOpt", ".egg",
tmpPyFileDirYarnOpt)
+
+ val tmpArchiveDir = Utils.createTempDir()
+ val archive1 = File.createTempFile("archive1", ".zip", tmpArchiveDir)
+ val archive2 = File.createTempFile("archive2", ".zip", tmpArchiveDir)
+
+ val tmpArchiveDirYarnOpt = Utils.createTempDir()
+ val archive1YarnOpt = File.createTempFile("archive1YarnOpt", ".zip",
tmpArchiveDirYarnOpt)
+ val archive2YarnOpt = File.createTempFile("archive2YarnOpt", ".zip",
tmpArchiveDirYarnOpt)
+
+ val tempPyFile = File.createTempFile("tmpApp", ".py")
+ tempPyFile.deleteOnExit()
+
+ val args = Seq(
+ "--class", UserClasspathFirstTest.getClass.getName.stripPrefix("$"),
+ "--name", "testApp",
+ "--master", "yarn",
+ "--deploy-mode", "client",
+ "--jars", s"${tmpJarDir.getAbsolutePath}/*.jar",
+ "--files", s"${tmpFileDir.getAbsolutePath}/tmpFile*",
+ "--py-files", s"${tmpPyFileDir.getAbsolutePath}/tmpPy*",
+ "--archives", s"${tmpArchiveDir.getAbsolutePath}/*.zip",
+ "--conf", "spark.yarn.dist.files=" +
+ s"${Seq(file1YarnOpt,
file2YarnOpt).map(_.getAbsolutePath).mkString(",")}",
+ "--conf", "spark.yarn.dist.pyFiles=" +
+ s"${Seq(pyFile1YarnOpt,
pyFile2YarnOpt).map(_.getAbsolutePath).mkString(",")}",
+ "--conf", "spark.yarn.dist.jars=" +
+ s"${Seq(jar1YarnOpt,
jar2YarnOpt).map(_.toURI.toString).mkString(",")}",
+ "--conf", "spark.yarn.dist.archives=" +
+ s"${Seq(archive1YarnOpt,
archive2YarnOpt).map(_.toURI.toString).mkString(",")}",
+ tempPyFile.toURI().toString())
+
+ val appArgs = new SparkSubmitArguments(args)
+ val (_, _, conf, _) = submit.prepareSubmitEnvironment(appArgs)
+ conf.get("spark.yarn.dist.jars").split(",").toSet should be
+ (Set(Seq(jar1, jar2, jar1YarnOpt,
jar2YarnOpt).map(_.toURI.toString).toList))
Review comment:
Actually I also prefer assert in this case: just didn't want to touch
existing approach. I'll make a change given we both think it's better.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
