Repository: spark Updated Branches: refs/heads/master 1665b5f72 -> 06c054411
[SPARK-20981][SPARKSUBMIT] Add new configuration spark.jars.repositories as equivalence of --repositories ## What changes were proposed in this pull request? In our use case of launching Spark applications via REST APIs (Livy), there's no way for user to specify command line arguments, all Spark configurations are set through configurations map. For "--repositories" because there's no equivalent Spark configuration, so we cannot specify the custom repository through configuration. So here propose to add "--repositories" equivalent configuration in Spark. ## How was this patch tested? New UT added. Author: jerryshao <[email protected]> Closes #18201 from jerryshao/SPARK-20981. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/06c05441 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/06c05441 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/06c05441 Branch: refs/heads/master Commit: 06c0544113ba77857c5cb1bbf94dcaf21d0b01af Parents: 1665b5f Author: jerryshao <[email protected]> Authored: Mon Jun 5 11:06:50 2017 -0700 Committer: Marcelo Vanzin <[email protected]> Committed: Mon Jun 5 11:06:50 2017 -0700 ---------------------------------------------------------------------- .../spark/deploy/SparkSubmitArguments.scala | 2 ++ .../apache/spark/deploy/SparkSubmitSuite.scala | 20 ++++++++++++++++++++ docs/configuration.md | 13 +++++++++++-- 3 files changed, 33 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/06c05441/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 5100a17..b76a3d2 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -187,6 +187,8 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S packages = Option(packages).orElse(sparkProperties.get("spark.jars.packages")).orNull packagesExclusions = Option(packagesExclusions) .orElse(sparkProperties.get("spark.jars.excludes")).orNull + repositories = Option(repositories) + .orElse(sparkProperties.get("spark.jars.repositories")).orNull deployMode = Option(deployMode) .orElse(sparkProperties.get("spark.submit.deployMode")) .orElse(env.get("DEPLOY_MODE")) http://git-wip-us.apache.org/repos/asf/spark/blob/06c05441/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index 6e9721c..de71999 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -477,6 +477,26 @@ class SparkSubmitSuite } } + test("includes jars passed through spark.jars.packages and spark.jars.repositories") { + val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + val main = MavenCoordinate("my.great.lib", "mylib", "0.1") + val dep = MavenCoordinate("my.great.dep", "mylib", "0.1") + // Test using "spark.jars.packages" and "spark.jars.repositories" configurations. + IvyTestUtils.withRepository(main, Some(dep.toString), None) { repo => + val args = Seq( + "--class", JarCreationTest.getClass.getName.stripSuffix("$"), + "--name", "testApp", + "--master", "local-cluster[2,1,1024]", + "--conf", "spark.jars.packages=my.great.lib:mylib:0.1,my.great.dep:mylib:0.1", + "--conf", s"spark.jars.repositories=$repo", + "--conf", "spark.ui.enabled=false", + "--conf", "spark.master.rest.enabled=false", + unusedJar.toString, + "my.great.lib.MyLib", "my.great.dep.MyLib") + runSparkSubmit(args) + } + } + // TODO(SPARK-9603): Building a package is flaky on Jenkins Maven builds. // See https://gist.github.com/shivaram/3a2fecce60768a603dac for a error log ignore("correctly builds R packages included in a jar with --packages") { http://git-wip-us.apache.org/repos/asf/spark/blob/06c05441/docs/configuration.md ---------------------------------------------------------------------- diff --git a/docs/configuration.md b/docs/configuration.md index 0771e36..f777811 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -474,11 +474,20 @@ Apart from these, the following properties are also available, and may be useful <td> Path to an Ivy settings file to customize resolution of jars specified using <code>spark.jars.packages</code> instead of the built-in defaults, such as maven central. Additional repositories given by the command-line - option <code>--repositories</code> will also be included. Useful for allowing Spark to resolve artifacts from behind - a firewall e.g. via an in-house artifact server like Artifactory. Details on the settings file format can be + option <code>--repositories</code> or <code>spark.jars.repositories</code> will also be included. + Useful for allowing Spark to resolve artifacts from behind a firewall e.g. via an in-house + artifact server like Artifactory. Details on the settings file format can be found at http://ant.apache.org/ivy/history/latest-milestone/settings.html </td> </tr> + <tr> + <td><code>spark.jars.repositories</code></td> + <td></td> + <td> + Comma-separated list of additional remote repositories to search for the maven coordinates + given with <code>--packages</code> or <code>spark.jars.packages</code>. + </td> +</tr> <tr> <td><code>spark.pyspark.driver.python</code></td> <td></td> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
