Repository: spark Updated Branches: refs/heads/master d92cd227c -> fa6347938
[SPARK-17178][SPARKR][SPARKSUBMIT] Allow to set sparkr shell command through --conf ## What changes were proposed in this pull request? Allow user to set sparkr shell command through --conf spark.r.shell.command ## How was this patch tested? Unit test is added and also verify it manually through ``` bin/sparkr --master yarn-client --conf spark.r.shell.command=/usr/local/bin/R ``` Author: Jeff Zhang <[email protected]> Closes #14744 from zjffdu/SPARK-17178. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fa634793 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fa634793 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fa634793 Branch: refs/heads/master Commit: fa6347938fc1c72ddc03a5f3cd2e929b5694f0a6 Parents: d92cd22 Author: Jeff Zhang <[email protected]> Authored: Wed Aug 31 00:20:41 2016 -0700 Committer: Felix Cheung <[email protected]> Committed: Wed Aug 31 00:20:41 2016 -0700 ---------------------------------------------------------------------- docs/configuration.md | 11 ++++++++++- .../org/apache/spark/launcher/SparkLauncher.java | 2 ++ .../spark/launcher/SparkSubmitCommandBuilder.java | 3 ++- .../launcher/SparkSubmitCommandBuilderSuite.java | 18 ++++++++++++++++++ 4 files changed, 32 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/fa634793/docs/configuration.md ---------------------------------------------------------------------- diff --git a/docs/configuration.md b/docs/configuration.md index d0c76aa..6e98f67 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1786,6 +1786,14 @@ showDF(properties, numRows = 200, truncate = FALSE) Executable for executing R scripts in client modes for driver. Ignored in cluster modes. </td> </tr> +<tr> + <td><code>spark.r.shell.command</code></td> + <td>R</td> + <td> + Executable for executing sparkR shell in client modes for driver. Ignored in cluster modes. It is the same as environment variable <code>SPARKR_DRIVER_R</code>, but take precedence over it. + <code>spark.r.shell.command</code> is used for sparkR shell while <code>spark.r.driver.command</code> is used for running R script. + </td> +</tr> </table> #### Deploy @@ -1852,7 +1860,8 @@ The following variables can be set in `spark-env.sh`: </tr> <tr> <td><code>SPARKR_DRIVER_R</code></td> - <td>R binary executable to use for SparkR shell (default is <code>R</code>).</td> + <td>R binary executable to use for SparkR shell (default is <code>R</code>). + Property <code>spark.r.shell.command</code> take precedence if it is set</td> </tr> <tr> <td><code>SPARK_LOCAL_IP</code></td> http://git-wip-us.apache.org/repos/asf/spark/blob/fa634793/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java ---------------------------------------------------------------------- diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java index 7b7a7bf..ea56214 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkLauncher.java @@ -68,6 +68,8 @@ public class SparkLauncher { static final String PYSPARK_PYTHON = "spark.pyspark.python"; + static final String SPARKR_R_SHELL = "spark.r.shell.command"; + /** Logger name to use when launching a child process. */ public static final String CHILD_PROCESS_LOGGER_NAME = "spark.launcher.childProcLoggerName"; http://git-wip-us.apache.org/repos/asf/spark/blob/fa634793/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java ---------------------------------------------------------------------- diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java index f6da644..29c6d82 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java @@ -336,7 +336,8 @@ class SparkSubmitCommandBuilder extends AbstractCommandBuilder { join(File.separator, sparkHome, "R", "lib", "SparkR", "profile", "shell.R")); List<String> args = new ArrayList<>(); - args.add(firstNonEmpty(System.getenv("SPARKR_DRIVER_R"), "R")); + args.add(firstNonEmpty(conf.get(SparkLauncher.SPARKR_R_SHELL), + System.getenv("SPARKR_DRIVER_R"), "R")); return args; } http://git-wip-us.apache.org/repos/asf/spark/blob/fa634793/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java ---------------------------------------------------------------------- diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java index 16e5a22..ad2e7a7 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java @@ -173,6 +173,24 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite { } @Test + public void testSparkRShell() throws Exception { + List<String> sparkSubmitArgs = Arrays.asList( + SparkSubmitCommandBuilder.SPARKR_SHELL, + "--master=foo", + "--deploy-mode=bar", + "--conf", "spark.r.shell.command=/usr/bin/R"); + + Map<String, String> env = new HashMap<>(); + List<String> cmd = buildCommand(sparkSubmitArgs, env); + assertEquals("/usr/bin/R", cmd.get(cmd.size() - 1)); + assertEquals( + String.format( + "\"%s\" \"foo\" \"%s\" \"bar\" \"--conf\" \"spark.r.shell.command=/usr/bin/R\" \"%s\"", + parser.MASTER, parser.DEPLOY_MODE, SparkSubmitCommandBuilder.SPARKR_SHELL_RESOURCE), + env.get("SPARKR_SUBMIT_ARGS")); + } + + @Test public void testExamplesRunner() throws Exception { List<String> sparkSubmitArgs = Arrays.asList( SparkSubmitCommandBuilder.RUN_EXAMPLE, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
