Repository: incubator-zeppelin Updated Branches: refs/heads/master c7645abee -> c9db780a1
[ZEPPELIN-17] PySpark Interpreter should allow starting with a specific version of Python Add PYSPARK_PYTHON. We could also add PYSPARK_DRIVER_PYTHON (Zeppelin Interpreter is the driver) but it doesn't seem to be documented, and more importantly running different Python between driver and worker can cause errors. http://spark.apache.org/docs/1.3.0/configuration.html http://spark.apache.org/docs/1.3.0/configuration.html#environment-variables Author: Felix Cheung <[email protected]> Closes #19 from felixcheung/master and squashes the following commits: e89ba08 [Felix Cheung] PySpark Interpreter should allow starting with a specific version of Python, as PySpark does. 65ba046 [Felix Cheung] Merge commit 'a007a9b5f235ebd9c608a005c5243503291d94d5' 7a30a14 [Felix Cheung] minor doc update for running on YARN Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/c9db780a Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/c9db780a Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/c9db780a Branch: refs/heads/master Commit: c9db780a1744aa3900c6d08f2727187731839cd3 Parents: c7645ab Author: Felix Cheung <[email protected]> Authored: Tue Mar 31 13:24:16 2015 -0700 Committer: Lee moon soo <[email protected]> Committed: Thu Apr 2 01:01:11 2015 +0900 ---------------------------------------------------------------------- .../java/com/nflabs/zeppelin/spark/PySparkInterpreter.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/c9db780a/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java index 071fcea..36cdcae 100644 --- a/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java +++ b/spark/src/main/java/com/nflabs/zeppelin/spark/PySparkInterpreter.java @@ -64,7 +64,10 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand new InterpreterPropertyBuilder() .add("spark.home", SparkInterpreter.getSystemDefault("SPARK_HOME", "spark.home", ""), - "Spark home path. Should be provided for pyspark").build()); + "Spark home path. Should be provided for pyspark") + .add("zeppelin.pyspark.python", + SparkInterpreter.getSystemDefault("PYSPARK_PYTHON", null, "python"), + "Python command to run pyspark with").build()); } public PySparkInterpreter(Properties property) { @@ -115,7 +118,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand gatewayServer.start(); // Run python shell - CommandLine cmd = CommandLine.parse("python"); + CommandLine cmd = CommandLine.parse(getProperty("zeppelin.pyspark.python")); cmd.addArgument(scriptPath, false); cmd.addArgument(Integer.toString(port), false); executor = new DefaultExecutor();
