This is an automated email from the ASF dual-hosted git repository. yaniv pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit 2e7904db9eb14903bf6ef7921f4e309a78cf650b Author: Yaniv Rodenski <[email protected]> AuthorDate: Fri Apr 26 18:31:17 2019 +1000 fixing master --- .../dispatcher/runners/providers/PySparkRunnerProvider.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala index 919d8e3..7cac602 100644 --- a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala +++ b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala @@ -11,9 +11,14 @@ class PySparkRunnerProvider(val env: String, val conf: ClusterConfig) extends Py log.info(s"===> Cluster manager: ${conf.mode}") command + //s" $$SPARK_HOME/conf/spark-env.sh" + - // s" && env PYSPARK_PYTHON=$getVirtualPythonPath" + + // s" && env PYSPARK_PYTHON=$getVirtualPythonPath" + //s" env PYSPARK_DRIVER_PYTHON=$getVirtualPythonPath" + d - s" && $$SPARK_HOME/bin/spark-submit --master yarn-cluster --conf spark.pyspark.virtualenv.enabled=true --conf spark.pyspark.virtualenv.type=native --conf spark.pyspark.virtualenv.bin.path=$getVirtualPythonBin --conf spark.pyspark.python=$getVirtualPythonPath --files $$SPARK_HOME/conf/hive-site.xml ${actionData.getSrc}" + s" && $$SPARK_HOME/bin/spark-submit --master yarn-client " + + s"--conf spark.pyspark.virtualenv.enabled=true " + + s"--conf spark.pyspark.virtualenv.type=native " + + s"--conf spark.pyspark.virtualenv.bin.path=$getVirtualPythonBin " + + s"--conf spark.pyspark.python=$getVirtualPythonPath " + + s"--files $$SPARK_HOME/conf/hive-site.xml ${actionData.getSrc}" } override def getRunnerResources: Array[String] = {
