This is an automated email from the ASF dual-hosted git repository. yaniv pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit 74e24bf1930bd6ffa7e41ba228807778483bc3f4 Author: Yaniv Rodenski <[email protected]> AuthorDate: Sun Apr 21 23:21:40 2019 +1000 fixed YARN java encoding --- .../runners/providers/PythonRunnerProviderBase.kt | 2 +- .../dist/amaterasu_pandas-0.2.0-incubating-rc4.zip | Bin 8304 -> 8304 bytes .../dist/amaterasu_python-0.2.0-incubating-rc4.zip | Bin 6167 -> 6167 bytes .../runners/providers/PySparkRunnerProvider.scala | 3 ++- .../dist/amaterasu_pyspark-0.2.0-incubating-rc4.zip | Bin 14488 -> 14488 bytes .../amaterasu/leader/yarn/ApplicationMaster.kt | 2 +- .../dist/amaterasu-sdk-0.2.0-incubating-rc4.zip | Bin 14898 -> 14898 bytes 7 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt b/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt index d6e9f42..0f91d3a 100644 --- a/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt +++ b/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt @@ -32,7 +32,7 @@ abstract class PythonRunnerProviderBase(val env: String, val conf: ClusterConfig get() = arrayOf("amaterasu-sdk-${conf.version()}.zip") override fun getCommand(jobId: String, actionData: ActionData, env: String, executorId: String, callbackAddress: String): String { - val pythonPath = conf!!.pythonPath() + val pythonPath = conf.pythonPath() val virtualEnvCmd = "$pythonPath -m venv amaterasu_env" val installBaseRequirementsCmd = "$virtualPythonPath -m pip install --upgrade --force-reinstall -r $requirementsFileName" var cmd = "$virtualEnvCmd && $installBaseRequirementsCmd" diff --git a/frameworks/python/pandas_runtime/dist/amaterasu_pandas-0.2.0-incubating-rc4.zip b/frameworks/python/pandas_runtime/dist/amaterasu_pandas-0.2.0-incubating-rc4.zip index f4aa8c6..7a324a3 100644 Binary files a/frameworks/python/pandas_runtime/dist/amaterasu_pandas-0.2.0-incubating-rc4.zip and b/frameworks/python/pandas_runtime/dist/amaterasu_pandas-0.2.0-incubating-rc4.zip differ diff --git a/frameworks/python/python_runtime/dist/amaterasu_python-0.2.0-incubating-rc4.zip b/frameworks/python/python_runtime/dist/amaterasu_python-0.2.0-incubating-rc4.zip index f62dc5a..02c0e7b 100644 Binary files a/frameworks/python/python_runtime/dist/amaterasu_python-0.2.0-incubating-rc4.zip and b/frameworks/python/python_runtime/dist/amaterasu_python-0.2.0-incubating-rc4.zip differ diff --git a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala index 2c718d9..90d7adb 100644 --- a/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala +++ b/frameworks/spark/dispatcher/src/main/scala/org/apache/amaterasu/frameworks/spark/dispatcher/runners/providers/PySparkRunnerProvider.scala @@ -12,7 +12,8 @@ class PySparkRunnerProvider(val env: String, val conf: ClusterConfig) extends Py command + //s" $$SPARK_HOME/conf/spark-env.sh" + s" && env PYSPARK_PYTHON=$getVirtualPythonPath" + - s" && $$SPARK_HOME/bin/spark-submit ${actionData.getSrc}" + s" env PYSPARK_DRIVER_PYTHON=$getVirtualPythonPath" + + s" $$SPARK_HOME/bin/spark-submit ${actionData.getSrc}" } override def getRunnerResources: Array[String] = { diff --git a/frameworks/spark/pyspark_runtime/dist/amaterasu_pyspark-0.2.0-incubating-rc4.zip b/frameworks/spark/pyspark_runtime/dist/amaterasu_pyspark-0.2.0-incubating-rc4.zip index c8e66c6..7137321 100644 Binary files a/frameworks/spark/pyspark_runtime/dist/amaterasu_pyspark-0.2.0-incubating-rc4.zip and b/frameworks/spark/pyspark_runtime/dist/amaterasu_pyspark-0.2.0-incubating-rc4.zip differ diff --git a/leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/ApplicationMaster.kt b/leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/ApplicationMaster.kt index 77c17d1..437de28 100644 --- a/leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/ApplicationMaster.kt +++ b/leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/ApplicationMaster.kt @@ -332,7 +332,7 @@ class ApplicationMaster : KLogging(), AMRMClientAsync.CallbackHandler { val targetPath = Path.mergePaths(yarnJarPath, actionDistPath) val outputStream = fs.create(targetPath) - outputStream.writeUTF(content) + outputStream.write(content.toByteArray()) outputStream.close() log.info("written file $targetPath") diff --git a/sdk_python/dist/amaterasu-sdk-0.2.0-incubating-rc4.zip b/sdk_python/dist/amaterasu-sdk-0.2.0-incubating-rc4.zip index 9ce27b8..14345a0 100644 Binary files a/sdk_python/dist/amaterasu-sdk-0.2.0-incubating-rc4.zip and b/sdk_python/dist/amaterasu-sdk-0.2.0-incubating-rc4.zip differ
