This is an automated email from the ASF dual-hosted git repository. yaniv pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit d39a4c460a5aa141e193b6e40b737aaa7948794c Author: Yaniv Rodenski <[email protected]> AuthorDate: Sun May 5 14:21:57 2019 +1000 fixing ama-requierments location in YARN --- .../python/dispatcher/runners/providers/PythonRunnerProviderBase.kt | 6 ++++++ leader-yarn/build.gradle | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt b/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt index daea452..01ef770 100644 --- a/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt +++ b/frameworks/python/dispatcher/src/main/kotlin/org/apache/amaterasu/frameworks/python/dispatcher/runners/providers/PythonRunnerProviderBase.kt @@ -22,6 +22,8 @@ import org.apache.amaterasu.common.dataobjects.ActionData import org.apache.amaterasu.leader.common.utilities.DataLoader import org.apache.amaterasu.sdk.frameworks.RunnerSetupProvider import java.io.File +import java.nio.file.Files +import java.nio.file.Paths abstract class PythonRunnerProviderBase(val env: String, val conf: ClusterConfig) : RunnerSetupProvider() { @@ -47,7 +49,11 @@ abstract class PythonRunnerProviderBase(val env: String, val conf: ClusterConfig override fun getActionDependencies(jobId: String, actionData: ActionData): Array<String> { val reqFile = File("dist/$requirementsFileName") + val dist = Paths.get("dist/") + if (reqFile.exists()) reqFile.delete() + if (Files.notExists(dist)) Files.createDirectories(dist) + val dependencies = runnerResources + mandatoryPYPIPackages dependencies.forEach { resource -> diff --git a/leader-yarn/build.gradle b/leader-yarn/build.gradle index 2c617b9..65c33dc 100644 --- a/leader-yarn/build.gradle +++ b/leader-yarn/build.gradle @@ -34,7 +34,6 @@ buildscript { plugins { id "com.github.johnrengelman.shadow" version "2.0.4" - id 'scala' } apply plugin: 'kotlin'
