This is an automated email from the ASF dual-hosted git repository. yaniv pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git
commit 0543daec22886557142f1d743128ac8fa69c6786 Author: Yaniv Rodenski <[email protected]> AuthorDate: Mon Apr 22 11:36:36 2019 +1000 datasets.ymal is available for yarn --- .../dist/amaterasu_pandas-0.2.0-incubating-rc4.zip | Bin 8304 -> 8304 bytes .../dist/amaterasu_python-0.2.0-incubating-rc4.zip | Bin 6167 -> 6167 bytes .../dist/amaterasu_pyspark-0.2.0-incubating-rc4.zip | Bin 14488 -> 14488 bytes .../amaterasu/leader/common/utilities/DataLoader.kt | 15 +++++++++++++++ .../amaterasu/leader/yarn/ApplicationMaster.kt | 6 ++++-- .../amaterasu/sdk/frameworks/RunnerSetupProvider.kt | 2 +- sdk_python/amaterasu/base/datasets.py | 2 ++ sdk_python/amaterasu/base/runtime.py | 2 +- .../dist/amaterasu-sdk-0.2.0-incubating-rc4.zip | Bin 14898 -> 14921 bytes 9 files changed, 23 insertions(+), 4 deletions(-) diff --git a/frameworks/python/pandas_runtime/dist/amaterasu_pandas-0.2.0-incubating-rc4.zip b/frameworks/python/pandas_runtime/dist/amaterasu_pandas-0.2.0-incubating-rc4.zip index 7a324a3..cb05407 100644 Binary files a/frameworks/python/pandas_runtime/dist/amaterasu_pandas-0.2.0-incubating-rc4.zip and b/frameworks/python/pandas_runtime/dist/amaterasu_pandas-0.2.0-incubating-rc4.zip differ diff --git a/frameworks/python/python_runtime/dist/amaterasu_python-0.2.0-incubating-rc4.zip b/frameworks/python/python_runtime/dist/amaterasu_python-0.2.0-incubating-rc4.zip index 02c0e7b..d3ac8a3 100644 Binary files a/frameworks/python/python_runtime/dist/amaterasu_python-0.2.0-incubating-rc4.zip and b/frameworks/python/python_runtime/dist/amaterasu_python-0.2.0-incubating-rc4.zip differ diff --git a/frameworks/spark/pyspark_runtime/dist/amaterasu_pyspark-0.2.0-incubating-rc4.zip b/frameworks/spark/pyspark_runtime/dist/amaterasu_pyspark-0.2.0-incubating-rc4.zip index 7137321..5972ee1 100644 Binary files a/frameworks/spark/pyspark_runtime/dist/amaterasu_pyspark-0.2.0-incubating-rc4.zip and b/frameworks/spark/pyspark_runtime/dist/amaterasu_pyspark-0.2.0-incubating-rc4.zip differ diff --git a/leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/utilities/DataLoader.kt b/leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/utilities/DataLoader.kt index dc76c62..73c66fd 100644 --- a/leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/utilities/DataLoader.kt +++ b/leader-common/src/main/kotlin/org/apache/amaterasu/leader/common/utilities/DataLoader.kt @@ -73,6 +73,21 @@ object DataLoader : KLogging() { } @JvmStatic + fun getDatasets(env: String): String { + var file = File("repo/env/$env/datasets.yml") + return if (file.exists()) { + file.readText() + } else { + file = File("repo/env/$env/datasets.yaml") + if (file.exists()) { + file.readText() + } else { + "" + } + } + } + + @JvmStatic fun getTaskDataString(actionData: ActionData, env: String): String { return mapper.writeValueAsString(getTaskData(actionData, env)) } diff --git a/leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/ApplicationMaster.kt b/leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/ApplicationMaster.kt index 437de28..5aea775 100644 --- a/leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/ApplicationMaster.kt +++ b/leader-yarn/src/main/kotlin/org/apache/amaterasu/leader/yarn/ApplicationMaster.kt @@ -249,7 +249,7 @@ class ApplicationMaster : KLogging(), AMRMClientAsync.CallbackHandler { jobManager.actionStarted(actionData.id) containersIdsToTask[container.id.containerId] = actionData notifier.info("created container for ${actionData.name} created") - ctx.localResources.forEach { t: String, u: LocalResource -> notifier.info("resource: $t = ${u.resource}") } + //ctx.localResources.forEach { t: String, u: LocalResource -> notifier.info("resource: $t = ${u.resource}") } log.info("launching container succeeded: ${container.id.containerId}; task: ${actionData.id}") } } @@ -318,9 +318,11 @@ class ApplicationMaster : KLogging(), AMRMClientAsync.CallbackHandler { val dataStores = DataLoader.getTaskData(actionData, env).exports val dataStoresYaml = yamlMapper.writeValueAsString(dataStores) - writeConfigFile(dataStoresYaml, jobManager.jobId, actionData.name, "datastores.yaml") + val datesets = DataLoader.getDatasets(env) + writeConfigFile(datesets, jobManager.jobId, actionData.name, "datasets.yaml") + writeConfigFile("jobId: ${jobManager.jobId}\nactionName: ${actionData.name}", jobManager.jobId, actionData.name, "runtime.yaml") } diff --git a/sdk/src/main/kotlin/org/apache/amaterasu/sdk/frameworks/RunnerSetupProvider.kt b/sdk/src/main/kotlin/org/apache/amaterasu/sdk/frameworks/RunnerSetupProvider.kt index eb75b2d..499ce10 100644 --- a/sdk/src/main/kotlin/org/apache/amaterasu/sdk/frameworks/RunnerSetupProvider.kt +++ b/sdk/src/main/kotlin/org/apache/amaterasu/sdk/frameworks/RunnerSetupProvider.kt @@ -24,7 +24,7 @@ import org.apache.amaterasu.common.logging.Logging abstract class RunnerSetupProvider : Logging() { - private val actionFiles = arrayOf("env.yaml", "runtime.yaml", "datastores.yaml") + private val actionFiles = arrayOf("env.yaml", "runtime.yaml", "datastores.yaml", "datasets.yaml") abstract val runnerResources: Array<String> diff --git a/sdk_python/amaterasu/base/datasets.py b/sdk_python/amaterasu/base/datasets.py index 26d6cfc..0df9f8a 100644 --- a/sdk_python/amaterasu/base/datasets.py +++ b/sdk_python/amaterasu/base/datasets.py @@ -70,7 +70,9 @@ class BaseDatasetManager(abc.ABC): def _find_dataset_config(self, dataset_name: str) -> Dict: for dataset_type, dataset_configurations in self._datasets_conf.items(): + print("-->" + dataset_type) for config in dataset_configurations: + print("---->" + config['name']) if config['name'] == dataset_name: dataset_config = config.copy() dataset_config['type'] = dataset_type diff --git a/sdk_python/amaterasu/base/runtime.py b/sdk_python/amaterasu/base/runtime.py index b6861c8..68ed836 100644 --- a/sdk_python/amaterasu/base/runtime.py +++ b/sdk_python/amaterasu/base/runtime.py @@ -75,7 +75,7 @@ class BaseAmaContextBuilder(abc.ABC): def __init__(self): self.env_conf_path = _get_absolute_file_path('env.yaml') self.runtime_conf_path = _get_absolute_file_path('runtime.yaml') - self.datasets_conf_path = _get_absolute_file_path('datastores.yaml') + self.datasets_conf_path = _get_absolute_file_path('datasets.yaml') self.ama_conf = self._create_env() self._frameworks = self._resolve_supported_frameworks() diff --git a/sdk_python/dist/amaterasu-sdk-0.2.0-incubating-rc4.zip b/sdk_python/dist/amaterasu-sdk-0.2.0-incubating-rc4.zip index 14345a0..08611c8 100644 Binary files a/sdk_python/dist/amaterasu-sdk-0.2.0-incubating-rc4.zip and b/sdk_python/dist/amaterasu-sdk-0.2.0-incubating-rc4.zip differ
