Repository: airavata Updated Branches: refs/heads/feature-workload-mgmt ead87d8fe -> f37895ca5
Yaml implementation for sub types in data staging Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/f37895ca Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/f37895ca Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/f37895ca Branch: refs/heads/feature-workload-mgmt Commit: f37895ca57e46d23e76b292eb796b171e64059f3 Parents: ead87d8 Author: Ajinkya Dhamnaskar <[email protected]> Authored: Tue Jun 6 13:38:23 2017 -0400 Committer: Ajinkya Dhamnaskar <[email protected]> Committed: Tue Jun 6 13:38:23 2017 -0400 ---------------------------------------------------------------------- .../config/DataStagingConfigException.java | 31 +++++++++ .../config/DataStagingYamlConfig.java | 68 ++++++++++++++++++++ .../config/DataTransferTaskConfig.java | 60 +++++++++++++++++ .../datastaging/facade/DataStagingFacade.java | 34 ++++++++++ .../datastaging/utils/DataStagingFactory.java | 34 ++++++++++ .../src/main/resources/data-staging-config.yaml | 27 ++++++++ 6 files changed, 254 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/f37895ca/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataStagingConfigException.java ---------------------------------------------------------------------- diff --git a/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataStagingConfigException.java b/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataStagingConfigException.java new file mode 100644 index 0000000..a0b7ea7 --- /dev/null +++ b/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataStagingConfigException.java @@ -0,0 +1,31 @@ +package org.apache.airavata.worker.task.datastaging.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Created by Ajinkya on 6/1/17. + */ +public class DataStagingConfigException extends Exception { + + private static final Logger log = LoggerFactory.getLogger(DataStagingConfigException.class); + + /** + * + */ + private static final long serialVersionUID = 1L; + + public DataStagingConfigException(String s) { + super(s); + } + + public DataStagingConfigException(Exception e) { + super(e); + log.error(e.getMessage(),e); + } + + public DataStagingConfigException(String s, Throwable throwable) { + super(s, throwable); + log.error(s,throwable); + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/f37895ca/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataStagingYamlConfig.java ---------------------------------------------------------------------- diff --git a/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataStagingYamlConfig.java b/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataStagingYamlConfig.java new file mode 100644 index 0000000..3db3144 --- /dev/null +++ b/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataStagingYamlConfig.java @@ -0,0 +1,68 @@ +package org.apache.airavata.worker.task.datastaging.config; + +import org.apache.airavata.model.data.movement.DataMovementProtocol; +import org.yaml.snakeyaml.Yaml; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Created by Ajinkya on 6/1/17. + */ +public class DataStagingYamlConfig { + + private static final String TASK_CLASS = "taskClass"; + private static final String FILE_TRANSFER_TASKS = "fileTransferTasks"; + private static final String TRANSFER_PROTOCOL = "transferProtocol"; + private static final String PROPERTIES = "properties"; + private List<DataTransferTaskConfig> fileTransferTasks = new ArrayList<>(); + + public DataStagingYamlConfig() throws DataStagingConfigException { + InputStream resourceAsStream = DataStagingYamlConfig.class.getClassLoader(). + getResourceAsStream("data-staging-config.yaml"); + parse(resourceAsStream); + } + + private void parse(InputStream resourceAsStream) throws DataStagingConfigException { + if (resourceAsStream == null) { + throw new DataStagingConfigException("Configuration file{data-staging-config.yaml} is not fund"); + } + Yaml yaml = new Yaml(); + Object load = yaml.load(resourceAsStream); + if (load == null) { + throw new DataStagingConfigException("Yaml configuration object null"); + } + + if (load instanceof Map) { + Map<String, Object> loadMap = (Map<String, Object>) load; + String identifier; + + List<Map<String, Object>> fileTransYamls = (List<Map<String, Object>>) loadMap.get(FILE_TRANSFER_TASKS); + DataTransferTaskConfig dataTransferTaskConfig; + if (fileTransYamls != null) { + for (Map<String, Object> fileTransConfig : fileTransYamls) { + dataTransferTaskConfig = new DataTransferTaskConfig(); + identifier = ((String) fileTransConfig.get(TRANSFER_PROTOCOL)); + dataTransferTaskConfig.setTransferProtocol(DataMovementProtocol.valueOf(identifier)); + dataTransferTaskConfig.setTaskClass(((String) fileTransConfig.get(TASK_CLASS))); + Object propertiesObj = fileTransConfig.get(PROPERTIES); + List propertiesList; + if (propertiesObj instanceof List) { + propertiesList = (List) propertiesObj; + if (propertiesList.size() > 0) { + Map<String, String> props = (Map<String, String>) propertiesList.get(0); + dataTransferTaskConfig.addProperties(props); + } + } + fileTransferTasks.add(dataTransferTaskConfig); + } + } + } + } + + public List<DataTransferTaskConfig> getFileTransferTasks() { + return fileTransferTasks; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/f37895ca/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataTransferTaskConfig.java ---------------------------------------------------------------------- diff --git a/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataTransferTaskConfig.java b/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataTransferTaskConfig.java new file mode 100644 index 0000000..9a238e8 --- /dev/null +++ b/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/config/DataTransferTaskConfig.java @@ -0,0 +1,60 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.airavata.worker.task.datastaging.config; + +import org.apache.airavata.model.data.movement.DataMovementProtocol; + +import java.util.HashMap; +import java.util.Map; + +public class DataTransferTaskConfig { + private DataMovementProtocol transferProtocol; + private String taskClass; + private Map<String,String> properties = new HashMap<>(); + + + public DataMovementProtocol getTransferProtocol() { + return transferProtocol; + } + + public void setTransferProtocol(DataMovementProtocol transferProtocol) { + this.transferProtocol = transferProtocol; + } + + public String getTaskClass() { + return taskClass; + } + + public void setTaskClass(String taskClass) { + this.taskClass = taskClass; + } + + public void addProperty(String key, String value) { + properties.put(key, value); + } + + public void addProperties(Map<String, String> propMap) { + propMap.forEach(properties::put); + } + + public Map<String,String> getProperties(){ + return properties; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/f37895ca/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/facade/DataStagingFacade.java ---------------------------------------------------------------------- diff --git a/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/facade/DataStagingFacade.java b/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/facade/DataStagingFacade.java new file mode 100644 index 0000000..a8889ad --- /dev/null +++ b/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/facade/DataStagingFacade.java @@ -0,0 +1,34 @@ +package org.apache.airavata.worker.task.datastaging.facade; + +import org.apache.airavata.model.status.TaskStatus; +import org.apache.airavata.model.task.TaskTypes; +import org.apache.airavata.worker.core.context.TaskContext; +import org.apache.airavata.worker.core.task.Task; +import org.apache.airavata.worker.core.task.TaskException; + +import java.util.Map; + +/** + * Created by Ajinkya on 6/1/17. + */ +public class DataStagingFacade implements Task { + @Override + public void init(Map<String, String> propertyMap) throws TaskException { + + } + + @Override + public TaskStatus execute(TaskContext taskContext) { + return null; + } + + @Override + public TaskStatus recover(TaskContext taskContext) { + return null; + } + + @Override + public TaskTypes getType() { + return null; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/f37895ca/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/utils/DataStagingFactory.java ---------------------------------------------------------------------- diff --git a/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/utils/DataStagingFactory.java b/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/utils/DataStagingFactory.java index 51996b8..f1a0df8 100644 --- a/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/utils/DataStagingFactory.java +++ b/modules/worker/task-datastaging/src/main/java/org/apache/airavata/worker/task/datastaging/utils/DataStagingFactory.java @@ -2,13 +2,25 @@ package org.apache.airavata.worker.task.datastaging.utils; import org.apache.airavata.common.exception.ApplicationSettingsException; import org.apache.airavata.credential.store.store.CredentialStoreException; +import org.apache.airavata.model.data.movement.DataMovementProtocol; +import org.apache.airavata.model.task.TaskTypes; import org.apache.airavata.worker.core.authentication.SSHKeyAuthentication; +import org.apache.airavata.worker.core.config.TaskImplementationConfig; +import org.apache.airavata.worker.core.config.WorkerYamlConfigruation; import org.apache.airavata.worker.core.context.ProcessContext; import org.apache.airavata.worker.core.exceptions.WorkerException; +import org.apache.airavata.worker.core.task.Task; import org.apache.airavata.worker.core.utils.WorkerFactory; +import org.apache.airavata.worker.task.datastaging.config.DataStagingConfigException; +import org.apache.airavata.worker.task.datastaging.config.DataStagingYamlConfig; +import org.apache.airavata.worker.task.datastaging.config.DataTransferTaskConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.reflect.Constructor; +import java.util.HashMap; +import java.util.Map; + /** * Created by Ajinkya on 4/13/17. */ @@ -16,6 +28,28 @@ public class DataStagingFactory { private static final Logger log = LoggerFactory.getLogger(DataStagingFactory.class); + private static boolean isDataStagingConfigurationLoaded = false; + private static Map<DataMovementProtocol, Task> dataMovementTask = new HashMap<>(); + + public static void loadConfiguration() throws DataStagingConfigException { + if (!isDataStagingConfigurationLoaded) { + DataStagingYamlConfig config = new DataStagingYamlConfig(); + try { + for (DataTransferTaskConfig dataTransferTaskConfig : config.getFileTransferTasks()) { + String taskClass = dataTransferTaskConfig.getTaskClass(); + Class<?> aClass = Class.forName(taskClass); + Constructor<?> constructor = aClass.getConstructor(); + Task task = (Task) constructor.newInstance(); + task.init(dataTransferTaskConfig.getProperties()); + dataMovementTask.put(dataTransferTaskConfig.getTransferProtocol(), task); + } + }catch (Exception e) { + throw new DataStagingConfigException("Data staging config issue", e); + } + isDataStagingConfigurationLoaded = true; + } + } + public static SSHKeyAuthentication getComputerResourceSSHKeyAuthentication(ProcessContext pc) throws WorkerException, CredentialStoreException { try { http://git-wip-us.apache.org/repos/asf/airavata/blob/f37895ca/modules/worker/task-datastaging/src/main/resources/data-staging-config.yaml ---------------------------------------------------------------------- diff --git a/modules/worker/task-datastaging/src/main/resources/data-staging-config.yaml b/modules/worker/task-datastaging/src/main/resources/data-staging-config.yaml new file mode 100644 index 0000000..5d6f805 --- /dev/null +++ b/modules/worker/task-datastaging/src/main/resources/data-staging-config.yaml @@ -0,0 +1,27 @@ +################################################################## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +################################################################ + +fileTransferTasks: + - transferProtocol: SCP + taskClass: org.apache.airavata.worker.task.datastaging.impl.SCPDataStageTask + - transferProtocol: LOCAL + taskClass: org.apache.airavata.worker.task.datastaging.impl.SCPDataStageTask + +
