more code for AIRAVATA-1011
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/176acc23 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/176acc23 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/176acc23 Branch: refs/heads/master Commit: 176acc23770d860dff1fc42ae32408e1f02d3944 Parents: a1b1ff9 Author: lahiru <[email protected]> Authored: Wed Feb 5 15:29:55 2014 -0500 Committer: lahiru <[email protected]> Committed: Wed Feb 5 15:29:55 2014 -0500 ---------------------------------------------------------------------- .../orchestrator/core/Orchestrator.java | 86 ---------- .../core/OrchestratorConfiguration.java | 10 ++ .../core/validator/JobMetadataValidator.java | 32 ++++ .../impl/AbstractJobMetadataValidator.java | 30 ++++ .../validator/impl/SimpleAppDataValidator.java | 51 ++++++ .../airavata/orchestrator/cpi/Orchestrator.java | 2 +- .../cpi/impl/AbstractOrchestrator.java | 172 +++++++++++++++++++ .../cpi/impl/SimpleOrchestratorImpl.java | 11 +- .../src/main/resources/orchestrator.properties | 1 + 9 files changed, 306 insertions(+), 89 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/176acc23/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/Orchestrator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/Orchestrator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/Orchestrator.java deleted file mode 100644 index 0a4f913..0000000 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/Orchestrator.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * -*/ -package org.apache.airavata.orchestrator.core; - -import org.apache.airavata.orchestrator.core.exception.OrchestratorException; -import org.apache.airavata.orchestrator.core.model.ExperimentRequest; -import org.apache.airavata.registry.api.JobRequest; - -/* - This is the interface for orchestrator functionality exposed to the out side of the - module -*/ -public interface Orchestrator { - - - /** - * This method will initialize the Orchestrator, during restart this will - * get called and do init tasks - * @return - * @throws OrchestratorException - */ - boolean initialize() throws OrchestratorException; - - - /** - * This method is the very first method which create an entry in - * database for a given experiment, this return the experiment ID, so - * user have full control for the experiment - * @param request - * @return - * @throws OrchestratorException - */ - String createExperiment(ExperimentRequest request) throws OrchestratorException; - - /** - * After creating the experiment user has the experimentID, then user - * can create the JobRequest and send the Job input parameters to Orchestrator - * @param request - * @return - * @throws OrchestratorException - */ - boolean launchExperiment(JobRequest request) throws OrchestratorException; - - /** - * This method can be used to cancel a running experiment, if job is already finished it - * throws an exception. If job is not yet submitted it will just change the status to cancelled, - * if Job is running it will be killed from the resource and make the status to cancelled - * @param experimentID - * @return - * @throws OrchestratorException - */ - boolean terminateExperiment(String experimentID)throws OrchestratorException; - - /** - * This is like a cron job which runs continuously and take available jobs to - * submit to GFAC and submit them to GFAC - * @throws OrchestratorException - */ - void startJobSubmitter() throws OrchestratorException; - - /** - * This method will get called during graceful shutdown of Orchestrator - * This can be used to handle the shutdown of orchestrator gracefully. - * @return - * @throws OrchestratorException - */ - void shutdown() throws OrchestratorException; -} http://git-wip-us.apache.org/repos/asf/airavata/blob/176acc23/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java index c35dacf..ef36692 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java @@ -46,6 +46,16 @@ public class OrchestratorConfiguration { private boolean embeddedMode; + private String validatorClass; + + + public String getValidatorClass() { + return validatorClass; + } + + public void setValidatorClass(String validatorClass) { + this.validatorClass = validatorClass; + } public boolean isEmbeddedMode() { return embeddedMode; http://git-wip-us.apache.org/repos/asf/airavata/blob/176acc23/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java new file mode 100644 index 0000000..d365c87 --- /dev/null +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java @@ -0,0 +1,32 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.orchestrator.core.validator; + +import org.apache.airavata.orchestrator.core.exception.OrchestratorException; + +public interface JobMetadataValidator { + + boolean runBasicValidation(String experimentID)throws OrchestratorException; + + boolean runAppSpecificValidation(String experimentID)throws OrchestratorException; + + boolean validate(String experimentID) throws OrchestratorException; +} http://git-wip-us.apache.org/repos/asf/airavata/blob/176acc23/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java new file mode 100644 index 0000000..e83d5c9 --- /dev/null +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java @@ -0,0 +1,30 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.orchestrator.core.validator.impl; + +import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator; + +public abstract class AbstractJobMetadataValidator implements JobMetadataValidator { + public boolean runBasicValidation(String experimentID) { + //todo implement basic validation which is not an application specific + return false; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/176acc23/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java new file mode 100644 index 0000000..0d16062 --- /dev/null +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java @@ -0,0 +1,51 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.orchestrator.core.validator.impl; + +import org.apache.airavata.orchestrator.core.exception.OrchestratorException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SimpleAppDataValidator extends AbstractJobMetadataValidator { + private final static Logger logger = LoggerFactory.getLogger(SimpleAppDataValidator.class); + + public boolean runAppSpecificValidation(String experimentID) throws OrchestratorException{ + // implement simple application specific validator to be used for + // all the applications. + return true; //To change body of implemented methods use File | Settings | File Templates. + } + + public boolean validate(String experimentID) throws OrchestratorException { + boolean result = false; + if (super.runBasicValidation(experimentID)) { + + if (runAppSpecificValidation(experimentID)) { + return true; + } + String error = "Application data validation steps failed"; + logger.error(error); + return false; + } + String error = "Basic validation steps failed"; + logger.error(error); + return false; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/176acc23/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java index ab237af..3da9874 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java @@ -91,5 +91,5 @@ public interface Orchestrator { * @param experimentID * @return */ - boolean validateExperiment(String experimentID); + boolean validateExperiment(String experimentID)throws OrchestratorException; } http://git-wip-us.apache.org/repos/asf/airavata/blob/176acc23/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/AbstractOrchestrator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/AbstractOrchestrator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/AbstractOrchestrator.java new file mode 100644 index 0000000..4c0fca3 --- /dev/null +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/AbstractOrchestrator.java @@ -0,0 +1,172 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.orchestrator.cpi.impl; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; + +import org.apache.airavata.client.AiravataAPIFactory; +import org.apache.airavata.client.api.AiravataAPI; +import org.apache.airavata.client.api.exception.AiravataAPIInvocationException; +import org.apache.airavata.common.exception.AiravataConfigurationException; +import org.apache.airavata.orchestrator.core.OrchestratorConfiguration; +import org.apache.airavata.orchestrator.core.context.OrchestratorContext; +import org.apache.airavata.orchestrator.core.exception.OrchestratorException; +import org.apache.airavata.orchestrator.core.gfac.GFACInstance; +import org.apache.airavata.orchestrator.core.model.ExperimentRequest; +import org.apache.airavata.orchestrator.core.utils.OrchestratorConstants; +import org.apache.airavata.orchestrator.core.utils.OrchestratorUtils; +import org.apache.airavata.orchestrator.cpi.Orchestrator; +import org.apache.airavata.registry.api.AiravataRegistry2; +import org.apache.airavata.registry.api.AiravataRegistryFactory; +import org.apache.airavata.registry.api.AiravataUser; +import org.apache.airavata.registry.api.Gateway; +import org.apache.airavata.registry.api.exception.RegistryException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class AbstractOrchestrator implements Orchestrator { + private final static Logger logger = LoggerFactory.getLogger(AbstractOrchestrator.class); + protected AiravataRegistry2 airavataRegistry; + protected AiravataAPI airavataAPI; + protected OrchestratorContext orchestratorContext; + protected OrchestratorConfiguration orchestratorConfiguration; + + private String registryURL; + private String gatewayName; + private String airavataUserName; + + public String getRegistryURL() { + return registryURL; + } + + public void setRegistryURL(String registryURL) { + this.registryURL = registryURL; + } + + public String getGatewayName() { + return gatewayName; + } + + public void setGatewayName(String gatewayName) { + this.gatewayName = gatewayName; + } + public String getAiravataUserName() { + return airavataUserName; + } + + public void setAiravataUserName(String airavataUserName) { + this.airavataUserName = airavataUserName; + } + public boolean initialize() throws OrchestratorException { + try { + /* Initializing the OrchestratorConfiguration object */ + orchestratorConfiguration = OrchestratorUtils.loadOrchestratorConfiguration(); + setGatewayProperties(); + /* initializing the Orchestratorcontext object */ + airavataRegistry = AiravataRegistryFactory.getRegistry(new Gateway(getGatewayName()), new AiravataUser(getAiravataUserName())); + // todo move this code to gfac service mode Jobsubmitter, + // todo this is ugly, SHOULD fix these isEmbedded mode code from Orchestrator + if (!orchestratorConfiguration.isEmbeddedMode()) { + Map<String, Integer> gfacNodeList = airavataRegistry.getGFACNodeList(); + if (gfacNodeList.size() == 0) { + String error = "No GFAC instances available in the system, Can't initialize Orchestrator"; + logger.error(error); + throw new OrchestratorException(error); + } + Set<String> uriList = gfacNodeList.keySet(); + Iterator<String> iterator = uriList.iterator(); + List<GFACInstance> gfacInstanceList = new ArrayList<GFACInstance>(); + while (iterator.hasNext()) { + String uri = iterator.next(); + Integer integer = gfacNodeList.get(uri); + gfacInstanceList.add(new GFACInstance(uri, integer)); + } + } + orchestratorContext = new OrchestratorContext(); + orchestratorContext.setOrchestratorConfiguration(orchestratorConfiguration); + orchestratorConfiguration.setAiravataAPI(getAiravataAPI()); + orchestratorContext.setRegistry(airavataRegistry); + } catch (RegistryException e) { + logger.error("Failed to initializing Orchestrator"); + OrchestratorException orchestratorException = new OrchestratorException(e); + throw orchestratorException; + } catch (AiravataConfigurationException e) { + logger.error("Failed to initializing Orchestrator"); + OrchestratorException orchestratorException = new OrchestratorException(e); + throw orchestratorException; + } catch (IOException e) { + logger.error("Failed to initializing Orchestrator - Error parsing orchestrator.properties"); + OrchestratorException orchestratorException = new OrchestratorException(e); + throw orchestratorException; + } + return true; + } + + //get the registry URL and the credentials from the property file + protected void setGatewayProperties() { + InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(OrchestratorConstants.AIRAVATA_PROPERTIES); + Properties properties = new Properties(); + try { + properties.load(inputStream); + } catch (IOException e) { + e.printStackTrace(); + } + setAiravataUserName(properties.getProperty("system.user")); + setGatewayName(properties.getProperty("system.gateway")); + setRegistryURL(properties.getProperty("airavata.server.url")); + } + //todo decide whether to return an error or do what + //FIXME: (MEP) as posted on dev list, I think this should return a JobRequest with the experimentID set. This would simplify some of the validation in EmbeddedGFACJobSubmitter's launcGfacWithJobRequest--just throw the job away if the JobRequest is incomplete or malformed. + public String createExperiment(ExperimentRequest request) throws OrchestratorException { + //todo use a consistent method to create the experiment ID + //FIXME: (MEP) Should you trust the user to do this? What if the same experimentID is sent twice by the same gateway? + String experimentID = request.getUserExperimentID(); + if(experimentID == null){ + experimentID = UUID.randomUUID().toString(); + } + try { + airavataRegistry.storeExperiment(request.getSubmitterUserName(), experimentID, null, null); + } catch (RegistryException e) { + //todo put more meaningful error message + logger.error("Failed to create experiment for the request from " + request.getSubmitterUserName()); + throw new OrchestratorException(e); + } + return experimentID; + } + private AiravataAPI getAiravataAPI() { + if (airavataAPI == null) { + try { + airavataAPI = AiravataAPIFactory.getAPI(getGatewayName(), getAiravataUserName()); + } catch (AiravataAPIInvocationException e) { + logger.error("Unable to create Airavata API", e); + } + } + return airavataAPI; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/176acc23/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java index b75f226..40fecba 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java @@ -29,6 +29,7 @@ import org.apache.airavata.orchestrator.core.NewJobWorker; import org.apache.airavata.orchestrator.core.exception.OrchestratorException; import org.apache.airavata.orchestrator.core.job.JobSubmitter; import org.apache.airavata.orchestrator.core.utils.OrchestratorUtils; +import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator; import org.apache.airavata.registry.api.JobRequest; import org.apache.airavata.registry.api.exception.RegistryException; import org.slf4j.Logger; @@ -42,6 +43,8 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator { // this is going to be null unless the thread count is 0 private JobSubmitter jobSubmitter = null; + private JobMetadataValidator jobMetadataValidator = null; + public boolean initialize() throws OrchestratorException { super.initialize(); // we have a thread to run normal new jobs except to monitor hanged jobs @@ -55,6 +58,10 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator { Class<? extends JobSubmitter> aClass = Class.forName(submitterClass.trim()).asSubclass(JobSubmitter.class); jobSubmitter = aClass.newInstance(); jobSubmitter.initialize(this.orchestratorContext); + + String validatorClzz = this.orchestratorContext.getOrchestratorConfiguration().getValidatorClass(); + Class<? extends JobMetadataValidator> vClass = Class.forName(validatorClzz.trim()).asSubclass(JobMetadataValidator.class); + jobMetadataValidator = vClass.newInstance(); } catch (Exception e) { String error = "Error creating JobSubmitter in non threaded mode "; logger.error(error); @@ -160,7 +167,7 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator { return true; } - public boolean validateExperiment(String experimentID) { - return false; + public boolean validateExperiment(String experimentID) throws OrchestratorException{ + return jobMetadataValidator.validate(experimentID); } } http://git-wip-us.apache.org/repos/asf/airavata/blob/176acc23/modules/orchestrator/orchestrator-core/src/main/resources/orchestrator.properties ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/resources/orchestrator.properties b/modules/orchestrator/orchestrator-core/src/main/resources/orchestrator.properties index 728f215..85568a4 100644 --- a/modules/orchestrator/orchestrator-core/src/main/resources/orchestrator.properties +++ b/modules/orchestrator/orchestrator-core/src/main/resources/orchestrator.properties @@ -18,6 +18,7 @@ # under the License. # job.submitter=org.apache.airavata.orchestrator.core.impl.EmbeddedGFACJobSubmitter +job.validator=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator submitter.interval=10000 threadpool.size=10 start.submitter=true
