fixing build error after removing JobRequest and cleaning the OrchAPI - AIRAVATA-1011
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/f78c0e15 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/f78c0e15 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/f78c0e15 Branch: refs/heads/master Commit: f78c0e15c568cd604a2f7dc02301a263a64b888a Parents: 47a7c15 Author: lahiru <[email protected]> Authored: Wed Feb 12 14:57:12 2014 -0500 Committer: lahiru <[email protected]> Committed: Wed Feb 12 14:57:12 2014 -0500 ---------------------------------------------------------------------- .../core/OrchestratorConfiguration.java | 10 + .../core/impl/EmbeddedGFACJobSubmitter.java | 49 ++-- .../orchestrator/core/job/JobSubmitter.java | 6 +- .../core/utils/OrchestratorConstants.java | 1 + .../core/utils/OrchestratorUtils.java | 20 +- .../airavata/orchestrator/cpi/Orchestrator.java | 45 +-- .../cpi/impl/AbstractOrchestrator.java | 24 +- .../cpi/impl/SimpleOrchestratorImpl.java | 111 +------- .../src/main/resources/orchestrator.properties | 1 + .../orchestrator/core/NewOrchestratorTest.java | 275 +++++++++---------- .../src/test/resources/orchestrator.properties | 3 +- 11 files changed, 181 insertions(+), 364 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/f78c0e15/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java index ef36692..fcb3552 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java @@ -48,6 +48,8 @@ public class OrchestratorConfiguration { private String validatorClass; + private boolean enableValidation; + public String getValidatorClass() { return validatorClass; @@ -120,4 +122,12 @@ public class OrchestratorConfiguration { public void setHangedJobSubmitterClass(String hangedJobSubmitterClass) { this.hangedJobSubmitterClass = hangedJobSubmitterClass; } + + public boolean isEnableValidation() { + return enableValidation; + } + + public void setEnableValidation(boolean enableValidation) { + this.enableValidation = enableValidation; + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/f78c0e15/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/EmbeddedGFACJobSubmitter.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/EmbeddedGFACJobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/EmbeddedGFACJobSubmitter.java index 63b1135..d65eeb5 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/EmbeddedGFACJobSubmitter.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/impl/EmbeddedGFACJobSubmitter.java @@ -46,7 +46,6 @@ import org.apache.airavata.orchestrator.core.gfac.GFACInstance; import org.apache.airavata.orchestrator.core.job.JobSubmitter; import org.apache.airavata.orchestrator.core.utils.OrchestratorUtils; import org.apache.airavata.registry.api.AiravataRegistry2; -import org.apache.airavata.registry.api.JobRequest; import org.apache.airavata.registry.cpi.DataType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -71,14 +70,12 @@ public class EmbeddedGFACJobSubmitter implements JobSubmitter { } - //FIXME: (MEP) why are you passing in a GFACInstance? It isn't used. public boolean submitJob(GFACInstance gfac, List<String> experimentIDList) throws OrchestratorException { for (int i = 0; i < experimentIDList.size(); i++) { try { // once its fetched it's status will changed to fetched state - JobRequest jobRequest = orchestratorContext.getRegistry().fetchAcceptedJob(experimentIDList.get(i)); - launchGfacWithJobRequest(jobRequest); + launchGfacWithJobRequest(experimentIDList.get(i)); } catch (Exception e) { logger.error("Error getting job related information"); throw new OrchestratorException(e); @@ -87,10 +84,9 @@ public class EmbeddedGFACJobSubmitter implements JobSubmitter { return true; } - //FIXME: (MEP) This method is pretty gruesome. If we really expect multiple implementations of the JobSubmitter - // interface and at least some of them will need to do the stuff in this method, then we need a parent class GenericJobSubmitterImpl.java (maybe abstract) that includes launchGfacWithJobRequest() so that subclasses can inherit it. - private void launchGfacWithJobRequest(JobRequest jobRequest) throws OrchestratorException { - String experimentID = OrchestratorUtils.getUniqueID(jobRequest); + //FIXME: (MEP) This method is pretty gruesome. If we really expect multiple implementations of the JobSubmitter + // interface and at least some of them will need to do the stuff in this method, then we need a parent class GenericJobSubmitterImpl.java (maybe abstract) that includes launchGfacWithJobRequest() so that subclasses can inherit it. + private void launchGfacWithJobRequest(String experimentID) throws OrchestratorException { ConfigurationData configurationData = (ConfigurationData) orchestratorContext.getNewRegistry().get(DataType.EXPERIMENT_CONFIGURATION_DATA, experimentID); @@ -101,32 +97,20 @@ public class EmbeddedGFACJobSubmitter implements JobSubmitter { } AiravataAPI airavataAPI = null; try { - airavataAPI = orchestratorContext.getOrchestratorConfiguration().getAiravataAPI(); - //FIXME: (MEP) Why do all of this validation here? Is it needed? Why would you get an empty job request? - //FIXME: (MEP) If you do need this, it should go into a utility class or something similar that does the validation. - HostDescription hostDescription = jobRequest.getHostDescription(); AiravataRegistry2 registry = orchestratorContext.getRegistry(); - if (hostDescription == null) { - List<HostDescription> registeredHosts = new ArrayList<HostDescription>(); - Map<String, ApplicationDescription> applicationDescriptors = registry.getApplicationDescriptors(serviceName); - for (String hostDescName : applicationDescriptors.keySet()) { - registeredHosts.add(registry.getHostDescriptor(hostDescName)); - } - Class<? extends HostScheduler> aClass = Class.forName(ServerSettings.getHostScheduler()).asSubclass(HostScheduler.class); - HostScheduler hostScheduler = aClass.newInstance(); - hostDescription = hostScheduler.schedule(registeredHosts); + List<HostDescription> registeredHosts = new ArrayList<HostDescription>(); + Map<String, ApplicationDescription> applicationDescriptors = registry.getApplicationDescriptors(serviceName); + for (String hostDescName : applicationDescriptors.keySet()) { + registeredHosts.add(registry.getHostDescriptor(hostDescName)); } + Class<? extends HostScheduler> aClass = Class.forName(ServerSettings.getHostScheduler()).asSubclass(HostScheduler.class); + HostScheduler hostScheduler = aClass.newInstance(); + HostDescription hostDescription = hostScheduler.schedule(registeredHosts); - ServiceDescription serviceDescription = jobRequest.getServiceDescription(); - if (serviceDescription == null) { - serviceDescription = registry.getServiceDescriptor(serviceName); - } + ServiceDescription serviceDescription = registry.getServiceDescriptor(serviceName); - ApplicationDescription applicationDescription = jobRequest.getApplicationDescription(); - if (applicationDescription == null) { - applicationDescription = registry.getApplicationDescriptors(serviceName, hostDescription.getType().getHostName()); - } + ApplicationDescription applicationDescription = registry.getApplicationDescriptors(serviceName, hostDescription.getType().getHostName()); // When we run getInParameters we set the actualParameter object, this has to be fixed //FIXME: will these class loaders work correctly in Thrift? //FIXME: gfac-config.xml is only under src/test. @@ -167,12 +151,11 @@ public class EmbeddedGFACJobSubmitter implements JobSubmitter { } } - //FIXME: (MEP) I suggest putting this into a separate JobSubmitter implementation. If so, launchGfacWithJobRequest() needs to be in an inherited parent. - public boolean directJobSubmit(JobRequest request) throws OrchestratorException { + public boolean directJobSubmit(String experimentID) throws OrchestratorException { try { - launchGfacWithJobRequest(request); + launchGfacWithJobRequest(experimentID); } catch (Exception e) { - String error = "Error launching the job : " + OrchestratorUtils.getUniqueID(request); + String error = "Error launching the job : " + experimentID; logger.error(error); throw new OrchestratorException(error); } http://git-wip-us.apache.org/repos/asf/airavata/blob/f78c0e15/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/job/JobSubmitter.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/job/JobSubmitter.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/job/JobSubmitter.java index 6ad9960..6405c4b 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/job/JobSubmitter.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/job/JobSubmitter.java @@ -25,8 +25,6 @@ import java.util.List; import org.apache.airavata.orchestrator.core.context.OrchestratorContext; import org.apache.airavata.orchestrator.core.exception.OrchestratorException; import org.apache.airavata.orchestrator.core.gfac.GFACInstance; -import org.apache.airavata.registry.api.AiravataRegistry2; -import org.apache.airavata.registry.api.JobRequest; /** * This is the submitter interface, orchestrator can @@ -55,8 +53,8 @@ public interface JobSubmitter { /** * This can be used when user doesn't want to run in a threaded pull mode * just get the request data and do the submission - * @param request + * @param experimentID * @return */ - boolean directJobSubmit(JobRequest request) throws OrchestratorException; + boolean directJobSubmit(String experimentID) throws OrchestratorException; } http://git-wip-us.apache.org/repos/asf/airavata/blob/f78c0e15/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorConstants.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorConstants.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorConstants.java index 93e97b1..bb242aa 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorConstants.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorConstants.java @@ -34,4 +34,5 @@ public class OrchestratorConstants { public static final String THREAD_POOL_SIZE = "threadpool.size"; public static final String START_SUBMITTER = "start.submitter"; public static final String EMBEDDED_MODE = "embedded.mode"; + public static final String ENABLE_VALIDATION = "enable.validation"; } http://git-wip-us.apache.org/repos/asf/airavata/blob/f78c0e15/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java index afe856c..ba574a5 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java @@ -20,13 +20,10 @@ */ package org.apache.airavata.orchestrator.core.utils; -import org.apache.airavata.gfac.GFacException; import org.apache.airavata.gfac.utils.GFacUtils; import org.apache.airavata.orchestrator.core.NewJobWorker; import org.apache.airavata.orchestrator.core.OrchestratorConfiguration; import org.apache.airavata.orchestrator.core.exception.OrchestratorException; -import org.apache.airavata.registry.api.JobRequest; -import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType; import org.apache.airavata.schemas.gfac.Parameter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -58,25 +55,10 @@ public class OrchestratorUtils { orchestratorConfiguration.setThreadPoolSize(Integer.parseInt((String) orchestratorProps.get(OrchestratorConstants.THREAD_POOL_SIZE))); orchestratorConfiguration.setStartSubmitter(Boolean.valueOf(orchestratorProps.getProperty(OrchestratorConstants.START_SUBMITTER))); orchestratorConfiguration.setEmbeddedMode(Boolean.valueOf(orchestratorProps.getProperty(OrchestratorConstants.EMBEDDED_MODE))); + orchestratorConfiguration.setEnableValidation(Boolean.valueOf(orchestratorProps.getProperty(OrchestratorConstants.ENABLE_VALIDATION))); return orchestratorConfiguration; } - public static boolean validateJobRequest(JobRequest request) { - /* todo implement a job request validation */ - - return true; - } - - public static String getUniqueID(JobRequest jobRequest) { - if (jobRequest.getUserExperimentID() != null) { - return jobRequest.getUserExperimentID(); - } else if (jobRequest.getSystemExperimentID() != null) { - return jobRequest.getSystemExperimentID(); - } else { - return null; - } - - } public static Map<String, Object> getMessageContext(Map<String, String> experimentData, Parameter[] parameters) throws OrchestratorException { http://git-wip-us.apache.org/repos/asf/airavata/blob/f78c0e15/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java index a4f5a90..5adde42 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java @@ -21,8 +21,6 @@ package org.apache.airavata.orchestrator.cpi; import org.apache.airavata.orchestrator.core.exception.OrchestratorException; -import org.apache.airavata.orchestrator.core.model.ExperimentRequest; -import org.apache.airavata.registry.api.JobRequest; /* This is the interface for orchestrator functionality exposed to the out side of the @@ -30,47 +28,18 @@ import org.apache.airavata.registry.api.JobRequest; */ public interface Orchestrator { - - /** - * This method will initialize the Orchestrator, during restart this will - * get called and do init tasks - * @return - * @throws OrchestratorException - */ - boolean initialize() throws OrchestratorException; - - /** - * After creating the experiment user has the experimentID, then user - * can create the JobRequest and send the Job input parameters to Orchestrator - * @param request - * @return - * @throws OrchestratorException - */ - boolean launchExperiment(JobRequest request) throws OrchestratorException; - /** - * This method can be used to cancel a running experiment, if job is already finished it - * throws an exception. If job is not yet submitted it will just change the status to cancelled, - * if Job is running it will be killed from the resource and make the status to cancelled + * After creating the experiment Data user have the + * experimentID as the handler to the experiment, during the launchExperiment + * We just have to give the experimentID + * * @param experimentID * @return * @throws OrchestratorException */ - boolean terminateExperiment(String experimentID)throws OrchestratorException; + boolean launchExperiment(String experimentID) throws OrchestratorException; - /** - * This is like a cron job which runs continuously and take available jobs to - * submit to GFAC and submit them to GFAC - * @throws OrchestratorException - */ - void startJobSubmitter() throws OrchestratorException; + //todo have to add another method to handle failed or jobs to be recovered by orchestrator + //todo if you don't add these this is not an orchestrator, its just an intemediate component which invoke gfac - /** - * This method can be used to parse the current job data configured in - * Registry and validate its status, if it has minimum required parameters to - * submit the job this method returns true otherwise this returns false - * @param experimentID - * @return - */ - boolean validateExperiment(String experimentID)throws OrchestratorException; } http://git-wip-us.apache.org/repos/asf/airavata/blob/f78c0e15/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/AbstractOrchestrator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/AbstractOrchestrator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/AbstractOrchestrator.java index ce0ff47..b7a3d25 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/AbstractOrchestrator.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/AbstractOrchestrator.java @@ -84,7 +84,7 @@ public abstract class AbstractOrchestrator implements Orchestrator { public void setAiravataUserName(String airavataUserName) { this.airavataUserName = airavataUserName; } - public boolean initialize() throws OrchestratorException { + public AbstractOrchestrator() throws OrchestratorException { try { /* Initializing the OrchestratorConfiguration object */ orchestratorConfiguration = OrchestratorUtils.loadOrchestratorConfiguration(); @@ -102,6 +102,7 @@ public abstract class AbstractOrchestrator implements Orchestrator { } Set<String> uriList = gfacNodeList.keySet(); Iterator<String> iterator = uriList.iterator(); + // todo consume these data to List<GFACInstance> gfacInstanceList = new ArrayList<GFACInstance>(); while (iterator.hasNext()) { String uri = iterator.next(); @@ -111,7 +112,6 @@ public abstract class AbstractOrchestrator implements Orchestrator { } orchestratorContext = new OrchestratorContext(); orchestratorContext.setOrchestratorConfiguration(orchestratorConfiguration); - AiravataAPI airavataAPI = getAiravataAPI(); orchestratorConfiguration.setAiravataAPI(getAiravataAPI()); orchestratorContext.setRegistry(airavataRegistry); @@ -130,7 +130,6 @@ public abstract class AbstractOrchestrator implements Orchestrator { OrchestratorException orchestratorException = new OrchestratorException(e); throw orchestratorException; } - return true; } //get the registry URL and the credentials from the property file @@ -146,24 +145,7 @@ public abstract class AbstractOrchestrator implements Orchestrator { setGatewayName(properties.getProperty("system.gateway")); setRegistryURL(properties.getProperty("airavata.server.url")); } - //todo decide whether to return an error or do what - //FIXME: (MEP) as posted on dev list, I think this should return a JobRequest with the experimentID set. This would simplify some of the validation in EmbeddedGFACJobSubmitter's launcGfacWithJobRequest--just throw the job away if the JobRequest is incomplete or malformed. - public String createExperiment(ExperimentRequest request) throws OrchestratorException { - //todo use a consistent method to create the experiment ID - //FIXME: (MEP) Should you trust the user to do this? What if the same experimentID is sent twice by the same gateway? - String experimentID = request.getUserExperimentID(); - if(experimentID == null){ - experimentID = UUID.randomUUID().toString(); - } - try { - airavataRegistry.storeExperiment(request.getSubmitterUserName(), experimentID, null, null); - } catch (RegistryException e) { - //todo put more meaningful error message - logger.error("Failed to create experiment for the request from " + request.getSubmitterUserName()); - throw new OrchestratorException(e); - } - return experimentID; - } + private AiravataAPI getAiravataAPI() { if (airavataAPI == null) { try { http://git-wip-us.apache.org/repos/asf/airavata/blob/f78c0e15/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java index 40fecba..5f1aaa0 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java @@ -24,13 +24,9 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.airavata.common.utils.AiravataJobState; -import org.apache.airavata.orchestrator.core.HangedJobWorker; -import org.apache.airavata.orchestrator.core.NewJobWorker; import org.apache.airavata.orchestrator.core.exception.OrchestratorException; import org.apache.airavata.orchestrator.core.job.JobSubmitter; -import org.apache.airavata.orchestrator.core.utils.OrchestratorUtils; import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator; -import org.apache.airavata.registry.api.JobRequest; import org.apache.airavata.registry.api.exception.RegistryException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,15 +41,10 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator { private JobMetadataValidator jobMetadataValidator = null; - public boolean initialize() throws OrchestratorException { - super.initialize(); - // we have a thread to run normal new jobs except to monitor hanged jobs - if (orchestratorConfiguration.getThreadPoolSize() != 0) { - executor = Executors.newFixedThreadPool(orchestratorConfiguration.getThreadPoolSize() + 1); - this.startJobSubmitter(); - } else { - try { + public SimpleOrchestratorImpl() throws OrchestratorException { + try { + try { String submitterClass = this.orchestratorContext.getOrchestratorConfiguration().getNewJobSubmitterClass(); Class<? extends JobSubmitter> aClass = Class.forName(submitterClass.trim()).asSubclass(JobSubmitter.class); jobSubmitter = aClass.newInstance(); @@ -67,107 +58,29 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator { logger.error(error); throw new OrchestratorException(error, e); } + } catch (OrchestratorException e) { + logger.error("Error Constructing the Orchestrator"); + throw e; } - return true; } - - public void shutdown() throws OrchestratorException { - executor.shutdown(); - - } - - public boolean launchExperiment(JobRequest request) throws OrchestratorException { - // validate the jobRequest first - if (!OrchestratorUtils.validateJobRequest(request)) { - logger.error("Invalid Job request sent, Experiment creation failed"); - return false; - } - String experimentID = OrchestratorUtils.getUniqueID(request); + public boolean launchExperiment(String experimentID) throws OrchestratorException { // we give higher priority to userExperimentID if (experimentID == null) { - logger.error("Invalid Experiment ID given: " + request.getUserName()); + logger.error("Invalid Experiment ID given: " + experimentID); return false; } - //todo use a more concrete user type in to this - //FIXME: (MEP) Why don't we pass the JobRequest to the registry and let it do all of this? Or just store the JobRequest as an object directly in the registry? try { - if (request.getHostDescription() != null) { - if (!airavataRegistry.isHostDescriptorExists(request.getHostDescription().getType().getHostName())) { - airavataRegistry.addHostDescriptor(request.getHostDescription()); - } - } - if (request.getServiceDescription() != null) { - if (!airavataRegistry.isServiceDescriptorExists(request.getServiceDescription().getType().getName())) { - airavataRegistry.addServiceDescriptor(request.getServiceDescription()); - } - } - if (request.getApplicationDescription() != null) { - if (request.getServiceDescription() != null && request.getHostDescription() != null) { - if (!airavataRegistry.isApplicationDescriptorExists(request.getServiceDescription().getType().getName(), - request.getHostDescription().getType().getHostName(), request.getApplicationDescription().getType().getApplicationName().getStringValue())) { - airavataRegistry.addApplicationDescriptor(request.getServiceDescription(), - request.getHostDescription(), request.getApplicationDescription()); - } - } else { - String error = "Providing just Application Descriptor is not sufficient to save to Registry"; - logger.error(error); - throw new OrchestratorException(error); - } - } airavataRegistry.changeStatus(experimentID, AiravataJobState.State.ACCEPTED); if (orchestratorContext.getOrchestratorConfiguration().getThreadPoolSize() == 0) { - jobSubmitter.directJobSubmit(request); + jobSubmitter.directJobSubmit(experimentID); } - - //todo save jobRequest data in to the database - } catch (RegistryException e) { + } catch (RegistryException e) + { //todo put more meaningful error message - logger.error("Failed to create experiment for the request from " + request.getUserName()); + logger.error("Failed to create experiment for the request from " + experimentID); return false; } return true; } - - public void startJobSubmitter() throws OrchestratorException { - //FIXME: (MEP) Why create a new thread for jobSubmittedWorker but use the pool for HangedJobWorker? - //FIXME: (MEP) As discussed on the dev list, we need to improve this - NewJobWorker jobSubmitterWorker = new NewJobWorker(orchestratorContext); - executor.execute(jobSubmitterWorker); - - for (int i = 0; i < orchestratorContext.getOrchestratorConfiguration().getThreadPoolSize() - 1; i++) { - HangedJobWorker hangedJobWorker = new HangedJobWorker(orchestratorContext); - executor.execute(hangedJobWorker); - } - } - - public boolean terminateExperiment(String experimentID) throws OrchestratorException { - try { - AiravataJobState state = orchestratorContext.getRegistry().getState(experimentID); - if (state.getJobState().equals(AiravataJobState.State.RUNNING) || state.getJobState().equals(AiravataJobState.State.PENDING) || - state.getJobState().equals(AiravataJobState.State.ACTIVE) || state.getJobState().equals(AiravataJobState.State.SUBMITTED)) { - - //todo perform cancelling and last peform the database update - - orchestratorContext.getRegistry().changeStatus(experimentID, AiravataJobState.State.CANCELLED); - } else if (state.getJobState().equals(AiravataJobState.State.DONE)) { - String error = "Job is already Finished so cannot cancel the job " + experimentID; - logger.error(error); - new OrchestratorException(error); - } else { - // do nothing but simply change the job state to cancelled because job is not yet submitted to the resource - orchestratorContext.getRegistry().changeStatus(experimentID, AiravataJobState.State.CANCELLED); - } - - } catch (RegistryException e) { - String error = "Error reading the job state for the given Experiment ID: " + experimentID; - logger.error(error); - throw new OrchestratorException(error, e); - } - return true; - } - - public boolean validateExperiment(String experimentID) throws OrchestratorException{ - return jobMetadataValidator.validate(experimentID); - } } http://git-wip-us.apache.org/repos/asf/airavata/blob/f78c0e15/modules/orchestrator/orchestrator-core/src/main/resources/orchestrator.properties ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/resources/orchestrator.properties b/modules/orchestrator/orchestrator-core/src/main/resources/orchestrator.properties index 85568a4..54147e9 100644 --- a/modules/orchestrator/orchestrator-core/src/main/resources/orchestrator.properties +++ b/modules/orchestrator/orchestrator-core/src/main/resources/orchestrator.properties @@ -23,3 +23,4 @@ submitter.interval=10000 threadpool.size=10 start.submitter=true embedded.mode=true +enable.validation=false http://git-wip-us.apache.org/repos/asf/airavata/blob/f78c0e15/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/NewOrchestratorTest.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/NewOrchestratorTest.java b/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/NewOrchestratorTest.java index 99654d4..7402103 100644 --- a/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/NewOrchestratorTest.java +++ b/modules/orchestrator/orchestrator-core/src/test/java/org/apache/airavata/orchestrator/core/NewOrchestratorTest.java @@ -48,156 +48,133 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; -public class NewOrchestratorTest extends AbstractOrchestratorTest{ +public class NewOrchestratorTest extends AbstractOrchestratorTest { private static final Logger log = LoggerFactory.getLogger(NewOrchestratorTest.class); - private Orchestrator orchestrator; - private String experimentID; - @BeforeTest - public void setUp() throws Exception { - AiravataUtils.setExecutionAsServer(); - super.setUp(); - orchestrator = new SimpleOrchestratorImpl(); - orchestrator.initialize(); - createJobRequestWithDocuments(getAiravataAPI()); - } - - private void createJobRequestWithDocuments(AiravataAPI airavataAPI) { - // creating host description - HostDescription descriptor = new HostDescription(); - descriptor.getType().setHostName("localhost"); - descriptor.getType().setHostAddress("127.0.0.1"); - try { - airavataAPI.getApplicationManager().saveHostDescription(descriptor); - } catch (AiravataAPIInvocationException e) { - e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. - } - - ServiceDescription serviceDescription = new ServiceDescription(); - List<InputParameterType> inputParameters = new ArrayList<InputParameterType>(); - List<OutputParameterType> outputParameters = new ArrayList<OutputParameterType>(); - serviceDescription.getType().setName("Echo"); - serviceDescription.getType().setDescription("Echo service"); - // Creating input parameters - InputParameterType parameter = InputParameterType.Factory.newInstance(); - parameter.setParameterName("echo_input"); - parameter.setParameterDescription("echo input"); - ParameterType parameterType = parameter.addNewParameterType(); - parameterType.setType(DataType.STRING); - parameterType.setName("String"); - inputParameters.add(parameter); - - // Creating output parameters - OutputParameterType outputParameter = OutputParameterType.Factory.newInstance(); - outputParameter.setParameterName("echo_output"); - outputParameter.setParameterDescription("Echo output"); - ParameterType outputParaType = outputParameter.addNewParameterType(); - outputParaType.setType(DataType.STRING); - outputParaType.setName("String"); - outputParameters.add(outputParameter); - - // Setting input and output parameters to serviceDescriptor - serviceDescription.getType().setInputParametersArray(inputParameters.toArray(new InputParameterType[]{})); - serviceDescription.getType().setOutputParametersArray(outputParameters.toArray(new OutputParameterType[]{})); - - try { - airavataAPI.getApplicationManager().saveServiceDescription(serviceDescription); - } catch (AiravataAPIInvocationException e) { - e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. - } - - ApplicationDescription applicationDeploymentDescription = new ApplicationDescription(); - ApplicationDeploymentDescriptionType applicationDeploymentDescriptionType = applicationDeploymentDescription - .getType(); - applicationDeploymentDescriptionType.addNewApplicationName().setStringValue("EchoApplication"); - applicationDeploymentDescriptionType.setExecutableLocation("/bin/echo"); - applicationDeploymentDescriptionType.setScratchWorkingDirectory("/tmp"); - - try { - airavataAPI.getApplicationManager().saveApplicationDescription("Echo", "localhost", applicationDeploymentDescription); - } catch (AiravataAPIInvocationException e) { - e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. - } - - //Using new airavata-api methods to store experiment metadata - BasicMetadata basicMetadata = new BasicMetadata(); - basicMetadata.setExperimentName("test123"); - basicMetadata.setUserName("admin"); - basicMetadata.setUserNameIsSet(true); - basicMetadata.setProjectID("default"); - - AdvancedInputDataHandling advancedInputDataHandling = new AdvancedInputDataHandling(); - AdvancedOutputDataHandling advancedOutputDataHandling = new AdvancedOutputDataHandling(); - ComputationalResourceScheduling computationalResourceScheduling = new ComputationalResourceScheduling(); - QualityOfServiceParams qualityOfServiceParams = new QualityOfServiceParams(); - ConfigurationData configurationData = new ConfigurationData(); - - - HashMap<String, String> exInputs = new HashMap<String, String>(); - exInputs.put("echo_input", "echo_output=hello"); - - configurationData.setExperimentInputs(exInputs); - configurationData.setAdvanceInputDataHandling(advancedInputDataHandling); - configurationData.setAdvanceOutputDataHandling(advancedOutputDataHandling); - configurationData.setComputationalResourceScheduling(computationalResourceScheduling); - configurationData.setQosParams(qualityOfServiceParams); - configurationData.setApplicationId("Echo"); - - Registry registry = new RegistryImpl(); - experimentID = (String) registry.add(ParentDataType.EXPERIMENT_BASIC_DATA, basicMetadata); - registry.add(ChildDataType.EXPERIMENT_CONFIGURATION_DATA, configurationData, experimentID); - } - - @Test - public void noDescriptorTest() throws Exception { - - JobRequest jobRequest = createJobRequestWithoutDocuments(experimentID); - - boolean b = orchestrator.launchExperiment(jobRequest); - - if (b) { - // This means orchestrator successfully accepted the job - Assert.assertTrue(true); - } else { - Assert.assertFalse(true); - } - } - - - private JobRequest createJobRequestWithoutDocuments(String systemExpID) { - JobRequest jobRequest = new JobRequest(); - jobRequest.setServiceName("Echo"); -// -// HashMap<String, Object> inputData = new HashMap<String, Object>(); -// ActualParameter echo_input = new ActualParameter(); -// ((StringParameterType) echo_input.getType()).setValue("echo_output=hello"); -// inputData.put("echo_input", echo_input); - - HashMap<String, Object> outputData = new HashMap<String, Object>(); - - - // setting all the parameters to jobRequest - jobRequest.setSystemExperimentID(systemExpID); -// jobRequest.setInputParameters(inputData); - jobRequest.setOutputParameters(outputData); - - return jobRequest; - } - - private AiravataAPI getAiravataAPI() { - AiravataAPI airavataAPI = null; - if (airavataAPI == null) { - try { - String systemUserName = ServerSettings.getSystemUser(); - String gateway = ServerSettings.getSystemUserGateway(); - airavataAPI = AiravataAPIFactory.getAPI(gateway, systemUserName); - } catch (ApplicationSettingsException e) { - e.printStackTrace(); - } catch (AiravataAPIInvocationException e) { - e.printStackTrace(); - } - } - return airavataAPI; - } + private Orchestrator orchestrator; + private String experimentID; + + @BeforeTest + public void setUp() throws Exception { + AiravataUtils.setExecutionAsServer(); + super.setUp(); + orchestrator = new SimpleOrchestratorImpl(); + createJobRequestWithDocuments(getAiravataAPI()); + } + + private void createJobRequestWithDocuments(AiravataAPI airavataAPI) { + // creating host description + HostDescription descriptor = new HostDescription(); + descriptor.getType().setHostName("localhost"); + descriptor.getType().setHostAddress("127.0.0.1"); + try { + airavataAPI.getApplicationManager().saveHostDescription(descriptor); + } catch (AiravataAPIInvocationException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + + ServiceDescription serviceDescription = new ServiceDescription(); + List<InputParameterType> inputParameters = new ArrayList<InputParameterType>(); + List<OutputParameterType> outputParameters = new ArrayList<OutputParameterType>(); + serviceDescription.getType().setName("Echo"); + serviceDescription.getType().setDescription("Echo service"); + // Creating input parameters + InputParameterType parameter = InputParameterType.Factory.newInstance(); + parameter.setParameterName("echo_input"); + parameter.setParameterDescription("echo input"); + ParameterType parameterType = parameter.addNewParameterType(); + parameterType.setType(DataType.STRING); + parameterType.setName("String"); + inputParameters.add(parameter); + + // Creating output parameters + OutputParameterType outputParameter = OutputParameterType.Factory.newInstance(); + outputParameter.setParameterName("echo_output"); + outputParameter.setParameterDescription("Echo output"); + ParameterType outputParaType = outputParameter.addNewParameterType(); + outputParaType.setType(DataType.STRING); + outputParaType.setName("String"); + outputParameters.add(outputParameter); + + // Setting input and output parameters to serviceDescriptor + serviceDescription.getType().setInputParametersArray(inputParameters.toArray(new InputParameterType[]{})); + serviceDescription.getType().setOutputParametersArray(outputParameters.toArray(new OutputParameterType[]{})); + + try { + airavataAPI.getApplicationManager().saveServiceDescription(serviceDescription); + } catch (AiravataAPIInvocationException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + + ApplicationDescription applicationDeploymentDescription = new ApplicationDescription(); + ApplicationDeploymentDescriptionType applicationDeploymentDescriptionType = applicationDeploymentDescription + .getType(); + applicationDeploymentDescriptionType.addNewApplicationName().setStringValue("EchoApplication"); + applicationDeploymentDescriptionType.setExecutableLocation("/bin/echo"); + applicationDeploymentDescriptionType.setScratchWorkingDirectory("/tmp"); + + try { + airavataAPI.getApplicationManager().saveApplicationDescription("Echo", "localhost", applicationDeploymentDescription); + } catch (AiravataAPIInvocationException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + + //Using new airavata-api methods to store experiment metadata + BasicMetadata basicMetadata = new BasicMetadata(); + basicMetadata.setExperimentName("test123"); + basicMetadata.setUserName("admin"); + basicMetadata.setUserNameIsSet(true); + basicMetadata.setProjectID("default"); + + AdvancedInputDataHandling advancedInputDataHandling = new AdvancedInputDataHandling(); + AdvancedOutputDataHandling advancedOutputDataHandling = new AdvancedOutputDataHandling(); + ComputationalResourceScheduling computationalResourceScheduling = new ComputationalResourceScheduling(); + QualityOfServiceParams qualityOfServiceParams = new QualityOfServiceParams(); + ConfigurationData configurationData = new ConfigurationData(); + + + HashMap<String, String> exInputs = new HashMap<String, String>(); + exInputs.put("echo_input", "echo_output=hello"); + + configurationData.setExperimentInputs(exInputs); + configurationData.setAdvanceInputDataHandling(advancedInputDataHandling); + configurationData.setAdvanceOutputDataHandling(advancedOutputDataHandling); + configurationData.setComputationalResourceScheduling(computationalResourceScheduling); + configurationData.setQosParams(qualityOfServiceParams); + configurationData.setApplicationId("Echo"); + + Registry registry = new RegistryImpl(); + experimentID = (String) registry.add(ParentDataType.EXPERIMENT_BASIC_DATA, basicMetadata); + registry.add(ChildDataType.EXPERIMENT_CONFIGURATION_DATA, configurationData, experimentID); + } + + @Test + public void noDescriptorTest() throws Exception { + + boolean b = orchestrator.launchExperiment(experimentID); + + if (b) { + // This means orchestrator successfully accepted the job + Assert.assertTrue(true); + } else { + Assert.assertFalse(true); + } + } + + private AiravataAPI getAiravataAPI() { + AiravataAPI airavataAPI = null; + if (airavataAPI == null) { + try { + String systemUserName = ServerSettings.getSystemUser(); + String gateway = ServerSettings.getSystemUserGateway(); + airavataAPI = AiravataAPIFactory.getAPI(gateway, systemUserName); + } catch (ApplicationSettingsException e) { + e.printStackTrace(); + } catch (AiravataAPIInvocationException e) { + e.printStackTrace(); + } + } + return airavataAPI; + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/f78c0e15/modules/orchestrator/orchestrator-core/src/test/resources/orchestrator.properties ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/test/resources/orchestrator.properties b/modules/orchestrator/orchestrator-core/src/test/resources/orchestrator.properties index 766d9b1..264a28e 100644 --- a/modules/orchestrator/orchestrator-core/src/test/resources/orchestrator.properties +++ b/modules/orchestrator/orchestrator-core/src/test/resources/orchestrator.properties @@ -22,4 +22,5 @@ job.validator=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppData submitter.interval=10000 threadpool.size=1 start.submitter=true -embedded.mode=true \ No newline at end of file +embedded.mode=true +enable.validation=false \ No newline at end of file
