Repository: airavata Updated Branches: refs/heads/master 867778bf1 -> c072a3965
Added WRF sample and fixed outut handling Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/3e08f214 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/3e08f214 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/3e08f214 Branch: refs/heads/master Commit: 3e08f21494c50859c169bf735585950d0d3744fc Parents: a6de79b Author: raminder <[email protected]> Authored: Wed May 28 15:10:08 2014 -0400 Committer: raminder <[email protected]> Committed: Wed May 28 15:10:08 2014 -0400 ---------------------------------------------------------------------- .../client/samples/CreateLaunchExperiment.java | 81 +++++++++++++++++- .../airavata/client/tools/DocumentCreator.java | 10 +-- .../apache/airavata/gfac/core/cpi/GFacImpl.java | 5 +- .../airavata/gfac/core/utils/OutputUtils.java | 86 ++++++++++---------- .../gsissh/handler/GSISSHOutputHandler.java | 4 + 5 files changed, 135 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/3e08f214/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java index b728c7f..4493552 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java @@ -35,6 +35,9 @@ import org.apache.airavata.client.api.exception.AiravataAPIInvocationException; import org.apache.airavata.client.tools.DocumentCreator; import org.apache.airavata.common.utils.AiravataUtils; import org.apache.airavata.model.util.ExperimentModelUtil; +import org.apache.airavata.schemas.gfac.InputParameterType; +import org.apache.airavata.schemas.gfac.OutputParameterType; +import org.apache.airavata.schemas.gfac.ParameterType; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,6 +67,8 @@ public class CreateLaunchExperiment { // final String expId = createExperimentForTrestles(airavata); // final String expId = createExperimentForStampede(airavata); final String expId = createExperimentForLocalHost(airavata); +// final String expId = createExperimentForLonestar(airavata); +// final String expId = createExperimentWRFTrestles(airavata); System.out.println("Experiment ID : " + expId); // updateExperiment(airavata, expId); launchExperiment(airavata, expId); @@ -137,7 +142,11 @@ public class CreateLaunchExperiment { } } } - System.out.println(airavata.getExperimentStatus(expId)); + ExperimentStatus experimentStatus = airavata.getExperimentStatus(expId); + if(experimentStatus.getExperimentState().equals(ExperimentState.FAILED)){ + return; + } + System.out.println(experimentStatus); Thread.sleep(5000); } catch (Exception e) { e.printStackTrace(); @@ -251,6 +260,71 @@ public class CreateLaunchExperiment { throw new TException(e); } } + + public static String createExperimentWRFTrestles(Airavata.Client client) throws TException { + try{ + List<DataObjectType> exInputs = new ArrayList<DataObjectType>(); + DataObjectType input = new DataObjectType(); + input.setKey("WRF_Namelist"); + input.setType(DataType.URI); + input.setValue("/Users/raminder/Downloads/wrf_sample_inputs/namelist.input"); + + DataObjectType input1 = new DataObjectType(); + input1.setKey("WRF_Input_File"); + input1.setType(DataType.URI); + input1.setValue("/Users/raminder/Downloads/wrf_sample_inputs/wrfinput_d01"); + + DataObjectType input2 = new DataObjectType(); + input2.setKey("WRF_Boundary_File"); + input2.setType(DataType.URI); + input2.setValue("/Users/raminder/Downloads/wrf_sample_inputs/wrfbdy_d01"); + + exInputs.add(input); + exInputs.add(input1); + exInputs.add(input2); + + + List<DataObjectType> exOut = new ArrayList<DataObjectType>(); + DataObjectType output = new DataObjectType(); + output.setKey("WRF_Output"); + output.setType(DataType.URI); + output.setValue(""); + + DataObjectType output1 = new DataObjectType(); + output1.setKey("WRF_Execution_Log"); + output1.setType(DataType.URI); + output1.setValue(""); + + + exOut.add(output); + exOut.add(output1); + + + Experiment simpleExperiment = + ExperimentModelUtil.createSimpleExperiment("default", "admin", "WRFExperiment", "Testing", "WRF", exInputs); + simpleExperiment.setExperimentOutputs(exOut); + + ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("trestles.sdsc.edu", 32, 2, 1, "normal", 0, 0, 1, "sds128"); + UserConfigurationData userConfigurationData = new UserConfigurationData(); + userConfigurationData.setAiravataAutoSchedule(false); + userConfigurationData.setOverrideManualScheduledParams(false); + userConfigurationData.setComputationalResourceScheduling(scheduling); + simpleExperiment.setUserConfigurationData(userConfigurationData); + return client.createExperiment(simpleExperiment); + } catch (AiravataSystemException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new AiravataSystemException(e); + } catch (InvalidRequestException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new InvalidRequestException(e); + } catch (AiravataClientException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new AiravataClientException(e); + }catch (TException e) { + logger.error("Error occured while creating the experiment...", e.getMessage()); + throw new TException(e); + } + } public static String cloneExperiment(Airavata.Client client, String expId) throws TException { try{ @@ -428,12 +502,11 @@ public class CreateLaunchExperiment { String projectId = client.createProject(project); Experiment simpleExperiment = - ExperimentModelUtil.createSimpleExperiment(projectId, "admin", "echoExperiment", "SimpleEcho4", "SimpleEcho4", exInputs); + ExperimentModelUtil.createSimpleExperiment(projectId, "admin", "echoExperiment", "SimpleEcho4", "Echo", exInputs); simpleExperiment.setExperimentOutputs(exOut); ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("lonestar.tacc.utexas.edu", 1, 1, 1, "normal", 0, 0, 1, "TG-STA110014S"); - scheduling.setResourceHostId("lonestar-host"); UserConfigurationData userConfigurationData = new UserConfigurationData(); userConfigurationData.setAiravataAutoSchedule(false); userConfigurationData.setOverrideManualScheduledParams(false); @@ -463,6 +536,8 @@ public class CreateLaunchExperiment { throw new TException(e); } } + + public static void launchExperiment (Airavata.Client client, String expId) throws TException{ try { http://git-wip-us.apache.org/repos/asf/airavata/blob/3e08f214/modules/airavata-client/src/main/java/org/apache/airavata/client/tools/DocumentCreator.java ---------------------------------------------------------------------- diff --git a/modules/airavata-client/src/main/java/org/apache/airavata/client/tools/DocumentCreator.java b/modules/airavata-client/src/main/java/org/apache/airavata/client/tools/DocumentCreator.java index 1e50832..801c6a0 100644 --- a/modules/airavata-client/src/main/java/org/apache/airavata/client/tools/DocumentCreator.java +++ b/modules/airavata-client/src/main/java/org/apache/airavata/client/tools/DocumentCreator.java @@ -358,7 +358,7 @@ public class DocumentCreator { */ String wrfserviceName = "WRF"; ServiceDescription wrfServ = new ServiceDescription(); - wrfServ.getType().setName(serviceName); + wrfServ.getType().setName(wrfserviceName); List<InputParameterType> wrfinputList = new ArrayList<InputParameterType>(); List<OutputParameterType> wrfoutputList = new ArrayList<OutputParameterType>(); @@ -425,11 +425,11 @@ public class DocumentCreator { QueueType wrfQueueType = wrfApp.addNewQueue(); wrfQueueType.setQueueName("normal"); - wrfApp.setCpuCount(1); - wrfApp.setJobType(JobTypeType.SERIAL); - wrfApp.setNodeCount(1); + wrfApp.setCpuCount(32); + wrfApp.setJobType(JobTypeType.MPI); + wrfApp.setNodeCount(2); wrfApp.setProcessorsPerNode(1); - wrfApp.setMaxWallTime(10); + wrfApp.setMaxWallTime(30); /* * Use bat file if it is compiled on Windows */ http://git-wip-us.apache.org/repos/asf/airavata/blob/3e08f214/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/GFacImpl.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/GFacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/GFacImpl.java index e5f1864..f1fa244 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/GFacImpl.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/GFacImpl.java @@ -212,10 +212,13 @@ public class GFacImpl implements GFac { // 2. Add another property to jobExecutionContext and read them inside the provider and use it. String serviceName = taskData.getApplicationId(); if (serviceName == null) { - throw new GFacException("Error executing the job because there is not Application Name in this Experiment: " + serviceName); + throw new GFacException("Error executing the job because there is not Application Name in this Experiment: " + serviceName ); } ServiceDescription serviceDescription = airavataRegistry2.getServiceDescriptor(serviceName); + if (serviceDescription == null ) { + throw new GFacException("Error executing the job because there is not Application Name in this Experiment: " + serviceName ); + } String hostName; HostDescription hostDescription = null; if(taskData.getTaskScheduling().getResourceHostId() != null){ http://git-wip-us.apache.org/repos/asf/airavata/blob/3e08f214/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java index f9cc10b..75b03ad 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java @@ -60,9 +60,10 @@ public class OutputUtils { } else if ("StdErr".equals(actual.getType().getType().toString())) { ((StdErrParameterType) actual.getType()).setValue(stderr); result.put(paramName, actual); - }else if("URI".equals(actual.getType().getType().toString())){ - continue; - } + } +// else if("URI".equals(actual.getType().getType().toString())){ +// continue; +// } else { String parseStdout = parseStdout(stdout, paramName); if (parseStdout != null) { @@ -75,47 +76,48 @@ public class OutputUtils { return result; } - public static void fillOutputFromStdout1(Map<String, Object> output, String stdout, String stderr,List<DataObjectType> outputArray) throws Exception { + public static void fillOutputFromStdout1(Map<String, Object> output, String stdout, String stderr, List<DataObjectType> outputArray) throws Exception { - if (stdout == null || stdout.equals("")){ - throw new GFacHandlerException("Standard output is empty."); - } + if (stdout == null || stdout.equals("")) { + throw new GFacHandlerException("Standard output is empty."); + } - Set<String> keys = output.keySet(); - for (String paramName : keys) { - ActualParameter actual = (ActualParameter) output.get(paramName); - // if parameter value is not already set, we let it go - - if (actual == null) { - continue; - } - if ("StdOut".equals(actual.getType().getType().toString())) { - DataObjectType out = new DataObjectType(); - out.setKey(paramName); - out.setType(DataType.STDOUT); - out.setValue(stdout); - outputArray.add(out); - } else if ("StdErr".equals(actual.getType().getType().toString())) { - DataObjectType out = new DataObjectType(); - out.setKey(paramName); - out.setType(DataType.STDERR); - out.setValue(stderr); - outputArray.add(out); - }else if("URI".equals(actual.getType().getType().toString())){ - continue; - } - else { - String parseStdout = parseStdout(stdout, paramName); - if (parseStdout != null) { - DataObjectType out = new DataObjectType(); - out.setKey(paramName); - out.setType(DataType.STRING); - out.setValue(parseStdout); - outputArray.add(out); - } - } - } - } + Set<String> keys = output.keySet(); + for (String paramName : keys) { + ActualParameter actual = (ActualParameter) output.get(paramName); + // if parameter value is not already set, we let it go + + if (actual == null) { + continue; + } + if ("StdOut".equals(actual.getType().getType().toString())) { + DataObjectType out = new DataObjectType(); + out.setKey(paramName); + out.setType(DataType.STDOUT); + out.setValue(stdout); + outputArray.add(out); + } else if ("StdErr".equals(actual.getType().getType().toString())) { + DataObjectType out = new DataObjectType(); + out.setKey(paramName); + out.setType(DataType.STDERR); + out.setValue(stderr); + outputArray.add(out); + } +// else if ("URI".equals(actual.getType().getType().toString())) { +// continue; +// } + else { + String parseStdout = parseStdout(stdout, paramName); + if (parseStdout != null) { + DataObjectType out = new DataObjectType(); + out.setKey(paramName); + out.setType(DataType.STRING); + out.setValue(parseStdout); + outputArray.add(out); + } + } + } + } private static String parseStdout(String stdout, String outParam) throws Exception { String regex = Pattern.quote(outParam) + regexPattern; http://git-wip-us.apache.org/repos/asf/airavata/blob/3e08f214/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java index d48f92b..b59625d 100644 --- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java +++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java @@ -166,6 +166,10 @@ public class GSISSHOutputHandler extends AbstractHandler { List<String> outputList = cluster.listDirectory(app.getOutputDataDirectory()); if (outputList.size() == 0 || outputList.get(0).isEmpty()) { OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr, outputArray); + for (DataObjectType outputLocation : outputArray){ + jobExecutionContext.addOutputFile(outputLocation.getValue()); + + } OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); break; } else {
