Repository: airavata Updated Branches: refs/heads/master 98c9a34ea -> 8ff835fdd
treating STDOUT and STDERR as outputs Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/8ff835fd Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/8ff835fd Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/8ff835fd Branch: refs/heads/master Commit: 8ff835fdde359c5edf8e34f426f76ff6f50c1d26 Parents: 98c9a34 Author: Chathuri Wimalasena <[email protected]> Authored: Thu Jan 8 14:38:17 2015 -0500 Committer: Chathuri Wimalasena <[email protected]> Committed: Thu Jan 8 14:38:17 2015 -0500 ---------------------------------------------------------------------- .../airavata/gfac/core/cpi/BetterGfacImpl.java | 34 +++++++++++------ .../apache/airavata/job/GFacConfigXmlTest.java | 6 --- .../gsissh/handler/GSISSHOutputHandler.java | 40 +++++++++++++++++++- .../gfac/gsissh/util/GFACGSISSHUtils.java | 4 +- .../ssh/handler/AdvancedSCPOutputHandler.java | 26 ++++++++++++- 5 files changed, 86 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/8ff835fd/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java index 77c9d60..006319c 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java @@ -51,9 +51,7 @@ import org.apache.airavata.gfac.core.utils.GFacUtils; import org.apache.airavata.messaging.core.Publisher; import org.apache.airavata.messaging.core.PublisherFactory; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; -import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription; -import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType; -import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType; +import org.apache.airavata.model.appcatalog.appinterface.*; import org.apache.airavata.model.appcatalog.computeresource.*; import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference; import org.apache.airavata.model.messaging.event.*; @@ -303,13 +301,6 @@ public class BetterGfacImpl implements GFac,Watcher { List<InputDataObjectType> taskInputs = taskData.getApplicationInputs(); jobExecutionContext.setInMessageContext(new MessageContext(GFacUtils.getInputParamMap(taskInputs))); -// List<OutputDataObjectType> outputData = experiment.getExperimentOutputs(); - List<OutputDataObjectType> taskOutputs = taskData.getApplicationOutputs(); - if (taskOutputs == null || taskOutputs.isEmpty() ){ - taskOutputs = applicationInterface.getApplicationOutputs(); - } - jobExecutionContext.setOutMessageContext(new MessageContext(GFacUtils.getOuputParamMap(taskOutputs))); - jobExecutionContext.setProperty(Constants.PROP_TOPIC, experimentID); jobExecutionContext.setGfac(this); jobExecutionContext.setZk(zk); @@ -384,6 +375,23 @@ public class BetterGfacImpl implements GFac,Watcher { } else { setUpWorkingLocation(jobExecutionContext, applicationInterface, "/tmp"); } + List<OutputDataObjectType> taskOutputs = taskData.getApplicationOutputs(); + if (taskOutputs == null || taskOutputs.isEmpty() ){ + taskOutputs = applicationInterface.getApplicationOutputs(); + } + + for (OutputDataObjectType objectType : taskOutputs){ + if (objectType.getAddedToCommandLine() != null && objectType.getAddedToCommandLine()== CommandLineType.EXPLICIT){ + objectType.setValue(jobExecutionContext.getOutputDir() + File.separator + objectType.getName()); + } + if (objectType.getType() == DataType.STDOUT){ + objectType.setValue(jobExecutionContext.getOutputDir() + File.separator + jobExecutionContext.getApplicationName() + ".stdout"); + } + if (objectType.getType() == DataType.STDERR){ + objectType.setValue(jobExecutionContext.getOutputDir() + File.separator + jobExecutionContext.getApplicationName() + ".stderr"); + } + } + jobExecutionContext.setOutMessageContext(new MessageContext(GFacUtils.getOuputParamMap(taskOutputs))); return jobExecutionContext; } @@ -402,8 +410,10 @@ public class BetterGfacImpl implements GFac,Watcher { /* * Input and Output Directory */ - jobExecutionContext.setInputDir(workingDir + File.separator + Constants.INPUT_DATA_DIR_VAR_NAME); - jobExecutionContext.setOutputDir(workingDir + File.separator + Constants.OUTPUT_DATA_DIR_VAR_NAME); +// jobExecutionContext.setInputDir(workingDir + File.separator + Constants.INPUT_DATA_DIR_VAR_NAME); + jobExecutionContext.setInputDir(workingDir); +// jobExecutionContext.setOutputDir(workingDir + File.separator + Constants.OUTPUT_DATA_DIR_VAR_NAME); + jobExecutionContext.setOutputDir(workingDir); /* * Stdout and Stderr for Shell http://git-wip-us.apache.org/repos/asf/airavata/blob/8ff835fd/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/GFacConfigXmlTest.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/GFacConfigXmlTest.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/GFacConfigXmlTest.java index 237173f..507ab9e 100644 --- a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/GFacConfigXmlTest.java +++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/GFacConfigXmlTest.java @@ -21,14 +21,8 @@ package org.apache.airavata.job; import junit.framework.Assert; -import org.airavata.appcatalog.cpi.AppCatalog; -import org.airavata.appcatalog.cpi.AppCatalogException; -import org.apache.aiaravata.application.catalog.data.impl.AppCatalogFactory; import org.apache.airavata.common.utils.AiravataUtils; -import org.apache.airavata.gfac.ExecutionMode; import org.apache.airavata.gfac.GFacConfiguration; -import org.apache.airavata.gfac.GFacException; -import org.apache.airavata.gfac.Scheduler; import org.apache.airavata.gfac.core.context.ApplicationContext; import org.apache.airavata.gfac.core.context.JobExecutionContext; import org.apache.airavata.gfac.core.cpi.BetterGfacImpl; http://git-wip-us.apache.org/repos/asf/airavata/blob/8ff835fd/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java index 107f1fe..a29028d 100644 --- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java +++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java @@ -129,7 +129,7 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler { data.append(oldFiles.get(index++)).append(","); } else { int i = 0; - localStdOutFile = new File(outputDataDir + File.separator + timeStampedExperimentID + "stdout"); + localStdOutFile = new File(outputDataDir + File.separator + jobExecutionContext.getApplicationName() + ".stdout"); while(stdOutStr.isEmpty()){ try { cluster.scpFrom(jobExecutionContext.getStandardOutput(), localStdOutFile.getAbsolutePath()); @@ -149,7 +149,7 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler { localStdErrFile = new File(oldFiles.get(index)); data.append(oldFiles.get(index++)).append(","); } else { - localStdErrFile = new File(outputDataDir + File.separator + timeStampedExperimentID + "stderr"); + localStdErrFile = new File(outputDataDir + File.separator + jobExecutionContext.getApplicationName() + ".stderr"); cluster.scpFrom(jobExecutionContext.getStandardError(), localStdErrFile.getAbsolutePath()); StringBuffer temp = new StringBuffer(data.append(localStdErrFile.getAbsolutePath()).append(",").toString()); GFacUtils.savePluginData(jobExecutionContext, temp.insert(0, ++index), this.getClass().getName()); @@ -216,6 +216,42 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler { dataObjectType.setName(key); dataObjectType.setType(DataType.URI); outputArray.add(dataObjectType); + }else if (DataType.STDOUT == outputDataObjectType1.getType()) { + String localFile; + if (index < oldIndex) { + localFile = oldFiles.get(index); + data.append(oldFiles.get(index++)).append(","); + } else { + String fileName = localStdOutFile.getName(); + localFile = outputDataDir + File.separator + fileName; + StringBuffer temp = new StringBuffer(data.append(localFile).append(",").toString()); + GFacUtils.savePluginData(jobExecutionContext, temp.insert(0, ++index), this.getClass().getName()); + } + jobExecutionContext.addOutputFile(localFile); + outputDataObjectType1.setValue(localFile); + OutputDataObjectType dataObjectType = new OutputDataObjectType(); + dataObjectType.setValue(localFile); + dataObjectType.setName(key); + dataObjectType.setType(DataType.STDOUT); + outputArray.add(dataObjectType); + }else if (DataType.STDERR == outputDataObjectType1.getType()) { + String localFile; + if (index < oldIndex) { + localFile = oldFiles.get(index); + data.append(oldFiles.get(index++)).append(","); + } else { + String fileName = localStdErrFile.getName(); + localFile = outputDataDir + File.separator + fileName; + StringBuffer temp = new StringBuffer(data.append(localFile).append(",").toString()); + GFacUtils.savePluginData(jobExecutionContext, temp.insert(0, ++index), this.getClass().getName()); + } + jobExecutionContext.addOutputFile(localFile); + outputDataObjectType1.setValue(localFile); + OutputDataObjectType dataObjectType = new OutputDataObjectType(); + dataObjectType.setValue(localFile); + dataObjectType.setName(key); + dataObjectType.setType(DataType.STDERR); + outputArray.add(dataObjectType); } } break; http://git-wip-us.apache.org/repos/asf/airavata/blob/8ff835fd/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java index 6ef3395..1e03668 100644 --- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java +++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java @@ -214,7 +214,7 @@ public class GFACGSISSHUtils { } for (InputDataObjectType inputDataObjectType : sortedInputSet) { if (inputDataObjectType.getAddedToCommandLine() != null - && inputDataObjectType.getAddedToCommandLine() == CommandLineType.EXPLICIT) { + && inputDataObjectType.getAddedToCommandLine() == CommandLineType.IMPLICIT) { continue; } if (inputDataObjectType.getApplicationArgument() != null @@ -233,7 +233,7 @@ public class GFACGSISSHUtils { if (outputParam instanceof OutputDataObjectType) { OutputDataObjectType output = (OutputDataObjectType) outputParam; if (output.getValue() != null && !output.getValue().equals("") && output.getAddedToCommandLine() != null - && output.getAddedToCommandLine() == CommandLineType.IMPLICIT) { + && output.getAddedToCommandLine() == CommandLineType.EXPLICIT) { inputValues.add(output.getValue()); } } http://git-wip-us.apache.org/repos/asf/airavata/blob/8ff835fd/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java index 154041f..29091d5 100644 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java +++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java @@ -34,8 +34,10 @@ import org.apache.airavata.gsi.ssh.api.SSHApiException; import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo; import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo; import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPublicKeyFileAuthentication; +import org.apache.airavata.model.appcatalog.appinterface.CommandLineType; import org.apache.airavata.model.appcatalog.appinterface.DataType; import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType; +import org.apache.airavata.model.appcatalog.appinterface.ValidityType; import org.apache.airavata.model.workspace.experiment.CorrectiveAction; import org.apache.airavata.model.workspace.experiment.ErrorCategory; import org.apache.airavata.registry.cpi.ChildDataType; @@ -146,9 +148,9 @@ public class AdvancedSCPOutputHandler extends AbstractHandler { OutputDataObjectType outputDataObjectType = (OutputDataObjectType) output.get(paramName); if (outputDataObjectType.getType() == DataType.URI) { String downloadFile = outputDataObjectType.getValue(); - if(downloadFile == null || !(new File(downloadFile).isFile())){ + if(downloadFile == null || !(new File(downloadFile).isFile())){ GFacUtils.saveErrorDetails(jobExecutionContext, "Empty Output returned from the application", CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR); - throw new GFacHandlerException("Empty Output returned from the application"); + throw new GFacHandlerException("Empty Output returned from the application.." ); } pbsCluster.scpTo(outputPath, downloadFile); String fileName = downloadFile.substring(downloadFile.lastIndexOf(File.separatorChar)+1, downloadFile.length()); @@ -157,6 +159,26 @@ public class AdvancedSCPOutputHandler extends AbstractHandler { dataObjectType.setName(paramName); dataObjectType.setType(DataType.URI); outputArray.add(dataObjectType); + }else if (outputDataObjectType.getType() == DataType.STDOUT) { + pbsCluster.scpTo(outputPath, standardOutput); + String fileName = standardOutput.substring(standardOutput.lastIndexOf(File.separatorChar)+1, standardOutput.length()); + OutputDataObjectType dataObjectType = new OutputDataObjectType(); + dataObjectType.setValue(outputPath + File.separatorChar + fileName); + dataObjectType.setName(paramName); + dataObjectType.setType(DataType.STDOUT); + dataObjectType.setValidityType(ValidityType.REQUIRED); + dataObjectType.setAddedToCommandLine(CommandLineType.IMPLICIT); + outputArray.add(dataObjectType); + }else if (outputDataObjectType.getType() == DataType.STDERR) { + pbsCluster.scpTo(outputPath, standardError); + String fileName = standardError.substring(standardError.lastIndexOf(File.separatorChar)+1, standardError.length()); + OutputDataObjectType dataObjectType = new OutputDataObjectType(); + dataObjectType.setValue(outputPath + File.separatorChar + fileName); + dataObjectType.setName(paramName); + dataObjectType.setType(DataType.STDERR); + dataObjectType.setValidityType(ValidityType.REQUIRED); + dataObjectType.setAddedToCommandLine(CommandLineType.IMPLICIT); + outputArray.add(dataObjectType); } } registry.add(ChildDataType.EXPERIMENT_OUTPUT, outputArray, jobExecutionContext.getExperimentID());
