Repository: airavata Updated Branches: refs/heads/master ace68a6e7 -> c83330e98
Cleanup the output handling to register outputs. AIRAVATA-1284 Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/33fafe05 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/33fafe05 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/33fafe05 Branch: refs/heads/master Commit: 33fafe05d392e691b983a4b148eb6cd0dbb365f1 Parents: e3b0bf2 Author: raminder <[email protected]> Authored: Wed Jun 4 11:17:16 2014 -0400 Committer: raminder <[email protected]> Committed: Wed Jun 4 11:17:16 2014 -0400 ---------------------------------------------------------------------- .../airavata/gfac/core/utils/OutputUtils.java | 51 +++----------------- .../gfac/gram/handler/GridFTPOutputHandler.java | 12 ++--- .../gsissh/handler/GSISSHOutputHandler.java | 14 +++--- .../gfac/local/provider/impl/LocalProvider.java | 3 +- .../ssh/handler/AdvancedSCPOutputHandler.java | 30 ++++++------ .../gfac/ssh/handler/SSHOutputHandler.java | 6 +-- 6 files changed, 37 insertions(+), 79 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/33fafe05/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java index 75b03ad..3649946 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/utils/OutputUtils.java @@ -20,7 +20,6 @@ */ package org.apache.airavata.gfac.core.utils; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -30,8 +29,8 @@ import java.util.regex.Pattern; import org.apache.airavata.common.utils.StringUtil; import org.apache.airavata.commons.gfac.type.ActualParameter; import org.apache.airavata.commons.gfac.type.MappingFactory; -import org.apache.airavata.model.workspace.experiment.DataObjectType; import org.apache.airavata.gfac.core.handler.GFacHandlerException; +import org.apache.airavata.model.workspace.experiment.DataObjectType; import org.apache.airavata.model.workspace.experiment.DataType; import org.apache.airavata.schemas.gfac.StdErrParameterType; import org.apache.airavata.schemas.gfac.StdOutParameterType; @@ -39,44 +38,7 @@ import org.apache.airavata.schemas.gfac.StdOutParameterType; public class OutputUtils { private static String regexPattern = "\\s*=\\s*(.*)\\r?\\n"; - public static Map<String, ActualParameter> fillOutputFromStdout(Map<String, Object> output, String stdout, String stderr) throws Exception { - - if (stdout == null || stdout.equals("")){ - throw new GFacHandlerException("Standard output is empty."); - } - - Map<String, ActualParameter> result = new HashMap<String, ActualParameter>(); - Set<String> keys = output.keySet(); - for (String paramName : keys) { - ActualParameter actual = (ActualParameter) output.get(paramName); - // if parameter value is not already set, we let it go - - if (actual == null) { - continue; - } - if ("StdOut".equals(actual.getType().getType().toString())) { - ((StdOutParameterType) actual.getType()).setValue(stdout); - result.put(paramName, actual); - } else if ("StdErr".equals(actual.getType().getType().toString())) { - ((StdErrParameterType) actual.getType()).setValue(stderr); - result.put(paramName, actual); - } -// else if("URI".equals(actual.getType().getType().toString())){ -// continue; -// } - else { - String parseStdout = parseStdout(stdout, paramName); - if (parseStdout != null) { - MappingFactory.fromString(actual, parseStdout); - result.put(paramName, actual); - } - } - } - - return result; - } - - public static void fillOutputFromStdout1(Map<String, Object> output, String stdout, String stderr, List<DataObjectType> outputArray) throws Exception { + public static void fillOutputFromStdout(Map<String, Object> output, String stdout, String stderr, List<DataObjectType> outputArray) throws Exception { if (stdout == null || stdout.equals("")) { throw new GFacHandlerException("Standard output is empty."); @@ -91,13 +53,15 @@ public class OutputUtils { continue; } if ("StdOut".equals(actual.getType().getType().toString())) { - DataObjectType out = new DataObjectType(); + ((StdOutParameterType) actual.getType()).setValue(stdout); + DataObjectType out = new DataObjectType(); out.setKey(paramName); out.setType(DataType.STDOUT); out.setValue(stdout); outputArray.add(out); } else if ("StdErr".equals(actual.getType().getType().toString())) { - DataObjectType out = new DataObjectType(); + ((StdErrParameterType) actual.getType()).setValue(stderr); + DataObjectType out = new DataObjectType(); out.setKey(paramName); out.setType(DataType.STDERR); out.setValue(stderr); @@ -114,7 +78,8 @@ public class OutputUtils { out.setType(DataType.STRING); out.setValue(parseStdout); outputArray.add(out); - } + MappingFactory.fromString(actual, parseStdout); + } } } } http://git-wip-us.apache.org/repos/asf/airavata/blob/33fafe05/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPOutputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPOutputHandler.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPOutputHandler.java index 2e1f9c9..edc1b11 100644 --- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPOutputHandler.java +++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPOutputHandler.java @@ -48,6 +48,7 @@ import org.apache.airavata.gfac.gram.security.GSISecurityContext; import org.apache.airavata.gfac.gram.external.GridFtp; import org.apache.airavata.gfac.gram.util.GramProviderUtils; import org.apache.airavata.model.workspace.experiment.CorrectiveAction; +import org.apache.airavata.model.workspace.experiment.DataObjectType; import org.apache.airavata.model.workspace.experiment.DataTransferDetails; import org.apache.airavata.model.workspace.experiment.ErrorCategory; import org.apache.airavata.model.workspace.experiment.TaskDetails; @@ -156,7 +157,7 @@ public class GridFTPOutputHandler extends AbstractHandler { log.error("Cannot download stdout/err files. One reason could be the job is not successfully finished: "+e.getMessage()); } - + List<DataObjectType> outputArray = new ArrayList<DataObjectType>(); Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters(); Set<String> keys = output.keySet(); for (String paramName : keys) { @@ -166,19 +167,16 @@ public class GridFTPOutputHandler extends AbstractHandler { List<String> outputList = ftp.listDir(outputURI, gssCred); String[] valueList = outputList.toArray(new String[outputList.size()]); ((URIArrayType) actualParameter.getType()).setValueArray(valueList); - // why to instantiate new instance? -// stringMap = new HashMap<String, ActualParameter>(); stringMap.put(paramName, actualParameter); }else if ("StringArray".equals(actualParameter.getType().getType().toString())) { String[] valueList = OutputUtils.parseStdoutArray(stdout, paramName); ((StringArrayType) actualParameter.getType()).setValueArray(valueList); -// stringMap = new HashMap<String, ActualParameter>(); stringMap.put(paramName, actualParameter); } else if ("URI".equals(actualParameter.getType().getType().toString())) { URI outputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory()); List<String> outputList = ftp.listDir(outputURI, gssCred); if (outputList.size() == 0 || outputList.get(0).isEmpty()) { - stringMap = OutputUtils.fillOutputFromStdout(output, stdout, stderr); + OutputUtils.fillOutputFromStdout(output, stdout, stderr,outputArray); } else { String valueList = outputList.get(0); ((URIParameterType) actualParameter.getType()).setValue(valueList); @@ -188,7 +186,7 @@ public class GridFTPOutputHandler extends AbstractHandler { } else { // This is to handle exception during the output parsing. - stringMap = OutputUtils.fillOutputFromStdout(output, stdout, stderr); + OutputUtils.fillOutputFromStdout(output, stdout, stderr,outputArray); } status.setTransferState(TransferState.DOWNLOAD); detail.setTransferStatus(status); @@ -196,7 +194,7 @@ public class GridFTPOutputHandler extends AbstractHandler { registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID()); } - if (stringMap == null || stringMap.isEmpty()) { + if (outputArray == null || outputArray.isEmpty()) { throw new GFacHandlerException("Empty Output returned from the Application, Double check the application" + "and ApplicationDescriptor output Parameter Names"); } http://git-wip-us.apache.org/repos/asf/airavata/blob/33fafe05/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java index f4872ce..f3736f2 100644 --- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java +++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java @@ -166,20 +166,19 @@ public class GSISSHOutputHandler extends AbstractHandler { List<String> outputList = cluster.listDirectory(app.getOutputDataDirectory()); if (outputList.size() == 0 || outputList.get(0).isEmpty()) { - OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr, outputArray); - Map<String, ActualParameter> stringActualParameterMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); - Set<String> strings = stringActualParameterMap.keySet(); + OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr, outputArray); + Set<String> strings = output.keySet(); outputArray.clear(); for(String key:strings) { - ActualParameter actualParameter1 = stringActualParameterMap.get(key); + ActualParameter actualParameter1 = (ActualParameter) output.get(key); if("URI".equals(actualParameter1.getType().getType().toString())){ String downloadFile = MappingFactory.toString(actualParameter1); cluster.scpFrom(downloadFile, outputDataDir); String fileName = downloadFile.substring(downloadFile.lastIndexOf(File.separatorChar)+1, downloadFile.length()); String localFile = outputDataDir + File.separator +fileName; jobExecutionContext.addOutputFile(localFile); - - DataObjectType dataObjectType = new DataObjectType(); + MappingFactory.fromString(actualParameter1, localFile); + DataObjectType dataObjectType = new DataObjectType(); dataObjectType.setValue(localFile); dataObjectType.setKey(key); dataObjectType.setType(DataType.URI); @@ -198,8 +197,7 @@ public class GSISSHOutputHandler extends AbstractHandler { outputArray.add(dataObjectType); } } else { - OutputUtils.fillOutputFromStdout1(output, stdOutStr,stdErrStr, outputArray); - OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); + OutputUtils.fillOutputFromStdout(output, stdOutStr,stdErrStr, outputArray); break; } } http://git-wip-us.apache.org/repos/asf/airavata/blob/33fafe05/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/provider/impl/LocalProvider.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/provider/impl/LocalProvider.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/provider/impl/LocalProvider.java index 5337a1e..d1ffa9d 100644 --- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/provider/impl/LocalProvider.java +++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/provider/impl/LocalProvider.java @@ -214,9 +214,8 @@ public class LocalProvider extends AbstractProvider { String stdOutStr = GFacUtils.readFileToString(app.getStandardOutput()); String stdErrStr = GFacUtils.readFileToString(app.getStandardError()); Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters(); - OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr, outputArray); + OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr, outputArray); registry.add(ChildDataType.EXPERIMENT_OUTPUT, outputArray, jobExecutionContext.getExperimentID()); - OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); } catch (XmlException e) { throw new GFacProviderException("Cannot read output:" + e.getMessage(), e); } catch (IOException io) { http://git-wip-us.apache.org/repos/asf/airavata/blob/33fafe05/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java index 936528b..9b9f7b2 100644 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java +++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java @@ -110,7 +110,6 @@ public class AdvancedSCPOutputHandler extends AbstractHandler { String outputDataDirectory = app.getOutputDataDirectory(); super.invoke(jobExecutionContext); AuthenticationInfo authenticationInfo = null; - System.out.println("Testing"); if (password != null) { authenticationInfo = new DefaultPasswordAuthenticationInfo(this.password); } else { @@ -127,21 +126,22 @@ public class AdvancedSCPOutputHandler extends AbstractHandler { pbsCluster.scpTo(outputPath, standardError); pbsCluster.scpTo(outputPath, standardOutput); List<DataObjectType> outputArray = new ArrayList<DataObjectType>(); - //FIXME: this will not work for if all the parameters are not URI Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters(); - List<String> list = new ArrayList<String>(output.keySet()); - int i = 0; - for (String files : jobExecutionContext.getOutputFiles()) { - pbsCluster.scpTo(outputPath, files); - String fileName = files.substring(files.lastIndexOf(File.separatorChar)+1, files.length()); - DataObjectType dataObjectType = new DataObjectType(); - dataObjectType.setValue(outputPath + File.separatorChar + fileName); - dataObjectType.setKey(list.get(i)); - dataObjectType.setType(DataType.URI); - outputArray.add(dataObjectType); - i++; - } - registry.add(ChildDataType.EXPERIMENT_OUTPUT, outputArray, jobExecutionContext.getExperimentID()); + Set<String> keys = output.keySet(); + for (String paramName : keys) { + ActualParameter actualParameter = (ActualParameter) output.get(paramName); + if ("URI".equals(actualParameter.getType().getType().toString())) { + String downloadFile = MappingFactory.toString(actualParameter); + pbsCluster.scpTo(outputPath, downloadFile); + String fileName = downloadFile.substring(downloadFile.lastIndexOf(File.separatorChar)+1, downloadFile.length()); + DataObjectType dataObjectType = new DataObjectType(); + dataObjectType.setValue(outputPath + File.separatorChar + fileName); + dataObjectType.setKey(paramName); + dataObjectType.setType(DataType.URI); + outputArray.add(dataObjectType); + } + } + registry.add(ChildDataType.EXPERIMENT_OUTPUT, outputArray, jobExecutionContext.getExperimentID()); } catch (SSHApiException e) { log.error("Error transfering files to remote host : " + hostName + " with the user: " + userName); log.error(e.getMessage()); http://git-wip-us.apache.org/repos/asf/airavata/blob/33fafe05/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java index bbc8b1d..1dc48cd 100644 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java +++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java @@ -163,8 +163,7 @@ public class SSHOutputHandler extends AbstractHandler { List<String> outputList = cluster.listDirectory(app.getOutputDataDirectory()); if (outputList.size() == 0 || outputList.get(0).isEmpty()) { - OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr,outputArray); - OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); + OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr,outputArray); break; } else { String valueList = outputList.get(0); @@ -177,8 +176,7 @@ public class SSHOutputHandler extends AbstractHandler { outputArray.add(dataObjectType); } } else { - OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr,outputArray); - OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr); + OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr,outputArray); } } if (outputArray == null || outputArray.isEmpty()) {
