Repository: airavata Updated Branches: refs/heads/master e54313c5e -> 97ad96b27
adding relative path when job creation, save file uri to data transfer details Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/97ad96b2 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/97ad96b2 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/97ad96b2 Branch: refs/heads/master Commit: 97ad96b27c8eddb9a10e20d192b20374857db3e4 Parents: e54313c Author: Chathuri Wimalasena <[email protected]> Authored: Mon Jan 12 10:11:48 2015 -0500 Committer: Chathuri Wimalasena <[email protected]> Committed: Mon Jan 12 10:11:48 2015 -0500 ---------------------------------------------------------------------- .../gsissh/handler/GSISSHOutputHandler.java | 4 +- .../gfac/gsissh/util/GFACGSISSHUtils.java | 12 ++- .../gfac/ssh/handler/SSHOutputHandler.java | 26 +++++- .../airavata/gfac/ssh/util/GFACSSHUtils.java | 85 ++++++++++++++++++-- 4 files changed, 115 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/97ad96b2/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java index a29028d..2d6e387 100644 --- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java +++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java @@ -158,12 +158,12 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler { String stdErrStr = GFacUtils.readFileToString(localStdErrFile.getAbsolutePath()); status.setTransferState(TransferState.STDOUT_DOWNLOAD); detail.setTransferStatus(status); - detail.setTransferDescription("STDOUT:" + stdOutStr); + detail.setTransferDescription("STDOUT:" + localStdOutFile.getAbsolutePath()); registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); status.setTransferState(TransferState.STDERROR_DOWNLOAD); detail.setTransferStatus(status); - detail.setTransferDescription("STDERR:" + stdErrStr); + detail.setTransferDescription("STDERR:" + localStdErrFile.getAbsolutePath()); registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); //todo this is a mess we have to fix this http://git-wip-us.apache.org/repos/asf/airavata/blob/97ad96b2/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java index 1e03668..1bc0683 100644 --- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java +++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java @@ -41,6 +41,7 @@ import org.apache.airavata.gsi.ssh.impl.PBSCluster; import org.apache.airavata.gsi.ssh.util.CommonUtils; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; import org.apache.airavata.model.appcatalog.appinterface.CommandLineType; +import org.apache.airavata.model.appcatalog.appinterface.DataType; import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType; import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType; import org.apache.airavata.model.appcatalog.computeresource.*; @@ -50,6 +51,7 @@ import org.apache.airavata.model.workspace.experiment.TaskDetails; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.*; @@ -224,7 +226,15 @@ public class GFACGSISSHUtils { if (inputDataObjectType.getValue() != null && !inputDataObjectType.getValue().equals("")) { - inputValues.add(inputDataObjectType.getValue()); + if (inputDataObjectType.getType() == DataType.URI) { + // set only the relative path + String filePath = inputDataObjectType.getValue(); + filePath = filePath.substring(filePath.lastIndexOf(File.separatorChar) + 1, filePath.length()); + inputValues.add(filePath); + }else { + inputValues.add(inputDataObjectType.getValue()); + } + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/97ad96b2/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java index e2469fd..b574540 100644 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java +++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java @@ -122,12 +122,12 @@ public class SSHOutputHandler extends AbstractHandler { String stdErrStr = GFacUtils.readFileToString(localStdErrFile.getAbsolutePath()); status.setTransferState(TransferState.STDOUT_DOWNLOAD); detail.setTransferStatus(status); - detail.setTransferDescription("STDOUT:" + stdOutStr); + detail.setTransferDescription("STDOUT:" + localStdOutFile.getAbsolutePath()); registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); status.setTransferState(TransferState.STDERROR_DOWNLOAD); detail.setTransferStatus(status); - detail.setTransferDescription("STDERR:" + stdErrStr); + detail.setTransferDescription("STDERR:" + localStdErrFile.getAbsolutePath()); registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID()); @@ -166,9 +166,28 @@ public class SSHOutputHandler extends AbstractHandler { dataObjectType.setName(key); dataObjectType.setType(DataType.URI); outputArray.add(dataObjectType); + }else if (DataType.STDOUT == actualParameter.getType()) { + String fileName = localStdOutFile.getName(); + String localFile = outputDataDir + File.separator + fileName; + jobExecutionContext.addOutputFile(localFile); + actualParameter.setValue(localFile); + OutputDataObjectType dataObjectType = new OutputDataObjectType(); + dataObjectType.setValue(localFile); + dataObjectType.setName(key); + dataObjectType.setType(DataType.STDOUT); + outputArray.add(dataObjectType); + }else if (DataType.STDERR == actualParameter.getType()) { + String fileName = localStdErrFile.getName(); + String localFile = outputDataDir + File.separator + fileName; + jobExecutionContext.addOutputFile(localFile); + actualParameter.setValue(localFile); + OutputDataObjectType dataObjectType = new OutputDataObjectType(); + dataObjectType.setValue(localFile); + dataObjectType.setName(key); + dataObjectType.setType(DataType.STDERR); + outputArray.add(dataObjectType); } } - break; } else if (outputList.size() == 1) {//FIXME: Ultrascan case String valueList = outputList.get(0); @@ -194,7 +213,6 @@ public class SSHOutputHandler extends AbstractHandler { + "and ApplicationDescriptor output Parameter Names"); } } - // FIXME: why we set standard error ouput and outputDirectory again ? jobExecutionContext.setStandardError(localStdErrFile.getAbsolutePath()); jobExecutionContext.setStandardOutput(localStdOutFile.getAbsolutePath()); jobExecutionContext.setOutputDir(outputDataDir); http://git-wip-us.apache.org/repos/asf/airavata/blob/97ad96b2/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java index ab698ff..d649ade 100644 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java +++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java @@ -41,7 +41,11 @@ import org.apache.airavata.gsi.ssh.api.job.JobDescriptor; import org.apache.airavata.gsi.ssh.impl.GSISSHAbstractCluster; import org.apache.airavata.gsi.ssh.impl.PBSCluster; import org.apache.airavata.gsi.ssh.util.CommonUtils; +import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; +import org.apache.airavata.model.appcatalog.appinterface.CommandLineType; +import org.apache.airavata.model.appcatalog.appinterface.DataType; import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType; +import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType; import org.apache.airavata.model.appcatalog.computeresource.*; import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference; import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling; @@ -51,6 +55,7 @@ import org.apache.airavata.model.workspace.experiment.TaskDetails; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.*; public class GFACSSHUtils { @@ -248,14 +253,58 @@ public class GFACSSHUtils { int i = random.nextInt(Integer.MAX_VALUE); jobDescriptor.setJobName(String.valueOf(i + 99999999)); jobDescriptor.setWorkingDirectory(jobExecutionContext.getWorkingDir()); + List<String> inputValues = new ArrayList<String>(); MessageContext input = jobExecutionContext.getInMessageContext(); - Map<String, Object> inputs = input.getParameters(); - Set<String> keys = inputs.keySet(); - for (String paramName : keys) { - InputDataObjectType inputDataObjectType = (InputDataObjectType) inputs.get(paramName); - inputValues.add(inputDataObjectType.getValue()); + + // sort the inputs first and then build the command List + Comparator<InputDataObjectType> inputOrderComparator = new Comparator<InputDataObjectType>() { + @Override + public int compare(InputDataObjectType inputDataObjectType, InputDataObjectType t1) { + return inputDataObjectType.getInputOrder() - t1.getInputOrder(); + } + }; + Set<InputDataObjectType> sortedInputSet = new TreeSet<InputDataObjectType>(inputOrderComparator); + for (Object object : input.getParameters().values()) { + if (object instanceof InputDataObjectType) { + InputDataObjectType inputDOT = (InputDataObjectType) object; + sortedInputSet.add(inputDOT); + } + } + for (InputDataObjectType inputDataObjectType : sortedInputSet) { + if (inputDataObjectType.getAddedToCommandLine() != null + && inputDataObjectType.getAddedToCommandLine() == CommandLineType.IMPLICIT) { + continue; + } + if (inputDataObjectType.getApplicationArgument() != null + && !inputDataObjectType.getApplicationArgument().equals("")) { + inputValues.add(inputDataObjectType.getApplicationArgument()); + } + + if (inputDataObjectType.getValue() != null + && !inputDataObjectType.getValue().equals("")) { + if (inputDataObjectType.getType() == DataType.URI) { + // set only the relative path + String filePath = inputDataObjectType.getValue(); + filePath = filePath.substring(filePath.lastIndexOf(File.separatorChar) + 1, filePath.length()); + inputValues.add(filePath); + }else { + inputValues.add(inputDataObjectType.getValue()); + } + + } } + Map<String, Object> outputParams = jobExecutionContext.getOutMessageContext().getParameters(); + for (Object outputParam : outputParams.values()) { + if (outputParam instanceof OutputDataObjectType) { + OutputDataObjectType output = (OutputDataObjectType) outputParam; + if (output.getValue() != null && !output.getValue().equals("") && output.getAddedToCommandLine() != null + && output.getAddedToCommandLine() == CommandLineType.EXPLICIT) { + inputValues.add(output.getValue()); + } + } + } + jobDescriptor.setInputValues(inputValues); jobDescriptor.setUserName(((GSISSHAbstractCluster) cluster).getServerInfo().getUserName()); jobDescriptor.setShellName("/bin/bash"); @@ -296,9 +345,35 @@ public class GFACSSHUtils { } else { logger.error("Task scheduling cannot be null at this point.."); } + ApplicationDeploymentDescription appDepDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription(); + List<String> moduleCmds = appDepDescription.getModuleLoadCmds(); + if (moduleCmds != null) { + for (String moduleCmd : moduleCmds) { + jobDescriptor.addModuleLoadCommands(moduleCmd); + } + } + List<String> preJobCommands = appDepDescription.getPreJobCommands(); + if (preJobCommands != null) { + for (String preJobCommand : preJobCommands) { + jobDescriptor.addPreJobCommand(parseCommand(preJobCommand, jobExecutionContext)); + } + } + + List<String> postJobCommands = appDepDescription.getPostJobCommands(); + if (postJobCommands != null) { + for (String postJobCommand : postJobCommands) { + jobDescriptor.addPostJobCommand(parseCommand(postJobCommand, jobExecutionContext)); + } + } return jobDescriptor; } + private static String parseCommand(String value, JobExecutionContext jobExecutionContext) { + String parsedValue = value.replaceAll("\\$workingDir", jobExecutionContext.getWorkingDir()); + parsedValue = parsedValue.replaceAll("\\$inputDir", jobExecutionContext.getInputDir()); + parsedValue = parsedValue.replaceAll("\\$outputDir", jobExecutionContext.getOutputDir()); + return parsedValue; + } /** * This method can be used to set the Security Context if its not set and later use it in other places * @param jobExecutionContext
