fixing bug in moving checkpoint files
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/1108b57c Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/1108b57c Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/1108b57c Branch: refs/heads/develop Commit: 1108b57c961ad2ffd684ed4bef6a8e7f84bf7088 Parents: 868fc55 Author: scnakandala <[email protected]> Authored: Wed May 3 16:04:16 2017 -0400 Committer: scnakandala <[email protected]> Committed: Wed May 3 17:19:16 2017 -0400 ---------------------------------------------------------------------- .../gfac/core/cluster/RemoteCluster.java | 12 ++++++++- .../airavata/gfac/impl/BESRemoteCluster.java | 22 +++++++++++++--- .../airavata/gfac/impl/HPCRemoteCluster.java | 27 ++++++++++++++++++++ .../airavata/gfac/impl/LocalRemoteCluster.java | 18 ++++++++++--- .../gfac/impl/task/SCPDataStageTask.java | 18 ++++++++++--- 5 files changed, 87 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java index 9a5c757..ebf5f6b 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java @@ -23,7 +23,6 @@ import com.jcraft.jsch.Session; import org.apache.airavata.gfac.core.GFacException; import org.apache.airavata.gfac.core.SSHApiException; import org.apache.airavata.gfac.core.authentication.AuthenticationInfo; -import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication; import org.apache.airavata.model.status.JobStatus; import java.util.List; @@ -79,6 +78,17 @@ public interface RemoteCluster { // FIXME: replace SSHApiException with suitable boolean ignoreEmptyFile) throws GFacException; /** + * This method can be used to get the file name of a file giving the extension. It assumes that there will be only + * one file with that extension. In case if there are more than one file one random file name from the matching ones + * will be returned. + * @param fileExtension + * @param parentPath + * @param session + * @return + */ + public String getFileNameFromExtension(String fileExtension, String parentPath, Session session) throws GFacException; + + /** * This will create directories in computing resources * * @param directoryPath the full qualified path for the directory user wants to create http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java index de0c70f..4f22c6a 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java @@ -22,10 +22,11 @@ package org.apache.airavata.gfac.impl; import com.jcraft.jsch.Session; import org.apache.airavata.gfac.core.GFacException; import org.apache.airavata.gfac.core.JobManagerConfiguration; -import org.apache.airavata.gfac.core.SSHApiException; import org.apache.airavata.gfac.core.authentication.AuthenticationInfo; -import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication; -import org.apache.airavata.gfac.core.cluster.*; +import org.apache.airavata.gfac.core.cluster.AbstractRemoteCluster; +import org.apache.airavata.gfac.core.cluster.CommandInfo; +import org.apache.airavata.gfac.core.cluster.JobSubmissionOutput; +import org.apache.airavata.gfac.core.cluster.ServerInfo; import org.apache.airavata.model.status.JobStatus; import java.util.List; @@ -56,6 +57,21 @@ public class BESRemoteCluster extends AbstractRemoteCluster{ } + /** + * This method can be used to get the file name of a file giving the extension. It assumes that there will be only + * one file with that extension. In case if there are more than one file one random file name from the matching ones + * will be returned. + * + * @param fileExtension + * @param parentPath + * @param session + * @return + */ + @Override + public String getFileNameFromExtension(String fileExtension, String parentPath, Session session) { + return null; + } + @Override public void makeDirectory(String directoryPath) throws GFacException { http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java index c1bca29..3d972b0 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java @@ -190,6 +190,33 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{ } } + /** + * This method can be used to get the file name of a file giving the extension. It assumes that there will be only + * one file with that extension. In case if there are more than one file one random file name from the matching ones + * will be returned. + * + * @param fileExtension + * @param parentPath + * @param session + * @return + */ + @Override + public String getFileNameFromExtension(String fileExtension, String parentPath, Session session) throws GFacException { + try { + List<String> fileNames = SSHUtils.listDirectory(parentPath, session); + for(String fileName : fileNames){ + if(fileName.endsWith(fileExtension)){ + return fileName; + } + } + log.warn("No matching file found for extension: " + fileExtension + " in the " + parentPath + " directory"); + return null; + } catch (Exception e) { + e.printStackTrace(); + throw new GFacException("Failed to list directory " + parentPath); + } + } + @Override public void makeDirectory(String directoryPath) throws GFacException { int retryCount = 0; http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java index 878e3b6..64a96db 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java @@ -21,12 +21,9 @@ package org.apache.airavata.gfac.impl; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; -import org.apache.airavata.common.exception.AiravataException; import org.apache.airavata.gfac.core.GFacException; import org.apache.airavata.gfac.core.JobManagerConfiguration; -import org.apache.airavata.gfac.core.SSHApiException; import org.apache.airavata.gfac.core.authentication.AuthenticationInfo; -import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication; import org.apache.airavata.gfac.core.cluster.*; import org.apache.airavata.model.status.JobStatus; import org.slf4j.Logger; @@ -131,6 +128,21 @@ public class LocalRemoteCluster extends AbstractRemoteCluster { } } + /** + * This method can be used to get the file name of a file giving the extension. It assumes that there will be only + * one file with that extension. In case if there are more than one file one random file name from the matching ones + * will be returned. + * + * @param fileExtension + * @param parentPath + * @param session + * @return + */ + @Override + public String getFileNameFromExtension(String fileExtension, String parentPath, Session session) { + throw new UnsupportedOperationException(); + } + @Override public void makeDirectory(String directoryPath) throws GFacException { Path dirPath = Paths.get(directoryPath); http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java index 4e4aa86..9a42124 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java @@ -133,6 +133,20 @@ public class SCPDataStageTask implements Task { URI sourceURI = new URI(subTaskModel.getSource()); String fileName = sourceURI.getPath().substring(sourceURI.getPath().lastIndexOf(File.separator) + 1, sourceURI.getPath().length()); + + authenticationInfo = Factory.getComputerResourceSSHKeyAuthentication(processContext); + ServerInfo serverInfo = processContext.getComputeResourceServerInfo(); + Session sshSession = Factory.getSSHSession(authenticationInfo, serverInfo); + + //Wildcard for file name. Has to find the correct name. + if(fileName.startsWith("*.")){ + String temp = taskContext.getParentProcessContext().getDataMovementRemoteCluster() + .getFileNameFromExtension(fileName.substring(2), inputPath, sshSession); + if(temp != null && temp != ""){ + fileName = temp; + } + } + URI destinationURI = null; if (subTaskModel.getDestination().startsWith("dummy")) { destinationURI = TaskUtils.getDestinationURI(taskContext, hostName, inputPath, fileName); @@ -149,11 +163,9 @@ public class SCPDataStageTask implements Task { return status; } - authenticationInfo = Factory.getComputerResourceSSHKeyAuthentication(processContext); + status = new TaskStatus(TaskState.COMPLETED); - ServerInfo serverInfo = processContext.getComputeResourceServerInfo(); - Session sshSession = Factory.getSSHSession(authenticationInfo, serverInfo); if (processState == ProcessState.INPUT_DATA_STAGING) { inputDataStaging(taskContext, sshSession, sourceURI, destinationURI); status.setReason("Successfully staged input data");
