ignoring empty output files when output staging
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/462843a4 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/462843a4 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/462843a4 Branch: refs/heads/master Commit: 462843a47c276806a57bd197400f0664eef13dcd Parents: d79055a Author: Chathuri Wimalasena <[email protected]> Authored: Wed Jan 20 14:44:27 2016 -0500 Committer: Chathuri Wimalasena <[email protected]> Committed: Wed Jan 20 14:44:27 2016 -0500 ---------------------------------------------------------------------- .../apache/airavata/gfac/core/cluster/RemoteCluster.java | 2 +- .../org/apache/airavata/gfac/impl/BESRemoteCluster.java | 2 +- .../org/apache/airavata/gfac/impl/HPCRemoteCluster.java | 6 +++--- .../apache/airavata/gfac/impl/LocalRemoteCluster.java | 2 +- .../java/org/apache/airavata/gfac/impl/SSHUtils.java | 4 ++-- .../apache/airavata/gfac/impl/task/SCPDataStageTask.java | 11 ++--------- .../apache/airavata/gfac/impl/task/utils/StreamData.java | 2 +- 7 files changed, 11 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java index 9e4544c..f50d660 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java @@ -70,7 +70,7 @@ public interface RemoteCluster { // FIXME: replace SSHApiException with suitable * @param inOrOut direction to file transfer , to the remote cluster(DIRECTION.IN) or from the remote cluster(DIRECTION.OUT) * */ - public void scpThirdParty(String sourceFile, String destinationFile ,Session session , DIRECTION inOrOut) throws SSHApiException; + public void scpThirdParty(String sourceFile, String destinationFile ,Session session , DIRECTION inOrOut, boolean ignoreEmptyFile) throws SSHApiException; /** * This will create directories in computing resources http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java index 1e41eda..0dd69fe 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java @@ -52,7 +52,7 @@ public class BESRemoteCluster extends AbstractRemoteCluster{ } @Override - public void scpThirdParty(String sourceFile, String destinationFile, Session session, DIRECTION inOrOut) throws SSHApiException { + public void scpThirdParty(String sourceFile, String destinationFile, Session session, DIRECTION inOrOut, boolean ignoreEmptyFile) throws SSHApiException { } http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java index 022c8bc..d4988b5 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java @@ -165,16 +165,16 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{ } @Override - public void scpThirdParty(String sourceFile, String destinationFile, Session clientSession, DIRECTION direction) throws SSHApiException { + public void scpThirdParty(String sourceFile, String destinationFile, Session clientSession, DIRECTION direction, boolean ignoreEmptyFile) throws SSHApiException { try { if(!session.isConnected()){ session = getOpenSession(); } log.info("Transferring from:" + sourceFile + " To: " + destinationFile); if (direction == DIRECTION.TO) { - SSHUtils.scpThirdParty(sourceFile, clientSession, destinationFile, session); + SSHUtils.scpThirdParty(sourceFile, clientSession, destinationFile, session, ignoreEmptyFile); } else { - SSHUtils.scpThirdParty(sourceFile, session, destinationFile, clientSession); + SSHUtils.scpThirdParty(sourceFile, session, destinationFile, clientSession, ignoreEmptyFile); } } catch (IOException | JSchException e) { throw new SSHApiException("Failed scp file:" + sourceFile + " to remote file " http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java index d1aa6e0..1bab250 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java @@ -71,7 +71,7 @@ public class LocalRemoteCluster extends AbstractRemoteCluster { } @Override - public void scpThirdParty(String sourceFile, String destinationFile, Session session, DIRECTION inOrOut) throws SSHApiException { + public void scpThirdParty(String sourceFile, String destinationFile, Session session, DIRECTION inOrOut, boolean ignoreEmptyFile) throws SSHApiException { throw new UnsupportedOperationException("Scp third party is not support with LocalRemoteCluster"); } http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java index 2281ebc..a8e1017 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java @@ -272,7 +272,7 @@ public class SSHUtils { * @param destinationSession JSch Session for target * @return returns the final local file path of the new file came from the remote resource */ - public static void scpThirdParty(String sourceFile, Session sourceSession, String destinationFile, Session destinationSession) throws + public static void scpThirdParty(String sourceFile, Session sourceSession, String destinationFile, Session destinationSession, boolean ignoreEmptyFile) throws IOException, JSchException { OutputStream sout = null; InputStream sin = null; @@ -346,7 +346,7 @@ public class SSHUtils { break; } } - if (fileSize == 0L){ + if (fileSize == 0L && !ignoreEmptyFile){ String error = "Input file is empty..."; log.error(error); throw new JSchException(error); http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java index 605b972..71f689a 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java @@ -25,16 +25,10 @@ import com.jcraft.jsch.Session; import org.apache.airavata.common.exception.AiravataException; import org.apache.airavata.common.exception.ApplicationSettingsException; import org.apache.airavata.common.utils.ThriftUtils; -import org.apache.airavata.credential.store.credential.Credential; -import org.apache.airavata.credential.store.credential.impl.ssh.SSHCredential; -import org.apache.airavata.credential.store.store.CredentialReader; -import org.apache.airavata.credential.store.store.CredentialStoreException; import org.apache.airavata.gfac.core.GFacException; import org.apache.airavata.gfac.core.GFacUtils; import org.apache.airavata.gfac.core.SSHApiException; import org.apache.airavata.gfac.core.authentication.AuthenticationInfo; -import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication; -import org.apache.airavata.gfac.core.authentication.SSHPasswordAuthentication; import org.apache.airavata.gfac.core.cluster.CommandInfo; import org.apache.airavata.gfac.core.cluster.RawCommandInfo; import org.apache.airavata.gfac.core.cluster.RemoteCluster; @@ -60,7 +54,6 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -258,7 +251,7 @@ public class SCPDataStageTask implements Task { * scp third party file transfer 'to' compute resource. */ taskContext.getParentProcessContext().getDataMovementRemoteCluster().scpThirdParty(sourceURI.getPath(), - destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.TO); + destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.TO, false); } private void outputDataStaging(TaskContext taskContext, Session sshSession, URI sourceURI, URI destinationURI) @@ -268,7 +261,7 @@ public class SCPDataStageTask implements Task { * scp third party file transfer 'from' comute resource. */ taskContext.getParentProcessContext().getDataMovementRemoteCluster().scpThirdParty(sourceURI.getPath(), - destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM); + destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM, true); // update output locations GFacUtils.saveExperimentOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(), destinationURI.getPath()); GFacUtils.saveProcessOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(), destinationURI.getPath()); http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java index d89c7d3..76b678a 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java @@ -158,7 +158,7 @@ public class StreamData extends TimerTask { * scp third party file transfer 'from' comute resource. */ taskContext.getParentProcessContext().getDataMovementRemoteCluster().scpThirdParty(sourceURI.getPath(), - destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM); + destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM, true); // update output locations GFacUtils.saveExperimentOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(), destinationURI.getPath()); GFacUtils.saveProcessOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(), destinationURI.getPath());
