Fixed concurrent issue with sessions
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/77a51182 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/77a51182 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/77a51182 Branch: refs/heads/master Commit: 77a511829a42171658bc095a165f5137bf35ae89 Parents: 4a1e79f Author: Shameera Rathnayaka <[email protected]> Authored: Fri Apr 22 14:14:44 2016 -0400 Committer: Shameera Rathnayaka <[email protected]> Committed: Fri Apr 22 14:14:44 2016 -0400 ---------------------------------------------------------------------- .../org/apache/airavata/gfac/impl/Factory.java | 2 +- .../airavata/gfac/impl/GFacEngineImpl.java | 3 +- .../airavata/gfac/impl/HPCRemoteCluster.java | 50 +++++++++++++++----- 3 files changed, 42 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/77a51182/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java index 9e1cbe5..fbeb1d8 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java @@ -455,7 +455,7 @@ public abstract class Factory { return new CancelRequestWatcherImpl(experimentId, processId); } - public static Session getSSHSession(AuthenticationInfo authenticationInfo, ServerInfo serverInfo) throws AiravataException { + public static synchronized Session getSSHSession(AuthenticationInfo authenticationInfo, ServerInfo serverInfo) throws AiravataException { SSHKeyAuthentication authentication = null; String key = serverInfo.getUserName() + "_" + serverInfo.getHost() + "_" + serverInfo.getPort(); Session session = sessionMap.get(key); http://git-wip-us.apache.org/repos/asf/airavata/blob/77a51182/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java index fbd8095..c0e2526 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java @@ -559,7 +559,8 @@ public class GFacEngineImpl implements GFacEngine { errorModel.setUserFriendlyMessage("Error while executing " + task.getType() + " task" ); errorModel.setActualErrorMessage(errorMsg); GFacUtils.saveTaskError(taskContext, errorModel); - throw new GFacException("Error while executing " + task.getType() + " task"); + throw new GFacException("Error: userFriendly msg :" + errorModel.getUserFriendlyMessage() + ", actual msg :" + + errorModel.getActualErrorMessage()); } } http://git-wip-us.apache.org/repos/asf/airavata/blob/77a51182/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java index 677ba9b..468249e 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java @@ -47,6 +47,7 @@ import java.util.UUID; */ public class HPCRemoteCluster extends AbstractRemoteCluster{ private static final Logger log = LoggerFactory.getLogger(HPCRemoteCluster.class); + private static final int MAX_RETRY_COUNT = 3; private final SSHKeyAuthentication authentication; private final JSch jSch; private Session session; @@ -164,15 +165,28 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{ @Override public void scpThirdParty(String sourceFile, String destinationFile, Session clientSession, DIRECTION direction, boolean ignoreEmptyFile) throws SSHApiException { + int retryCount= 0; try { - session = Factory.getSSHSession(authenticationInfo, serverInfo); - log.info("Transferring from:" + sourceFile + " To: " + destinationFile); - if (direction == DIRECTION.TO) { - SSHUtils.scpThirdParty(sourceFile, clientSession, destinationFile, session, ignoreEmptyFile); - } else { - SSHUtils.scpThirdParty(sourceFile, session, destinationFile, clientSession, ignoreEmptyFile); - } - } catch (IOException | AiravataException | JSchException e) { + while (retryCount < MAX_RETRY_COUNT) { + retryCount++; + session = Factory.getSSHSession(authenticationInfo, serverInfo); + log.info("Transferring from:" + sourceFile + " To: " + destinationFile); + try { + if (direction == DIRECTION.TO) { + SSHUtils.scpThirdParty(sourceFile, clientSession, destinationFile, session, ignoreEmptyFile); + } else { + SSHUtils.scpThirdParty(sourceFile, session, destinationFile, clientSession, ignoreEmptyFile); + } + } catch (JSchException e) { + if (retryCount == MAX_RETRY_COUNT) { + log.error("Retry count " + MAX_RETRY_COUNT + " exceeded for transferring from:" + + sourceFile + " To: " + destinationFile, e); + throw e; + } + log.error("Issue with jsch, Retry transferring from:" + sourceFile + " To: " + destinationFile, e); + } + } + } catch (IOException | AiravataException| JSchException e) { throw new SSHApiException("Failed scp file:" + sourceFile + " to remote file " +destinationFile , e); } @@ -180,10 +194,24 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{ @Override public void makeDirectory(String directoryPath) throws SSHApiException { + int retryCount = 0; try { - session = Factory.getSSHSession(authenticationInfo, serverInfo); - log.info("Creating directory: " + serverInfo.getHost() + ":" + directoryPath); - SSHUtils.makeDirectory(directoryPath, session); + while (retryCount < MAX_RETRY_COUNT) { + retryCount++; + session = Factory.getSSHSession(authenticationInfo, serverInfo); + log.info("Creating directory: " + serverInfo.getHost() + ":" + directoryPath); + try { + SSHUtils.makeDirectory(directoryPath, session); + } catch (JSchException e) { + if (retryCount == MAX_RETRY_COUNT) { + log.error("Retry count " + MAX_RETRY_COUNT + " exceeded for creating directory: " + + serverInfo.getHost() + ":" + directoryPath, e); + + throw e; + } + log.error("Issue with jsch, Retry creating directory: " + serverInfo.getHost() + ":" + directoryPath); + } + } } catch (JSchException | AiravataException | IOException e) { throw new SSHApiException("Failed to create directory " + serverInfo.getHost() + ":" + directoryPath, e); }
