Repository: airavata Updated Branches: refs/heads/master bc20a06d1 -> 64582600a
Made changes to SCP connection creator and add third-party transfer.AIRAVATA-1300 Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/64582600 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/64582600 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/64582600 Branch: refs/heads/master Commit: 64582600a429fc71fdf6d823a3a83cba0d880578 Parents: bc20a06 Author: raminder <[email protected]> Authored: Mon Jun 23 17:56:33 2014 -0400 Committer: raminder <[email protected]> Committed: Mon Jun 23 17:56:33 2014 -0400 ---------------------------------------------------------------------- .../gfac/ssh/handler/SSHInputHandler.java | 5 + .../apache/airavata/gsi/ssh/api/Cluster.java | 8 ++ .../gsi/ssh/impl/GSISSHAbstractCluster.java | 18 ++- .../apache/airavata/gsi/ssh/util/SSHUtils.java | 117 +++++++++++++++++++ 4 files changed, 147 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/64582600/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java index b237a81..4c1345d 100644 --- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java +++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java @@ -123,10 +123,15 @@ public class SSHInputHandler extends AbstractHandler { String substring = paramValue.substring(i + 1); try { String targetFile = app.getInputDataDirectory() + File.separator + substring; + if(paramValue.startsWith("scp:")){ + paramValue = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length()); + cluster.scpThirdParty(paramValue, targetFile); + }else{ if(paramValue.startsWith("file")){ paramValue = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length()); } cluster.scpTo(targetFile, paramValue); + } return targetFile; } catch (SSHApiException e) { throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage()); http://git-wip-us.apache.org/repos/asf/airavata/blob/64582600/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java index 4454624..b3ecc16 100644 --- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java +++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java @@ -72,6 +72,14 @@ public interface Cluster { public void scpFrom(String remoteFile, String localFile) throws SSHApiException; /** + * This will copy a remote file in path rFile to local file lFile + * @param remoteFile remote file path, this has to be a full qualified path + * @param localFile This is the local file to copy, this can be a directory too + * @throws SSHApiException + */ + public void scpThirdParty(String remoteFileSorce, String remoteFileTarget) throws SSHApiException; + + /** * This will create directories in computing resources * @param directoryPath the full qualified path for the directory user wants to create * @throws SSHApiException throws during error http://git-wip-us.apache.org/repos/asf/airavata/blob/64582600/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java index 0ddea0f..118bf91 100644 --- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java +++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java @@ -135,7 +135,7 @@ public class GSISSHAbstractCluster implements Cluster { logDebug("The private key file for vanilla SSH " + privateKeyFile); String publicKeyFile = sshPublicKeyFileAuthentication. - getPrivateKeyFile(serverInfo.getUserName(), serverInfo.getHost()); + getPublicKeyFile(serverInfo.getUserName(), serverInfo.getHost()); logDebug("The public key file for vanilla SSH " + publicKeyFile); @@ -374,6 +374,22 @@ public class GSISSHAbstractCluster implements Cluster { + serverInfo.getHost() + ":rFile", e); } } + + public void scpThirdParty(String remoteFileSource, String remoteFileTarget) throws SSHApiException { + try { + if(!session.isConnected()){ + session.connect(); + } + log.info("Transfering from:" + remoteFileSource + " To: " + remoteFileTarget); + SSHUtils.scpThirdParty(remoteFileSource, remoteFileTarget, session); + } catch (IOException e) { + throw new SSHApiException("Failed during scping file:" + remoteFileSource + " to remote file " + +remoteFileTarget , e); + } catch (JSchException e) { + throw new SSHApiException("Failed during scping file:" + remoteFileSource + " to remote file " + +remoteFileTarget, e); + } + } public void makeDirectory(String directoryPath) throws SSHApiException { try { http://git-wip-us.apache.org/repos/asf/airavata/blob/64582600/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java index 02547a0..f36d2d0 100644 --- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java +++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java @@ -583,6 +583,123 @@ public class SSHUtils { } } + /** + * This method will copy a remote file to a local directory + * + * @param remoteFile remote file path, this has to be a full qualified path + * @param localFile This is the local file to copy, this can be a directory too + * @param session + * @return returns the final local file path of the new file came from the remote resource + */ + public static void scpThirdParty(String remoteFileSource, String remoteFileTarget, Session session) throws IOException, JSchException, SSHApiException { + FileOutputStream fos = null; + try { + String prefix = null; + + // exec 'scp -f remotefile' remotely + String command = "scp -3 " + remoteFileSource + " " + remoteFileTarget; + Channel channel = session.openChannel("exec"); + ((ChannelExec) channel).setCommand(command); + + StandardOutReader stdOutReader = new StandardOutReader(); + ((ChannelExec) channel).setErrStream(stdOutReader.getStandardError()); + // get I/O streams for remote scp + OutputStream out = channel.getOutputStream(); + InputStream in = channel.getInputStream(); + + channel.connect(); + + byte[] buf = new byte[1024]; + + // send '\0' + buf[0] = 0; + out.write(buf, 0, 1); + out.flush(); + + while (true) { + int c = checkAck(in); + if (c != 'C') { + break; + } + + // read '0644 ' + in.read(buf, 0, 5); + + long filesize = 0L; + while (true) { + if (in.read(buf, 0, 1) < 0) { + // error + break; + } + if (buf[0] == ' ') break; + filesize = filesize * 10L + (long) (buf[0] - '0'); + } + +// String file = null; +// for (int i = 0; ; i++) { +// in.read(buf, i, 1); +// if (buf[i] == (byte) 0x0a) { +// file = new String(buf, 0, i); +// break; +// } +// } + + //System.out.println("filesize="+filesize+", file="+file); + + // send '\0' + buf[0] = 0; + out.write(buf, 0, 1); + out.flush(); + + while (true) { + int len = in.read(buf, 0, buf.length); + if (len <= 0) break; + out.write(buf, 0, len); + } +// // read a content of lfile +// fos = new FileOutputStream(prefix == null ? localFile : prefix + file); +// int foo; +// while (true) { +// if (buf.length < filesize) foo = buf.length; +// else foo = (int) filesize; +// foo = in.read(buf, 0, foo); +// if (foo < 0) { +// // error +// break; +// } +// fos.write(buf, 0, foo); +// filesize -= foo; +// if (filesize == 0L) break; +// } +// fos.close(); +// fos = null; + + if (checkAck(in) != 0) { + String error = "Error transfering the file content"; + log.error(error); + throw new SSHApiException(error); + } + + // send '\0' + buf[0] = 0; + out.write(buf, 0, 1); + out.flush(); + } + stdOutReader.onOutput(channel); + if (!stdOutReader.getStdErrorString().equals("")) { + throw new SSHApiException(stdOutReader.getStdErrorString()); + } + + } catch (Exception e) { + log.error(e.getMessage(), e); + } finally { + try { + if (fos != null) fos.close(); + } catch (Exception ee) { + } + } + } + public static void makeDirectory(String path, Session session) throws IOException, JSchException, SSHApiException { // exec 'scp -t rfile' remotely
