http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/local/LocalUseCaseInvocation.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/local/LocalUseCaseInvocation.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/local/LocalUseCaseInvocation.java new file mode 100755 index 0000000..b57be14 --- /dev/null +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/local/LocalUseCaseInvocation.java @@ -0,0 +1,560 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.taverna.activities.externaltool.local; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.UnsupportedEncodingException; +import java.io.Writer; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; + +import org.apache.taverna.activities.externaltool.desc.ScriptInput; +import org.apache.taverna.activities.externaltool.desc.ScriptOutput; +import org.apache.taverna.activities.externaltool.desc.UseCaseDescription; +import org.apache.taverna.activities.externaltool.invocation.InvocationException; +import org.apache.taverna.activities.externaltool.invocation.UseCaseInvocation; +import org.apache.taverna.activities.externaltool.ssh.SshNode; +import org.apache.taverna.activities.externaltool.ssh.SshNodeFactory; +import org.apache.taverna.activities.externaltool.ssh.SshUrl; +import org.apache.taverna.reference.AbstractExternalReference; +import org.apache.taverna.reference.ErrorDocument; +import org.apache.taverna.reference.ExternalReferenceSPI; +import org.apache.taverna.reference.Identified; +import org.apache.taverna.reference.ReferenceService; +import org.apache.taverna.reference.ReferenceSet; +import org.apache.taverna.reference.ReferencedDataNature; +import org.apache.taverna.reference.T2Reference; +import org.apache.taverna.reference.impl.external.file.FileReference; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.log4j.Logger; + +/** + * The job is executed locally, i.e. not via the grid. + * @author Hajo Krabbenhoeft + */ +public class LocalUseCaseInvocation extends UseCaseInvocation { + + private static Logger logger = Logger.getLogger(LocalUseCaseInvocation.class); + + private final File tempDir; + + public static String LOCAL_USE_CASE_INVOCATION_TYPE = "789663B8-DA91-428A-9F7D-B3F3DA185FD4"; + + private Process running; + + private final String shellPrefix; + + private final String linkCommand; + + private Reader stdInReader = null; + + private static Map<String, Set<String>> runIdToTempDir = Collections.synchronizedMap(new HashMap<String, Set<String>> ()); + + private static String LOCAL_INVOCATION_FILE = "localInvocations"; + + public LocalUseCaseInvocation(UseCaseDescription desc, boolean retrieveData, String mainTempDirectory, String shellPrefix, String linkCommand) throws IOException { + + usecase = desc; + setRetrieveData(retrieveData); + this.shellPrefix = shellPrefix; + this.linkCommand = linkCommand; + + if (mainTempDirectory != null) { + + File mainTempDir = new File(mainTempDirectory); + + tempDir = File.createTempFile("usecase", "dir", mainTempDir); + } else { + tempDir = File.createTempFile("usecase", "dir"); + } + tempDir.delete(); + tempDir.mkdir(); + logger.info("mainTempDirectory is " + mainTempDirectory); + logger.info("Using tempDir " + tempDir.getAbsolutePath()); + + } + + void recDel(File c) { + File[] files = c.listFiles(); + if (files != null) { + for (File cc : files) + recDel(cc); + } + c.delete(); + } + + private String setOneBinaryInput(ReferenceService referenceService, + T2Reference t2Reference, ScriptInput input, String targetSuffix) + throws InvocationException { + + if (input.isFile() || input.isTempFile()) { + // Try to get it as a file + String target = tempDir.getAbsolutePath() + "/" + targetSuffix; + FileReference fileRef = getAsFileReference(referenceService, + t2Reference); + if (fileRef != null) { + + if (!input.isForceCopy()) { + if (linkCommand != null) { + String source = fileRef.getFile().getAbsolutePath(); + String actualLinkCommand = getActualOsCommand( + linkCommand, source, targetSuffix, target); + logger.info("Link command is " + actualLinkCommand); + String[] splitCmds = actualLinkCommand.split(" "); + ProcessBuilder builder = new ProcessBuilder(splitCmds); + builder.directory(tempDir); + try { + int code = builder.start().waitFor(); + if (code == 0) { + return target; + } else { + logger.error("Link command gave errorcode: " + + code); + } + + } catch (InterruptedException e) { + // go through + } catch (IOException e) { + // go through + } + + } + } + } + + InputStream is = null; + OutputStream os = null; + is = getAsStream(referenceService, t2Reference); + + try { + os = new FileOutputStream(target); + } catch (FileNotFoundException e) { + throw new InvocationException(e); + } + + try { + IOUtils.copyLarge(is, os); + } catch (IOException e) { + throw new InvocationException(e); + } + try { + is.close(); + os.close(); + } catch (IOException e) { + throw new InvocationException(e); + } + return target; + } else { + String value = (String) referenceService.renderIdentifier( + t2Reference, String.class, this.getContext()); + return value; + } + } + + @Override + public String setOneInput(ReferenceService referenceService, + T2Reference t2Reference, ScriptInput input) + throws InvocationException { + + if (input.getCharsetName() == null) { + input.setCharsetName(Charset.defaultCharset().name()); + } + String target = null; + String targetSuffix = null; + if (input.isFile()) { + targetSuffix = input.getTag(); + } else if (input.isTempFile()) { + targetSuffix = "tempfile." + (nTempFiles++) + ".tmp"; + } + + if (input.isBinary()) { + return setOneBinaryInput(referenceService, t2Reference, input, + targetSuffix); + } + + logger.info("Target is " + target); + if (input.isFile() || input.isTempFile()) { + target = tempDir.getAbsolutePath() + "/" + targetSuffix; + // Try to get it as a file + Reader r; + Writer w; + FileReference fileRef = getAsFileReference(referenceService, + t2Reference); + if (fileRef != null) { + + if (!input.isForceCopy()) { + if (linkCommand != null) { + String source = fileRef.getFile().getAbsolutePath(); + String actualLinkCommand = getActualOsCommand( + linkCommand, source, targetSuffix, target); + logger.info("Link command is " + actualLinkCommand); + String[] splitCmds = actualLinkCommand.split(" "); + ProcessBuilder builder = new ProcessBuilder(splitCmds); + builder.directory(tempDir); + try { + int code = builder.start().waitFor(); + if (code == 0) { + return target; + } else { + logger.error("Link command gave errorcode: " + + code); + } + + } catch (InterruptedException e) { + // go through + } catch (IOException e) { + // go through + } + + } + } + + if (fileRef.getDataNature().equals(ReferencedDataNature.TEXT)) { + r = new InputStreamReader(fileRef.openStream(this + .getContext()), Charset.forName(fileRef + .getCharset())); + } else { + try { + r = new FileReader(fileRef.getFile()); + } catch (FileNotFoundException e) { + throw new InvocationException(e); + } + } + } else { + r = new InputStreamReader(getAsStream(referenceService, + t2Reference)); + } + try { + w = new OutputStreamWriter(new FileOutputStream(target), input + .getCharsetName()); + } catch (UnsupportedEncodingException e) { + throw new InvocationException(e); + } catch (FileNotFoundException e) { + throw new InvocationException(e); + } + try { + IOUtils.copyLarge(r, w); + } catch (IOException e) { + throw new InvocationException(e); + } + try { + r.close(); + w.close(); + } catch (IOException e) { + throw new InvocationException(e); + } + return target; + } else { + String value = (String) referenceService.renderIdentifier( + t2Reference, String.class, this.getContext()); + return value; + } + } + + private void forgetRun() { + Set<String> directories = runIdToTempDir.get(getRunId()); + try { + directories.remove(tempDir.getCanonicalPath()); + } catch (IOException e) { + logger.error(e); + } + } + + private static void deleteDirectory(String location) { + try { + FileUtils.deleteDirectory(new File(location)); + } catch (IOException e) { + logger.error("Problem deleting " + location, e); + } + } + + public static void cleanup(String runId) { + Set<String> tempDirectories = runIdToTempDir.get(runId); + if (tempDirectories != null) { + for (String tempDir : tempDirectories) { + deleteDirectory(tempDir); + } + runIdToTempDir.remove(runId); + } + } + + @Override + protected void submit_generate_job_inner() throws InvocationException { + tags.put("uniqueID", "" + getSubmissionID()); + String command = usecase.getCommand(); + for (String cur : tags.keySet()) { + command = command.replaceAll("\\Q%%" + cur + "%%\\E", Matcher.quoteReplacement(tags.get(cur))); + } + + List<String> cmds = new ArrayList<String>(); + if ((shellPrefix != null) && !shellPrefix.isEmpty()) { + String[] prefixCmds = shellPrefix.split(" "); + for (int i = 0; i < prefixCmds.length; i++) { + cmds.add(prefixCmds[i]); + } + cmds.add(command); + } else { + String[] splitCmds = command.split(" "); + for (int i = 0; i < splitCmds.length; i++) { + cmds.add(splitCmds[i]); + } + } + + ProcessBuilder builder = new ProcessBuilder(cmds); + builder.directory(tempDir); + + for (int i = 0; i < cmds.size(); i++) { + logger.info("cmds[" + i + "] = " + cmds.get(i)); + } + logger.info("Command is " + command + " in directory " + tempDir); + try { + running = builder.start(); + if (stdInReader != null) { + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(running.getOutputStream())); + IOUtils.copyLarge(stdInReader, writer); + writer.close(); + } + } catch (IOException e) { + throw new InvocationException(e); + } + } + + private void copy_stream(InputStream read, OutputStream write) throws IOException { + int a = read.available(); + if (a > 0) { + byte[] buf = new byte[a]; + read.read(buf); + write.write(buf); + } + } + + @Override + public HashMap<String, Object> submit_wait_fetch_results(ReferenceService referenceService) throws InvocationException { + ByteArrayOutputStream stdout_buf = new ByteArrayOutputStream(); + ByteArrayOutputStream stderr_buf = new ByteArrayOutputStream(); + while (true) { + try { + copy_stream(running.getInputStream(), stdout_buf); + copy_stream(running.getErrorStream(), stderr_buf); + } catch (IOException e1) { + throw new InvocationException(e1); + } + try { + int exitcode = running.exitValue(); + if (!usecase.getValidReturnCodes().contains(exitcode)) { + try { + throw new InvocationException("Invalid exit code " + exitcode + ":" + stderr_buf.toString("US-ASCII")); + } catch (UnsupportedEncodingException e) { + throw new InvocationException("Invalid exit code " + exitcode + ":" + stderr_buf.toString()); + } + } + else + break; + } catch (IllegalThreadStateException e) { + + try { + Thread.sleep(100); + } catch (InterruptedException e1) { + throw new InvocationException(e); + } + + } + } + + HashMap<String, Object> results = new HashMap<String, Object>(); + results.put("STDOUT", stdout_buf.toByteArray()); + results.put("STDERR", stderr_buf.toByteArray()); + + for (Map.Entry<String, ScriptOutput> cur : usecase.getOutputs().entrySet()) { + ScriptOutput scriptOutput = cur.getValue(); + File result = new File(tempDir.getAbsoluteFile() + "/" + cur.getValue().getPath()); + if (result.exists()) { + AbstractExternalReference ref; + if (isRetrieveData()) { + FileInputStream is; + try { + is = new FileInputStream(result); + } catch (FileNotFoundException e) { + throw new InvocationException(e); + } + if (scriptOutput.isBinary()) { + ref = inlineByteArrayReferenceBuilder.createReference(is, null); + } else { + ref = inlineStringReferenceBuilder.createReference(is, null); + } + try { + is.close(); + } catch (IOException e) { + throw new InvocationException(e); + } + } + else { + ref = new FileReference(result); + if (scriptOutput.isBinary()) { + ((FileReference) ref) + .setDataNature(ReferencedDataNature.BINARY); + } else { + ((FileReference) ref) + .setDataNature(ReferencedDataNature.TEXT); + ((FileReference) ref).setCharset("UTF-8"); + } + } + results.put(cur.getKey(), ref); + } else { + ErrorDocument ed = referenceService.getErrorDocumentService().registerError("No result for " + cur.getKey(), 0, getContext()); + results.put(cur.getKey(), ed); + } + } + + if (isRetrieveData()) { + forgetRun(); + try { + deleteDirectory(tempDir.getCanonicalPath()); + } catch (IOException e) { + throw new InvocationException(e); + } + } + + return results; + } + + private FileReference getAsFileReference(ReferenceService referenceService, T2Reference t2Reference) { + Identified identified = referenceService.resolveIdentifier(t2Reference, null, null); + if (identified instanceof ReferenceSet) { + for (ExternalReferenceSPI ref : ((ReferenceSet) identified).getExternalReferences()) { + if (ref instanceof FileReference) { + return (FileReference) ref; + } + } + } + return null; + } + + @Override + public void setStdIn(ReferenceService referenceService, + T2Reference t2Reference) { + stdInReader = new BufferedReader(new InputStreamReader(getAsStream(referenceService, t2Reference))); + } + + @Override + public void rememberRun(String runId) { + this.setRunId(runId); + Set<String> directories = runIdToTempDir.get(runId); + if (directories == null) { + directories = Collections.synchronizedSet(new HashSet<String> ()); + runIdToTempDir.put(runId, directories); + } + try { + directories.add(tempDir.getCanonicalPath()); + } catch (IOException e) { + logger.error("Unable to record temporary directory: " + tempDir, e); + } + } + + public static void load(File directory) { + File invocationsFile = new File(directory, LOCAL_INVOCATION_FILE); + if (!invocationsFile.exists()) { + return; + } + BufferedReader reader = null; + try { + reader = new BufferedReader(new FileReader(invocationsFile)); + String line = reader.readLine(); + while (line != null) { + String[] parts = line.split(" "); + if (parts.length != 2) { + break; + } + String runId = parts[0]; + String tempDirString = parts[1]; + Set<String> tempDirs = runIdToTempDir.get(runId); + if (tempDirs == null) { + tempDirs = new HashSet<String>(); + runIdToTempDir.put(runId, tempDirs); + } + tempDirs.add(tempDirString); + line = reader.readLine(); + } + } catch (FileNotFoundException e) { + logger.error(e); + } catch (IOException e) { + logger.error(e); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException e) { + logger.error(e); + } + } + } + } + + public static void persist(File directory) { + File invocationsFile = new File(directory, LOCAL_INVOCATION_FILE); + BufferedWriter writer = null; + try { + writer = new BufferedWriter(new FileWriter(invocationsFile)); + for (String runId : runIdToTempDir.keySet()) { + for (String tempDir : runIdToTempDir.get(runId)) { + writer.write(runId); + writer.write(" "); + writer.write(tempDir); + writer.newLine(); + } + } + } catch (IOException e) { + logger.error(e); + } finally { + if (writer != null) { + try { + writer.close(); + } catch (IOException e) { + logger.error(e); + } + } + } + } + +}
http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/ExternalToolSshInvocationMechanism.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/ExternalToolSshInvocationMechanism.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/ExternalToolSshInvocationMechanism.java index c373879..94cc466 100644 --- a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/ExternalToolSshInvocationMechanism.java +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/ExternalToolSshInvocationMechanism.java @@ -24,13 +24,8 @@ import java.util.List; import org.jdom.Element; import org.jdom.Text; - -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshNodeFactory; -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshUseCaseInvocation; import org.apache.taverna.activities.externaltool.manager.InvocationMechanism; -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshNode; - /** * @author alanrw * http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshAutoLoginTrustEveryone.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshAutoLoginTrustEveryone.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshAutoLoginTrustEveryone.java new file mode 100644 index 0000000..dfee8bb --- /dev/null +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshAutoLoginTrustEveryone.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.taverna.activities.externaltool.ssh; + +import org.apache.log4j.Logger; +import org.apache.taverna.activities.externaltool.invocation.AskUserForPw; + +import com.jcraft.jsch.UIKeyboardInteractive; +import com.jcraft.jsch.UserInfo; + +final class SshAutoLoginTrustEveryone implements UserInfo, UIKeyboardInteractive { + + private static Logger logger = Logger.getLogger(SshAutoLoginTrustEveryone.class); + + private final AskUserForPw askUserForPw; + + public SshAutoLoginTrustEveryone(AskUserForPw askUserForPw) { + super(); + this.askUserForPw = askUserForPw; + } + + public void showMessage(String arg0) { + logger.info(arg0); + } + + public boolean promptYesNo(String arg0) { + if (arg0.startsWith("The authenticity of host")) + return true; + return false; + } + + public boolean promptPassword(String arg0) { + return true; + } + + public boolean promptPassphrase(String arg0) { + return true; + } + + public String getPassword() { + return askUserForPw.getPassword(); + } + + public String getPassphrase() { + return askUserForPw.getPassphrase(); + } + + public String[] promptKeyboardInteractive(String destination, String name, String instruction, String[] prompt, boolean[] echo) { + if (prompt.length >= 1 && prompt[0].toLowerCase().startsWith("password")) + return new String[] { askUserForPw.getPassword() }; + return null; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshInvocationCreator.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshInvocationCreator.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshInvocationCreator.java index 88f6ceb..b876ec8 100644 --- a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshInvocationCreator.java +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshInvocationCreator.java @@ -25,20 +25,13 @@ import java.util.Map; import org.apache.taverna.activities.externaltool.InvocationCreator; import org.apache.taverna.activities.externaltool.RetrieveLoginFromTaverna; - +import org.apache.taverna.activities.externaltool.desc.UseCaseDescription; +import org.apache.taverna.activities.externaltool.invocation.UseCaseInvocation; import org.apache.log4j.Logger; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.SftpException; -import de.uni_luebeck.inb.knowarc.usecases.UseCaseDescription; -import de.uni_luebeck.inb.knowarc.usecases.invocation.UseCaseInvocation; -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshNode; -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshNodeFactory; -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshReference; -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshUrl; -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshUseCaseInvocation; - import org.apache.taverna.activities.externaltool.manager.InvocationMechanism; import org.apache.taverna.reference.ExternalReferenceSPI; import org.apache.taverna.reference.Identified; http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshInvocationPersister.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshInvocationPersister.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshInvocationPersister.java index 1884cbe..bad5c2e 100644 --- a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshInvocationPersister.java +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshInvocationPersister.java @@ -21,6 +21,7 @@ package org.apache.taverna.activities.externaltool.ssh; import java.io.File; +import org.apache.taverna.activities.externaltool.invocation.InvocationException; import org.apache.taverna.activities.externaltool.manager.InvocationPersister; /* * Licensed to the Apache Software Foundation (ASF) under one @@ -45,9 +46,6 @@ import org.apache.taverna.security.credentialmanager.CredentialManager; import org.apache.log4j.Logger; -import de.uni_luebeck.inb.knowarc.usecases.invocation.InvocationException; -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshUseCaseInvocation; - /** * @author alanrw * http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshMechanismCreator.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshMechanismCreator.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshMechanismCreator.java index 2a15d38..097f8db 100644 --- a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshMechanismCreator.java +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshMechanismCreator.java @@ -27,10 +27,6 @@ import org.apache.taverna.activities.externaltool.manager.MechanismCreator; import org.apache.log4j.Logger; import org.jdom.Element; -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshNode; -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshNodeFactory; -import de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshUseCaseInvocation; - /** * @author alanrw * http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshNode.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshNode.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshNode.java new file mode 100644 index 0000000..378696f --- /dev/null +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshNode.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.taverna.activities.externaltool.ssh; + +import org.apache.taverna.activities.externaltool.manager.InvocationMechanism; + +public class SshNode { + + public static String DEFAULT_HOST = "127.0.0.1"; + public static int DEFAULT_PORT = 22; + public static String DEFAULT_DIRECTORY = "/tmp/"; + + private String host = DEFAULT_HOST; + private int port = DEFAULT_PORT; + private String directory = DEFAULT_DIRECTORY; + + private SshUrl url; + + private String linkCommand = null; + private String copyCommand = null; + private boolean retrieveData = false; + + /** + * + */ + SshNode() { + super(); + linkCommand = InvocationMechanism.UNIX_LINK; + copyCommand = InvocationMechanism.UNIX_COPY; + + } + /** + * @param directory the directory to set + */ + public void setDirectory(String directory) { + if ((directory != null) && !directory.isEmpty()) { + if (!directory.endsWith("/")) { + directory = directory + "/"; + } + this.directory = directory; + } + } + + /** + * @return the directory + */ + public String getDirectory() { + return directory; + } + + /** + * @param host the host to set + */ + public void setHost(String host) { + this.host = host; + } + + /** + * @return the host + */ + public String getHost() { + return host; + } + + /** + * @param port the port to set + */ + public void setPort(int port) { + this.port = port; + } + + /** + * @return the port + */ + public int getPort() { + return port; + } + + SshUrl getUrl() { + if (url == null) { + url = new SshUrl(this); + } + return url; + } + + public int hashCode() { + return getUrl().hashCode(); + } + + public boolean equals(Object obj) { + if ((obj == null) || !(obj instanceof SshNode)) { + return false; + } + return (this.hashCode() == obj.hashCode()); + } + /** + * @return the linkCommand + */ + public String getLinkCommand() { + return linkCommand; + } + /** + * @param linkCommand the linkCommand to set + */ + public void setLinkCommand(String linkCommand) { + if ((linkCommand != null) && linkCommand.isEmpty()) { + this.linkCommand = null; + } else { + this.linkCommand = linkCommand; + } } + /** + * @return the copyCommand + */ + public String getCopyCommand() { + return copyCommand; + } + /** + * @param copyCommand the copyCommand to set + */ + public void setCopyCommand(String copyCommand) { + if ((copyCommand != null) && copyCommand.isEmpty()) { + this.copyCommand = null; + } else { + this.copyCommand = copyCommand; + } + } + + /** + * @return the retrieveData + */ + public boolean isRetrieveData() { + return retrieveData; + } + /** + * @param retrieveData the retrieveData to set + */ + public void setRetrieveData(boolean retrieveData) { + this.retrieveData = retrieveData; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshNodeFactory.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshNodeFactory.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshNodeFactory.java new file mode 100644 index 0000000..a697bd7 --- /dev/null +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshNodeFactory.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.taverna.activities.externaltool.ssh; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class SshNodeFactory { + + private Map<String, SshNode> nodeMap = Collections.synchronizedMap(new HashMap<String, SshNode> ()); + + private static SshNodeFactory INSTANCE = new SshNodeFactory(); + + private SshNode defaultNode; + + private SshNodeFactory() { + defaultNode = getSshNode(SshNode.DEFAULT_HOST, SshNode.DEFAULT_PORT, SshNode.DEFAULT_DIRECTORY); + } + + public SshNode getDefaultNode() { + return defaultNode; + } + + public static SshNodeFactory getInstance() { + return INSTANCE; + } + + public SshNode getSshNode(String host, int port, String directory) { + String url = makeUrl(host, port, directory); + if (nodeMap.containsKey(url)) { + return nodeMap.get(url); + } + else { + SshNode newNode = new SshNode(); + newNode.setHost(host); + newNode.setPort(port); + newNode.setDirectory(directory); + nodeMap.put(url, newNode); + return newNode; + } + } + + public boolean containsSshNode(String host, int port, String directory) { + return nodeMap.containsKey(makeUrl(host, port, directory)); + } + + public static String makeUrl(String host, int port, String directory) { + return ("ssh://" + host + ":" + port + directory); + } +} http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshPool.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshPool.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshPool.java new file mode 100755 index 0000000..e7f9787 --- /dev/null +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshPool.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.taverna.activities.externaltool.ssh; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.apache.taverna.activities.externaltool.desc.RuntimeEnvironmentConstraint; +import org.apache.taverna.activities.externaltool.invocation.AskUserForPw; + +import com.jcraft.jsch.ChannelExec; +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; + +public class SshPool { + + private static Logger logger = Logger.getLogger(SshPool.class); + + + private static JSch jsch = new JSch(); + + private static int CONNECT_TIMEOUT = 10000; // milliseconds + + private static Map<SshNode, Session> sessionMap = Collections.synchronizedMap(new HashMap<SshNode, Session> ()); + private static Map<Session, ChannelSftp> sftpGetMap = Collections.synchronizedMap(new HashMap<Session, ChannelSftp> ()); + private static Map<Session, ChannelSftp> sftpPutMap = Collections.synchronizedMap(new HashMap<Session, ChannelSftp> ()); + + public static Session getSshSession(final SshUrl sshUrl, final AskUserForPw askUserForPw) throws JSchException { + return getSshSession(sshUrl.getSshNode(), askUserForPw); + } + + public static synchronized Session getSshSession(final SshNode sshNode, final AskUserForPw askUserForPw) throws JSchException { + + Session s = sessionMap.get(sshNode); + if ((s != null) && s.isConnected()) { + logger.info("Reusing session"); + return s; + } + if (s != null) { + logger.info("Session was not connected"); + } + if (s == null) { + logger.info("No session found for " + sshNode.toString()); + } + + if (askUserForPw.getKeyfile().length() > 0) { + jsch.addIdentity(askUserForPw.getKeyfile()); + } + logger.info("Using host is " + sshNode.getHost() + " and port " + sshNode.getPort()); + Session sshSession = jsch.getSession(askUserForPw.getUsername(), sshNode.getHost(), sshNode.getPort()); + sshSession.setUserInfo(new SshAutoLoginTrustEveryone(askUserForPw)); + sshSession.connect(CONNECT_TIMEOUT); + + askUserForPw.authenticationSucceeded(); + sessionMap.put(sshNode, sshSession); + if (sshSession == null) { + logger.error("Returning a null session"); + } + return sshSession; + } + + public static ChannelSftp getSftpGetChannel(SshNode sshNode, final AskUserForPw askUserForPw) throws JSchException { + return getSftpGetChannel(getSshSession(sshNode, askUserForPw)); + } + + private static synchronized ChannelSftp getSftpGetChannel(Session session) throws JSchException { + ChannelSftp result = sftpGetMap.get(session); + if (!session.isConnected()) { + logger.warn("Session is not connected"); + } + if (result == null) { + logger.info("Creating new sftp channel"); + result = (ChannelSftp) session.openChannel("sftp"); + sftpGetMap.put(session, result); + } + else { + logger.info("Reusing sftp channel"); + } + if (!result.isConnected()) { + logger.info("Connecting"); + result.connect(); + } else { + logger.info("Already connected"); + } + return result; + } + + public static ChannelSftp getSftpPutChannel(SshNode sshNode, final AskUserForPw askUserForPw) throws JSchException { + return getSftpPutChannel(getSshSession(sshNode, askUserForPw)); + } + + private static synchronized ChannelSftp getSftpPutChannel(Session session) throws JSchException { + ChannelSftp result = null; + synchronized(sftpPutMap) { + result = sftpPutMap.get(session); + if (!session.isConnected()) { + logger.info("Session is not connected"); + } + if (result == null) { + logger.info("Creating new sftp channel"); + result = (ChannelSftp) session.openChannel("sftp"); + sftpPutMap.put(session, result); + } + else { + logger.info("Reusing sftp channel"); + } + } + if (!result.isConnected()) { + logger.info("Connecting"); + result.connect(CONNECT_TIMEOUT); + } else { + logger.info("Already connected"); + } + return result; + } + + public static synchronized ChannelExec openExecChannel(SshNode sshNode, final AskUserForPw askUserForPw) throws JSchException { + return (ChannelExec) getSshSession(sshNode, askUserForPw).openChannel("exec"); + } + + private static synchronized ChannelExec openExecChannel(Session session) throws JSchException { + return (ChannelExec) session.openChannel("exec"); + } + +} + + + + + + + + + + + + + http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshReference.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshReference.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshReference.java new file mode 100644 index 0000000..091bd0c --- /dev/null +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshReference.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.taverna.activities.externaltool.ssh; + +import java.io.InputStream; + +import org.apache.taverna.activities.externaltool.RetrieveLoginFromTaverna; +import org.apache.taverna.reference.AbstractExternalReference; +import org.apache.taverna.reference.DereferenceException; +import org.apache.taverna.reference.ExternalReferenceSPI; +import org.apache.taverna.reference.ReferenceContext; +import org.apache.taverna.security.credentialmanager.CredentialManager; +import org.apache.taverna.reference.ReferencedDataNature; + +import org.apache.log4j.Logger; + +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.SftpException; + +/** + * @author alanrw + * + */ +public class SshReference extends AbstractExternalReference implements + ExternalReferenceSPI { + + private static Logger logger = Logger.getLogger(SshReference.class); + + + private String host = "127.0.0.1"; + private int port = 22; + private String directory = "/tmp/"; + private String subDirectory; + private String fileName; + + private CredentialManager credentialManager; + + private int dataNatureInteger = ReferencedDataNature.UNKNOWN.ordinal(); + private String charset = "UTF-8"; + + public SshReference() { + super(); + } + + public SshReference(SshUrl url) { + super(); + this.host = url.getSshNode().getHost(); + this.port = url.getSshNode().getPort(); + this.directory = url.getSshNode().getDirectory(); + this.subDirectory = url.getSubDirectory(); + this.fileName = url.getFileName(); + this.setDataNature(url.getDataNature()); + this.setCharset(url.getCharset()); + } + + /* (non-Javadoc) + * @see net.sf.taverna.t2.reference.ExternalReferenceSPI#getApproximateSizeInBytes() + */ + @Override + public Long getApproximateSizeInBytes() { + return 10000L; + } + + /* (non-Javadoc) + * @see net.sf.taverna.t2.reference.ExternalReferenceSPI#openStream(net.sf.taverna.t2.reference.ReferenceContext) + */ + @Override + public InputStream openStream(ReferenceContext context) + throws DereferenceException { + try { + SshNode node = SshNodeFactory.getInstance().getSshNode(this.getHost(), this.getPort(), this.getDirectory()); + String fullPath = getDirectory() + getSubDirectory() + "/" + getFileName(); + ChannelSftp channel = SshPool.getSftpGetChannel(node, new RetrieveLoginFromTaverna(new SshUrl(node).toString(), credentialManager)); + logger.info("Opening stream on " + fullPath); + return (channel.get(fullPath)); + } catch (JSchException e) { + //TODO + logger.error(e); + } catch (SftpException e) { + // TODO Auto-generated catch block + logger.error(e); + } + return null; + } + + /** + * @return the host + */ + public String getHost() { + return host; + } + + /** + * @param host the host to set + */ + public void setHost(String host) { + this.host = host; + } + + /** + * @return the port + */ + public int getPort() { + return port; + } + + /** + * @param port the port to set + */ + public void setPort(int port) { + this.port = port; + } + + /** + * @return the directory + */ + public String getDirectory() { + return directory; + } + + /** + * @param directory the directory to set + */ + public void setDirectory(String directory) { + this.directory = directory; + } + + /** + * @return the subDirectory + */ + public String getSubDirectory() { + return subDirectory; + } + + /** + * @param subDirectory the subDirectory to set + */ + public void setSubDirectory(String subDirectory) { + this.subDirectory = subDirectory; + } + + /** + * @return the fileName + */ + public String getFileName() { + return fileName; + } + + /** + * @param fileName the fileName to set + */ + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getFullPath() { + return getDirectory() + "/" + getSubDirectory() + "/" + getFileName(); + } + + public ReferencedDataNature getDataNature() { + return ReferencedDataNature.values()[dataNatureInteger]; + } + + public void setDataNature(ReferencedDataNature dataNature) { + this.dataNatureInteger = dataNature.ordinal(); + } + + public String getCharset() { + return charset; + } + + public void setCredentialManager(CredentialManager credentialManager) { + this.credentialManager = credentialManager; + } + + public void setCharset(String charset) { + this.charset = charset; + } + + /** + * @return the dataNatureInteger + */ + public int getDataNatureInteger() { + return dataNatureInteger; + } + + /** + * @param dataNatureInteger the dataNatureInteger to set + */ + public void setDataNatureInteger(int dataNatureInteger) { + this.dataNatureInteger = dataNatureInteger; + } + + public SshReference clone() { + SshReference result = new SshReference(); + result.setHost(this.getHost()); + result.setPort(this.getPort()); + result.setDirectory(this.getDirectory()); + result.setSubDirectory(this.getSubDirectory()); + result.setFileName(this.getFileName()); + result.setDataNature(this.getDataNature()); + result.setCharset(this.getCharset()); + return result; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUrl.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUrl.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUrl.java new file mode 100644 index 0000000..4c39d66 --- /dev/null +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUrl.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.taverna.activities.externaltool.ssh; + +import org.apache.taverna.reference.ReferencedDataNature; + +public class SshUrl { + + private SshNode sshNode; + private String subDirectory; + private String fileName; + + private ReferencedDataNature dataNature = ReferencedDataNature.UNKNOWN; + private String charset = "UTF-8"; + + + public SshUrl(SshNode sshNode) { + this.setSshNode(sshNode); + } + + + /** + * @return the host + */ + public String getHost() { + return getSshNode().getHost(); + } + /** + * @param host the host to set + */ + public void setHost(String host) { + getSshNode().setHost(host); + } + /** + * @return the port + */ + public int getPort() { + return getSshNode().getPort(); + } + /** + * @param port the port to set + */ + public void setPort(int port) { + getSshNode().setPort(port); + } + /** + * @return the directory + */ + public String getDirectory() { + return getSshNode().getDirectory(); + } + /** + * @param directory the directory to set + */ + public void setDirectory(String directory) { + getSshNode().setDirectory(directory); + } + /** + * @return the subDirectory + */ + public String getSubDirectory() { + return subDirectory; + } + /** + * @param subDirectory the subDirectory to set + */ + public void setSubDirectory(String subDirectory) { + this.subDirectory = subDirectory; + } + /** + * @return the fileName + */ + public String getFileName() { + return fileName; + } + /** + * @param fileName the fileName to set + */ + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String toString() { + String result = SshNodeFactory.makeUrl(getHost(), getPort(), getDirectory()); + if (getSubDirectory() != null) { + result += getSubDirectory(); + } + if (getFileName() != null) { + result += "/" + getFileName(); + } + return result; + } + + public int hashCode() { + return toString().hashCode(); + + } + + public boolean equals(Object obj) { + if ((obj == null) || !(obj instanceof SshUrl)) { + return false; + } + return (this.hashCode() == obj.hashCode()); + } + + public SshUrl getBaseUrl() { + SshUrl result = new SshUrl(this.getSshNode()); + return result; + } + + + /** + * @return the sshNode + */ + public SshNode getSshNode() { + return sshNode; + } + + + /** + * @param sshNode the sshNode to set + */ + public void setSshNode(SshNode sshNode) { + this.sshNode = sshNode; + } + + public ReferencedDataNature getDataNature() { + return dataNature; + } + + + public void setDataNature(ReferencedDataNature dataNature) { + this.dataNature = dataNature; + } + + + public String getCharset() { + return charset; + } + + + public void setCharset(String charset) { + this.charset = charset; + } + + + +} http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUrlToSshReference.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUrlToSshReference.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUrlToSshReference.java new file mode 100644 index 0000000..74b01e1 --- /dev/null +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUrlToSshReference.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.taverna.activities.externaltool.ssh; + +import org.apache.taverna.reference.ExternalReferenceSPI; +import org.apache.taverna.reference.ReferenceContext; +import org.apache.taverna.reference.ValueToReferenceConversionException; +import org.apache.taverna.reference.ValueToReferenceConverterSPI; +import org.apache.taverna.security.credentialmanager.CredentialManager; + +public class SshUrlToSshReference implements ValueToReferenceConverterSPI { + + private CredentialManager credentialManager; + + /* (non-Javadoc) + * @see net.sf.taverna.t2.reference.ValueToReferenceConverterSPI#canConvert(java.lang.Object, net.sf.taverna.t2.reference.ReferenceContext) + */ + @Override + public boolean canConvert(Object o, ReferenceContext context) { + return (o instanceof SshUrl); + } + + /* (non-Javadoc) + * @see net.sf.taverna.t2.reference.ValueToReferenceConverterSPI#convert(java.lang.Object, net.sf.taverna.t2.reference.ReferenceContext) + */ + @Override + public ExternalReferenceSPI convert(Object o, ReferenceContext context) + throws ValueToReferenceConversionException { + SshReference result = new SshReference((SshUrl) o); + result.setCredentialManager(credentialManager); + return result; + } + + public void setCredentialManager(CredentialManager credentialManager) { + this.credentialManager = credentialManager; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUseCaseInvocation.java ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUseCaseInvocation.java b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUseCaseInvocation.java new file mode 100755 index 0000000..fcbc33f --- /dev/null +++ b/taverna-external-tool-activity/src/main/java/org/apache/taverna/activities/externaltool/ssh/SshUseCaseInvocation.java @@ -0,0 +1,560 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.taverna.activities.externaltool.ssh; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.Vector; +import java.util.regex.Matcher; + +import org.apache.taverna.activities.externaltool.RetrieveLoginFromTaverna; +import org.apache.taverna.activities.externaltool.desc.ScriptInput; +import org.apache.taverna.activities.externaltool.desc.ScriptOutput; +import org.apache.taverna.activities.externaltool.desc.UseCaseDescription; +import org.apache.taverna.activities.externaltool.invocation.AskUserForPw; +import org.apache.taverna.activities.externaltool.invocation.InvocationException; +import org.apache.taverna.activities.externaltool.invocation.UseCaseInvocation; +import org.apache.taverna.reference.AbstractExternalReference; +import org.apache.taverna.reference.ErrorDocument; +import org.apache.taverna.reference.ErrorDocumentServiceException; +import org.apache.taverna.reference.ExternalReferenceSPI; +import org.apache.taverna.reference.Identified; +import org.apache.taverna.reference.ReferenceService; +import org.apache.taverna.reference.ReferenceSet; +import org.apache.taverna.reference.ReferencedDataNature; +import org.apache.taverna.reference.T2Reference; +import org.apache.taverna.security.credentialmanager.CredentialManager; + +import org.apache.log4j.Logger; + +import com.jcraft.jsch.ChannelExec; +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import com.jcraft.jsch.SftpException; +import com.jcraft.jsch.ChannelSftp.LsEntry; + +/** + * The job is executed by connecting to a worker pc using ssh, i.e. not via the + * grid. + * + * @author Hajo Krabbenhoeft + */ +public class SshUseCaseInvocation extends UseCaseInvocation { + + private static Logger logger = Logger.getLogger(SshUseCaseInvocation.class); + + private SshUrl location = null; + + private InputStream stdInputStream = null; + + public static final String SSH_USE_CASE_INVOCATION_TYPE = "D0A4CDEB-DD10-4A8E-A49C-8871003083D8"; + private String tmpname; + private final SshNode workerNode; + private final AskUserForPw askUserForPw; + + private ChannelExec running; + + private List<String> precedingCommands = new ArrayList<String>(); + + private final ByteArrayOutputStream stdout_buf = new ByteArrayOutputStream(); + private final ByteArrayOutputStream stderr_buf = new ByteArrayOutputStream(); + + private static Map<String, Object> nodeLock = Collections + .synchronizedMap(new HashMap<String, Object>()); + + private static Map<String, Set<SshUrl>> runIdToTempDir = Collections + .synchronizedMap(new HashMap<String, Set<SshUrl>>()); + + private static String SSH_INVOCATION_FILE = "sshInvocations"; + + private final CredentialManager credentialManager; + + public static String test(final SshNode workerNode, + final AskUserForPw askUserForPw) { + try { + Session sshSession = SshPool + .getSshSession(workerNode, askUserForPw); + + ChannelSftp sftpTest = (ChannelSftp) sshSession.openChannel("sftp"); + sftpTest.connect(); + sftpTest.cd(workerNode.getDirectory()); + sftpTest.disconnect(); + sshSession.disconnect(); + } catch (JSchException e) { + return e.toString(); + } catch (SftpException e) { + return e.toString(); + } + return null; + } + + public SshUseCaseInvocation(UseCaseDescription desc, SshNode workerNodeA, + AskUserForPw askUserForPwA, CredentialManager credentialManager) + throws JSchException, SftpException { + this.workerNode = workerNodeA; + this.credentialManager = credentialManager; + + setRetrieveData(workerNodeA.isRetrieveData()); + this.askUserForPw = askUserForPwA; + usecase = desc; + + ChannelSftp sftp = SshPool.getSftpPutChannel(workerNode, askUserForPw); + synchronized (getNodeLock(workerNode)) { + + logger.info("Changing remote directory to " + + workerNode.getDirectory()); + sftp.cd(workerNode.getDirectory()); + Random rnd = new Random(); + while (true) { + tmpname = "usecase" + rnd.nextLong(); + try { + sftp.lstat(workerNode.getDirectory() + tmpname); + continue; + } catch (Exception e) { + // file seems to not exist :) + } + sftp.mkdir(workerNode.getDirectory() + tmpname); + sftp.cd(workerNode.getDirectory() + tmpname); + break; + } + } + } + + private static void recursiveDelete(ChannelSftp sftp, String path) + throws SftpException, JSchException { + Vector<?> entries = sftp.ls(path); + for (Object object : entries) { + LsEntry entry = (LsEntry) object; + if (entry.getFilename().equals(".") + || entry.getFilename().equals("..")) { + continue; + } + if (entry.getAttrs().isDir()) { + recursiveDelete(sftp, path + entry.getFilename() + "/"); + } else { + sftp.rm(path + entry.getFilename()); + } + } + sftp.rmdir(path); + } + + private static void deleteDirectory(SshUrl directory, + CredentialManager credentialManager) throws InvocationException { + URI uri; + try { + uri = new URI(directory.toString()); + + ChannelSftp sftp; + SshNode workerNode; + String fullPath = uri.getPath(); + String path = fullPath.substring(0, fullPath.lastIndexOf("/")); + String tempDir = fullPath.substring(fullPath.lastIndexOf("/")); + try { + workerNode = SshNodeFactory.getInstance().getSshNode( + uri.getHost(), uri.getPort(), path); + + sftp = SshPool.getSftpPutChannel(workerNode, + new RetrieveLoginFromTaverna(workerNode.getUrl() + .toString(), credentialManager)); + } catch (JSchException e) { + throw new InvocationException(e); + } + synchronized (getNodeLock(workerNode)) { + try { + sftp.cd(path); + recursiveDelete(sftp, path + "/" + tempDir + "/"); + } catch (SftpException e) { + throw new InvocationException(e); + } catch (JSchException e) { + throw new InvocationException(e); + } + } + } catch (URISyntaxException e1) { + throw new InvocationException(e1); + } + } + + public static void cleanup(String runId, CredentialManager credentialManager) + throws InvocationException { + Set<SshUrl> tempDirectories = runIdToTempDir.get(runId); + if (tempDirectories != null) { + for (SshUrl tempUrl : tempDirectories) { + deleteDirectory(tempUrl, credentialManager); + } + runIdToTempDir.remove(runId); + } + } + + @Override + protected void submit_generate_job_inner() throws InvocationException { + tags.put("uniqueID", "" + getSubmissionID()); + String command = usecase.getCommand(); + for (String cur : tags.keySet()) { + command = command.replaceAll("\\Q%%" + cur + "%%\\E", + Matcher.quoteReplacement(tags.get(cur))); + } + String fullCommand = "cd " + workerNode.getDirectory() + tmpname; + for (String preceding : precedingCommands) { + fullCommand += " && " + preceding; + } + fullCommand += " && " + command; + + logger.info("Full command is " + fullCommand); + + try { + running = SshPool.openExecChannel(workerNode, askUserForPw); + running.setCommand(fullCommand); + running.setOutputStream(stdout_buf); + running.setErrStream(stderr_buf); + if (stdInputStream != null) { + running.setInputStream(stdInputStream); + } + running.connect(); + } catch (JSchException e) { + throw new InvocationException(e); + } + + } + + @Override + public HashMap<String, Object> submit_wait_fetch_results( + ReferenceService referenceService) throws InvocationException { + while (!running.isClosed()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new InvocationException("Invocation interrupted:" + + e.getMessage()); + } + } + + int exitcode = running.getExitStatus(); + if (!usecase.getValidReturnCodes().contains(exitcode)) { + try { + throw new InvocationException("Invalid exit code " + exitcode + + ":" + stderr_buf.toString("US-ASCII")); + } catch (UnsupportedEncodingException e) { + throw new InvocationException("Invalid exit code " + exitcode + + ":" + stderr_buf.toString()); + } + } + + HashMap<String, Object> results = new HashMap<String, Object>(); + + results.put("STDOUT", stdout_buf.toByteArray()); + results.put("STDERR", stderr_buf.toByteArray()); + try { + stdout_buf.close(); + stderr_buf.close(); + } catch (IOException e2) { + throw new InvocationException(e2); + } + + try { + ChannelSftp sftp = SshPool.getSftpPutChannel(workerNode, + askUserForPw); + synchronized (getNodeLock(workerNode)) { + for (Map.Entry<String, ScriptOutput> cur : usecase.getOutputs() + .entrySet()) { + ScriptOutput scriptOutput = cur.getValue(); + String fullPath = workerNode.getDirectory() + tmpname + "/" + + scriptOutput.getPath(); + try { + if (sftp.stat(fullPath) != null) { + SshUrl url = new SshUrl(workerNode); + url.setSubDirectory(tmpname); + url.setFileName(scriptOutput.getPath()); + if (scriptOutput.isBinary()) { + url.setDataNature(ReferencedDataNature.BINARY); + } else { + url.setDataNature(ReferencedDataNature.TEXT); + url.setCharset("UTF-8"); + } + if (isRetrieveData()) { + SshReference urlRef = new SshReference(url); + InputStream is = urlRef.openStream(null); + AbstractExternalReference ref; + if (scriptOutput.isBinary()) { + ref = inlineByteArrayReferenceBuilder + .createReference(is, null); + } else { + ref = inlineStringReferenceBuilder + .createReference(is, null); + } + try { + is.close(); + } catch (IOException e) { + throw new InvocationException(e); + } + results.put(cur.getKey(), ref); + } else { + results.put(cur.getKey(), url); + } + } else { + ErrorDocument ed = referenceService + .getErrorDocumentService().registerError( + "No result for " + cur.getKey(), 0, + getContext()); + results.put(cur.getKey(), ed); + } + } catch (SftpException e) { + ErrorDocument ed = referenceService + .getErrorDocumentService().registerError( + "No result for " + cur.getKey(), 0, + getContext()); + results.put(cur.getKey(), ed); + + } + } + } + } catch (JSchException e1) { + throw new InvocationException(e1); + } catch (ErrorDocumentServiceException e) { + throw new InvocationException(e); + } + + if (running != null) { + running.disconnect(); + } + if (stdInputStream != null) { + try { + stdInputStream.close(); + } catch (IOException e) { + throw new InvocationException(e); + } + } + + if (isRetrieveData()) { + forgetRun(); + deleteDirectory(location, credentialManager); + + } + return results; + } + + @Override + public String setOneInput(ReferenceService referenceService, + T2Reference t2Reference, ScriptInput input) + throws InvocationException { + String target = null; + String remoteName = null; + if (input.isFile()) { + remoteName = input.getTag(); + } else if (input.isTempFile()) { + remoteName = "tempfile." + (nTempFiles++) + ".tmp"; + + } + if (input.isFile() || input.isTempFile()) { + SshReference sshRef = getAsSshReference(referenceService, + t2Reference, workerNode); + target = workerNode.getDirectory() + tmpname + "/" + remoteName; + logger.info("Target is " + target); + if (sshRef != null) { + if (!input.isForceCopy()) { + String linkCommand = workerNode.getLinkCommand(); + if (linkCommand != null) { + String actualLinkCommand = getActualOsCommand( + linkCommand, sshRef.getFullPath(), remoteName, + target); + precedingCommands.add(actualLinkCommand); + return target; + + } + } + String copyCommand = workerNode.getCopyCommand(); + if (copyCommand != null) { + String actualCopyCommand = getActualOsCommand(copyCommand, + sshRef.getFullPath(), remoteName, target); + precedingCommands.add(actualCopyCommand); + return target; + } + } + try { + ChannelSftp sftp = SshPool.getSftpPutChannel(workerNode, + askUserForPw); + synchronized (getNodeLock(workerNode)) { + InputStream r = getAsStream(referenceService, t2Reference); + sftp.put(r, target); + r.close(); + } + } catch (SftpException e) { + throw new InvocationException(e); + } catch (JSchException e) { + throw new InvocationException(e); + } catch (IOException e) { + throw new InvocationException(e); + } + return target; + } else { + String value = (String) referenceService.renderIdentifier( + t2Reference, String.class, this.getContext()); + return value; + + } + } + + public SshReference getAsSshReference(ReferenceService referenceService, + T2Reference t2Reference, SshNode workerNode) { + Identified identified = referenceService.resolveIdentifier(t2Reference, + null, null); + if (identified instanceof ReferenceSet) { + for (ExternalReferenceSPI ref : ((ReferenceSet) identified) + .getExternalReferences()) { + if (ref instanceof SshReference) { + SshReference sshRef = (SshReference) ref; + if (sshRef.getHost().equals(workerNode.getHost())) { + return sshRef; + } + } + } + } + return null; + } + + private static Object getNodeLock(final SshNode node) { + return getNodeLock(node.getHost()); + } + + private static synchronized Object getNodeLock(String hostName) { + if (!nodeLock.containsKey(hostName)) { + nodeLock.put(hostName, new Object()); + } + return nodeLock.get(hostName); + } + + @Override + public void setStdIn(ReferenceService referenceService, + T2Reference t2Reference) { + stdInputStream = new BufferedInputStream(getAsStream(referenceService, + t2Reference)); + } + + @Override + public void rememberRun(String runId) { + this.setRunId(runId); + Set<SshUrl> directories = runIdToTempDir.get(runId); + if (directories == null) { + directories = Collections.synchronizedSet(new HashSet<SshUrl>()); + runIdToTempDir.put(runId, directories); + } + location = new SshUrl(workerNode); + location.setSubDirectory(tmpname); + directories.add(location); + } + + private void forgetRun() { + Set<SshUrl> directories = runIdToTempDir.get(getRunId()); + directories.remove(location); + } + + public static void load(File directory) { + File invocationsFile = new File(directory, SSH_INVOCATION_FILE); + if (!invocationsFile.exists()) { + return; + } + BufferedReader reader = null; + try { + reader = new BufferedReader(new FileReader(invocationsFile)); + String line = reader.readLine(); + while (line != null) { + String[] parts = line.split(" "); + if (parts.length != 2) { + break; + } + String runId = parts[0]; + String urlString = parts[1]; + Set<SshUrl> urls = runIdToTempDir.get(runId); + if (urls == null) { + urls = new HashSet<SshUrl>(); + runIdToTempDir.put(runId, urls); + } + URI uri = new URI(urlString); + String fullPath = uri.getPath(); + String path = fullPath.substring(0, fullPath.lastIndexOf("/")); + String tempDir = fullPath.substring(fullPath.lastIndexOf("/")); + SshNode node = SshNodeFactory.getInstance().getSshNode( + uri.getHost(), uri.getPort(), path); + SshUrl newUrl = new SshUrl(node); + newUrl.setSubDirectory(tempDir); + urls.add(newUrl); + line = reader.readLine(); + } + } catch (FileNotFoundException e) { + logger.error(e); + } catch (URISyntaxException e) { + logger.error(e); + } catch (IOException e) { + logger.error(e); + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException e) { + logger.error(e); + } + } + } + } + + public static void persist(File directory) { + File invocationsFile = new File(directory, SSH_INVOCATION_FILE); + BufferedWriter writer = null; + try { + writer = new BufferedWriter(new FileWriter(invocationsFile)); + for (String runId : runIdToTempDir.keySet()) { + for (SshUrl url : runIdToTempDir.get(runId)) { + writer.write(runId); + writer.write(" "); + writer.write(url.toString()); + writer.newLine(); + } + } + } catch (IOException e) { + logger.error(e); + } finally { + if (writer != null) { + try { + writer.close(); + } catch (IOException e) { + logger.error(e); + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/resources/META-INF/services/org.apache.taverna.reference.ExternalReferenceSPI ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/resources/META-INF/services/org.apache.taverna.reference.ExternalReferenceSPI b/taverna-external-tool-activity/src/main/resources/META-INF/services/org.apache.taverna.reference.ExternalReferenceSPI index 6a18935..d63d896 100644 --- a/taverna-external-tool-activity/src/main/resources/META-INF/services/org.apache.taverna.reference.ExternalReferenceSPI +++ b/taverna-external-tool-activity/src/main/resources/META-INF/services/org.apache.taverna.reference.ExternalReferenceSPI @@ -1,2 +1,2 @@ # Implementation classes of ExternalReferenceSPI go here, one per line -de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshReference +org.apache.taverna.activities.externaltool.invocation.ssh.SshReference http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/resources/META-INF/services/org.apache.taverna.reference.ValueToReferenceConverterSPI ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/resources/META-INF/services/org.apache.taverna.reference.ValueToReferenceConverterSPI b/taverna-external-tool-activity/src/main/resources/META-INF/services/org.apache.taverna.reference.ValueToReferenceConverterSPI index 53ca06c..50c5a16 100644 --- a/taverna-external-tool-activity/src/main/resources/META-INF/services/org.apache.taverna.reference.ValueToReferenceConverterSPI +++ b/taverna-external-tool-activity/src/main/resources/META-INF/services/org.apache.taverna.reference.ValueToReferenceConverterSPI @@ -1,3 +1,3 @@ # Implementation classes of ValueToReferenceConverterSPI go here, one per line -de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshUrlToSshReference +org.apache.taverna.activities.externaltool.invocation.ssh.SshUrlToSshReference http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/resources/META-INF/spring/external-tool-activity-context.xml ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/resources/META-INF/spring/external-tool-activity-context.xml b/taverna-external-tool-activity/src/main/resources/META-INF/spring/external-tool-activity-context.xml index d933de9..297fc8d 100644 --- a/taverna-external-tool-activity/src/main/resources/META-INF/spring/external-tool-activity-context.xml +++ b/taverna-external-tool-activity/src/main/resources/META-INF/spring/external-tool-activity-context.xml @@ -21,7 +21,7 @@ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd";> - <bean id="SshReference" class="de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshReference"> + <bean id="SshReference" class="org.apache.taverna.activities.externaltool.invocation.ssh.SshReference"> <property name="credentialManager" ref="credentialManager" /> </bean> @@ -37,7 +37,7 @@ <property name="invocationGroupManager" ref="invocationGroupManager" /> </bean> - <bean id="SshUrlToSshReference" class="de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshUrlToSshReference"> + <bean id="SshUrlToSshReference" class="org.apache.taverna.activities.externaltool.invocation.ssh.SshUrlToSshReference"> <property name="credentialManager" ref="credentialManager" /> </bean> http://git-wip-us.apache.org/repos/asf/incubator-taverna-common-activities/blob/98468d30/taverna-external-tool-activity/src/main/resources/de/uni_luebeck/inb/knowarc/usecases/invocation/ssh/SshReference.hbm.xml ---------------------------------------------------------------------- diff --git a/taverna-external-tool-activity/src/main/resources/de/uni_luebeck/inb/knowarc/usecases/invocation/ssh/SshReference.hbm.xml b/taverna-external-tool-activity/src/main/resources/de/uni_luebeck/inb/knowarc/usecases/invocation/ssh/SshReference.hbm.xml index 4f0a737..b67f6be 100644 --- a/taverna-external-tool-activity/src/main/resources/de/uni_luebeck/inb/knowarc/usecases/invocation/ssh/SshReference.hbm.xml +++ b/taverna-external-tool-activity/src/main/resources/de/uni_luebeck/inb/knowarc/usecases/invocation/ssh/SshReference.hbm.xml @@ -23,7 +23,7 @@ <!-- Hibernate mapping for ssh reference bean --> <hibernate-mapping> <joined-subclass - name="de.uni_luebeck.inb.knowarc.usecases.invocation.ssh.SshReference" + name="org.apache.taverna.activities.externaltool.invocation.ssh.SshReference" extends="org.apache.taverna.reference.AbstractExternalReference"> <!-- Link to primary key from abstract superclass --> <key column="bean_id" />
