http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/GSISSHAbstractCluster.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/GSISSHAbstractCluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/GSISSHAbstractCluster.java new file mode 100644 index 0000000..c7d6279 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/GSISSHAbstractCluster.java @@ -0,0 +1,767 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.impl; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.StringWriter; +import java.net.URL; +import java.security.SecureRandom; +import java.util.List; +import java.util.Map; + +import javax.xml.transform.Source; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.stream.StreamResult; +import javax.xml.transform.stream.StreamSource; + +import org.apache.airavata.gfac.ssh.api.Cluster; +import org.apache.airavata.gfac.ssh.api.CommandExecutor; +import org.apache.airavata.gfac.ssh.api.SSHApiException; +import org.apache.airavata.gfac.ssh.api.ServerInfo; +import org.apache.airavata.gfac.ssh.api.authentication.AuthenticationInfo; +import org.apache.airavata.gfac.ssh.api.authentication.GSIAuthenticationInfo; +import org.apache.airavata.gfac.ssh.api.authentication.SSHKeyAuthentication; +import org.apache.airavata.gfac.ssh.api.authentication.SSHPasswordAuthentication; +import org.apache.airavata.gfac.ssh.api.authentication.SSHPublicKeyAuthentication; +import org.apache.airavata.gfac.ssh.api.authentication.SSHPublicKeyFileAuthentication; +import org.apache.airavata.gfac.ssh.api.job.JobDescriptor; +import org.apache.airavata.gfac.ssh.api.job.JobManagerConfiguration; +import org.apache.airavata.gfac.ssh.api.job.OutputParser; +import org.apache.airavata.gfac.ssh.config.ConfigReader; +import org.apache.airavata.gfac.ssh.jsch.ExtendedJSch; +import org.apache.airavata.gfac.ssh.util.SSHAPIUIKeyboardInteractive; +import org.apache.airavata.gfac.ssh.util.SSHKeyPasswordHandler; +import org.apache.airavata.gfac.ssh.util.SSHUtils; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.jcraft.jsch.ExtendedSession; +import com.jcraft.jsch.GSISSHIdentityFile; +import com.jcraft.jsch.GSISSHIdentityRepository; +import com.jcraft.jsch.Identity; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; + +public class GSISSHAbstractCluster implements Cluster { + + private static final Logger log = LoggerFactory.getLogger(GSISSHAbstractCluster.class); + public static final String X509_CERT_DIR = "X509_CERT_DIR"; + public static final String SSH_SESSION_TIMEOUT = "ssh.session.timeout"; + + public JobManagerConfiguration jobManagerConfiguration; + + private ServerInfo serverInfo; + + private AuthenticationInfo authenticationInfo; + + private Session session; + + private ConfigReader configReader; + + private JSch defaultJSch; + + private static Identity identityFile = null; + + public GSISSHAbstractCluster(ServerInfo serverInfo, AuthenticationInfo authenticationInfo, JobManagerConfiguration config) throws SSHApiException { + this(serverInfo, authenticationInfo); + this.jobManagerConfiguration = config; + } + + public GSISSHAbstractCluster(ServerInfo serverInfo, AuthenticationInfo authenticationInfo) throws SSHApiException { + + reconnect(serverInfo, authenticationInfo); + } + + public GSISSHAbstractCluster(JobManagerConfiguration config) { + this.jobManagerConfiguration = config; + } + private synchronized void reconnect(ServerInfo serverInfo, AuthenticationInfo authenticationInfo) throws SSHApiException { + this.serverInfo = serverInfo; + + this.authenticationInfo = authenticationInfo; + + if (authenticationInfo instanceof GSIAuthenticationInfo) { + JSch.setConfig("gssapi-with-mic.x509", "org.apache.airavata.gfac.ssh.GSSContextX509"); + JSch.setConfig("userauth.gssapi-with-mic", "com.jcraft.jsch.UserAuthGSSAPIWithMICGSSCredentials"); + System.setProperty(X509_CERT_DIR, (String) ((GSIAuthenticationInfo) authenticationInfo).getProperties(). + get("X509_CERT_DIR")); + } + + + try { + this.configReader = new ConfigReader(); + } catch (IOException e) { + throw new SSHApiException("Unable to load system configurations.", e); + } + try { + if(defaultJSch == null){ + defaultJSch = createJSch(authenticationInfo); + } + log.debug("Connecting to server - " + serverInfo.getHost() + ":" + serverInfo.getPort() + " with user name - " + + serverInfo.getUserName()); + + session = createSession(defaultJSch,serverInfo.getUserName(), serverInfo.getHost(), serverInfo.getPort()); + } + catch (Exception e) { + throw new SSHApiException("An exception occurred while creating SSH session." + + "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() + + " connecting user name - " + + serverInfo.getUserName(), e); + } + + //============================================================= + // Handling vanilla SSH pieces + //============================================================= + if (authenticationInfo instanceof SSHPasswordAuthentication) { + String password = ((SSHPasswordAuthentication) authenticationInfo). + getPassword(serverInfo.getUserName(), serverInfo.getHost()); + + session.setUserInfo(new SSHAPIUIKeyboardInteractive(password)); + + // TODO figure out why we need to set password to session + session.setPassword(password); + + } else if (authenticationInfo instanceof SSHPublicKeyFileAuthentication) { + + SSHPublicKeyFileAuthentication sshPublicKeyFileAuthentication + = (SSHPublicKeyFileAuthentication) authenticationInfo; + String privateKeyFile = sshPublicKeyFileAuthentication. + getPrivateKeyFile(serverInfo.getUserName(), serverInfo.getHost()); + + logDebug("The private key file for vanilla SSH " + privateKeyFile); + + String publicKeyFile = sshPublicKeyFileAuthentication. + getPublicKeyFile(serverInfo.getUserName(), serverInfo.getHost()); + + logDebug("The public key file for vanilla SSH " + publicKeyFile); + + try { + identityFile = GSISSHIdentityFile.newInstance(privateKeyFile, null, defaultJSch); + } catch (JSchException e) { + throw new SSHApiException("An exception occurred while initializing keys using files. " + + "(private key and public key)." + + "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() + + " connecting user name - " + + serverInfo.getUserName() + " private key file - " + privateKeyFile + ", public key file - " + + publicKeyFile, e); + } + + // Add identity to identity repository + GSISSHIdentityRepository identityRepository = new GSISSHIdentityRepository(defaultJSch); + identityRepository.add(identityFile); + + // Set repository to session + session.setIdentityRepository(identityRepository); + + // Set the user info + SSHKeyPasswordHandler sshKeyPasswordHandler + = new SSHKeyPasswordHandler((SSHKeyAuthentication) authenticationInfo); + + session.setUserInfo(sshKeyPasswordHandler); + + } else if (authenticationInfo instanceof SSHPublicKeyAuthentication) { + + SSHPublicKeyAuthentication sshPublicKeyAuthentication + = (SSHPublicKeyAuthentication) authenticationInfo; + try { + String name = serverInfo.getUserName() + "_" + serverInfo.getHost(); + identityFile = GSISSHIdentityFile.newInstance(name, + sshPublicKeyAuthentication.getPrivateKey(serverInfo.getUserName(), serverInfo.getHost()), + sshPublicKeyAuthentication.getPublicKey(serverInfo.getUserName(), serverInfo.getHost()), defaultJSch); + } catch (JSchException e) { + throw new SSHApiException("An exception occurred while initializing keys using byte arrays. " + + "(private key and public key)." + + "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() + + " connecting user name - " + + serverInfo.getUserName(), e); + } + + // Add identity to identity repository + GSISSHIdentityRepository identityRepository = new GSISSHIdentityRepository(defaultJSch); + identityRepository.add(identityFile); + + // Set repository to session + session.setIdentityRepository(identityRepository); + + // Set the user info + SSHKeyPasswordHandler sshKeyPasswordHandler + = new SSHKeyPasswordHandler((SSHKeyAuthentication) authenticationInfo); + + session.setUserInfo(sshKeyPasswordHandler); + + } + + // Not a good way, but we dont have any choice + if (session instanceof ExtendedSession) { + if (authenticationInfo instanceof GSIAuthenticationInfo) { + ((ExtendedSession) session).setAuthenticationInfo((GSIAuthenticationInfo) authenticationInfo); + } + } + + try { + session.connect(Integer.parseInt(configReader.getConfiguration(SSH_SESSION_TIMEOUT))); + } catch (Exception e) { + throw new SSHApiException("An exception occurred while connecting to server." + + "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() + + " connecting user name - " + + serverInfo.getUserName(), e); + } + } + + public synchronized JobDescriptor cancelJob(String jobID) throws SSHApiException { + JobStatus jobStatus = getJobStatus(jobID); + if (jobStatus == null || jobStatus == JobStatus.U) { + log.info("Validation before cancel is failed, couldn't found job in remote host to cancel. Job may be already completed|failed|canceled"); + return null; + } + RawCommandInfo rawCommandInfo = jobManagerConfiguration.getCancelCommand(jobID); + + StandardOutReader stdOutReader = new StandardOutReader(); + log.info("Executing RawCommand : " + rawCommandInfo.getCommand()); + CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader); + String outputifAvailable = getOutputifAvailable(stdOutReader, "Error reading output of job submission", jobManagerConfiguration.getBaseCancelCommand()); + // this might not be the case for all teh resources, if so Cluster implementation can override this method + // because here after cancelling we try to get the job description and return it back + try { + return this.getJobDescriptorById(jobID); + } catch (Exception e) { + //its ok to fail to get status when the job is gone + return null; + } + } + + public synchronized String submitBatchJobWithScript(String scriptPath, String workingDirectory) throws SSHApiException { + this.scpTo(workingDirectory, scriptPath); + + // since this is a constant we do not ask users to fill this + +// RawCommandInfo rawCommandInfo = new RawCommandInfo(this.installedPath + this.jobManagerConfiguration.getSubmitCommand() + " " + +// workingDirectory + File.separator + FilenameUtils.getName(scriptPath)); + + RawCommandInfo rawCommandInfo = jobManagerConfiguration.getSubmitCommand(workingDirectory,scriptPath); + StandardOutReader standardOutReader = new StandardOutReader(); + log.info("Executing RawCommand : " + rawCommandInfo.getCommand()); + CommandExecutor.executeCommand(rawCommandInfo, this.session, standardOutReader); + + //Check whether pbs submission is successful or not, if it failed throw and exception in submitJob method + // with the error thrown in qsub command + // + String outputifAvailable = getOutputifAvailable(standardOutReader,"Error reading output of job submission",jobManagerConfiguration.getBaseSubmitCommand()); + OutputParser outputParser = jobManagerConfiguration.getParser(); + return outputParser.parseJobSubmission(outputifAvailable); + } + + public synchronized String submitBatchJob(JobDescriptor jobDescriptor) throws SSHApiException { + TransformerFactory factory = TransformerFactory.newInstance(); + URL resource = this.getClass().getClassLoader().getResource(jobManagerConfiguration.getJobDescriptionTemplateName()); + + if (resource == null) { + String error = "System configuration file '" + jobManagerConfiguration.getJobDescriptionTemplateName() + + "' not found in the classpath"; + throw new SSHApiException(error); + } + + Source xslt = new StreamSource(new File(resource.getPath())); + Transformer transformer; + StringWriter results = new StringWriter(); + File tempPBSFile = null; + try { + // generate the pbs script using xslt + transformer = factory.newTransformer(xslt); + Source text = new StreamSource(new ByteArrayInputStream(jobDescriptor.toXML().getBytes())); + transformer.transform(text, new StreamResult(results)); + String scriptContent = results.toString().replaceAll("^[ |\t]*\n$", ""); + if (scriptContent.startsWith("\n")) { + scriptContent = scriptContent.substring(1); + } +// log.debug("generated PBS:" + results.toString()); + + // creating a temporary file using pbs script generated above + int number = new SecureRandom().nextInt(); + number = (number < 0 ? -number : number); + + tempPBSFile = new File(Integer.toString(number) + jobManagerConfiguration.getScriptExtension()); + FileUtils.writeStringToFile(tempPBSFile, scriptContent); + + //reusing submitBatchJobWithScript method to submit a job + String jobID = null; + int retry = 3; + while(retry>0) { + try { + jobID = this.submitBatchJobWithScript(tempPBSFile.getAbsolutePath(), + jobDescriptor.getWorkingDirectory()); + retry=0; + } catch (SSHApiException e) { + retry--; + if(retry==0) { + throw e; + }else{ + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + log.error(e1.getMessage(), e1); + } + log.error("Error occured during job submission but doing a retry"); + } + } + } + log.debug("Job has successfully submitted, JobID : " + jobID); + if (jobID != null) { + return jobID.replace("\n", ""); + } else { + return null; + } + } catch (TransformerConfigurationException e) { + throw new SSHApiException("Error parsing PBS transformation", e); + } catch (TransformerException e) { + throw new SSHApiException("Error generating PBS script", e); + } catch (IOException e) { + throw new SSHApiException("An exception occurred while connecting to server." + + "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() + + " connecting user name - " + + serverInfo.getUserName(), e); + } finally { + if (tempPBSFile != null) { + tempPBSFile.delete(); + } + } + } + + + public void generateJobScript(JobDescriptor jobDescriptor) throws SSHApiException { + TransformerFactory factory = TransformerFactory.newInstance(); + URL resource = this.getClass().getClassLoader().getResource(jobManagerConfiguration.getJobDescriptionTemplateName()); + + if (resource == null) { + String error = "System configuration file '" + jobManagerConfiguration.getJobDescriptionTemplateName() + + "' not found in the classpath"; + throw new SSHApiException(error); + } + + Source xslt = new StreamSource(new File(resource.getPath())); + Transformer transformer; + StringWriter results = new StringWriter(); + File tempPBSFile = null; + try { + // generate the pbs script using xslt + transformer = factory.newTransformer(xslt); + Source text = new StreamSource(new ByteArrayInputStream(jobDescriptor.toXML().getBytes())); + transformer.transform(text, new StreamResult(results)); + String scriptContent = results.toString().replaceAll("^[ |\t]*\n$", ""); + if (scriptContent.startsWith("\n")) { + scriptContent = scriptContent.substring(1); + } +// log.debug("generated PBS:" + results.toString()); + + // creating a temporary file using pbs script generated above + int number = new SecureRandom().nextInt(); + number = (number < 0 ? -number : number); + + tempPBSFile = new File(Integer.toString(number) + jobManagerConfiguration.getScriptExtension()); + log.info("File Path: " + tempPBSFile.getAbsolutePath()); + log.info("File Content: " + scriptContent); + FileUtils.writeStringToFile(tempPBSFile, scriptContent); + } catch (TransformerConfigurationException e) { + throw new SSHApiException("Error parsing PBS transformation", e); + } catch (TransformerException e) { + throw new SSHApiException("Error generating PBS script", e); + } catch (IOException e) { + throw new SSHApiException("An exception occurred while connecting to server." + + "Connecting server - " + serverInfo.getHost() + ":" + serverInfo.getPort() + + " connecting user name - " + + serverInfo.getUserName(), e); + } finally { + if (tempPBSFile != null) { + tempPBSFile.delete(); + } + } + } + + + + public synchronized JobDescriptor getJobDescriptorById(String jobID) throws SSHApiException { + RawCommandInfo rawCommandInfo = jobManagerConfiguration.getMonitorCommand(jobID); + StandardOutReader stdOutReader = new StandardOutReader(); + log.info("Executing RawCommand : " + rawCommandInfo.getCommand()); + CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader); + String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !",jobManagerConfiguration.getBaseMonitorCommand()); + JobDescriptor jobDescriptor = new JobDescriptor(); + jobManagerConfiguration.getParser().parseSingleJob(jobDescriptor, result); + return jobDescriptor; + } + + public synchronized JobStatus getJobStatus(String jobID) throws SSHApiException { + RawCommandInfo rawCommandInfo = jobManagerConfiguration.getMonitorCommand(jobID); + StandardOutReader stdOutReader = new StandardOutReader(); + log.info("Executing RawCommand : " + rawCommandInfo.getCommand()); + CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader); + String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !", jobManagerConfiguration.getBaseMonitorCommand()); + return jobManagerConfiguration.getParser().parseJobStatus(jobID, result); + } + + @Override + public String getJobIdByJobName(String jobName, String userName) throws SSHApiException { + RawCommandInfo rawCommandInfo = jobManagerConfiguration.getJobIdMonitorCommand(jobName, userName); + StandardOutReader stdOutReader = new StandardOutReader(); + log.info("Executing RawCommand : " + rawCommandInfo.getCommand()); + CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader); + String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !", + jobManagerConfiguration.getJobIdMonitorCommand(jobName,userName).getCommand()); + return jobManagerConfiguration.getParser().parseJobId(jobName, result); + } + + private static void logDebug(String message) { + if (log.isDebugEnabled()) { + log.debug(message); + } + } + + public JobManagerConfiguration getJobManagerConfiguration() { + return jobManagerConfiguration; + } + + public void setJobManagerConfiguration(JobManagerConfiguration jobManagerConfiguration) { + this.jobManagerConfiguration = jobManagerConfiguration; + } + + public synchronized void scpTo(String remoteFile, String localFile) throws SSHApiException { + int retry = 3; + while (retry > 0) { + try { + if (!session.isConnected()) { + session.connect(); + } + log.info("Transfering file:/" + localFile + " To:" + serverInfo.getHost() + ":" + remoteFile); + SSHUtils.scpTo(remoteFile, localFile, session); + retry = 0; + } catch (IOException e) { + retry--; + reconnect(serverInfo, authenticationInfo); + if (retry == 0) { + throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file " + + serverInfo.getHost() + ":rFile : " + remoteFile, e); + } + } catch (JSchException e) { + retry--; + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + log.error(e1.getMessage(), e1); + } + reconnect(serverInfo, authenticationInfo); + if (retry == 0) { + throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file " + + serverInfo.getHost() + ":rFile : " + remoteFile, e); + } + } + } + } + + public synchronized void scpFrom(String remoteFile, String localFile) throws SSHApiException { + int retry = 3; + while(retry>0) { + try { + if (!session.isConnected()) { + session.connect(); + } + log.info("Transfering from:" + serverInfo.getHost() + ":" + remoteFile + " To:" + "file:/" + localFile); + SSHUtils.scpFrom(remoteFile, localFile, session); + retry=0; + } catch (IOException e) { + retry--; + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + log.error(e1.getMessage(), e1); + } + reconnect(serverInfo, authenticationInfo); + if (retry == 0) { + throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file " + + serverInfo.getHost() + ":rFile", e); + }else{ + log.error("Error performing scp but doing a retry"); + } + } catch (JSchException e) { + retry--; + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + log.error(e1.getMessage(), e1); + } + reconnect(serverInfo, authenticationInfo); + if(retry==0) { + throw new SSHApiException("Failed during scping local file:" + localFile + " to remote file " + + serverInfo.getHost() + ":rFile", e); + }else{ + log.error("Error performing scp but doing a retry"); + } + } + } + } + + public synchronized void scpThirdParty(String remoteFileSource, String remoteFileTarget) throws SSHApiException { + try { + if(!session.isConnected()){ + session.connect(); + } + log.info("Transfering from:" + remoteFileSource + " To: " + remoteFileTarget); + SSHUtils.scpThirdParty(remoteFileSource, remoteFileTarget, session); + } catch (IOException e) { + throw new SSHApiException("Failed during scping file:" + remoteFileSource + " to remote file " + +remoteFileTarget , e); + } catch (JSchException e) { + throw new SSHApiException("Failed during scping file:" + remoteFileSource + " to remote file " + +remoteFileTarget, e); + } + } + + public synchronized void makeDirectory(String directoryPath) throws SSHApiException { + int retry = 3; + while (retry > 0) { + try { + if (!session.isConnected()) { + session.connect(); + } + log.info("Creating directory: " + serverInfo.getHost() + ":" + directoryPath); + SSHUtils.makeDirectory(directoryPath, session); + retry = 0; + } catch (IOException e) { + throw new SSHApiException("Failed during creating directory:" + directoryPath + " to remote file " + + serverInfo.getHost() + ":rFile", e); + } catch (JSchException e) { + retry--; + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + log.error(e1.getMessage(), e1); + } + reconnect(serverInfo, authenticationInfo); + if (retry == 0) { + throw new SSHApiException("Failed during creating directory :" + directoryPath + " to remote file " + + serverInfo.getHost() + ":rFile", e); + } + } catch (SSHApiException e) { + retry--; + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + log.error(e1.getMessage(), e1); + } + reconnect(serverInfo, authenticationInfo); + if (retry == 0) { + throw new SSHApiException("Failed during creating directory :" + directoryPath + " to remote file " + + serverInfo.getHost() + ":rFile", e); + } + } + } + } + + public synchronized List<String> listDirectory(String directoryPath) throws SSHApiException { + int retry = 3; + List<String> files = null; + while (retry > 0) { + try { + if (!session.isConnected()) { + session.connect(); + } + log.info("Listing directory: " + serverInfo.getHost() + ":" + directoryPath); + files = SSHUtils.listDirectory(directoryPath, session); + retry=0; + } catch (IOException e) { + log.error(e.getMessage(), e); + retry--; + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + log.error(e1.getMessage(), e1); + } + reconnect(serverInfo, authenticationInfo); + if (retry == 0) { + throw new SSHApiException("Failed during listing directory:" + directoryPath + " to remote file ", e); + } + } catch (JSchException e) { + retry--; + reconnect(serverInfo, authenticationInfo); + if (retry == 0) { + throw new SSHApiException("Failed during listing directory :" + directoryPath + " to remote file ", e); + } + }catch (SSHApiException e) { + retry--; + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + log.error(e1.getMessage(), e1); + } + reconnect(serverInfo, authenticationInfo); + if (retry == 0) { + throw new SSHApiException("Failed during listing directory :" + directoryPath + " to remote file " + + serverInfo.getHost() + ":rFile", e); + } + } + } + return files; + } + + public synchronized void getJobStatuses(String userName, Map<String,JobStatus> jobIDs)throws SSHApiException { + int retry = 3; + RawCommandInfo rawCommandInfo = jobManagerConfiguration.getUserBasedMonitorCommand(userName); + StandardOutReader stdOutReader = new StandardOutReader(); + while (retry > 0){ + try { + log.info("Executing RawCommand : " + rawCommandInfo.getCommand()); + CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader); + retry=0; + } catch (SSHApiException e) { + retry--; + try { + Thread.sleep(5000); + } catch (InterruptedException e1) { + log.error(e1.getMessage(), e1); + } + reconnect(serverInfo, authenticationInfo); + if (retry == 0) { + throw new SSHApiException("Failed Getting statuses to remote file", e); + } + } + } + String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !", jobManagerConfiguration.getBaseMonitorCommand()); + jobManagerConfiguration.getParser().parseJobStatuses(userName, jobIDs, result); + } + + public ServerInfo getServerInfo() { + return serverInfo; + } + + public AuthenticationInfo getAuthenticationInfo() { + return authenticationInfo; + } + + /** + * This gaurantee to return a valid session + * + * @return + */ + public Session getSession() { + return this.session; + } + + /** + * This method will read standard output and if there's any it will be parsed + * + * @param jobIDReaderCommandOutput + * @param errorMsg + * @return + * @throws SSHApiException + */ + private String getOutputifAvailable(StandardOutReader jobIDReaderCommandOutput, String errorMsg, String command) throws SSHApiException { + String stdOutputString = jobIDReaderCommandOutput.getStdOutputString(); + String stdErrorString = jobIDReaderCommandOutput.getStdErrorString(); + log.info("StandardOutput Returned:" + stdOutputString); + log.info("StandardError Returned:" +stdErrorString); + String[] list = command.split(File.separator); + command = list[list.length - 1]; + // We are checking for stderr containing the command issued. Thus ignores the verbose logs in stderr. + if (stdErrorString != null && stdErrorString.contains(command.trim()) && !stdErrorString.contains("Warning")) { + log.error("Standard Error output : " + stdErrorString); + throw new SSHApiException(errorMsg + "\n\r StandardOutput: "+ stdOutputString + "\n\r StandardError: "+ stdErrorString); + }else if(stdOutputString.contains("error")){ + throw new SSHApiException(errorMsg + "\n\r StandardOutput: "+ stdOutputString + "\n\r StandardError: "+ stdErrorString); + } + return stdOutputString; + } + + public void disconnect() throws SSHApiException { + if(getSession().isConnected()){ + getSession().disconnect(); + } + } + /** + + * the file system abstraction which will be necessary to + * perform certain file system operations. + * @return the new default JSch implementation. + * @throws JSchException + * known host keys cannot be loaded. + */ + protected JSch createJSch(AuthenticationInfo authenticationInfo) throws JSchException { +// final File fs = new File(System.getProperty("user.home")); + if(authenticationInfo instanceof GSIAuthenticationInfo){ + final JSch jsch = new ExtendedJSch(); +// knownHosts(jsch, fs); + return jsch; + }else{ + final JSch jsch = new JSch(); +// knownHosts(jsch, fs); + return jsch; + } + + } + /** + * Create a new remote session for the requested address. + * + * @param user + * login to authenticate as. + * @param host + * server name to connect to. + * @param port + * port number of the SSH daemon (typically 22). + * @return new session instance, but otherwise unconfigured. + * @throws JSchException + * the session could not be created. + */ + private Session createSession(JSch jsch, String user, String host, int port) throws JSchException { + final Session session = jsch.getSession(user, host, port); + // We retry already in getSession() method. JSch must not retry + // on its own. + session.setConfig("MaxAuthTries", "1"); //$NON-NLS-1$ //$NON-NLS-2$ + session.setTimeout(Integer.parseInt(configReader.getConfiguration(SSH_SESSION_TIMEOUT))); + java.util.Properties config = this.configReader.getProperties(); + session.setConfig(config); + + return session; + } + private static void knownHosts(final JSch sch,final File home) throws JSchException { + if (home == null) + return; + final File known_hosts = new File(new File(home, ".ssh"), "known_hosts"); //$NON-NLS-1$ //$NON-NLS-2$ + try { + final FileInputStream in = new FileInputStream(known_hosts); + try { + sch.setKnownHosts(in); + } finally { + in.close(); + } + } catch (FileNotFoundException none) { + // Oh well. They don't have a known hosts in home. + } catch (IOException err) { + // Oh well. They don't have a known hosts in home. + } + } +}
http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/JobStatus.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/JobStatus.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/JobStatus.java new file mode 100644 index 0000000..648d955 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/JobStatus.java @@ -0,0 +1,110 @@ + /* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ + package org.apache.airavata.gfac.ssh.impl; + + /** + * This will contains all the PBS specific job statuses. + * C - Job is completed after having run/ + * E - Job is exiting after having run. + * H - Job is held. + * Q - job is queued, eligible to run or routed. + * R - job is running. + * T - job is being moved to new location. + * W - job is waiting for its execution time + * (-a option) to be reached. + * S - (Unicos only) job is suspend. + */ + public enum JobStatus { + C, E, H, Q, R, T, W, S,U,F,CA,CD,CF,CG,NF,PD,PR,TO,qw,t,r,h,Er,Eqw,PEND,RUN,PSUSP,USUSP,SSUSP,DONE,EXIT,UNKWN,ZOMBI; + + public static JobStatus fromString(String status){ + if(status != null){ + if("C".equals(status)){ + return JobStatus.C; + }else if("E".equals(status)){ + return JobStatus.E; + }else if("H".equals(status)){ + return JobStatus.H; + }else if("Q".equals(status)){ + return JobStatus.Q; + }else if("R".equals(status)){ + return JobStatus.R; + }else if("T".equals(status)){ + return JobStatus.T; + }else if("W".equals(status)){ + return JobStatus.W; + }else if("S".equals(status)){ + return JobStatus.S; + }else if("F".equals(status)){ + return JobStatus.F; + }else if("S".equals(status)){ + return JobStatus.S; + }else if("CA".equals(status)){ + return JobStatus.CA; + }else if("CF".equals(status)){ + return JobStatus.CF; + }else if("CD".equals(status)){ + return JobStatus.CD; + }else if("CG".equals(status)){ + return JobStatus.CG; + }else if("NF".equals(status)){ + return JobStatus.NF; + }else if("PD".equals(status)){ + return JobStatus.PD; + }else if("PR".equals(status)){ + return JobStatus.PR; + }else if("TO".equals(status)){ + return JobStatus.TO; + }else if("U".equals(status)){ + return JobStatus.U; + }else if("qw".equals(status)){ + return JobStatus.qw; + }else if("t".equals(status)){ + return JobStatus.t; + }else if("r".equals(status)){ + return JobStatus.r; + }else if("h".equals(status)){ + return JobStatus.h; + }else if("Er".equals(status)){ + return JobStatus.Er; + }else if("Eqw".equals(status)){ + return JobStatus.Er; + }else if("RUN".equals(status)){ // LSF starts here + return JobStatus.RUN; + }else if("PEND".equals(status)){ + return JobStatus.PEND; + }else if("DONE".equals(status)){ + return JobStatus.DONE; + }else if("PSUSP".equals(status)){ + return JobStatus.PSUSP; + }else if("USUSP".equals(status)){ + return JobStatus.USUSP; + }else if("SSUSP".equals(status)){ + return JobStatus.SSUSP; + }else if("EXIT".equals(status)){ + return JobStatus.EXIT; + }else if("ZOMBI".equals(status)){ + return JobStatus.ZOMBI; + } + } + return JobStatus.U; + } + } http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/PBSCluster.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/PBSCluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/PBSCluster.java new file mode 100644 index 0000000..def84d5 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/PBSCluster.java @@ -0,0 +1,45 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.impl; + +import org.apache.airavata.gfac.ssh.api.*; +import org.apache.airavata.gfac.ssh.api.authentication.*; +import org.apache.airavata.gfac.ssh.api.job.JobManagerConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * This is the default implementation of a cluster. + * this has most of the methods to be used by the end user of the + * library. + */ +public class PBSCluster extends GSISSHAbstractCluster { + private static final Logger log = LoggerFactory.getLogger(PBSCluster.class); + + + public PBSCluster(JobManagerConfiguration jobManagerConfiguration) { + super(jobManagerConfiguration); + } + public PBSCluster(ServerInfo serverInfo, AuthenticationInfo authenticationInfo, JobManagerConfiguration config) throws SSHApiException { + super(serverInfo, authenticationInfo,config); + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/RawCommandInfo.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/RawCommandInfo.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/RawCommandInfo.java new file mode 100644 index 0000000..9ac2ba0 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/RawCommandInfo.java @@ -0,0 +1,55 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.gfac.ssh.impl; + +import org.apache.airavata.gfac.ssh.api.CommandInfo; + +/** + * User: AmilaJ ([email protected]) + * Date: 8/14/13 + * Time: 5:18 PM + */ + +/** + * The raw command information. String returned by getCommand is directly executed in SSH + * shell. E.g :- getCommand return string set for rawCommand - "/opt/torque/bin/qsub /home/ogce/test.pbs". + */ +public class RawCommandInfo implements CommandInfo { + + private String rawCommand; + + public RawCommandInfo(String cmd) { + this.rawCommand = cmd; + } + + public String getCommand() { + return this.rawCommand; + } + + public String getRawCommand() { + return rawCommand; + } + + public void setRawCommand(String rawCommand) { + this.rawCommand = rawCommand; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/SSHUserInfo.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/SSHUserInfo.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/SSHUserInfo.java new file mode 100644 index 0000000..e878dff --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/SSHUserInfo.java @@ -0,0 +1,63 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.gfac.ssh.impl; + +import com.jcraft.jsch.UserInfo; + +/** + * User: AmilaJ ([email protected]) + * Date: 9/20/13 + * Time: 2:31 PM + */ + +public class SSHUserInfo implements UserInfo { + + private String password; + + public SSHUserInfo(String pwd) { + this.password = pwd; + } + + public String getPassphrase() { + return this.password; + } + + public String getPassword() { + return this.password; + } + + public boolean promptPassword(String message) { + return false; //To change body of implemented methods use File | Settings | File Templates. + } + + public boolean promptPassphrase(String message) { + return false; //To change body of implemented methods use File | Settings | File Templates. + } + + public boolean promptYesNo(String message) { + return false; //To change body of implemented methods use File | Settings | File Templates. + } + + public void showMessage(String message) { + //To change body of implemented methods use File | Settings | File Templates. + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/StandardOutReader.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/StandardOutReader.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/StandardOutReader.java new file mode 100644 index 0000000..265a57d --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/StandardOutReader.java @@ -0,0 +1,79 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.impl; + +import com.jcraft.jsch.Channel; + +import org.apache.airavata.gfac.ssh.api.CommandOutput; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public class StandardOutReader implements CommandOutput { + + private static final Logger logger = LoggerFactory.getLogger(StandardOutReader.class); + String stdOutputString = null; + ByteArrayOutputStream errorStream = new ByteArrayOutputStream(); + public void onOutput(Channel channel) { + try { + StringBuffer pbsOutput = new StringBuffer(""); + InputStream inputStream = channel.getInputStream(); + byte[] tmp = new byte[1024]; + do { + while (inputStream.available() > 0) { + int i = inputStream.read(tmp, 0, 1024); + if (i < 0) break; + pbsOutput.append(new String(tmp, 0, i)); + } + } while (!channel.isClosed()) ; + String output = pbsOutput.toString(); + this.setStdOutputString(output); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + + } + + + public void exitCode(int code) { + System.out.println("Program exit code - " + code); + } + + public String getStdOutputString() { + return stdOutputString; + } + + public void setStdOutputString(String stdOutputString) { + this.stdOutputString = stdOutputString; + } + + public String getStdErrorString() { + return errorStream.toString(); + } + + public OutputStream getStandardError() { + return errorStream; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/SystemCommandOutput.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/SystemCommandOutput.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/SystemCommandOutput.java new file mode 100644 index 0000000..24d218b --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/SystemCommandOutput.java @@ -0,0 +1,78 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.gfac.ssh.impl; + +import com.jcraft.jsch.Channel; +import org.apache.airavata.gfac.ssh.api.CommandOutput; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * User: AmilaJ ([email protected]) + * Date: 8/15/13 + * Time: 10:44 AM + */ + +public class SystemCommandOutput implements CommandOutput { + + private static final Logger logger = LoggerFactory.getLogger(SystemCommandOutput.class); + public void onOutput(Channel channel) { + try { + InputStream inputStream = channel.getInputStream(); + + byte[] tmp = new byte[1024]; + while (true) { + while (inputStream.available() > 0) { + int i = inputStream.read(tmp, 0, 1024); + if (i < 0) break; + System.out.print(new String(tmp, 0, i)); + } + if (channel.isClosed()) { + System.out.println("exit-status: " + channel.getExitStatus()); + break; + } + try { + Thread.sleep(1000); + } catch (Exception ignored) { + } + } + + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + + } + + public OutputStream getStandardError() { + return System.err; + } + + public void exitCode(int code) { + System.out.println("Program exit code - " + code); + } + + +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPasswordAuthenticationInfo.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPasswordAuthenticationInfo.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPasswordAuthenticationInfo.java new file mode 100644 index 0000000..8e76528 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPasswordAuthenticationInfo.java @@ -0,0 +1,48 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.gfac.ssh.impl.authentication; + +/** + * User: AmilaJ ([email protected]) + * Date: 9/20/13 + * Time: 12:15 PM + */ + +import org.apache.airavata.gfac.ssh.api.authentication.SSHPasswordAuthentication; + +/** + * An authenticator used for raw SSH sessions. Gives SSH user name, password + * directly. + * This is only an example implementation. + */ +public class DefaultPasswordAuthenticationInfo implements SSHPasswordAuthentication { + + private String password; + + public DefaultPasswordAuthenticationInfo(String pwd) { + this.password = pwd; + } + + public String getPassword(String userName, String hostName) { + return password; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPublicKeyAuthentication.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPublicKeyAuthentication.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPublicKeyAuthentication.java new file mode 100644 index 0000000..be8e1f9 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPublicKeyAuthentication.java @@ -0,0 +1,68 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.gfac.ssh.impl.authentication; + +import org.apache.airavata.gfac.ssh.api.authentication.SSHPublicKeyAuthentication; + +/** + * User: AmilaJ ([email protected]) + * Date: 10/4/13 + * Time: 11:44 AM + */ + +/** + * Default public key authentication. + * Note : This is only a sample implementation. + */ +public class DefaultPublicKeyAuthentication implements SSHPublicKeyAuthentication { + + private byte[] privateKey; + private byte[] publicKey; + private String passPhrase = null; + + public DefaultPublicKeyAuthentication(byte[] priv, byte[] pub) { + this.privateKey = priv; + this.publicKey = pub; + } + + public DefaultPublicKeyAuthentication(byte[] priv, byte[] pub, String pass) { + this.privateKey = priv; + this.publicKey = pub; + this.passPhrase = pass; + } + + public String getPassPhrase() { + return passPhrase; + } + + public void bannerMessage(String message) { + System.out.println(message); + } + + public byte[] getPrivateKey(String userName, String hostName) { + return privateKey; + } + + public byte[] getPublicKey(String userName, String hostName) { + return publicKey; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPublicKeyFileAuthentication.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPublicKeyFileAuthentication.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPublicKeyFileAuthentication.java new file mode 100644 index 0000000..5351dd2 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/DefaultPublicKeyFileAuthentication.java @@ -0,0 +1,70 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.gfac.ssh.impl.authentication; + +import org.apache.airavata.gfac.ssh.api.authentication.SSHPublicKeyFileAuthentication; + +/** + * User: AmilaJ ([email protected]) + * Date: 10/4/13 + * Time: 11:40 AM + */ + +/** + * Default public key authentication using files. + * Note : This is only a sample implementation. + */ +public class DefaultPublicKeyFileAuthentication implements SSHPublicKeyFileAuthentication { + + private String publicKeyFile; + private String privateKeyFile; + private String passPhrase = null; + + public DefaultPublicKeyFileAuthentication(String pubFile, String privFile) { + this.publicKeyFile = pubFile; + this.privateKeyFile = privFile; + + } + + public DefaultPublicKeyFileAuthentication(String pubFile, String privFile, String pass) { + this.publicKeyFile = pubFile; + this.privateKeyFile = privFile; + this.passPhrase = pass; + + } + + public String getPassPhrase() { + return passPhrase; + } + + public void bannerMessage(String message) { + System.out.println(message); + } + + public String getPublicKeyFile(String userName, String hostName) { + return publicKeyFile; + } + + public String getPrivateKeyFile(String userName, String hostName) { + return privateKeyFile; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/MyProxyAuthenticationInfo.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/MyProxyAuthenticationInfo.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/MyProxyAuthenticationInfo.java new file mode 100644 index 0000000..5e47a86 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/authentication/MyProxyAuthenticationInfo.java @@ -0,0 +1,108 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.gfac.ssh.impl.authentication; + +import org.apache.airavata.gfac.ssh.api.authentication.GSIAuthenticationInfo; +import org.globus.myproxy.MyProxy; +import org.globus.myproxy.MyProxyException; +import org.ietf.jgss.GSSCredential; + +/** + * User: AmilaJ ([email protected]) + * Date: 8/14/13 + * Time: 5:22 PM + */ + +public class MyProxyAuthenticationInfo extends GSIAuthenticationInfo { + + public static final String X509_CERT_DIR = "X509_CERT_DIR"; + private String userName; + private String password; + private String myProxyUrl; + private int myProxyPort; + private int lifeTime; + + public MyProxyAuthenticationInfo(String userName, String password, String myProxyUrl, int myProxyPort, + int life, String certificatePath) { + this.userName = userName; + this.password = password; + this.myProxyUrl = myProxyUrl; + this.myProxyPort = myProxyPort; + this.lifeTime = life; + properties.setProperty(X509_CERT_DIR, certificatePath); + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getMyProxyUrl() { + return myProxyUrl; + } + + public void setMyProxyUrl(String myProxyUrl) { + this.myProxyUrl = myProxyUrl; + } + + public int getMyProxyPort() { + return myProxyPort; + } + + public void setMyProxyPort(int myProxyPort) { + this.myProxyPort = myProxyPort; + } + + public int getLifeTime() { + return lifeTime; + } + + public void setLifeTime(int lifeTime) { + this.lifeTime = lifeTime; + } + + public GSSCredential getCredentials() throws SecurityException { + return getMyProxyCredentials(); + } + + private GSSCredential getMyProxyCredentials() throws SecurityException { + MyProxy myproxy = new MyProxy(this.myProxyUrl, this.myProxyPort); + try { + return myproxy.get(this.getUserName(), this.password, this.lifeTime); + } catch (MyProxyException e) { + throw new SecurityException("Error getting proxy credentials", e); + } + } + + +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/jsch/ExtendedJSch.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/jsch/ExtendedJSch.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/jsch/ExtendedJSch.java new file mode 100644 index 0000000..cee852f --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/jsch/ExtendedJSch.java @@ -0,0 +1,64 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.gfac.ssh.jsch; + +import com.jcraft.jsch.ExtendedSession; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import org.apache.airavata.gfac.ssh.api.authentication.GSIAuthenticationInfo; + +/** + * User: AmilaJ ([email protected]) + * Date: 8/15/13 + * Time: 10:03 AM + */ + +/** + * Extended JSch to incorporate authentication info. + */ +public class ExtendedJSch extends JSch { + + private GSIAuthenticationInfo authenticationInfo; + + public ExtendedJSch() { + super(); + } + + public GSIAuthenticationInfo getAuthenticationInfo() { + return authenticationInfo; + } + + public void setAuthenticationInfo(GSIAuthenticationInfo authenticationInfo) { + this.authenticationInfo = authenticationInfo; + } + + public Session getSession(String username, String host, int port) throws JSchException { + + if(host==null){ + throw new JSchException("host must not be null."); + } + Session s = new ExtendedSession(this, username, host, port); + return s; + + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/listener/JobSubmissionListener.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/listener/JobSubmissionListener.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/listener/JobSubmissionListener.java new file mode 100644 index 0000000..21aa1e3 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/listener/JobSubmissionListener.java @@ -0,0 +1,81 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.listener; + +import org.apache.airavata.gfac.ssh.api.SSHApiException; +import org.apache.airavata.gfac.ssh.api.job.JobDescriptor; +import org.apache.airavata.gfac.ssh.impl.JobStatus; + +/** + * This interface can be implemented by the end user of the API + * to do desired operations based on the job status change. API has a + * default joblistener which can be used by the end users, but its + * configurable and can be parseSingleJob to jobsubmission methods. + */ +public abstract class JobSubmissionListener { + + private JobStatus jobStatus = JobStatus.U; + + /** + * This can be usd to perform some operation during status change + * + * @param jobDescriptor + * @throws SSHApiException + */ + public abstract void statusChanged(JobDescriptor jobDescriptor) throws SSHApiException; + + /** + * This can be usd to perform some operation during status change + * @param jobStatus + * @throws SSHApiException + */ + public abstract void statusChanged(JobStatus jobStatus) throws SSHApiException; + + + public JobStatus getJobStatus() { + return jobStatus; + } + + public void setJobStatus(JobStatus jobStatus) { + this.jobStatus = jobStatus; + } + + /** + * This method is used to block the process until the currentStatus of the job is DONE or FAILED + */ + public void waitFor() throws SSHApiException{ + while (!isJobDone()) { + synchronized (this) { + try { + wait(); + } catch (InterruptedException e) {} + } + } + } + + /** + * BAsed on the implementation user can define how to decide the job done + * scenario + * @return + * @throws SSHApiException + */ + public abstract boolean isJobDone() throws SSHApiException; +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/CommonUtils.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/CommonUtils.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/CommonUtils.java new file mode 100644 index 0000000..6ff4fa6 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/CommonUtils.java @@ -0,0 +1,81 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.util; + +import org.apache.airavata.gfac.ssh.api.job.*; +import org.apache.airavata.gfac.ssh.impl.JobStatus; + +public class CommonUtils { + /** + * This returns true if the give job is finished + * otherwise false + * + * @param job + * @return + */ + public static boolean isJobFinished(JobDescriptor job) { + if (JobStatus.C.toString().equals(job.getStatus())) { + return true; + } else { + return false; + } + } + + /** + * This will read + * + * @param maxWalltime + * @return + */ + public static String maxWallTimeCalculator(int maxWalltime) { + if (maxWalltime < 60) { + return "00:" + maxWalltime + ":00"; + } else { + int minutes = maxWalltime % 60; + int hours = maxWalltime / 60; + return hours + ":" + minutes + ":00"; + } + } + public static String maxWallTimeCalculatorForLSF(int maxWalltime) { + if (maxWalltime < 60) { + return "00:" + maxWalltime; + } else { + int minutes = maxWalltime % 60; + int hours = maxWalltime / 60; + return hours + ":" + minutes; + } + } + public static JobManagerConfiguration getPBSJobManager(String installedPath) { + return new PBSJobConfiguration("PBSTemplate.xslt",".pbs", installedPath, new PBSOutputParser()); + } + + public static JobManagerConfiguration getSLURMJobManager(String installedPath) { + return new SlurmJobConfiguration("SLURMTemplate.xslt", ".slurm", installedPath, new SlurmOutputParser()); + } + + public static JobManagerConfiguration getUGEJobManager(String installedPath) { + return new UGEJobConfiguration("UGETemplate.xslt", ".pbs", installedPath, new UGEOutputParser()); + } + + public static JobManagerConfiguration getLSFJobManager(String installedPath) { + return new LSFJobConfiguration("LSFTemplate.xslt", ".lsf", installedPath, new LSFOutputParser()); + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/SSHAPIUIKeyboardInteractive.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/SSHAPIUIKeyboardInteractive.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/SSHAPIUIKeyboardInteractive.java new file mode 100644 index 0000000..bd700e9 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/SSHAPIUIKeyboardInteractive.java @@ -0,0 +1,73 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.gfac.ssh.util; + +import com.jcraft.jsch.UIKeyboardInteractive; +import com.jcraft.jsch.UserInfo; + +/** + * User: AmilaJ ([email protected]) + * Date: 10/4/13 + * Time: 8:34 AM + */ + +/** + * This is dummy class, the keyboard interactivity is not really used when acting as an API. + * But to get things working we have this. + */ +public class SSHAPIUIKeyboardInteractive implements UIKeyboardInteractive, UserInfo { + + private String password; + + public SSHAPIUIKeyboardInteractive(String pwd) { + this.password = pwd; + } + + public String[] promptKeyboardInteractive(String destination, String name, + String instruction, String[] prompt, boolean[] echo) { + return null; + } + + public String getPassphrase() { + return password; + } + + public String getPassword() { + return password; + } + + public boolean promptPassword(String message) { + return false; //To change body of implemented methods use File | Settings | File Templates. + } + + public boolean promptPassphrase(String message) { + return false; //To change body of implemented methods use File | Settings | File Templates. + } + + public boolean promptYesNo(String message) { + return false; //To change body of implemented methods use File | Settings | File Templates. + } + + public void showMessage(String message) { + //To change body of implemented methods use File | Settings | File Templates. + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/SSHKeyPasswordHandler.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/SSHKeyPasswordHandler.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/SSHKeyPasswordHandler.java new file mode 100644 index 0000000..569cd3c --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/util/SSHKeyPasswordHandler.java @@ -0,0 +1,68 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.gfac.ssh.util; + +import com.jcraft.jsch.UserInfo; +import org.apache.airavata.gfac.ssh.api.authentication.SSHKeyAuthentication; +import sun.reflect.generics.reflectiveObjects.NotImplementedException; + +/** + * User: AmilaJ ([email protected]) + * Date: 10/4/13 + * Time: 2:22 PM + */ + +/** + * This class is used to get the pass phrase to decrypt public/private keys. + */ +public class SSHKeyPasswordHandler implements UserInfo { + + private SSHKeyAuthentication keyAuthenticationHandler; + + public SSHKeyPasswordHandler(SSHKeyAuthentication handler) { + this.keyAuthenticationHandler = handler; + } + + public String getPassphrase() { + return keyAuthenticationHandler.getPassPhrase(); + } + + public String getPassword() { + throw new NotImplementedException(); + } + + public boolean promptPassword(String message) { + return false; + } + + public boolean promptPassphrase(String message) { + return true; + } + + public boolean promptYesNo(String message) { + return false; + } + + public void showMessage(String message) { + keyAuthenticationHandler.bannerMessage(message); + } +}
