http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/LSFJobConfiguration.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/LSFJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/LSFJobConfiguration.java new file mode 100644 index 0000000..f9c7f33 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/LSFJobConfiguration.java @@ -0,0 +1,121 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.api.job; + +import org.apache.airavata.gfac.ssh.impl.RawCommandInfo; +import org.apache.commons.io.FilenameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + +public class LSFJobConfiguration implements JobManagerConfiguration { + private final static Logger logger = LoggerFactory.getLogger(LSFJobConfiguration.class); + + private String jobDescriptionTemplateName; + + private String scriptExtension; + + private String installedPath; + + private OutputParser parser; + + public LSFJobConfiguration(){ + // this can be used to construct and use setter methods to set all the params in order + } + public LSFJobConfiguration(String jobDescriptionTemplateName, + String scriptExtension,String installedPath,OutputParser parser) { + this.jobDescriptionTemplateName = jobDescriptionTemplateName; + this.scriptExtension = scriptExtension; + this.parser = parser; + if (installedPath.endsWith("/") || installedPath.isEmpty()) { + this.installedPath = installedPath; + } else { + this.installedPath = installedPath + "/"; + } + } + + @Override + public RawCommandInfo getCancelCommand(String jobID) { + return new RawCommandInfo(this.installedPath + "bkill " + jobID); + } + + @Override + public String getJobDescriptionTemplateName() { + return jobDescriptionTemplateName; + } + + @Override + public RawCommandInfo getMonitorCommand(String jobID) { + return new RawCommandInfo(this.installedPath + "bjobs " + jobID); + } + + @Override + public RawCommandInfo getUserBasedMonitorCommand(String userName) { + return new RawCommandInfo(this.installedPath + "bjobs -u " + userName); + } + + @Override + public RawCommandInfo getJobIdMonitorCommand(String jobName, String userName) { + return new RawCommandInfo(this.installedPath + "bjobs -J " + jobName); + } + + @Override + public String getScriptExtension() { + return scriptExtension; + } + + @Override + public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) { + return new RawCommandInfo(this.installedPath + "bsub < " + + workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath)); + } + + @Override + public OutputParser getParser() { + return parser; + } + + public void setParser(OutputParser parser) { + this.parser = parser; + } + + @Override + public String getInstalledPath() { + return installedPath; + } + + + @Override + public String getBaseCancelCommand() { + return "bkill"; + } + + @Override + public String getBaseMonitorCommand() { + return "bjobs"; + } + + @Override + public String getBaseSubmitCommand() { + return "bsub"; + } +}
http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/LSFOutputParser.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/LSFOutputParser.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/LSFOutputParser.java new file mode 100644 index 0000000..c6dea17 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/LSFOutputParser.java @@ -0,0 +1,130 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.api.job; + +import org.apache.airavata.gfac.ssh.api.SSHApiException; +import org.apache.airavata.gfac.ssh.impl.JobStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class LSFOutputParser implements OutputParser { + private final static Logger logger = LoggerFactory.getLogger(LSFOutputParser.class); + + @Override + public void parseSingleJob(JobDescriptor jobDescriptor, String rawOutput) throws SSHApiException { + logger.debug(rawOutput); + //todo we need to implement this but we are not using it airavata runtime + // if someone is using the gsissh as a tool this will be useful to get a descriptive information about a single job + } + + @Override + public String parseJobSubmission(String rawOutput) throws SSHApiException { + logger.debug(rawOutput); + return rawOutput.substring(rawOutput.indexOf("<")+1,rawOutput.indexOf(">")); + } + + @Override + public JobStatus parseJobStatus(String jobID, String rawOutput) throws SSHApiException { + boolean jobFount = false; + logger.debug(rawOutput); + //todo this is not used anymore + return JobStatus.C; + } + + @Override + public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput) throws SSHApiException { + logger.debug(rawOutput); + + String[] info = rawOutput.split("\n"); +// int lastStop = 0; + for (String jobID : statusMap.keySet()) { + String jobName = jobID.split(",")[1]; + boolean found = false; + for (int i = 0; i < info.length; i++) { + if (info[i].contains(jobName.substring(0,8))) { + // now starts processing this line + logger.info(info[i]); + String correctLine = info[i]; + String[] columns = correctLine.split(" "); + List<String> columnList = new ArrayList<String>(); + for (String s : columns) { + if (!"".equals(s)) { + columnList.add(s); + } + } +// lastStop = i + 1; + try { + statusMap.put(jobID, JobStatus.valueOf(columnList.get(2))); + }catch(IndexOutOfBoundsException e){ + statusMap.put(jobID, JobStatus.valueOf("U")); + } + found = true; + break; + } + } + if(!found) + logger.error("Couldn't find the status of the Job with JobName: " + jobName + "Job Id: " + jobID.split(",")[0]); + } + } + + @Override + public String parseJobId(String jobName, String rawOutput) throws SSHApiException { + String regJobId = "jobId"; + Pattern pattern = Pattern.compile("(?=(?<" + regJobId + ">\\d+)\\s+\\w+\\s+" + jobName + ")"); // regex - look ahead and match + if (rawOutput != null) { + Matcher matcher = pattern.matcher(rawOutput); + if (matcher.find()) { + return matcher.group(regJobId); + } else { + logger.error("No match is found for JobName"); + return null; + } + } else { + logger.error("Error: RawOutput shouldn't be null"); + return null; + } + } + + public static void main(String[] args) { + String test = "Job <2477982> is submitted to queue <short>."; + System.out.println(test.substring(test.indexOf("<")+1, test.indexOf(">"))); + String test1 = "JOBID USER STAT QUEUE FROM_HOST EXEC_HOST JOB_NAME SUBMIT_TIME\n" + + "2636607 lg11w RUN long ghpcc06 c11b02 *069656647 Mar 7 00:58\n" + + "2636582 lg11w RUN long ghpcc06 c02b01 2134490944 Mar 7 00:48"; + Map<String, JobStatus> statusMap = new HashMap<String, JobStatus>(); + statusMap.put("2477983,2134490944", JobStatus.U); + LSFOutputParser lsfOutputParser = new LSFOutputParser(); + try { + lsfOutputParser.parseJobStatuses("cjh", statusMap, test1); + } catch (SSHApiException e) { + logger.error(e.getMessage(), e); + } + System.out.println(statusMap.get("2477983,2134490944")); + + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/OutputParser.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/OutputParser.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/OutputParser.java new file mode 100644 index 0000000..9730c33 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/OutputParser.java @@ -0,0 +1,67 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.api.job; + +import org.apache.airavata.gfac.ssh.api.SSHApiException; +import org.apache.airavata.gfac.ssh.impl.JobStatus; + +import java.util.Map; + +public interface OutputParser { + + /** + * Tihs can be used to fill a jobdescriptor based on the output + * @param descriptor + * @return + */ + public void parseSingleJob(JobDescriptor descriptor, String rawOutput)throws SSHApiException; + + /** + * This can be used to parseSingleJob the result of a job submission to get the JobID + * @param rawOutput + * @return + */ + public String parseJobSubmission(String rawOutput)throws SSHApiException; + + + /** + * This can be used to get the job status from the output + * @param jobID + * @param rawOutput + */ + public JobStatus parseJobStatus(String jobID, String rawOutput)throws SSHApiException; + + /** + * This can be used to parseSingleJob a big output and get multipleJob statuses + * @param statusMap list of status map will return and key will be the job ID + * @param rawOutput + */ + public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput)throws SSHApiException; + + /** + * filter the jobId value of given JobName from rawOutput + * @param jobName + * @param rawOutput + * @return + * @throws SSHApiException + */ + public String parseJobId(String jobName, String rawOutput) throws SSHApiException; +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/PBSJobConfiguration.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/PBSJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/PBSJobConfiguration.java new file mode 100644 index 0000000..0179e01 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/PBSJobConfiguration.java @@ -0,0 +1,119 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.api.job; + +import org.apache.airavata.gfac.ssh.impl.RawCommandInfo; +import org.apache.commons.io.FilenameUtils; + +import java.io.File; + +public class PBSJobConfiguration implements JobManagerConfiguration { + + private String jobDescriptionTemplateName; + + private String scriptExtension; + + private String installedPath; + + private OutputParser parser; + + public PBSJobConfiguration() { + // this can be used to construct and use setter methods to set all the params in order + } + + public PBSJobConfiguration(String jobDescriptionTemplateName, + String scriptExtension, String installedPath, OutputParser parser) { + this.jobDescriptionTemplateName = jobDescriptionTemplateName; + this.scriptExtension = scriptExtension; + this.parser = parser; + if (installedPath.endsWith("/")) { + this.installedPath = installedPath; + } else { + this.installedPath = installedPath + "/"; + } + } + + public RawCommandInfo getCancelCommand(String jobID) { + return new RawCommandInfo(this.installedPath + "qdel " + jobID); + } + + public String getJobDescriptionTemplateName() { + return jobDescriptionTemplateName; + } + + public void setJobDescriptionTemplateName(String jobDescriptionTemplateName) { + this.jobDescriptionTemplateName = jobDescriptionTemplateName; + } + + public RawCommandInfo getMonitorCommand(String jobID) { + return new RawCommandInfo(this.installedPath + "qstat -f " + jobID); + } + + public String getScriptExtension() { + return scriptExtension; + } + + public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) { + return new RawCommandInfo(this.installedPath + "qsub " + + workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath)); + } + + public String getInstalledPath() { + return installedPath; + } + + public void setInstalledPath(String installedPath) { + this.installedPath = installedPath; + } + + public OutputParser getParser() { + return parser; + } + + public void setParser(OutputParser parser) { + this.parser = parser; + } + + public RawCommandInfo getUserBasedMonitorCommand(String userName) { + return new RawCommandInfo(this.installedPath + "qstat -u " + userName); + } + + @Override + public RawCommandInfo getJobIdMonitorCommand(String jobName, String userName) { + // For PBS there is no option to get jobDetails by JobName, so we search with userName + return new RawCommandInfo(this.installedPath + "qstat -u " + userName); + } + + @Override + public String getBaseCancelCommand() { + return "qdel"; + } + + @Override + public String getBaseMonitorCommand() { + return "qstat"; + } + + @Override + public String getBaseSubmitCommand() { + return "qsub "; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/PBSOutputParser.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/PBSOutputParser.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/PBSOutputParser.java new file mode 100644 index 0000000..2f17787 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/PBSOutputParser.java @@ -0,0 +1,212 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.api.job; + +import org.apache.airavata.gfac.ssh.api.SSHApiException; +import org.apache.airavata.gfac.ssh.impl.JobStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class PBSOutputParser implements OutputParser { + private static final Logger log = LoggerFactory.getLogger(PBSOutputParser.class); + + public void parseSingleJob(JobDescriptor jobDescriptor, String rawOutput) { + log.debug(rawOutput); + String[] info = rawOutput.split("\n"); + String[] line; + for (int i = 0; i < info.length; i++) { + if (info[i].contains("=")) { + line = info[i].split("=", 2); + } else { + line = info[i].split(":", 2); + } + if (line.length >= 2) { + String header = line[0].trim(); + log.debug("Header = " + header); + String value = line[1].trim(); + log.debug("value = " + value); + + if (header.equals("Variable_List")) { + while (info[i + 1].startsWith("\t")) { + value += info[i + 1]; + i++; + } + value = value.replaceAll("\t", ""); + jobDescriptor.setVariableList(value); + } else if ("Job Id".equals(header)) { + jobDescriptor.setJobID(value); + } else if ("Job_Name".equals(header)) { + jobDescriptor.setJobName(value); + } else if ("Account_Name".equals(header)) { + jobDescriptor.setAcountString(value); + } else if ("job_state".equals(header)) { + jobDescriptor.setStatus(value); + } else if ("Job_Owner".equals(header)) { + jobDescriptor.setOwner(value); + } else if ("resources_used.cput".equals(header)) { + jobDescriptor.setUsedCPUTime(value); + } else if ("resources_used.mem".equals(header)) { + jobDescriptor.setUsedMemory(value); + } else if ("resources_used.walltime".equals(header)) { + jobDescriptor.setEllapsedTime(value); + } else if ("job_state".equals(header)) { + jobDescriptor.setStatus(value); + } else if ("queue".equals(header)) + jobDescriptor.setQueueName(value); + else if ("ctime".equals(header)) { + jobDescriptor.setCTime(value); + } else if ("qtime".equals(header)) { + jobDescriptor.setQTime(value); + } else if ("mtime".equals(header)) { + jobDescriptor.setMTime(value); + } else if ("start_time".equals(header)) { + jobDescriptor.setSTime(value); + } else if ("comp_time".equals(header)) { + jobDescriptor.setCompTime(value); + } else if ("exec_host".equals(header)) { + jobDescriptor.setExecuteNode(value); + } else if ("Output_Path".equals(header)) { + if (info[i + 1].contains("=") || info[i + 1].contains(":")) + jobDescriptor.setStandardOutFile(value); + else { + jobDescriptor.setStandardOutFile(value + info[i + 1].trim()); + i++; + } + } else if ("Error_Path".equals(header)) { + if (info[i + 1].contains("=") || info[i + 1].contains(":")) + jobDescriptor.setStandardErrorFile(value); + else { + String st = info[i + 1].trim(); + jobDescriptor.setStandardErrorFile(value + st); + i++; + } + + } else if ("submit_args".equals(header)) { + while (i + 1 < info.length) { + if (info[i + 1].startsWith("\t")) { + value += info[i + 1]; + i++; + } else + break; + } + value = value.replaceAll("\t", ""); + jobDescriptor.setSubmitArgs(value); + } + } + } + } + + public String parseJobSubmission(String rawOutput) { + log.debug(rawOutput); + return rawOutput; //In PBS stdout is going to be directly the jobID + } + + public JobStatus parseJobStatus(String jobID, String rawOutput) { + boolean jobFount = false; + log.debug(rawOutput); + String[] info = rawOutput.split("\n"); + String[] line = null; + int index = 0; + for (String anInfo : info) { + index++; + if (anInfo.contains("Job Id:")) { + if (anInfo.contains(jobID)) { + jobFount = true; + break; + } + } + } + if (jobFount) { + for (int i=index;i<info.length;i++) { + String anInfo = info[i]; + if (anInfo.contains("=")) { + line = anInfo.split("=", 2); + if (line.length != 0) { + if (line[0].contains("job_state")) { + return JobStatus.valueOf(line[1].replaceAll(" ", "")); + } + } + } + } + } + return null; + } + + public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput) { + log.debug(rawOutput); + String[] info = rawOutput.split("\n"); +// int lastStop = 0; + for (String jobID : statusMap.keySet()) { + String jobName = jobID.split(",")[1]; + boolean found = false; + for (int i = 0; i < info.length; i++) { + if (info[i].contains(jobName.substring(0,8))) { + // now starts processing this line + log.info(info[i]); + String correctLine = info[i]; + String[] columns = correctLine.split(" "); + List<String> columnList = new ArrayList<String>(); + for (String s : columns) { + if (!"".equals(s)) { + columnList.add(s); + } + } +// lastStop = i + 1; + try { + statusMap.put(jobID, JobStatus.valueOf(columnList.get(9))); + }catch(IndexOutOfBoundsException e){ + statusMap.put(jobID, JobStatus.valueOf("U")); + } + found = true; + break; + } + } + if(!found) + log.error("Couldn't find the status of the Job with JobName: " + jobName + "Job Id: " + jobID.split(",")[0]); + } + } + + @Override + public String parseJobId(String jobName, String rawOutput) throws SSHApiException { + String regJobId = "jobId"; + Pattern pattern = Pattern.compile("\\s*(?<" + regJobId + ">[^\\s]*).* " + jobName + " "); // regex , JOB_ID will come as first column + if (rawOutput != null) { + Matcher matcher = pattern.matcher(rawOutput); + if (matcher.find()) { + return matcher.group(regJobId); + } else { + log.error("No match is found for JobName"); + return null; + } + } else { + log.error("Error: RawOutput shouldn't be null"); + return null; + } + } + + +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/SlurmJobConfiguration.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/SlurmJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/SlurmJobConfiguration.java new file mode 100644 index 0000000..54d8f40 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/SlurmJobConfiguration.java @@ -0,0 +1,117 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.api.job; + +import org.apache.airavata.gfac.ssh.impl.RawCommandInfo; +import org.apache.commons.io.FilenameUtils; + +import java.io.File; + +public class SlurmJobConfiguration implements JobManagerConfiguration{ + + private String jobDescriptionTemplateName; + + private String scriptExtension; + + private String installedPath; + + private OutputParser parser; + + public SlurmJobConfiguration(){ + // this can be used to construct and use setter methods to set all the params in order + } + public SlurmJobConfiguration(String jobDescriptionTemplateName, + String scriptExtension,String installedPath,OutputParser parser) { + this.jobDescriptionTemplateName = jobDescriptionTemplateName; + this.scriptExtension = scriptExtension; + this.parser = parser; + if (installedPath.endsWith("/")) { + this.installedPath = installedPath; + } else { + this.installedPath = installedPath + "/"; + } + } + + public RawCommandInfo getCancelCommand(String jobID) { + return new RawCommandInfo(this.installedPath + "scancel " + jobID); + } + + public String getJobDescriptionTemplateName() { + return jobDescriptionTemplateName; + } + + public void setJobDescriptionTemplateName(String jobDescriptionTemplateName) { + this.jobDescriptionTemplateName = jobDescriptionTemplateName; + } + + public RawCommandInfo getMonitorCommand(String jobID) { + return new RawCommandInfo(this.installedPath + "squeue -j " + jobID); + } + + public String getScriptExtension() { + return scriptExtension; + } + + public RawCommandInfo getSubmitCommand(String workingDirectory,String pbsFilePath) { + return new RawCommandInfo(this.installedPath + "sbatch " + + workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath)); + } + + public String getInstalledPath() { + return installedPath; + } + + public void setInstalledPath(String installedPath) { + this.installedPath = installedPath; + } + + public OutputParser getParser() { + return parser; + } + + public void setParser(OutputParser parser) { + this.parser = parser; + } + + public RawCommandInfo getUserBasedMonitorCommand(String userName) { + return new RawCommandInfo(this.installedPath + "squeue -u " + userName); + } + + @Override + public RawCommandInfo getJobIdMonitorCommand(String jobName, String userName) { + return new RawCommandInfo(this.installedPath + "squeue -n " + jobName + " -u " + userName); + } + + @Override + public String getBaseCancelCommand() { + return "scancel"; + } + + @Override + public String getBaseMonitorCommand() { + return "squeue"; + } + + @Override + public String getBaseSubmitCommand() { + return "sbatch"; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/SlurmOutputParser.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/SlurmOutputParser.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/SlurmOutputParser.java new file mode 100644 index 0000000..11fb4ce --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/SlurmOutputParser.java @@ -0,0 +1,190 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.api.job; + +import org.apache.airavata.gfac.ssh.api.SSHApiException; +import org.apache.airavata.gfac.ssh.impl.JobStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SlurmOutputParser implements OutputParser { + private static final Logger log = LoggerFactory.getLogger(SlurmOutputParser.class); + public static final int JOB_NAME_OUTPUT_LENGTH = 8; + public static final String STATUS = "status"; + + public void parseSingleJob(JobDescriptor descriptor, String rawOutput) throws SSHApiException { + log.info(rawOutput); + String[] info = rawOutput.split("\n"); + String lastString = info[info.length - 1]; + if (lastString.contains("JOB ID")) { + // because there's no state + descriptor.setStatus("U"); + } else { + int column = 0; + System.out.println(lastString); + for (String each : lastString.split(" ")) { + if (each.trim().isEmpty()) { + continue; + } else { + switch (column) { + case 0: + descriptor.setJobID(each); + column++; + break; + case 1: + descriptor.setPartition(each); + column++; + break; + case 2: + descriptor.setJobName(each); + column++; + break; + case 3: + descriptor.setUserName(each); + column++; + break; + case 4: + descriptor.setStatus(each); + column++; + break; + case 5: + descriptor.setUsedCPUTime(each); + column++; + break; + case 6: + try { + int nodes = Integer.parseInt(each); + descriptor.setNodes(nodes); + }catch (Exception e){ + log.error("Node count read from command output is not an integer !!!"); + } + column++; + break; + case 7: + descriptor.setNodeList(each); + column++; + break; + } + } + } + } + + } + + /** + * This can be used to parseSingleJob the outpu of sbatch and extrac the jobID from the content + * + * @param rawOutput + * @return + */ + public String parseJobSubmission(String rawOutput) throws SSHApiException { + // FIXME : use regex to match correct jobId; + log.info(rawOutput); + String[] info = rawOutput.split("\n"); + for (String anInfo : info) { + if (anInfo.contains("Submitted batch job")) { + String[] split = anInfo.split("Submitted batch job"); + return split[1].trim(); + } + } + return ""; +// throw new SSHApiException(rawOutput); //todo//To change body of implemented methods use File | Settings | File Templates. + } + + public JobStatus parseJobStatus(String jobID, String rawOutput) throws SSHApiException { + log.info(rawOutput); + Pattern pattern = Pattern.compile(jobID + "(?=\\s+\\S+\\s+\\S+\\s+\\S+\\s+(?<" + STATUS + ">\\w+))"); + Matcher matcher = pattern.matcher(rawOutput); + if (matcher.find()) { + return JobStatus.valueOf(matcher.group(STATUS)); + } + return null; + } + + public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput) throws SSHApiException { + log.debug(rawOutput); + String[] info = rawOutput.split("\n"); + String lastString = info[info.length - 1]; + if (lastString.contains("JOBID") || lastString.contains("PARTITION")) { + log.info("There are no jobs with this username ... "); + return; + } +// int lastStop = 0; + for (String jobID : statusMap.keySet()) { + String jobId = jobID.split(",")[0]; + String jobName = jobID.split(",")[1]; + boolean found = false; + for (int i = 0; i < info.length; i++) { + if (info[i].contains(jobName.substring(0, 8))) { + // now starts processing this line + log.info(info[i]); + String correctLine = info[i]; + String[] columns = correctLine.split(" "); + List<String> columnList = new ArrayList<String>(); + for (String s : columns) { + if (!"".equals(s)) { + columnList.add(s); + } + } + try { + statusMap.put(jobID, JobStatus.valueOf(columnList.get(4))); + } catch (IndexOutOfBoundsException e) { + statusMap.put(jobID, JobStatus.valueOf("U")); + } + found = true; + break; + } + } + if (!found) { + log.error("Couldn't find the status of the Job with JobName: " + jobName + "Job Id: " + jobId); + } + } + } + + @Override + public String parseJobId(String jobName, String rawOutput) throws SSHApiException { + String regJobId = "jobId"; + if (jobName == null) { + return null; + } else if(jobName.length() > JOB_NAME_OUTPUT_LENGTH) { + jobName = jobName.substring(0, JOB_NAME_OUTPUT_LENGTH); + } + Pattern pattern = Pattern.compile("(?=(?<" + regJobId + ">\\d+)\\s+\\w+\\s+" + jobName + ")"); // regex - look ahead and match + if (rawOutput != null) { + Matcher matcher = pattern.matcher(rawOutput); + if (matcher.find()) { + return matcher.group(regJobId); + } else { + log.error("No match is found for JobName"); + return null; + } + } else { + log.error("Error: RawOutput shouldn't be null"); + return null; + } + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/UGEJobConfiguration.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/UGEJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/UGEJobConfiguration.java new file mode 100644 index 0000000..4fbbe30 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/UGEJobConfiguration.java @@ -0,0 +1,119 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.api.job; + +import org.apache.airavata.gfac.ssh.impl.RawCommandInfo; +import org.apache.commons.io.FilenameUtils; + +import java.io.File; + +public class UGEJobConfiguration implements JobManagerConfiguration { + + private String jobDescriptionTemplateName; + + private String scriptExtension; + + private String installedPath; + + private OutputParser parser; + + public UGEJobConfiguration() { + // this can be used to construct and use setter methods to set all the params in order + } + + public UGEJobConfiguration(String jobDescriptionTemplateName, + String scriptExtension, String installedPath, OutputParser parser) { + this.jobDescriptionTemplateName = jobDescriptionTemplateName; + this.scriptExtension = scriptExtension; + this.parser = parser; + if (installedPath.endsWith("/")) { + this.installedPath = installedPath; + } else { + this.installedPath = installedPath + "/"; + } + } + + public RawCommandInfo getCancelCommand(String jobID) { + return new RawCommandInfo(this.installedPath + "qdel " + jobID); + } + + public String getJobDescriptionTemplateName() { + return jobDescriptionTemplateName; + } + + public void setJobDescriptionTemplateName(String jobDescriptionTemplateName) { + this.jobDescriptionTemplateName = jobDescriptionTemplateName; + } + + public RawCommandInfo getMonitorCommand(String jobID) { + return new RawCommandInfo(this.installedPath + "qstat -j " + jobID); + } + + public String getScriptExtension() { + return scriptExtension; + } + + public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) { + return new RawCommandInfo(this.installedPath + "qsub " + + workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath)); + } + + public String getInstalledPath() { + return installedPath; + } + + public void setInstalledPath(String installedPath) { + this.installedPath = installedPath; + } + + public OutputParser getParser() { + return parser; + } + + public void setParser(OutputParser parser) { + this.parser = parser; + } + + public RawCommandInfo getUserBasedMonitorCommand(String userName) { + return new RawCommandInfo(this.installedPath + "qstat -u " + userName); + } + + @Override + public RawCommandInfo getJobIdMonitorCommand(String jobName, String userName) { + // For PBS there is no option to get jobDetails by JobName, so we search with userName + return new RawCommandInfo(this.installedPath + "qstat -u " + userName); + } + + @Override + public String getBaseCancelCommand() { + return "qdel"; + } + + @Override + public String getBaseMonitorCommand() { + return "qstat"; + } + + @Override + public String getBaseSubmitCommand() { + return "qsub "; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/UGEOutputParser.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/UGEOutputParser.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/UGEOutputParser.java new file mode 100644 index 0000000..a6cc3ed --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/api/job/UGEOutputParser.java @@ -0,0 +1,188 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.api.job; + +import org.apache.airavata.gfac.ssh.api.SSHApiException; +import org.apache.airavata.gfac.ssh.impl.JobStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class UGEOutputParser implements OutputParser{ + private static final Logger log = LoggerFactory.getLogger(PBSOutputParser.class); + public static final String JOB_ID = "jobId"; + + public void parseSingleJob(JobDescriptor jobDescriptor, String rawOutput) { + log.debug(rawOutput); + String[] info = rawOutput.split("\n"); + String[] line; + for (int i = 0; i < info.length; i++) { + if (info[i].contains("=")) { + line = info[i].split("=", 2); + } else { + line = info[i].split(":", 2); + } + if (line.length >= 2) { + String header = line[0].trim(); + log.debug("Header = " + header); + String value = line[1].trim(); + log.debug("value = " + value); + + if (header.equals("Variable_List")) { + while (info[i + 1].startsWith("\t")) { + value += info[i + 1]; + i++; + } + value = value.replaceAll("\t", ""); + jobDescriptor.setVariableList(value); + } else if ("Job Id".equals(header)) { + jobDescriptor.setJobID(value); + } else if ("Job_Name".equals(header)) { + jobDescriptor.setJobName(value); + } else if ("Account_Name".equals(header)) { + jobDescriptor.setAcountString(value); + } else if ("job_state".equals(header)) { + jobDescriptor.setStatus(value); + } else if ("Job_Owner".equals(header)) { + jobDescriptor.setOwner(value); + } else if ("resources_used.cput".equals(header)) { + jobDescriptor.setUsedCPUTime(value); + } else if ("resources_used.mem".equals(header)) { + jobDescriptor.setUsedMemory(value); + } else if ("resources_used.walltime".equals(header)) { + jobDescriptor.setEllapsedTime(value); + } else if ("job_state".equals(header)) { + jobDescriptor.setStatus(value); + } else if ("queue".equals(header)) + jobDescriptor.setQueueName(value); + else if ("ctime".equals(header)) { + jobDescriptor.setCTime(value); + } else if ("qtime".equals(header)) { + jobDescriptor.setQTime(value); + } else if ("mtime".equals(header)) { + jobDescriptor.setMTime(value); + } else if ("start_time".equals(header)) { + jobDescriptor.setSTime(value); + } else if ("comp_time".equals(header)) { + jobDescriptor.setCompTime(value); + } else if ("exec_host".equals(header)) { + jobDescriptor.setExecuteNode(value); + } else if ("Output_Path".equals(header)) { + if (info[i + 1].contains("=") || info[i + 1].contains(":")) + jobDescriptor.setStandardOutFile(value); + else { + jobDescriptor.setStandardOutFile(value + info[i + 1].trim()); + i++; + } + } else if ("Error_Path".equals(header)) { + if (info[i + 1].contains("=") || info[i + 1].contains(":")) + jobDescriptor.setStandardErrorFile(value); + else { + String st = info[i + 1].trim(); + jobDescriptor.setStandardErrorFile(value + st); + i++; + } + + } else if ("submit_args".equals(header)) { + while (i + 1 < info.length) { + if (info[i + 1].startsWith("\t")) { + value += info[i + 1]; + i++; + } else + break; + } + value = value.replaceAll("\t", ""); + jobDescriptor.setSubmitArgs(value); + } + } + } + } + + public String parseJobSubmission(String rawOutput) { + log.debug(rawOutput); + if (rawOutput != null && !rawOutput.isEmpty()) { + String[] info = rawOutput.split("\n"); + String lastLine = info[info.length - 1]; + return lastLine.split(" ")[2]; // In PBS stdout is going to be directly the jobID + } else { + return ""; + } + } + + public JobStatus parseJobStatus(String jobID, String rawOutput) { + Pattern pattern = Pattern.compile("job_number:[\\s]+" + jobID); + Matcher matcher = pattern.matcher(rawOutput); + if (matcher.find()) { + return JobStatus.Q; // fixme; return correct status. + } + return JobStatus.U; + } + + public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput) { + log.debug(rawOutput); + String[] info = rawOutput.split("\n"); + int lastStop = 0; + for (String jobID : statusMap.keySet()) { + for(int i=lastStop;i<info.length;i++){ + if(jobID.split(",")[0].contains(info[i].split(" ")[0]) && !"".equals(info[i].split(" ")[0])){ + // now starts processing this line + log.info(info[i]); + String correctLine = info[i]; + String[] columns = correctLine.split(" "); + List<String> columnList = new ArrayList<String>(); + for (String s : columns) { + if (!"".equals(s)) { + columnList.add(s); + } + } + lastStop = i+1; + if ("E".equals(columnList.get(4))) { + // There is another status with the same letter E other than error status + // to avoid that we make a small tweek to the job status + columnList.set(4, "Er"); + } + statusMap.put(jobID, JobStatus.valueOf(columnList.get(4))); + break; + } + } + } + } + + @Override + public String parseJobId(String jobName, String rawOutput) throws SSHApiException { + if (jobName.length() > 10) { + jobName = jobName.substring(0, 10); + } + Pattern pattern = Pattern.compile("(?<" + JOB_ID + ">\\S+)\\s+\\S+\\s+(" + jobName + ")"); + Matcher matcher = pattern.matcher(rawOutput); + if (matcher.find()) { + return matcher.group(JOB_ID); + } + return null; + } + + +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/config/ConfigReader.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/config/ConfigReader.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/config/ConfigReader.java new file mode 100644 index 0000000..9658fba --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/config/ConfigReader.java @@ -0,0 +1,76 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.airavata.gfac.ssh.config; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +/** + * Reads basic configurations. + */ +public class ConfigReader { + + private static final String CONFIGURATION_FILE = "gsissh.properties"; + + + private Properties properties; + + /** + * Reads configurations from the class path configuration file. + * @throws IOException If an error occurred while reading configurations. + */ + public ConfigReader() throws IOException { + this.properties = getPropertiesFromClasspath(CONFIGURATION_FILE); + } + + private Properties getPropertiesFromClasspath(String propFileName) throws IOException { + Properties props = new Properties(); + InputStream inputStream = this.getClass().getClassLoader() + .getResourceAsStream(propFileName); + + if (inputStream == null) { + throw new FileNotFoundException("System configuration file '" + propFileName + + "' not found in the classpath"); + } + + props.load(inputStream); + + return props; + } + + public String getConfiguration(String key) { + return this.properties.getProperty(key); + } + + + /** + * Gets all the SSH related properties used by JSch. + * @return All properties. + */ + public Properties getProperties() { + return this.properties; + } + + +} http://git-wip-us.apache.org/repos/asf/airavata/blob/7b809747/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/DefaultJobSubmissionListener.java ---------------------------------------------------------------------- diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/DefaultJobSubmissionListener.java b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/DefaultJobSubmissionListener.java new file mode 100644 index 0000000..d60ea32 --- /dev/null +++ b/tools/gsissh/src/main/java/org/apache/airavata/gfac/gsi/ssh/impl/DefaultJobSubmissionListener.java @@ -0,0 +1,42 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ +package org.apache.airavata.gfac.ssh.impl; + +import org.apache.airavata.gfac.ssh.api.SSHApiException; +import org.apache.airavata.gfac.ssh.api.job.JobDescriptor; +import org.apache.airavata.gfac.ssh.listener.JobSubmissionListener; + +public class DefaultJobSubmissionListener extends JobSubmissionListener { + + public void statusChanged(JobDescriptor jobDescriptor) throws SSHApiException { + System.out.println("Job status has changed to : " + jobDescriptor.getStatus()); + } + + @Override + public void statusChanged(JobStatus jobStatus) throws SSHApiException { + System.out.println("Job status has changed to : " + jobStatus.toString()); + } + + @Override + public boolean isJobDone() throws SSHApiException { + return getJobStatus().equals(JobStatus.C); + } +}
