Registered job management commands with resource. Fixed issues with Slurm output parser
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/706edf84 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/706edf84 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/706edf84 Branch: refs/heads/master Commit: 706edf8465ced1280e0940f5b73fa027958cd369 Parents: cce7be7 Author: Shameera Rathanyaka <[email protected]> Authored: Tue Jul 14 10:51:57 2015 -0400 Committer: Shameera Rathanyaka <[email protected]> Committed: Tue Jul 14 10:51:57 2015 -0400 ---------------------------------------------------------------------- .../tools/RegisterSampleApplications.java | 20 +++++-- .../apache/airavata/gfac/core/GFacUtils.java | 12 +--- .../airavata/gfac/impl/HPCRemoteCluster.java | 1 - .../apache/airavata/gfac/impl/job/JobUtil.java | 59 ++++++++++++++++++++ .../gfac/impl/job/SlurmOutputParser.java | 21 +++---- 5 files changed, 84 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/706edf84/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java ---------------------------------------------------------------------- diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java index 67a84cb..7294bbb 100644 --- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java +++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java @@ -1372,11 +1372,21 @@ public class RegisterSampleApplications { ResourceJobManager resourceJobManager = RegisterSampleApplicationsUtils. createResourceJobManager(resourceJobManagerType, monitoringEndPoint, jobMangerBinPath, null); - if (jobManagerCommand != null) { - Map<JobManagerCommand, String> jobManagerCommandStringMap = new HashMap<JobManagerCommand, String>(); - jobManagerCommandStringMap.put(JobManagerCommand.SUBMISSION, jobManagerCommand); - resourceJobManager.setJobManagerCommands(jobManagerCommandStringMap); - } + Map<JobManagerCommand, String> jobManagerCommandStringMap = new HashMap<JobManagerCommand, String>(); + if (resourceJobManagerType == ResourceJobManagerType.SLURM) { + jobManagerCommandStringMap.put(JobManagerCommand.SUBMISSION, "sbatch"); + jobManagerCommandStringMap.put(JobManagerCommand.JOB_MONITORING, "squeue"); + jobManagerCommandStringMap.put(JobManagerCommand.DELETION, "scancel"); + resourceJobManager.setJobManagerCommands(jobManagerCommandStringMap); + } else if (resourceJobManagerType == ResourceJobManagerType.PBS) { + jobManagerCommandStringMap.put(JobManagerCommand.SUBMISSION, "qsub"); + jobManagerCommandStringMap.put(JobManagerCommand.JOB_MONITORING, "qstat"); + jobManagerCommandStringMap.put(JobManagerCommand.DELETION, "qdel"); + resourceJobManager.setJobManagerCommands(jobManagerCommandStringMap); + } + // TODO - set job manage commands for UGE and LSF type compute resources. + + // TODO - set parallelism command SSHJobSubmission sshJobSubmission = new SSHJobSubmission(); sshJobSubmission.setResourceJobManager(resourceJobManager); http://git-wip-us.apache.org/repos/asf/airavata/blob/706edf84/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java index 1b58730..b1321d6 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java @@ -950,17 +950,7 @@ public class GFacUtils { ApplicationParallelismType parallelism = appDepDescription.getParallelism(); if (parallelism != null) { if (parallelism == ApplicationParallelismType.MPI || parallelism == ApplicationParallelismType.OPENMP || parallelism == ApplicationParallelismType.OPENMP_MPI) { - if (resourceJobManager != null) { - Map<JobManagerCommand, String> jobManagerCommands = resourceJobManager.getJobManagerCommands(); - if (jobManagerCommands != null && !jobManagerCommands.isEmpty()) { - for (JobManagerCommand command : jobManagerCommands.keySet()) { - if (command == JobManagerCommand.SUBMISSION) { - String commandVal = jobManagerCommands.get(command); - jobDescriptor.setJobSubmitter(commandVal); - } - } - } - } + jobDescriptor.setJobSubmitter("ibrun"); } } return jobDescriptor; http://git-wip-us.apache.org/repos/asf/airavata/blob/706edf84/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java index 69cf03e..fa73898 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java @@ -33,7 +33,6 @@ import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication; import org.apache.airavata.gfac.core.cluster.AbstractRemoteCluster; import org.apache.airavata.gfac.core.cluster.CommandInfo; import org.apache.airavata.gfac.core.cluster.CommandOutput; -import org.apache.airavata.gfac.core.cluster.OutputParser; import org.apache.airavata.gfac.core.cluster.RawCommandInfo; import org.apache.airavata.gfac.core.cluster.ServerInfo; import org.apache.airavata.model.status.JobStatus; http://git-wip-us.apache.org/repos/asf/airavata/blob/706edf84/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/JobUtil.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/JobUtil.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/JobUtil.java new file mode 100644 index 0000000..0cb49a9 --- /dev/null +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/JobUtil.java @@ -0,0 +1,59 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.gfac.impl.job; + +import org.apache.airavata.model.status.JobState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class JobUtil { + private static final Logger log = LoggerFactory.getLogger(JobUtil.class); + + public static JobState getJobState(String status) { + log.info("parsing the job status returned : " + status); + if (status != null) { + if ("C".equals(status) || "CD".equals(status) || "E".equals(status) || "CG".equals(status) || "DONE".equals(status)) { + return JobState.COMPLETE; +// } else if ("H".equals(status) || "h".equals(status)) { +// return JobState.HELD; + } else if ("Q".equals(status) || "qw".equals(status) || "PEND".equals(status)) { + return JobState.QUEUED; + } else if ("R".equals(status) || "CF".equals(status) || "r".equals(status) || "RUN".equals(status)) { + return JobState.ACTIVE; +// } else if ("T".equals(status)) { +// return JobState.HELD; + } else if ("W".equals(status) || "PD".equals(status)) { + return JobState.QUEUED; + } else if ("S".equals(status) || "PSUSP".equals(status) || "USUSP".equals(status) || "SSUSP".equals(status)) { + return JobState.SUSPENDED; + } else if ("CA".equals(status)) { + return JobState.CANCELED; + } else if ("F".equals(status) || "NF".equals(status) || "TO".equals(status) || "EXIT".equals(status)) { + return JobState.FAILED; + } else if ("PR".equals(status) || "Er".equals(status)) { + return JobState.FAILED; + } else if ("U".equals(status) || ("UNKWN".equals(status))) { + return JobState.UNKNOWN; + } + } + return JobState.UNKNOWN; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/706edf84/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmOutputParser.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmOutputParser.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmOutputParser.java index 1974843..fecb5e7 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmOutputParser.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmOutputParser.java @@ -38,6 +38,7 @@ public class SlurmOutputParser implements OutputParser { private static final Logger log = LoggerFactory.getLogger(SlurmOutputParser.class); public static final int JOB_NAME_OUTPUT_LENGTH = 8; public static final String STATUS = "status"; + public static final String JOBID = "jobId"; public void parseSingleJob(JobDescriptor descriptor, String rawOutput) throws SSHApiException { log.info(rawOutput); @@ -105,17 +106,13 @@ public class SlurmOutputParser implements OutputParser { * @return */ public String parseJobSubmission(String rawOutput) throws SSHApiException { - // FIXME : use regex to match correct jobId; - log.info(rawOutput); - String[] info = rawOutput.split("\n"); - for (String anInfo : info) { - if (anInfo.contains("Submitted batch job")) { - String[] split = anInfo.split("Submitted batch job"); - return split[1].trim(); - } - } - return ""; -// throw new SSHApiException(rawOutput); //todo//To change body of implemented methods use File | Settings | File Templates. + log.info(rawOutput); + Pattern pattern = Pattern.compile("Submitted batch job (?<" + JOBID + ">[^\\s]*)"); + Matcher matcher = pattern.matcher(rawOutput); + if (matcher.find()) { + return matcher.group(JOBID); + } + return ""; } public JobStatus parseJobStatus(String jobID, String rawOutput) throws SSHApiException { @@ -123,7 +120,7 @@ public class SlurmOutputParser implements OutputParser { Pattern pattern = Pattern.compile(jobID + "(?=\\s+\\S+\\s+\\S+\\s+\\S+\\s+(?<" + STATUS + ">\\w+))"); Matcher matcher = pattern.matcher(rawOutput); if (matcher.find()) { - return new JobStatus(JobState.valueOf(matcher.group(STATUS))); + return new JobStatus(JobUtil.getJobState(matcher.group(STATUS))); } return null; }
