Repository: airavata Updated Branches: refs/heads/master 6d0991553 -> 7523c1e91
Moved job status to failed if job submission command returns non zero exit code Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/7523c1e9 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/7523c1e9 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/7523c1e9 Branch: refs/heads/master Commit: 7523c1e91cdb4300a6904912b97ed185f4dc8353 Parents: 6d09915 Author: Shameera Rathnayaka <[email protected]> Authored: Mon Jan 4 14:38:20 2016 -0500 Committer: Shameera Rathnayaka <[email protected]> Committed: Mon Jan 4 14:38:20 2016 -0500 ---------------------------------------------------------------------- .../gfac/core/cluster/JobSubmissionOutput.java | 2 +- .../impl/task/DefaultJobSubmissionTask.java | 52 ++++++++++++++------ 2 files changed, 39 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/7523c1e9/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/JobSubmissionOutput.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/JobSubmissionOutput.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/JobSubmissionOutput.java index 7506707..6632ab5 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/JobSubmissionOutput.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/JobSubmissionOutput.java @@ -22,7 +22,7 @@ package org.apache.airavata.gfac.core.cluster; public class JobSubmissionOutput { - private int exitCode; + private int exitCode = Integer.MIN_VALUE; private String stdOut; private String stdErr; private String command; http://git-wip-us.apache.org/repos/asf/airavata/blob/7523c1e9/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java index b9805f8..7629f67 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java @@ -76,9 +76,43 @@ public class DefaultJobSubmissionTask implements JobSubmissionTask { jobModel.setJobDescription(FileUtils.readFileToString(jobFile)); JobSubmissionOutput jobSubmissionOutput = remoteCluster.submitBatchJob(jobFile.getPath(), processContext.getWorkingDir()); - jobModel.setExitCode(jobSubmissionOutput.getExitCode()); - jobModel.setStdErr(jobSubmissionOutput.getStdErr()); - jobModel.setStdOut(jobSubmissionOutput.getStdOut()); + int exitCode = jobSubmissionOutput.getExitCode(); + jobModel.setExitCode(exitCode); + jobModel.setStdErr(jobSubmissionOutput.getStdErr()); + jobModel.setStdOut(jobSubmissionOutput.getStdOut()); + GFacUtils.saveJobModel(processContext, jobModel); + if (exitCode != 0) { + String msg; + if (exitCode != Integer.MIN_VALUE) { + msg = "expId:" + processContext.getProcessModel().getExperimentId() + ", processId:" + + processContext.getProcessId() + ", taskId: " + taskContext.getTaskId() + + " return non zero exit code:" + exitCode + " for JobName:" + jobModel.getJobName() + + ", Hence changing job state to Failed"; + } else { + msg = "expId:" + processContext.getProcessModel().getExperimentId() + ", processId:" + + processContext.getProcessId() + ", taskId: " + taskContext.getTaskId() + + " doesn't return valid job submission exit code for JobName:" + jobModel.getJobName() + + ", Hence changing job state to Failed"; + } + log.error(msg); + ErrorModel errorModel = new ErrorModel(); + errorModel.setUserFriendlyMessage(msg); + errorModel.setActualErrorMessage(msg); + GFacUtils.saveJobModel(processContext, jobModel); + GFacUtils.saveExperimentError(processContext, errorModel); + GFacUtils.saveProcessError(processContext, errorModel); + GFacUtils.saveTaskError(taskContext, errorModel); + taskStatus.setState(TaskState.FAILED); + taskStatus.setReason("Job submission command exit with non zero exit code"); + taskStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime()); + taskContext.setTaskStatus(taskStatus); + try { + GFacUtils.saveAndPublishTaskStatus(taskContext); + } catch (GFacException e) { + log.error("Error while saving task status", e); + } + return taskStatus; + } String jobId = jobSubmissionOutput.getJobId(); if (jobId != null && !jobId.isEmpty()) { jobModel.setJobId(jobId); @@ -159,17 +193,7 @@ public class DefaultJobSubmissionTask implements JobSubmissionTask { errorModel.setActualErrorMessage(e.getMessage()); errorModel.setUserFriendlyMessage(msg); taskContext.getTaskModel().setTaskError(errorModel); - } catch (ApplicationSettingsException e) { - String msg = "Error occurred while creating job descriptor"; - log.error(msg, e); - taskStatus.setState(TaskState.FAILED); - taskStatus.setReason(msg); - taskStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime()); - ErrorModel errorModel = new ErrorModel(); - errorModel.setActualErrorMessage(e.getMessage()); - errorModel.setUserFriendlyMessage(msg); - taskContext.getTaskModel().setTaskError(errorModel); - } catch (GFacException e) { + } catch (ApplicationSettingsException | GFacException e) { String msg = "Error occurred while creating job descriptor"; log.error(msg, e); taskStatus.setState(TaskState.FAILED);
