Author: mona
Date: Tue Jul 16 16:30:01 2013
New Revision: 1503785
URL: http://svn.apache.org/r1503785
Log:
OOZIE-1450 Duplicate Coord_Action events on Waiting -> Timeout, and Coord
Materialize not removing actions on Failure (mona)
Modified:
oozie/branches/branch-4.0/client/src/main/java/org/apache/oozie/client/event/SLAEvent.java
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLAOperations.java
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/sla/TestSLAService.java
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
oozie/branches/branch-4.0/release-log.txt
Modified:
oozie/branches/branch-4.0/client/src/main/java/org/apache/oozie/client/event/SLAEvent.java
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-4.0/client/src/main/java/org/apache/oozie/client/event/SLAEvent.java?rev=1503785&r1=1503784&r2=1503785&view=diff
==============================================================================
---
oozie/branches/branch-4.0/client/src/main/java/org/apache/oozie/client/event/SLAEvent.java
(original)
+++
oozie/branches/branch-4.0/client/src/main/java/org/apache/oozie/client/event/SLAEvent.java
Tue Jul 16 16:30:01 2013
@@ -193,4 +193,10 @@ public abstract class SLAEvent extends E
*/
public abstract Date getLastModifiedTime();
+ @Override
+ public String toString() {
+ return "ID: " + getId() + ", MsgType:" + getMsgType() + ", SLAStatus:
" + getSLAStatus() + ", EventStatus: "
+ + getEventStatus() + " AppType " + getAppType();
+ }
+
}
Modified:
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java?rev=1503785&r1=1503784&r2=1503785&view=diff
==============================================================================
---
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java
(original)
+++
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/command/coord/CoordMaterializeTransitionXCommand.java
Tue Jul 16 16:30:01 2013
@@ -266,9 +266,11 @@ public class CoordMaterializeTransitionX
updateJobMaterializeInfo(coordJob);
}
catch (CommandException ex) {
- LOG.warn("Exception occurs:" + ex.getMessage() + " Making the job
failed ", ex);
+ LOG.warn("Exception occurred:" + ex.getMessage() + " Making the
job failed ", ex);
coordJob.setStatus(Job.Status.FAILED);
coordJob.resetPending();
+ // remove any materialized actions and slaEvents
+ insertList.clear();
}
catch (Exception e) {
LOG.error("Exception thrown :", e);
@@ -378,14 +380,14 @@ public class CoordMaterializeTransitionX
actionBean.setActionXml(actionXml);
insertList.add(actionBean);
- writeActionRegistration(actionXml, actionBean);
+ writeActionSlaRegistration(actionXml, actionBean);
// TODO: time 100s should be configurable
queue(new CoordActionNotificationXCommand(actionBean), 100);
queue(new CoordActionInputCheckXCommand(actionBean.getId(),
actionBean.getJobId()), 100);
}
- private void writeActionRegistration(String actionXml,
CoordinatorActionBean actionBean) throws Exception {
+ private void writeActionSlaRegistration(String actionXml,
CoordinatorActionBean actionBean) throws Exception {
Element eAction = XmlUtils.parseXml(actionXml);
Element eSla = eAction.getChild("action",
eAction.getNamespace()).getChild("info", eAction.getNamespace("sla"));
SLAEventBean slaEvent =
SLADbOperations.createSlaRegistrationEvent(eSla, actionBean.getId(),
SlaAppType.COORDINATOR_ACTION, coordJob
Modified:
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java?rev=1503785&r1=1503784&r2=1503785&view=diff
==============================================================================
---
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
(original)
+++
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java
Tue Jul 16 16:30:01 2013
@@ -264,9 +264,4 @@ public class SLACalcStatus extends SLAEv
return lastModifiedTime;
}
- @Override
- public String toString() {
- return "ID: " + getId() + " SLAStatus: " + slaStatus + " EventStatus:
"+eventStatus + " AppType " + getAppType();
- }
-
}
Modified:
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java?rev=1503785&r1=1503784&r2=1503785&view=diff
==============================================================================
---
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
(original)
+++
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java
Tue Jul 16 16:30:01 2013
@@ -600,17 +600,17 @@ public class SLACalculatorMemory impleme
* @throws JPAExecutorException
*/
private SLASummaryBean processJobEndFailureSLA(SLACalcStatus slaCalc, Date
actualStart, Date actualEnd) throws JPAExecutorException {
- if (actualStart == null) {
- // job failed before starting
- slaCalc.setEventProcessed(7);
- slaCalc.setActualEnd(actualEnd);
- slaCalc.setEventStatus(EventStatus.END_MISS);
- slaCalc.setSLAStatus(SLAStatus.MISS);
- eventHandler.queueEvent(new SLACalcStatus(slaCalc));
- return getSLASummaryBean(slaCalc);
- }
slaCalc.setActualStart(actualStart);
slaCalc.setActualEnd(actualEnd);
+ if (actualStart == null) { // job failed before starting
+ if (slaCalc.getEventProcessed() != 5) { // 101 = end+start already
processed
+ slaCalc.setEventStatus(EventStatus.END_MISS);
+ slaCalc.setSLAStatus(SLAStatus.MISS);
+ eventHandler.queueEvent(new SLACalcStatus(slaCalc));
+ slaCalc.setEventProcessed(7);
+ return getSLASummaryBean(slaCalc);
+ }
+ }
SLARegistrationBean reg = slaCalc.getSLARegistrationBean();
long expectedDuration = reg.getExpectedDuration();
long actualDuration = actualEnd.getTime() - actualStart.getTime();
Modified:
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLAOperations.java
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLAOperations.java?rev=1503785&r1=1503784&r2=1503785&view=diff
==============================================================================
---
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLAOperations.java
(original)
+++
oozie/branches/branch-4.0/core/src/main/java/org/apache/oozie/sla/SLAOperations.java
Tue Jul 16 16:30:01 2013
@@ -19,7 +19,6 @@ package org.apache.oozie.sla;
import java.text.ParseException;
import java.util.Date;
-
import org.apache.oozie.AppType;
import org.apache.oozie.ErrorCode;
import org.apache.oozie.client.event.SLAEvent.EventStatus;
@@ -118,8 +117,11 @@ public class SLAOperations {
EventStatus.valueOf(event);
}
catch (IllegalArgumentException iae) {
- throw new CommandException(ErrorCode.E0302, "'" + event +
"'",
- " for SLA Alert event. Should be one of " +
EventStatus.values());
+ XLog.getLog(SLAService.class).warn(
+ "Invalid value: [" + event + "]" + " for SLA
Alert-event. Should be one of "
+ + EventStatus.values() + ". Setting it to
default [" + EventStatus.END_MISS.name()
+ + "]");
+ event = EventStatus.END_MISS.name();
}
alertsStr.append(event).append(",");
}
Modified:
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java?rev=1503785&r1=1503784&r2=1503785&view=diff
==============================================================================
---
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
(original)
+++
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java
Tue Jul 16 16:30:01 2013
@@ -20,6 +20,7 @@ package org.apache.oozie.command.coord;
import java.sql.Timestamp;
import java.util.Date;
import java.util.List;
+
import org.apache.oozie.CoordinatorActionBean;
import org.apache.oozie.CoordinatorJobBean;
import org.apache.oozie.ErrorCode;
@@ -202,6 +203,35 @@ public class TestCoordMaterializeTransit
}
/**
+ * Test a coordinator does not materialize actions upon CommandException
+ * leading to FAILED state
+ *
+ * @throws Exception
+ */
+ public void testFailedJobNotMaterializeActions() throws Exception {
+ String coordXml = "<coordinator-app
xmlns=\"uri:oozie:coordinator:0.4\"" + " name=\"NAME\" frequency=\"5\""
+ + " start=\"#start\" end=\"#end\"
timezone=\"America/Los_Angeles\""
+ + " freq_timeunit=\"DAY\" end_of_duration=\"NONE\">" +
"<input-events>"
+ + "<data-in name=\"a\" dataset=\"a\">"
+ + "<dataset name=\"a\" frequency=\"7\"
initial-instance=\"2010-01-01T00:00Z\" timezone=\"UTC\" "
+ + "freq_timeunit=\"MINUTE\" end_of_duration=\"NONE\">"
+ +
"<uri-template>${hcatNode}/${db}/${table}/ds=${YEAR}-${MONTH}-${DAY};region=${region}</uri-template>"
+ + "</dataset>" +
"<start-instance>${coord:current(0)}</start-instance>"
+ + "<end-instance>${coord:latest(0)}</end-instance>" +
"</data-in>" + "</input-events>" + "<action>"
+ + "<workflow>" + "<app-path>hdfs:///tmp/workflows/</app-path>"
+ "</workflow>" + "</action>"
+ + "</coordinator-app>";
+ CoordinatorJobBean job = addRecordToCoordJobTable(coordXml);
+ new CoordMaterializeTransitionXCommand(job.getId(), 3600).call();
+ JPAService jpaService = Services.get().get(JPAService.class);
+ job = jpaService.execute(new CoordJobGetJPAExecutor(job.getId()));
+ assertEquals(CoordinatorJob.Status.FAILED, job.getStatus());
+ // GetActions for coord job, should be none
+ int actions = jpaService.execute(new
CoordJobGetActionsJPAExecutor(job.getId()));
+ assertEquals(0, actions);
+
+ }
+
+ /**
* Test a coordinator job that will run beyond 5 minutes from now,
* materilization should not happen.
* @throws Exception
Modified:
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/sla/TestSLAService.java
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/sla/TestSLAService.java?rev=1503785&r1=1503784&r2=1503785&view=diff
==============================================================================
---
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/sla/TestSLAService.java
(original)
+++
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/sla/TestSLAService.java
Tue Jul 16 16:30:01 2013
@@ -31,6 +31,7 @@ import org.apache.oozie.client.event.SLA
import org.apache.oozie.executor.jpa.WorkflowJobGetJPAExecutor;
import org.apache.oozie.executor.jpa.WorkflowJobInsertJPAExecutor;
import org.apache.oozie.executor.jpa.WorkflowJobUpdateJPAExecutor;
+import org.apache.oozie.executor.jpa.sla.SLARegistrationGetJPAExecutor;
import org.apache.oozie.executor.jpa.sla.SLASummaryGetJPAExecutor;
import org.apache.oozie.service.EventHandlerService;
import org.apache.oozie.service.JPAService;
@@ -38,7 +39,9 @@ import org.apache.oozie.service.Services
import org.apache.oozie.sla.listener.SLAEventListener;
import org.apache.oozie.sla.service.SLAService;
import org.apache.oozie.test.XDataTestCase;
+import org.apache.oozie.util.XmlUtils;
import org.apache.oozie.workflow.WorkflowInstance;
+import org.jdom.Element;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -267,6 +270,24 @@ public class TestSLAService extends XDat
}
+ /**
+ * Test SLAOperations handles unexpected alert-events in xml
+ * @throws Exception
+ */
+ public void testSLAOperations() throws Exception {
+ String slaXml = " <sla:info xmlns:sla='uri:oozie:sla:0.2'>"
+ + " <sla:nominal-time>2009-03-06T10:00Z</sla:nominal-time>" +
" <sla:should-start>5</sla:should-start>"
+ + " <sla:should-end>120</sla:should-end>" + "
<sla:max-duration>100</sla:max-duration>"
+ + "
<sla:alert-events>\"invalid_event_miss\'</sla:alert-events>"
+ + " <sla:alert-contact>[email protected]</sla:alert-contact>" +
"</sla:info>";
+ Element eSla = XmlUtils.parseXml(slaXml);
+ SLAOperations.createSlaRegistrationEvent(eSla, "job-id1",
"parent-id1", AppType.WORKFLOW_JOB, getTestUser(),
+ "test-appname", log, false);
+ SLARegistrationBean reg = Services.get().get(JPAService.class)
+ .execute(new SLARegistrationGetJPAExecutor("job-id1"));
+ assertEquals("END_MISS", reg.getAlertEvents());
+ }
+
static SLARegistrationBean _createSLARegistration(String jobId, AppType
appType) {
SLARegistrationBean bean = new SLARegistrationBean();
bean.setId(jobId);
Modified:
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/test/XDataTestCase.java?rev=1503785&r1=1503784&r2=1503785&view=diff
==============================================================================
---
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
(original)
+++
oozie/branches/branch-4.0/core/src/test/java/org/apache/oozie/test/XDataTestCase.java
Tue Jul 16 16:30:01 2013
@@ -168,6 +168,31 @@ public abstract class XDataTestCase exte
return coordJob;
}
+ protected CoordinatorJobBean addRecordToCoordJobTable(String appXml)
throws Exception {
+ Date start = DateUtils.parseDateOozieTZ("2009-02-01T01:00Z");
+ Date end = DateUtils.parseDateOozieTZ("2009-02-03T23:59Z");
+ appXml = appXml.replaceAll("#start",
DateUtils.formatDateOozieTZ(start));
+ appXml = appXml.replaceAll("#end", DateUtils.formatDateOozieTZ(end));
+ Path appPath = new Path(getFsTestCaseDir(), "coord");
+ writeToFile(appXml, appPath, "coordinator.xml");
+ CoordinatorJobBean coordJob = createCoordBean(appPath, appXml,
CoordinatorJob.Status.PREP, start, end, false,
+ false, 0);
+
+ try {
+ JPAService jpaService = Services.get().get(JPAService.class);
+ assertNotNull(jpaService);
+ CoordJobInsertJPAExecutor coordInsertCmd = new
CoordJobInsertJPAExecutor(coordJob);
+ jpaService.execute(coordInsertCmd);
+ }
+ catch (JPAExecutorException je) {
+ je.printStackTrace();
+ fail("Unable to insert the test coord job record to table");
+ throw je;
+ }
+
+ return coordJob;
+ }
+
/**
* Insert coord job for testing.
*
@@ -320,39 +345,12 @@ public abstract class XDataTestCase exte
Path appPath = new Path(getFsTestCaseDir(), "coord");
String appXml = writeCoordXml(appPath);
- CoordinatorJobBean coordJob = new CoordinatorJobBean();
-
coordJob.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.COORDINATOR));
- coordJob.setAppName("COORD-TEST");
- coordJob.setAppPath(appPath.toString());
- coordJob.setStatus(status);
- coordJob.setTimeZone("America/Los_Angeles");
- coordJob.setCreatedTime(new Date());
- coordJob.setLastModifiedTime(new Date());
- coordJob.setUser(getTestUser());
- coordJob.setGroup(getTestGroup());
- if (pending) {
- coordJob.setPending();
- }
- if (doneMatd) {
- coordJob.setDoneMaterialization();
- }
-
- Configuration conf = getCoordConf(appPath);
- coordJob.setConf(XmlUtils.prettyPrint(conf).toString());
- coordJob.setJobXml(appXml);
- coordJob.setLastActionNumber(0);
- coordJob.setFrequency("1");
- coordJob.setTimeUnit(Timeunit.DAY);
- coordJob.setExecution(Execution.FIFO);
- coordJob.setConcurrency(1);
- coordJob.setMatThrottling(1);
// Set the start and end time in future
String currentDatePlusMonth =
XDataTestCase.getCurrentDateafterIncrementingInMonths(1);
Date start = DateUtils.parseDateOozieTZ(currentDatePlusMonth);
Date end = DateUtils.parseDateOozieTZ(currentDatePlusMonth);
- coordJob.setStartTime(start);
- coordJob.setEndTime(end);
- return coordJob;
+
+ return createCoordBean(appPath, appXml, status, start, end, pending,
doneMatd, 0);
}
/**
@@ -372,41 +370,7 @@ public abstract class XDataTestCase exte
Path appPath = new Path(getFsTestCaseDir(), "coord");
String appXml = writeCoordXml(appPath, start, end);
- CoordinatorJobBean coordJob = new CoordinatorJobBean();
-
coordJob.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.COORDINATOR));
- coordJob.setAppName("COORD-TEST");
- coordJob.setAppPath(appPath.toString());
- coordJob.setStatus(status);
- coordJob.setTimeZone("America/Los_Angeles");
- coordJob.setCreatedTime(new Date());
- coordJob.setLastModifiedTime(new Date());
- coordJob.setUser(getTestUser());
- coordJob.setGroup(getTestGroup());
- if (pending) {
- coordJob.setPending();
- }
- if (doneMatd) {
- coordJob.setDoneMaterialization();
- }
- coordJob.setLastActionNumber(lastActionNum);
-
- Configuration conf = getCoordConf(appPath);
- coordJob.setConf(XmlUtils.prettyPrint(conf).toString());
- coordJob.setJobXml(appXml);
- coordJob.setFrequency("1");
- coordJob.setTimeUnit(Timeunit.DAY);
- coordJob.setExecution(Execution.FIFO);
- coordJob.setConcurrency(1);
- coordJob.setMatThrottling(1);
- try {
- coordJob.setStartTime(start);
- coordJob.setEndTime(end);
- }
- catch (Exception e) {
- e.printStackTrace();
- fail("Could not set Date/time");
- }
- return coordJob;
+ return createCoordBean(appPath, appXml, status, start, end, pending,
doneMatd, lastActionNum);
}
/**
@@ -427,6 +391,11 @@ public abstract class XDataTestCase exte
Path appPath = new Path(getFsTestCaseDir(), "coord");
String appXml = writeCoordXml(appPath, testFileName);
+ return createCoordBean(appPath, appXml, status, start, end, pending,
doneMatd, lastActionNum);
+ }
+
+ private CoordinatorJobBean createCoordBean(Path appPath, String appXml,
CoordinatorJob.Status status, Date start,
+ Date end, boolean pending, boolean doneMatd, int lastActionNum)
throws Exception {
CoordinatorJobBean coordJob = new CoordinatorJobBean();
coordJob.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.COORDINATOR));
coordJob.setAppName("COORD-TEST");
Modified: oozie/branches/branch-4.0/release-log.txt
URL:
http://svn.apache.org/viewvc/oozie/branches/branch-4.0/release-log.txt?rev=1503785&r1=1503784&r2=1503785&view=diff
==============================================================================
--- oozie/branches/branch-4.0/release-log.txt (original)
+++ oozie/branches/branch-4.0/release-log.txt Tue Jul 16 16:30:01 2013
@@ -1,5 +1,6 @@
-- Oozie 4.0.0 release
+OOZIE-1450 Duplicate Coord_Action events on Waiting -> Timeout, and Coord
Materialize not removing actions on Failure (mona)
OOZIE-1451 CoordActionInputCheckX does a redundant eagerLoadState (rohini)
OOZIE-1446 SLACalcStatus not updating the last modified time correctly and
duplicate DURATION_* event (virag,mona via mona)
OOZIE-1249 SLA Documentation (mona,virag,rohini via rohini)
