Repository: oozie Updated Branches: refs/heads/master 619952f3f -> 7bc738989
OOZIE-1933 SLACalculatorMemory HA changes assume SLARegistrationBean exists for all jobs (mona) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/7bc73898 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/7bc73898 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/7bc73898 Branch: refs/heads/master Commit: 7bc738989a5811fdd206b827f2c7acd428d8c4b5 Parents: 619952f Author: mona <[email protected]> Authored: Wed Jul 16 16:20:12 2014 -0700 Committer: mona <[email protected]> Committed: Wed Jul 16 16:20:12 2014 -0700 ---------------------------------------------------------------------- .../oozie/service/EventHandlerService.java | 28 ++------- .../apache/oozie/service/ShareLibService.java | 2 + .../oozie/service/WorkflowAppService.java | 2 + .../org/apache/oozie/sla/SLACalcStatus.java | 10 +--- .../apache/oozie/sla/SLACalculatorMemory.java | 29 +++++++--- .../java/org/apache/oozie/util/LogUtils.java | 54 +++++++++++++++++- .../oozie/service/TestEventHandlerService.java | 60 +++++++++++++++++++- .../apache/oozie/service/TestHASLAService.java | 1 + .../oozie/sla/TestSLACalculatorMemory.java | 38 +++++++++++-- .../org/apache/oozie/sla/TestSLAService.java | 2 +- .../java/org/apache/oozie/test/XTestCase.java | 30 ++++++++++ release-log.txt | 4 +- 12 files changed, 210 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/service/EventHandlerService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/EventHandlerService.java b/core/src/main/java/org/apache/oozie/service/EventHandlerService.java index 6c075ab..761537b 100644 --- a/core/src/main/java/org/apache/oozie/service/EventHandlerService.java +++ b/core/src/main/java/org/apache/oozie/service/EventHandlerService.java @@ -208,35 +208,19 @@ public class EventHandlerService implements Service { return listenerMap.toString(); } - public synchronized void queueEvent(Event event) { - setLogPrefix(LOG, event); + public void queueEvent(Event event) { + LOG = LogUtils.setLogPrefix(LOG, event); LOG.debug("Queueing event : {0}", event); LOG.trace("Stack trace while queueing event : {0}", event, new Throwable()); eventQueue.add(event); + LogUtils.clearLogPrefix(); } public EventQueue getEventQueue() { return eventQueue; } - private void setLogPrefix(XLog logObj, Event event) { - logObj = XLog.resetPrefix(logObj); - if (event instanceof JobEvent) { - JobEvent je = (JobEvent) event; - LogUtils.setLogPrefix(je.getId(), je.getAppName(), new XLog.Info()); - } - else if (event instanceof SLAEvent) { - SLAEvent se = (SLAEvent) event; - LogUtils.setLogPrefix(se.getId(), se.getAppName(), new XLog.Info()); - } - } - public class EventWorker implements Runnable { - private XLog workerLog; - - public EventWorker() { - workerLog = XLog.getLog(getClass()); - } @Override public void run() { @@ -247,10 +231,8 @@ public class EventHandlerService implements Service { if (!eventQueue.isEmpty()) { List<Event> work = eventQueue.pollBatch(); for (Event event : work) { - synchronized (workerLog) { - setLogPrefix(workerLog, event); - LOG.debug("Processing event : {0}", event); - } + LOG = LogUtils.setLogPrefix(LOG, event); + LOG.debug("Processing event : {0}", event); MessageType msgType = event.getMsgType(); List<?> listeners = listenerMap.get(msgType); if (listeners != null) { http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/service/ShareLibService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/ShareLibService.java b/core/src/main/java/org/apache/oozie/service/ShareLibService.java index 893b85b..7b8bc8c 100644 --- a/core/src/main/java/org/apache/oozie/service/ShareLibService.java +++ b/core/src/main/java/org/apache/oozie/service/ShareLibService.java @@ -472,6 +472,7 @@ public class ShareLibService implements Service, Instrumentable { else { Path shareLibpath = getLatestLibPath(services.get(WorkflowAppService.class).getSystemLibPath(), SHARED_LIB_PREFIX); + LOG.debug("MONA******* sharelibpath"+ shareLibpath); loadShareLibfromDFS(tempShareLibMap, shareLibpath); if (shareLibpath != null) { @@ -496,6 +497,7 @@ public class ShareLibService implements Service, Instrumentable { private void loadShareLibfromDFS(Map<String, List<Path>> shareLibMap, Path shareLibpath) throws IOException { if (shareLibpath == null) { + LOG.debug("MONA*******NULL? sharelibpath"+ shareLibpath); LOG.info("No share lib directory found"); return; http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java b/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java index 05efefb..26beaf6 100644 --- a/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java +++ b/core/src/main/java/org/apache/oozie/service/WorkflowAppService.java @@ -79,6 +79,7 @@ public abstract class WorkflowAppService implements Service { String path = conf.get(SYSTEM_LIB_PATH, " "); if (path.trim().length() > 0) { systemLibPath = new Path(path.trim()); + XLog.getLog(getClass()).debug("MONA******* systemlibpath="+systemLibPath); } maxWFLength = conf.getInt(CONFG_MAX_WF_LENGTH, 100000); @@ -329,6 +330,7 @@ public abstract class WorkflowAppService implements Service { * @return Oozie system libpath (sharelib) in HDFS if present, otherwise it returns <code>NULL</code>. */ public Path getSystemLibPath() { + XLog.getLog(getClass()).debug("MONA******* getting systemlibpath="+systemLibPath); return systemLibPath; } } http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java b/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java index 5349b33..67d6237 100644 --- a/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java +++ b/core/src/main/java/org/apache/oozie/sla/SLACalcStatus.java @@ -53,7 +53,7 @@ public class SLACalcStatus extends SLAEvent { public SLACalcStatus(SLARegistrationBean reg) { this(); setSLARegistrationBean(reg); - setLogPrefix(); + LOG = LogUtils.setLogPrefix(LOG, this); } public SLACalcStatus(SLASummaryBean summary, SLARegistrationBean regBean) { @@ -82,7 +82,7 @@ public class SLACalcStatus extends SLAEvent { setEventStatus(summary.getEventStatus()); setLastModifiedTime(summary.getLastModifiedTime()); setEventProcessed(summary.getEventProcessed()); - setLogPrefix(); + LOG = LogUtils.setLogPrefix(LOG, this); } /** @@ -99,7 +99,7 @@ public class SLACalcStatus extends SLAEvent { setActualEnd(a.getActualEnd()); setActualDuration(a.getActualDuration()); setEventProcessed(a.getEventProcessed()); - setLogPrefix(); + LOG = LogUtils.setLogPrefix(LOG, this); } public SLACalcStatus() { @@ -332,8 +332,4 @@ public class SLACalcStatus extends SLAEvent { return Services.get().getConf().getLong(SLAService.CONF_SLA_CALC_LOCK_TIMEOUT, 5 * 1000); } - private void setLogPrefix() { - LOG = XLog.resetPrefix(LOG); - LogUtils.setLogPrefix(this.getId(), this.getAppName(), new XLog.Info()); - } } http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java b/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java index 5b30fc0..3801325 100644 --- a/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java +++ b/core/src/main/java/org/apache/oozie/sla/SLACalculatorMemory.java @@ -72,6 +72,7 @@ import org.apache.oozie.service.ServiceException; import org.apache.oozie.service.Services; import org.apache.oozie.sla.service.SLAService; import org.apache.oozie.util.DateUtils; +import org.apache.oozie.util.LogUtils; import org.apache.oozie.util.XLog; import com.google.common.annotations.VisibleForTesting; @@ -83,7 +84,7 @@ import com.google.common.annotations.VisibleForTesting; */ public class SLACalculatorMemory implements SLACalculator { - private static final XLog LOG = XLog.getLog(SLACalculatorMemory.class); + private static XLog LOG = XLog.getLog(SLACalculatorMemory.class); // TODO optimization priority based insertion/processing/bumping up-down protected Map<String, SLACalcStatus> slaMap; protected Set<String> historySet; @@ -598,7 +599,9 @@ public class SLACalculatorMemory implements SLACalculator { updateJobSla(jobId); } catch (Exception e) { + setLogPrefix(jobId); LOG.error("Exception in SLA processing for job [{0}]", jobId, e); + LogUtils.clearLogPrefix(); } } } @@ -626,9 +629,11 @@ public class SLACalculatorMemory implements SLACalculator { return true; } else { + setLogPrefix(reg.getId()); LOG.error( "SLACalculator memory capacity reached. Cannot add or update new SLA Registration entry for job [{0}]", reg.getId()); + LogUtils.clearLogPrefix(); } } catch (JPAExecutorException jpa) { @@ -675,9 +680,11 @@ public class SLACalculatorMemory implements SLACalculator { return true; } else { + setLogPrefix(reg.getId()); LOG.error( "SLACalculator memory capacity reached. Cannot add or update new SLA Registration entry for job [{0}]", reg.getId()); + LogUtils.clearLogPrefix(); } } catch (JPAExecutorException jpa) { @@ -721,15 +728,19 @@ public class SLACalculatorMemory implements SLACalculator { slaInfo.setEventProcessed(8); historySet.remove(jobId); hasSla = true; - } else { + } + else if (Services.get().get(JobsConcurrencyService.class).isHighlyAvailableMode()) { // jobid might not exist in slaMap in HA Setting SLARegistrationBean slaRegBean = SLARegistrationQueryExecutor.getInstance().get( SLARegQuery.GET_SLA_REG_ALL, jobId); - SLASummaryBean slaSummaryBean = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, - jobId); - slaCalc = new SLACalcStatus(slaSummaryBean, slaRegBean); - if(slaCalc.getEventProcessed() < 7){ - slaMap.put(jobId, slaCalc); + if (slaRegBean != null) { // filter out jobs picked by SLA job event listener + // but not actually configured for SLA + SLASummaryBean slaSummaryBean = SLASummaryQueryExecutor.getInstance().get( + SLASummaryQuery.GET_SLA_SUMMARY, jobId); + slaCalc = new SLACalcStatus(slaSummaryBean, slaRegBean); + if (slaCalc.getEventProcessed() < 7) { + slaMap.put(jobId, slaCalc); + } } } } @@ -1080,4 +1091,8 @@ public class SLACalculatorMemory implements SLACalculator { public boolean isJobIdInHistorySet(String jobId) { return this.historySet.contains(jobId); } + + private void setLogPrefix(String jobId) { + LOG = LogUtils.setLogInfo(LOG, jobId, null, null); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/main/java/org/apache/oozie/util/LogUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/LogUtils.java b/core/src/main/java/org/apache/oozie/util/LogUtils.java index 723ac36..814caff 100644 --- a/core/src/main/java/org/apache/oozie/util/LogUtils.java +++ b/core/src/main/java/org/apache/oozie/util/LogUtils.java @@ -17,13 +17,19 @@ */ package org.apache.oozie.util; +import org.apache.oozie.AppType; import org.apache.oozie.BundleJobBean; import org.apache.oozie.CoordinatorActionBean; import org.apache.oozie.CoordinatorJobBean; import org.apache.oozie.WorkflowActionBean; import org.apache.oozie.WorkflowJobBean; import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.client.event.Event; +import org.apache.oozie.client.event.JobEvent; +import org.apache.oozie.client.event.SLAEvent; import org.apache.oozie.service.DagXLogInfoService; +import org.apache.oozie.service.Services; +import org.apache.oozie.service.UUIDService; import org.apache.oozie.service.XLogService; /** @@ -125,10 +131,52 @@ public class LogUtils { XLog.Info.get().setParameters(logInfo); } - public static void setLogPrefix(String jobId, String appName, XLog.Info logInfo) { + public static XLog setLogInfo(XLog logObj, String jobId, String actionId, String appName) { + clearLogPrefix(); + XLog.Info logInfo = XLog.Info.get(); logInfo.setParameter(DagXLogInfoService.JOB, jobId); - logInfo.setParameter(DagXLogInfoService.APP, appName); - XLog.Info.get().setParameters(logInfo); + if (actionId != null) { + logInfo.setParameter(DagXLogInfoService.ACTION, actionId); + } + if (appName != null) { + logInfo.setParameter(DagXLogInfoService.APP, appName); + } + return XLog.resetPrefix(logObj); + } + + public static XLog setLogPrefix(XLog logObj, Event event) { + String jobId = null, actionId = null, appName = null; + if (event instanceof JobEvent) { + JobEvent je = (JobEvent) event; + if (je.getAppType() == AppType.WORKFLOW_JOB || je.getAppType() == AppType.COORDINATOR_JOB + || je.getAppType() == AppType.BUNDLE_JOB) { + jobId = je.getId(); + } + else { + actionId = je.getId(); + jobId = Services.get().get(UUIDService.class).getId(actionId); + } + appName = je.getAppName(); + } + else if (event instanceof SLAEvent) { + SLAEvent se = (SLAEvent) event; + if (se.getAppType() == AppType.WORKFLOW_JOB || se.getAppType() == AppType.COORDINATOR_JOB + || se.getAppType() == AppType.BUNDLE_JOB) { + jobId = se.getId(); + } + else { + actionId = se.getId(); + jobId = Services.get().get(UUIDService.class).getId(actionId); + } + appName = se.getAppName(); + } + return LogUtils.setLogInfo(logObj, jobId, actionId, appName); + } + + public static void clearLogPrefix() { + XLog.Info.get().clearParameter(DagXLogInfoService.JOB); + XLog.Info.get().clearParameter(DagXLogInfoService.ACTION); + XLog.Info.get().clearParameter(DagXLogInfoService.APP); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/test/java/org/apache/oozie/service/TestEventHandlerService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestEventHandlerService.java b/core/src/test/java/org/apache/oozie/service/TestEventHandlerService.java index ffb25e7..8bedc95 100644 --- a/core/src/test/java/org/apache/oozie/service/TestEventHandlerService.java +++ b/core/src/test/java/org/apache/oozie/service/TestEventHandlerService.java @@ -17,9 +17,13 @@ */ package org.apache.oozie.service; +import java.util.List; import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; import org.apache.oozie.client.WorkflowJob; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.CoordinatorAction; @@ -107,7 +111,7 @@ public class TestEventHandlerService extends XDataTestCase { /* * Coordinator Action events */ - CoordinatorActionEvent event2 = new CoordinatorActionEvent("jobid", "parentid", + CoordinatorActionEvent event2 = new CoordinatorActionEvent("parentid@1", "parentid", CoordinatorAction.Status.WAITING, getTestUser(), "myapp", null, null, null); ehs.queueEvent(event2); ehs.new EventWorker().run(); @@ -147,7 +151,7 @@ public class TestEventHandlerService extends XDataTestCase { /* * Workflow Action events */ - WorkflowActionEvent event3 = new WorkflowActionEvent("waction-1", "parentid", + WorkflowActionEvent event3 = new WorkflowActionEvent("parentid@wfaction", "parentid", WorkflowAction.Status.RUNNING, getTestUser(), "myapp", null, null); ehs.queueEvent(event3); ehs.new EventWorker().run(); @@ -186,6 +190,58 @@ public class TestEventHandlerService extends XDataTestCase { } + @Test + public void testEventLogging() throws Exception { + EventHandlerService ehs = _testEventHandlerService(); + // job event + WorkflowJobEvent event = new WorkflowJobEvent("jobid", "parentid", WorkflowJob.Status.RUNNING, getTestUser(), + "myapp", null, null); + + TestLogAppender appender = null; + Logger logger = null; + try { + appender = getTestLogAppender(); + logger = Logger.getLogger(EventHandlerService.class); + logger.addAppender(appender); + logger.setLevel(Level.DEBUG); + ehs.queueEvent(event); + + List<LoggingEvent> log = appender.getLog(); + LoggingEvent logEntry = log.get(0); + assertEquals(Level.DEBUG, logEntry.getLevel()); + assertTrue(logEntry.getMessage().toString().contains("APP[myapp] JOB[jobid] ACTION[-] Queueing event : ID: jobid")); + assertEquals("org.apache.oozie.service.EventHandlerService", logEntry.getLoggerName()); + + ehs.new EventWorker().run(); + + log = appender.getLog(); + logEntry = log.get(1); + assertEquals(Level.DEBUG, logEntry.getLevel()); + assertTrue(logEntry.getMessage().toString().contains("APP[myapp] JOB[jobid] ACTION[-] Processing event : ID: jobid")); + + // action event + CoordinatorActionEvent event2 = new CoordinatorActionEvent("jobid2@1", "parentid", + CoordinatorAction.Status.WAITING, getTestUser(), "myapp", null, null, null); + ehs.queueEvent(event2); + + log = appender.getLog(); + logEntry = log.get(2); + assertTrue(logEntry.getMessage().toString().contains("APP[myapp] JOB[jobid2] ACTION[jobid2@1] Queueing event : ID: jobid2@1")); + + WorkflowJobEvent event3 = new WorkflowJobEvent("jobid-other", "parentid", WorkflowJob.Status.RUNNING, getTestUser(), + "myapp-other", null, null); + ehs.queueEvent(event3); + + log = appender.getLog(); + logEntry = log.get(3); + assertTrue(logEntry.getMessage().toString() + .contains("APP[myapp-other] JOB[jobid-other] ACTION[-] Queueing event : ID: jobid-other")); + } + finally { + logger.removeAppender(appender); + } + } + private EventHandlerService _testEventHandlerService() throws Exception { Services services = Services.get(); EventHandlerService ehs = services.get(EventHandlerService.class); http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/test/java/org/apache/oozie/service/TestHASLAService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestHASLAService.java b/core/src/test/java/org/apache/oozie/service/TestHASLAService.java index 419e98b..a472f1f 100644 --- a/core/src/test/java/org/apache/oozie/service/TestHASLAService.java +++ b/core/src/test/java/org/apache/oozie/service/TestHASLAService.java @@ -68,6 +68,7 @@ public class TestHASLAService extends ZKXTestCase { conf.setInt(SLAService.CONF_SLA_CHECK_INTERVAL, 100000); conf.setInt(EventHandlerService.CONF_WORKER_THREADS, 0); super.setUp(conf); + Services.get().setService(ZKJobsConcurrencyService.class); } @Override http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java b/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java index 438f2c2..210c99e 100644 --- a/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java +++ b/core/src/test/java/org/apache/oozie/sla/TestSLACalculatorMemory.java @@ -20,11 +20,12 @@ package org.apache.oozie.sla; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; -import java.util.HashSet; import java.util.List; -import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; import org.apache.oozie.AppType; import org.apache.oozie.CoordinatorActionBean; import org.apache.oozie.WorkflowActionBean; @@ -49,6 +50,7 @@ import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery; import org.apache.oozie.service.EventHandlerService; import org.apache.oozie.service.JPAService; import org.apache.oozie.service.Services; +import org.apache.oozie.sla.service.SLAService; import org.apache.oozie.test.XDataTestCase; import org.apache.oozie.workflow.WorkflowInstance; import org.junit.After; @@ -295,7 +297,7 @@ public class TestSLACalculatorMemory extends XDataTestCase { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-mm-dd"); SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory(); slaCalcMemory.init(new Configuration(false)); - SLARegistrationBean slaRegBean1 = _createSLARegistration("job-1", AppType.WORKFLOW_ACTION); + SLARegistrationBean slaRegBean1 = _createSLARegistration("job@1", AppType.WORKFLOW_ACTION); String jobId1 = slaRegBean1.getId(); slaRegBean1.setExpectedEnd(sdf.parse("2013-03-07")); slaRegBean1.setExpectedStart(sdf.parse("2012-03-07")); @@ -324,7 +326,7 @@ public class TestSLACalculatorMemory extends XDataTestCase { // As job succeeded, it should not be in memory assertEquals(0, slaCalcMemory.size()); SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1); - assertEquals("job-1", slaSummary.getId()); + assertEquals("job@1", slaSummary.getId()); assertEquals(8, slaSummary.getEventProcessed()); assertEquals(AppType.WORKFLOW_ACTION, slaSummary.getAppType()); assertEquals("OK", slaSummary.getJobStatus()); @@ -341,7 +343,7 @@ public class TestSLACalculatorMemory extends XDataTestCase { SimpleDateFormat sdf = new SimpleDateFormat("yyyy-mm-dd"); SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory(); slaCalcMemory.init(new Configuration(false)); - SLARegistrationBean slaRegBean1 = _createSLARegistration("job-1", AppType.COORDINATOR_ACTION); + SLARegistrationBean slaRegBean1 = _createSLARegistration("job@1", AppType.COORDINATOR_ACTION); String jobId1 = slaRegBean1.getId(); slaRegBean1.setExpectedEnd(sdf.parse("2013-03-07")); slaRegBean1.setExpectedStart(sdf.parse("2012-03-07")); @@ -376,7 +378,7 @@ public class TestSLACalculatorMemory extends XDataTestCase { assertEquals(0, slaCalcMemory.size()); SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1); - assertEquals("job-1", slaSummary.getId()); + assertEquals("job@1", slaSummary.getId()); assertEquals(8, slaSummary.getEventProcessed()); assertEquals(AppType.COORDINATOR_ACTION, slaSummary.getAppType()); assertEquals("FAILED", slaSummary.getJobStatus()); @@ -635,4 +637,28 @@ public class TestSLACalculatorMemory extends XDataTestCase { assertFalse(slaCalcMemory.isJobIdInHistorySet(job1.getId())); } + public void testErrorLoggingWithJobIdPrefix() throws Exception { + SLAService slaService = Services.get().get(SLAService.class); + SLACalculator slaCalculator = slaService.getSLACalculator(); + slaCalculator.addRegistration("dummy-id", _createSLARegistration("dummy-id", AppType.WORKFLOW_JOB)); + + TestLogAppender appender = getTestLogAppender(); + Logger logger = Logger.getLogger(SLACalculatorMemory.class); + logger.addAppender(appender); + logger.setLevel(Level.ERROR); + try { + slaService.runSLAWorker(); + } + finally { + logger.removeAppender(appender); + } + + List<LoggingEvent> log = appender.getLog(); + LoggingEvent firstLogEntry = log.get(0); + assertEquals(Level.ERROR, firstLogEntry.getLevel()); + assertTrue(firstLogEntry.getMessage().toString().contains("JOB[dummy-id]")); + assertEquals("org.apache.oozie.sla.SLACalculatorMemory", firstLogEntry.getLoggerName()); + + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/test/java/org/apache/oozie/sla/TestSLAService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/sla/TestSLAService.java b/core/src/test/java/org/apache/oozie/sla/TestSLAService.java index 205bcd1..e8f18cd 100644 --- a/core/src/test/java/org/apache/oozie/sla/TestSLAService.java +++ b/core/src/test/java/org/apache/oozie/sla/TestSLAService.java @@ -150,7 +150,7 @@ public class TestSLAService extends XDataTestCase { // As expected duration is not set, duration shall be processed and job removed from map assertEquals(2, slas.getSLACalculator().size()); // test same job multiple events (start-met, end-met) through job status event - sla1 = _createSLARegistration("action-1", AppType.COORDINATOR_ACTION); + sla1 = _createSLARegistration("action@1", AppType.COORDINATOR_ACTION); sla1.setExpectedStart(new Date(System.currentTimeMillis() + 1 * 3600 * 1000)); //1 hour ahead sla1.setExpectedEnd(new Date(System.currentTimeMillis() + 2 * 3600 * 1000)); //2 hours ahead slas.addRegistrationEvent(sla1); http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index e739ec3..7661ebe 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -26,6 +26,7 @@ import java.io.OutputStream; import java.io.IOException; import java.net.InetAddress; import java.net.URL; +import java.util.ArrayList; import java.net.UnknownHostException; import java.util.HashMap; import java.util.List; @@ -52,6 +53,8 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.util.Shell; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.spi.LoggingEvent; import org.apache.oozie.BundleActionBean; import org.apache.oozie.BundleJobBean; import org.apache.oozie.CoordinatorActionBean; @@ -1159,5 +1162,32 @@ public abstract class XTestCase extends TestCase { setupServicesForHCataLogImpl(services); return services; } + + protected class TestLogAppender extends AppenderSkeleton { + private final List<LoggingEvent> log = new ArrayList<LoggingEvent>(); + + @Override + public boolean requiresLayout() { + return false; + } + + @Override + protected void append(final LoggingEvent loggingEvent) { + log.add(loggingEvent); + } + + @Override + public void close() { + } + + public List<LoggingEvent> getLog() { + return new ArrayList<LoggingEvent>(log); + } + } + + public TestLogAppender getTestLogAppender() { + return new TestLogAppender(); + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/7bc73898/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 912e277..874e58a 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,9 +1,11 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1933 SLACalculatorMemory HA changes assume SLARegistrationBean exists for all jobs (mona) +OOZIE-1935 Log level (ActionStartXCommand) and Doc fix (CoordinatorFunctionalSpec) (mona) OOZIE-1586 upgrade oozie to hive 13.1 (including hcatalog) (raviprak via rkanter) OOZIE-1932 Services should load CallableQueueService after MemoryLocksService (mona) OOZIE-1740 Add a new function hadoop:conf() that can be invoked from the workflow.xml and will return a hadoop configuration option (sam liu via rkanter) -OOZIE-1911 SLA calculation in HA mode does wrong bit comparison for 'start' and 'duration' (mna) +OOZIE-1911 SLA calculation in HA mode does wrong bit comparison for 'start' and 'duration' (mona) OOZIE-1926 make gz blob compression as default (ryota) OOZIE-1916 Use Curator leader latch instead of checking the order of Oozie servers (rkanter) OOZIE-1886 Queue operation talking longer time (shwethags via rohini)
