Repository: oozie Updated Branches: refs/heads/master 2bce9e8f6 -> e5b0922d8
OOZIE-2146 Add option to filter sla information by bundle id or name (ryota) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/1b1ef47a Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/1b1ef47a Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/1b1ef47a Branch: refs/heads/master Commit: 1b1ef47ad35f893e1fa2a8245dc954c94570e937 Parents: 2bce9e8 Author: egashira <[email protected]> Authored: Wed Apr 1 12:52:29 2015 -0700 Committer: egashira <[email protected]> Committed: Wed Apr 1 12:52:29 2015 -0700 ---------------------------------------------------------------------- .../org/apache/oozie/client/OozieClient.java | 6 + .../org/apache/oozie/client/rest/JsonTags.java | 4 + ...rdinatorJobGetForUserAppnameJPAExecutor.java | 1 + .../sla/SLASummaryGetForFilterJPAExecutor.java | 212 +++++++++++++++++- .../org/apache/oozie/servlet/V2SLAServlet.java | 58 ++++- .../org/apache/oozie/sla/SLASummaryBean.java | 154 +++++++++---- .../oozie/servlet/DagServletTestCase.java | 4 +- .../apache/oozie/servlet/TestV2SLAServlet.java | 221 ++++++++++++++++--- docs/src/site/twiki/DG_SLAMonitoring.twiki | 65 +++++- release-log.txt | 1 + 10 files changed, 631 insertions(+), 95 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/1b1ef47a/client/src/main/java/org/apache/oozie/client/OozieClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java index 61d7923..ceb193a 100644 --- a/client/src/main/java/org/apache/oozie/client/OozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java @@ -148,6 +148,12 @@ public class OozieClient { public static final String FILTER_SLA_PARENT_ID = "parent_id"; + public static final String FILTER_BUNDLE = "bundle"; + + public static final String FILTER_SLA_EVENT_STATUS = "event_status"; + + public static final String FILTER_SLA_STATUS = "sla_status"; + public static final String FILTER_SLA_NOMINAL_START = "nominal_start"; public static final String FILTER_SLA_NOMINAL_END = "nominal_end"; http://git-wip-us.apache.org/repos/asf/oozie/blob/1b1ef47a/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java b/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java index 47da15e..ab6699f 100644 --- a/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java +++ b/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java @@ -170,12 +170,16 @@ public interface JsonTags { public static final String SLA_SUMMARY_NOMINAL_TIME = "nominalTime"; public static final String SLA_SUMMARY_EXPECTED_START = "expectedStart"; public static final String SLA_SUMMARY_ACTUAL_START = "actualStart"; + public static final String SLA_SUMMARY_START_DELAY = "startDelay"; public static final String SLA_SUMMARY_EXPECTED_END = "expectedEnd"; public static final String SLA_SUMMARY_ACTUAL_END = "actualEnd"; + public static final String SLA_SUMMARY_END_DELAY = "endDelay"; public static final String SLA_SUMMARY_EXPECTED_DURATION = "expectedDuration"; public static final String SLA_SUMMARY_ACTUAL_DURATION = "actualDuration"; + public static final String SLA_SUMMARY_DURATION_DELAY = "durationDelay"; public static final String SLA_SUMMARY_JOB_STATUS = "jobStatus"; public static final String SLA_SUMMARY_SLA_STATUS = "slaStatus"; + public static final String SLA_SUMMARY_EVENT_STATUS = "eventStatus"; public static final String SLA_SUMMARY_LAST_MODIFIED = "lastModified"; public static final String SLA_ALERT_STATUS = "slaAlertStatus"; http://git-wip-us.apache.org/repos/asf/oozie/blob/1b1ef47a/core/src/main/java/org/apache/oozie/executor/jpa/CoordinatorJobGetForUserAppnameJPAExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/executor/jpa/CoordinatorJobGetForUserAppnameJPAExecutor.java b/core/src/main/java/org/apache/oozie/executor/jpa/CoordinatorJobGetForUserAppnameJPAExecutor.java index 3bb2f88..5520ab5 100644 --- a/core/src/main/java/org/apache/oozie/executor/jpa/CoordinatorJobGetForUserAppnameJPAExecutor.java +++ b/core/src/main/java/org/apache/oozie/executor/jpa/CoordinatorJobGetForUserAppnameJPAExecutor.java @@ -24,6 +24,7 @@ import javax.persistence.Query; import org.apache.oozie.CoordinatorJobBean; import org.apache.oozie.ErrorCode; import org.apache.oozie.util.ParamChecker; +import org.apache.oozie.util.XLog; /** * DB query executor to fetch columns 'user' and 'appName' from Coordinator Job table http://git-wip-us.apache.org/repos/asf/oozie/blob/1b1ef47a/core/src/main/java/org/apache/oozie/executor/jpa/sla/SLASummaryGetForFilterJPAExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/executor/jpa/sla/SLASummaryGetForFilterJPAExecutor.java b/core/src/main/java/org/apache/oozie/executor/jpa/sla/SLASummaryGetForFilterJPAExecutor.java index b55cda3..3fd4793 100644 --- a/core/src/main/java/org/apache/oozie/executor/jpa/sla/SLASummaryGetForFilterJPAExecutor.java +++ b/core/src/main/java/org/apache/oozie/executor/jpa/sla/SLASummaryGetForFilterJPAExecutor.java @@ -19,17 +19,23 @@ package org.apache.oozie.executor.jpa.sla; import java.sql.Timestamp; +import java.util.ArrayList; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; + import javax.persistence.EntityManager; import javax.persistence.Query; import org.apache.oozie.ErrorCode; +import org.apache.oozie.client.event.SLAEvent; +import org.apache.oozie.client.event.SLAEvent.EventStatus; +import org.apache.oozie.client.event.SLAEvent.SLAStatus; import org.apache.oozie.executor.jpa.JPAExecutor; import org.apache.oozie.executor.jpa.JPAExecutorException; import org.apache.oozie.sla.SLASummaryBean; +import org.apache.oozie.util.XLog; /** * Load the list of SLASummaryBean (for dashboard) and return the list. @@ -37,9 +43,12 @@ import org.apache.oozie.sla.SLASummaryBean; public class SLASummaryGetForFilterJPAExecutor implements JPAExecutor<List<SLASummaryBean>> { private static final String selectStr = "SELECT OBJECT(s) FROM SLASummaryBean s WHERE "; - + private static final String bundleIdQuery = "SELECT a.coordId FROM BundleActionBean a WHERE a.bundleId=:bundleId"; + private static final String bundleNameQuery = "SELECT a.coordId FROM BundleActionBean a WHERE a.bundleId in " + + "(SELECT b.id from BundleJobBean b WHERE b.appName=:appName)"; private SLASummaryFilter filter; private int numMaxResults; + private XLog LOG = XLog.getLog(getClass()); public SLASummaryGetForFilterJPAExecutor(SLASummaryFilter filter, int numMaxResults) { @@ -55,10 +64,11 @@ public class SLASummaryGetForFilterJPAExecutor implements JPAExecutor<List<SLASu @SuppressWarnings("unchecked") @Override public List<SLASummaryBean> execute(EntityManager em) throws JPAExecutorException { - List<SLASummaryBean> ssBean; + List<SLASummaryBean> ssBean = null; StringBuilder sb = new StringBuilder(selectStr); Map<String, Object> queryParams = new LinkedHashMap<String, Object>(); boolean firstCondition = true; + boolean jobExists = true; if (filter.getJobId() != null) { firstCondition = false; if (filter.getParentId() != null) { @@ -76,8 +86,53 @@ public class SLASummaryGetForFilterJPAExecutor implements JPAExecutor<List<SLASu sb.append("s.parentId = :parentId"); queryParams.put("parentId", filter.getParentId()); } - if (filter.getAppName() != null && filter.getJobId() == null && filter.getParentId() == null) { + if (filter.getBundleId() != null || filter.getBundleName() != null) { firstCondition = false; + Query bq; + List<Object> returnList; + try { + if (filter.getBundleId() != null) { + bq = em.createQuery(bundleIdQuery); + bq.setParameter("bundleId", filter.getBundleId()); + } + else { + bq = em.createQuery(bundleNameQuery); + bq.setParameter("appName", filter.getBundleName()); + } + bq.setMaxResults(numMaxResults); + returnList = (List<Object>) bq.getResultList(); + } + catch (Exception e) { + throw new JPAExecutorException(ErrorCode.E0603, e.getMessage(), e); + } + StringBuilder sub = null; + int ind = 0; + if(returnList.size() == 0) { + jobExists = false; + } + for (Object obj : returnList) { + String coordId = (String) obj; + if (sub == null) { + sub = new StringBuilder(); + sub.append("s.parentId in (:parentId").append(ind); + } + else { + sub.append(",:parentId").append(ind); + } + queryParams.put("parentId" + ind, coordId); + ind++; + } + if(sub != null) { + sub.append(")"); + sb.append(sub.toString()); + } + } + if (filter.getAppName() != null) { + if (firstCondition ){ + firstCondition = false; + } else { + sb.append(" AND "); + } sb.append("s.appName = :appName"); queryParams.put("appName", filter.getAppName()); } @@ -103,26 +158,116 @@ public class SLASummaryGetForFilterJPAExecutor implements JPAExecutor<List<SLASu queryParams.put("nominalTimeEnd", new Timestamp(filter.getNominalEnd().getTime())); } - sb.append(" ORDER BY s.nominalTimeTS"); - try { - Query q = em.createQuery(sb.toString()); - for (Map.Entry<String, Object> entry : queryParams.entrySet()) { - q.setParameter(entry.getKey(), entry.getValue()); + if (filter.getEventStatus() != null) { + processEventStatusFilter(filter, queryParams,sb,firstCondition); + } + + if (filter.getSLAStatus() != null) { + StringBuilder sub = null; + int ind = 0; + if (firstCondition) { + firstCondition = false; + } + else { + sb.append(" AND "); + } + for (SLAStatus status : filter.getSLAStatus()) { + if (sub == null) { + sub = new StringBuilder(); + sub.append("s.slaStatus in (:slaStatus").append(ind); + } + else { + sub.append(",:slaStatus").append(ind); + } + queryParams.put("slaStatus" + ind, status.toString()); + ind++; + } + if(sub != null) { + sub.append(")"); + sb.append(sub.toString()); } - q.setMaxResults(numMaxResults); - ssBean = (List<SLASummaryBean>) q.getResultList(); } - catch (Exception e) { - throw new JPAExecutorException(ErrorCode.E0603, e.getMessage(), e); + + if (jobExists) { + sb.append(" ORDER BY s.nominalTimeTS"); + LOG.debug("Query String: " + sb.toString()); + try { + Query q = em.createQuery(sb.toString()); + for (Map.Entry<String, Object> entry : queryParams.entrySet()) { + q.setParameter(entry.getKey(), entry.getValue()); + } + q.setMaxResults(numMaxResults); + ssBean = (List<SLASummaryBean>) q.getResultList(); + } + catch (Exception e) { + throw new JPAExecutorException(ErrorCode.E0603, e.getMessage(), e); + } } return ssBean; } + private void processEventStatusFilter(SLASummaryFilter filter, Map<String, Object> queryParams, StringBuilder sb, + boolean firstCondition) { + if (firstCondition) { + firstCondition = false; + } + else { + sb.append(" AND "); + } + List<EventStatus> eventStatusList = filter.getEventStatus(); + int ind = 0; + Timestamp currentTime = new Timestamp(new Date().getTime()); + for (EventStatus status : eventStatusList) { + if (ind > 0) { + sb.append(" OR "); + } + if (status.equals(EventStatus.START_MET)) { + sb.append("(s.expectedStartTS IS NOT NULL AND s.actualStartTS IS NOT NULL ").append( + " AND s.expectedStartTS >= s.actualStartTS)"); + } + else if (status.equals(EventStatus.START_MISS)) { + sb.append("((s.expectedStartTS IS NOT NULL AND s.actualStartTS IS NOT NULL ") + .append(" AND s.expectedStartTS <= s.actualStartTS) ") + .append("OR (s.expectedStartTS IS NOT NULL AND s.actualStartTS IS NULL ") + .append(" AND s.expectedStartTS <= :currentTimeStamp))"); + queryParams.put("currentTimeStamp",currentTime); + } + else if (status.equals(EventStatus.DURATION_MET)) { + sb.append("(s.expectedDuration <> -1 AND s.actualDuration <> -1 ").append( + " AND s.expectedDuration >= s.actualDuration) "); + } + + else if (status.equals(EventStatus.DURATION_MISS)) { + sb.append("((s.expectedDuration <> -1 AND s.actualDuration <> -1 ") + .append("AND s.expectedDuration < s.actualDuration) ") + .append("OR s.eventStatus = 'DURATION_MISS')"); + } + else if (status.equals(EventStatus.END_MET)) { + sb.append("(s.expectedEndTS IS NOT NULL AND s.actualEndTS IS NOT NULL ").append( + " AND s.expectedEndTS <= s.actualEndTS) "); + } + else if (status.equals(EventStatus.END_MISS)) { + sb.append("((s.expectedEndTS IS NOT NULL AND s.actualEndTS IS NOT NULL ") + .append("AND s.expectedEndTS <= s.actualEndTS) ") + .append("OR (s.expectedEndTS IS NOT NULL AND s.actualEndTS IS NULL ") + .append("AND s.expectedEndTS <= :currentTimeStamp))"); + queryParams.put("currentTimeStamp",currentTime); + } + ind++; + } + } + public static class SLASummaryFilter { private String appName; private String jobId; private String parentId; + private String bundleId; + private String bundleName; + private List<SLAEvent.EventStatus> eventStatus; + private List<SLAEvent.SLAStatus> slaStatus; + private static String EventStatusSep = ","; + private static String SLAStatusSep = ","; private Date nominalStart; private Date nominalEnd; @@ -169,6 +314,49 @@ public class SLASummaryGetForFilterJPAExecutor implements JPAExecutor<List<SLASu this.nominalEnd = nominalEnd; } + public String getBundleId(){ + return this.bundleId; + } + + public void setBundleId(String bundleId) { + this.bundleId = bundleId; + } + + public String getBundleName(){ + return this.bundleName; + } + + public void setBundleName(String name){ + this.bundleName = name; + } + + public List<EventStatus> getEventStatus() { + return this.eventStatus; + } + + public void setEventStatus(String str) { + if (this.eventStatus == null) { + this.eventStatus = new ArrayList<EventStatus>(); + } + String[] statusArr = str.split(EventStatusSep); + for (String s : statusArr) { + this.eventStatus.add(SLAEvent.EventStatus.valueOf(s)); + } + } + + public List<SLAStatus> getSLAStatus() { + return this.slaStatus; + } + + public void setSLAStatus(String str) { + if (this.slaStatus == null) { + this.slaStatus = new ArrayList<SLAStatus>(); + } + String[] statusArr = str.split(SLAStatusSep); + for (String s : statusArr) { + this.slaStatus.add(SLAEvent.SLAStatus.valueOf(s)); + } + } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/1b1ef47a/core/src/main/java/org/apache/oozie/servlet/V2SLAServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V2SLAServlet.java b/core/src/main/java/org/apache/oozie/servlet/V2SLAServlet.java index 57170e1..d578eaf 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V2SLAServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V2SLAServlet.java @@ -25,10 +25,16 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.Calendar; +import java.util.Collections; +import java.util.Date; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -37,6 +43,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.oozie.ErrorCode; import org.apache.oozie.XException; import org.apache.oozie.client.OozieClient; +import org.apache.oozie.client.event.SLAEvent.EventStatus; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.command.CommandException; import org.apache.oozie.executor.jpa.SLARegistrationQueryExecutor; @@ -57,13 +64,17 @@ public class V2SLAServlet extends SLAServlet { private static final String INSTRUMENTATION_NAME = "v2sla"; private static final JsonRestServlet.ResourceInfo RESOURCES_INFO[] = new JsonRestServlet.ResourceInfo[1]; private static final Set<String> SLA_FILTER_NAMES = new HashSet<String>(); + private Pattern p = Pattern.compile("\\d{7}-\\d{15}-.*-B$"); static { SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_ID); SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_PARENT_ID); + SLA_FILTER_NAMES.add(OozieClient.FILTER_BUNDLE); SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_APPNAME); SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_NOMINAL_START); SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_NOMINAL_END); + SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_EVENT_STATUS); + SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_STATUS); } static { @@ -121,12 +132,44 @@ public class V2SLAServlet extends SLAServlet { try { Map<String, List<String>> filterList = parseFilter(URLDecoder.decode(filterString, "UTF-8"), SLA_FILTER_NAMES); SLASummaryFilter filter = new SLASummaryFilter(); + + if (!filterList.containsKey(OozieClient.FILTER_SLA_APPNAME) + && !filterList.containsKey(OozieClient.FILTER_SLA_ID) + && !filterList.containsKey(OozieClient.FILTER_SLA_PARENT_ID) + && !filterList.containsKey(OozieClient.FILTER_BUNDLE) + && !filterList.containsKey(OozieClient.FILTER_SLA_NOMINAL_START) + && !filterList.containsKey(OozieClient.FILTER_SLA_NOMINAL_END)) { + StringBuffer st = new StringBuffer(); + st.append("At least one of the filter parameters - ").append(OozieClient.FILTER_SLA_APPNAME) + .append(",").append(OozieClient.FILTER_SLA_ID).append(",") + .append(OozieClient.FILTER_SLA_PARENT_ID).append(",").append(OozieClient.FILTER_BUNDLE) + .append(",").append(OozieClient.FILTER_SLA_NOMINAL_START).append(" or ") + .append(OozieClient.FILTER_SLA_NOMINAL_END) + .append(" should be specified in the filter query parameter"); + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0305, st.toString()); + } + if (filterList.containsKey(OozieClient.FILTER_SLA_ID)) { filter.setJobId(filterList.get(OozieClient.FILTER_SLA_ID).get(0)); } if (filterList.containsKey(OozieClient.FILTER_SLA_PARENT_ID)) { filter.setParentId(filterList.get(OozieClient.FILTER_SLA_PARENT_ID).get(0)); } + if (filterList.containsKey(OozieClient.FILTER_BUNDLE)) { + String bundle = filterList.get(OozieClient.FILTER_BUNDLE).get(0); + if (isBundleId(bundle)) { + filter.setBundleId(bundle); + } + else { + filter.setBundleName(bundle); + } + } + if (filterList.containsKey(OozieClient.FILTER_SLA_EVENT_STATUS)) { + filter.setEventStatus(filterList.get(OozieClient.FILTER_SLA_EVENT_STATUS).get(0)); + } + if (filterList.containsKey(OozieClient.FILTER_SLA_STATUS)) { + filter.setSLAStatus(filterList.get(OozieClient.FILTER_SLA_STATUS).get(0)); + } if (filterList.containsKey(OozieClient.FILTER_SLA_APPNAME)) { filter.setAppName(filterList.get(OozieClient.FILTER_SLA_APPNAME).get(0)); } @@ -137,13 +180,6 @@ public class V2SLAServlet extends SLAServlet { filter.setNominalEnd(DateUtils.parseDateUTC(filterList.get(OozieClient.FILTER_SLA_NOMINAL_END).get(0))); } - if (filter.getAppName() == null && filter.getJobId() == null && filter.getParentId() == null) { - throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0305, - "At least one of the filter parameters - " + OozieClient.FILTER_SLA_ID + "," - + OozieClient.FILTER_SLA_PARENT_ID + " or " + OozieClient.FILTER_SLA_APPNAME - + " should be specified in the filter query parameter"); - } - JPAService jpaService = Services.get().get(JPAService.class); List<SLASummaryBean> slaSummaryList = null; if (jpaService != null) { @@ -181,4 +217,12 @@ public class V2SLAServlet extends SLAServlet { } + private boolean isBundleId(String id) { + boolean ret = false; + Matcher m = p.matcher(id); + if (m.matches()) { + return true; + } + return ret; + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/1b1ef47a/core/src/main/java/org/apache/oozie/sla/SLASummaryBean.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/sla/SLASummaryBean.java b/core/src/main/java/org/apache/oozie/sla/SLASummaryBean.java index a88dcf6..ef1ea98 100644 --- a/core/src/main/java/org/apache/oozie/sla/SLASummaryBean.java +++ b/core/src/main/java/org/apache/oozie/sla/SLASummaryBean.java @@ -20,6 +20,7 @@ package org.apache.oozie.sla; import java.sql.Timestamp; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,6 +35,7 @@ import javax.persistence.Table; import org.apache.oozie.AppType; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.event.SLAEvent; +import org.apache.oozie.client.event.SLAEvent.EventStatus; import org.apache.oozie.client.rest.JsonBean; import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.client.rest.JsonUtils; @@ -361,10 +363,25 @@ public class SLASummaryBean implements JsonBean { this.lastModifiedTS = DateUtils.convertDateToTimestamp(lastModified); } - @SuppressWarnings("unchecked") @Override public JSONObject toJSONObject() { + return toJSONObject(null); + } + + @Override + @SuppressWarnings("unchecked") + public JSONObject toJSONObject(String timeZoneId) { JSONObject json = new JSONObject(); + Map<EventStatus,Long> eventMap = calculateEventStatus(); + StringBuilder eventStatusStr = new StringBuilder(); + boolean first = true; + for(EventStatus e: eventMap.keySet()) { + if(!first) { + eventStatusStr.append(","); + } + eventStatusStr.append(e.toString()); + first = false; + } json.put(JsonTags.SLA_SUMMARY_ID, jobId); if (parentId != null) { json.put(JsonTags.SLA_SUMMARY_PARENT_ID, parentId); @@ -372,79 +389,127 @@ public class SLASummaryBean implements JsonBean { json.put(JsonTags.SLA_SUMMARY_APP_NAME, appName); json.put(JsonTags.SLA_SUMMARY_APP_TYPE, appType); json.put(JsonTags.SLA_SUMMARY_USER, user); - json.put(JsonTags.SLA_SUMMARY_NOMINAL_TIME, nominalTimeTS.getTime()); + json.put(JsonTags.SLA_SUMMARY_NOMINAL_TIME, getTimeOnTimeZone(nominalTimeTS, timeZoneId)); if (expectedStartTS != null) { - json.put(JsonTags.SLA_SUMMARY_EXPECTED_START, expectedStartTS.getTime()); - } - else { + json.put(JsonTags.SLA_SUMMARY_EXPECTED_START, getTimeOnTimeZone(expectedStartTS, timeZoneId)); + } else { json.put(JsonTags.SLA_SUMMARY_EXPECTED_START, null); } + if (actualStartTS != null) { - json.put(JsonTags.SLA_SUMMARY_ACTUAL_START, actualStartTS.getTime()); + json.put(JsonTags.SLA_SUMMARY_ACTUAL_START, getTimeOnTimeZone(actualStartTS, timeZoneId)); } else { json.put(JsonTags.SLA_SUMMARY_ACTUAL_START, null); } - json.put(JsonTags.SLA_SUMMARY_EXPECTED_END, expectedEndTS.getTime()); + Long startDelay = eventMap.get(EventStatus.START_MET) != null ? eventMap.get(EventStatus.START_MET) : eventMap + .get(EventStatus.START_MISS); + if (startDelay != null) { + json.put(JsonTags.SLA_SUMMARY_START_DELAY, startDelay); + } + if (expectedEndTS != null ) { + json.put(JsonTags.SLA_SUMMARY_EXPECTED_END, getTimeOnTimeZone(expectedEndTS,timeZoneId)); + } else { + json.put(JsonTags.SLA_SUMMARY_ACTUAL_END, null); + } if (actualEndTS != null) { - json.put(JsonTags.SLA_SUMMARY_ACTUAL_END, actualEndTS.getTime()); + json.put(JsonTags.SLA_SUMMARY_ACTUAL_END, getTimeOnTimeZone(actualEndTS,timeZoneId)); } else { json.put(JsonTags.SLA_SUMMARY_ACTUAL_END, null); } + Long endDelay = eventMap.get(EventStatus.END_MET) != null ? eventMap.get(EventStatus.END_MET) : eventMap + .get(EventStatus.END_MISS); + if (endDelay != null) { + json.put(JsonTags.SLA_SUMMARY_END_DELAY, endDelay); + } json.put(JsonTags.SLA_SUMMARY_EXPECTED_DURATION, expectedDuration); - json.put(JsonTags.SLA_SUMMARY_ACTUAL_DURATION, actualDuration); + if (actualDuration == -1 && expectedDuration != -1 && actualStartTS != null) { + long currentDur = (new Date().getTime() - actualStartTS.getTime()) / (1000 * 60); + json.put(JsonTags.SLA_SUMMARY_ACTUAL_DURATION, currentDur); + } + else { + json.put(JsonTags.SLA_SUMMARY_ACTUAL_DURATION, actualDuration); + } + Long durationDelay = eventMap.get(EventStatus.DURATION_MET) != null ? eventMap.get(EventStatus.DURATION_MET) + : eventMap.get(EventStatus.DURATION_MISS); + if (durationDelay != null) { + json.put(JsonTags.SLA_SUMMARY_DURATION_DELAY, durationDelay); + } json.put(JsonTags.SLA_SUMMARY_JOB_STATUS, jobStatus); json.put(JsonTags.SLA_SUMMARY_SLA_STATUS, slaStatus); - json.put(JsonTags.SLA_SUMMARY_LAST_MODIFIED, lastModifiedTS.getTime()); + json.put(JsonTags.SLA_SUMMARY_EVENT_STATUS, eventStatusStr.toString()); + json.put(JsonTags.SLA_SUMMARY_LAST_MODIFIED, getTimeOnTimeZone(lastModifiedTS, timeZoneId)); return json; } - @SuppressWarnings("unchecked") - @Override - public JSONObject toJSONObject(String timeZoneId) { - if (timeZoneId == null) { - return toJSONObject(); + private Object getTimeOnTimeZone(Timestamp ts, String timeZoneId) { + Object ret = null; + if(timeZoneId == null) { + ret = new Long(String.valueOf(ts.getTime())); + } else { + ret = JsonUtils.formatDateRfc822(ts, timeZoneId); } - else { - JSONObject json = new JSONObject(); - json.put(JsonTags.SLA_SUMMARY_ID, jobId); - if (parentId != null) { - json.put(JsonTags.SLA_SUMMARY_PARENT_ID, parentId); - } - json.put(JsonTags.SLA_SUMMARY_APP_NAME, appName); - json.put(JsonTags.SLA_SUMMARY_APP_TYPE, appType); - json.put(JsonTags.SLA_SUMMARY_USER, user); - json.put(JsonTags.SLA_SUMMARY_NOMINAL_TIME, JsonUtils.formatDateRfc822(nominalTimeTS, timeZoneId)); - if (expectedStartTS != null) { - json.put(JsonTags.SLA_SUMMARY_EXPECTED_START, JsonUtils.formatDateRfc822(expectedStartTS, timeZoneId)); + return ret; + } + + private Map<EventStatus, Long> calculateEventStatus() { + Map<EventStatus, Long> events = new HashMap<EventStatus, Long>(); + if (expectedStartTS != null) { + if (actualStartTS != null) { + long diff = (actualStartTS.getTime() - expectedStartTS.getTime()) / (1000 * 60); + if (diff > 0) { + events.put(EventStatus.START_MISS, diff); + } + else { + events.put(EventStatus.START_MET, diff); + } } else { - json.put(JsonTags.SLA_SUMMARY_EXPECTED_START, null); + long diff = (new Date().getTime() - expectedStartTS.getTime()) / (1000 * 60); + if (diff > 0) { + events.put(EventStatus.START_MISS, diff); + } } - if (actualStartTS != null) { - json.put(JsonTags.SLA_SUMMARY_ACTUAL_START, JsonUtils.formatDateRfc822(actualStartTS, timeZoneId)); + } + if (expectedDuration != -1) { + if (actualDuration != -1) { + long diff = actualDuration - expectedDuration; + if (diff > 0) { + events.put(EventStatus.DURATION_MISS, diff); + } + else { + events.put(EventStatus.DURATION_MET, diff); + } } else { - json.put(JsonTags.SLA_SUMMARY_ACTUAL_START, null); + if (actualStartTS != null) { + long currentDur = (new Date().getTime() - actualStartTS.getTime()) / (1000 * 60); + if (expectedDuration < currentDur) { + events.put(EventStatus.DURATION_MISS, (currentDur - expectedDuration)); + } + } } - json.put(JsonTags.SLA_SUMMARY_EXPECTED_END, JsonUtils.formatDateRfc822(expectedEndTS, timeZoneId)); + } + if (expectedEndTS != null) { if (actualEndTS != null) { - json.put(JsonTags.SLA_SUMMARY_ACTUAL_END, JsonUtils.formatDateRfc822(actualEndTS, timeZoneId)); + long diff = (actualEndTS.getTime() - expectedEndTS.getTime()) / (1000 * 60); + if (diff > 0) { + events.put(EventStatus.END_MISS, diff); + } + else { + events.put(EventStatus.END_MET, diff); + } } else { - json.put(JsonTags.SLA_SUMMARY_ACTUAL_END, null); + long diff = (new Date().getTime() - expectedEndTS.getTime()) / (1000 * 60); + if (diff > 0) { + events.put(EventStatus.END_MISS, diff); + } } - json.put(JsonTags.SLA_SUMMARY_EXPECTED_DURATION, expectedDuration); - json.put(JsonTags.SLA_SUMMARY_ACTUAL_DURATION, actualDuration); - json.put(JsonTags.SLA_SUMMARY_JOB_STATUS, jobStatus); - json.put(JsonTags.SLA_SUMMARY_SLA_STATUS, slaStatus); - json.put(JsonTags.SLA_SUMMARY_LAST_MODIFIED, JsonUtils.formatDateRfc822(lastModifiedTS, timeZoneId)); - - return json; } + return events; } - /** * Convert a sla summary list into a json object. * @@ -475,8 +540,8 @@ public class SLASummaryBean implements JsonBean { JSONObject slaJson = summary.toJSONObject(timeZoneId); String slaAlertStatus = ""; if (slaConfigMap.containsKey(summary.getId())) { - slaAlertStatus = slaConfigMap.get(summary.getId()).containsKey( - OozieClient.SLA_DISABLE_ALERT) ? "Disabled" : "Enabled"; + slaAlertStatus = slaConfigMap.get(summary.getId()).containsKey(OozieClient.SLA_DISABLE_ALERT) ? "Disabled" + : "Enabled"; } slaJson.put(JsonTags.SLA_ALERT_STATUS, slaAlertStatus); array.add(slaJson); @@ -485,5 +550,4 @@ public class SLASummaryBean implements JsonBean { json.put(JsonTags.SLA_SUMMARY_LIST, array); return json; } - } http://git-wip-us.apache.org/repos/asf/oozie/blob/1b1ef47a/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java b/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java index 48193c7..ce731a1 100644 --- a/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java +++ b/core/src/test/java/org/apache/oozie/servlet/DagServletTestCase.java @@ -19,12 +19,12 @@ package org.apache.oozie.servlet; import org.apache.oozie.service.AuthorizationService; - import org.apache.oozie.service.ProxyUserService; import org.apache.oozie.service.Services; import org.apache.oozie.service.ForTestAuthorizationService; import org.apache.oozie.service.ForTestWorkflowStoreService; import org.apache.oozie.test.EmbeddedServletContainer; +import org.apache.oozie.test.XDataTestCase; import org.apache.oozie.test.XFsTestCase; import java.net.URL; @@ -32,7 +32,7 @@ import java.net.URLEncoder; import java.util.Map; import java.util.concurrent.Callable; -public abstract class DagServletTestCase extends XFsTestCase { +public abstract class DagServletTestCase extends XDataTestCase { private EmbeddedServletContainer container; private String servletPath; http://git-wip-us.apache.org/repos/asf/oozie/blob/1b1ef47a/core/src/test/java/org/apache/oozie/servlet/TestV2SLAServlet.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/TestV2SLAServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestV2SLAServlet.java index 1886f48..db509ac 100644 --- a/core/src/test/java/org/apache/oozie/servlet/TestV2SLAServlet.java +++ b/core/src/test/java/org/apache/oozie/servlet/TestV2SLAServlet.java @@ -32,6 +32,11 @@ import java.util.concurrent.Callable; import javax.servlet.http.HttpServletResponse; import org.apache.oozie.AppType; +import org.apache.oozie.BundleActionBean; +import org.apache.oozie.BundleJobBean; +import org.apache.oozie.CoordinatorJobBean; +import org.apache.oozie.client.CoordinatorJob; +import org.apache.oozie.client.Job; import org.apache.oozie.client.event.SLAEvent.EventStatus; import org.apache.oozie.client.event.SLAEvent.SLAStatus; import org.apache.oozie.client.rest.JsonBean; @@ -39,6 +44,8 @@ import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.executor.jpa.BatchQueryExecutor; import org.apache.oozie.executor.jpa.JPAExecutorException; +import org.apache.oozie.executor.jpa.SLASummaryQueryExecutor; +import org.apache.oozie.service.Services; import org.apache.oozie.sla.SLASummaryBean; import org.apache.oozie.util.DateUtils; import org.json.simple.JSONArray; @@ -63,11 +70,11 @@ public class TestV2SLAServlet extends DagServletTestCase { final Date nominalTime2 = DateUtils.parseDateUTC("2012-06-02T10:20Z"); final Date nominalTime3 = DateUtils.parseDateUTC("2012-06-03T14:00Z"); insertEntriesIntoSLASummaryTable(2, "1-", "-W", "1-C", nominalTime1, "testapp-1", AppType.WORKFLOW_JOB, - currentTime); + currentTime, EventStatus.END_MISS, SLAStatus.IN_PROCESS); insertEntriesIntoSLASummaryTable(3, "2-", "-W", null, nominalTime2, "testapp-2", AppType.WORKFLOW_JOB, - currentTime); + currentTime, EventStatus.END_MISS, SLAStatus.IN_PROCESS); insertEntriesIntoSLASummaryTable(6, "3-", "-W", "2-C", nominalTime3, "testapp-3", AppType.WORKFLOW_JOB, - currentTime); + currentTime, EventStatus.END_MISS, SLAStatus.IN_PROCESS); Map<String, String> queryParams = new HashMap<String, String>(); JSONArray array = null; @@ -101,7 +108,164 @@ public class TestV2SLAServlet extends DagServletTestCase { // Matches 3rd and 4th element - 3-3-W 3-4-W assertSLAJSONResponse(array, 3, 4, "3-", "-W", "2-C", nominalTime3, "testapp-3", AppType.WORKFLOW_JOB, currentTime); + return null; + } + }); + } + + public void testBundleSLA() throws Exception { + runTest("/v2/sla", V2SLAServlet.class, IS_SECURITY_ENABLED, new Callable<Void>() { + public Void call() throws Exception { + + //insert Bundle Job/Action, Coord Job/Action + List<JsonBean> beans = new ArrayList<JsonBean> (); + String bundleId = "0000000-000000000000000-"+ Services.get().getSystemId() + "-B"; + BundleJobBean bjBean = createBundleJob(bundleId,Job.Status.RUNNING, false); + String bundleName = bjBean.getAppName(); + beans.add(bjBean); + CoordinatorJobBean cjBean1 = createCoordJob(CoordinatorJob.Status.SUCCEEDED, false, true); + beans.add(cjBean1); + CoordinatorJobBean cjBean2 = createCoordJob(CoordinatorJob.Status.SUCCEEDED, false, true); + beans.add(cjBean2); + + BundleActionBean baBean1 = createBundleAction(bundleId, cjBean1.getId(), "bundle-action-1", 0, + Job.Status.RUNNING); + beans.add(baBean1); + BundleActionBean baBean2 = createBundleAction(bundleId, cjBean2.getId(), "bundle-action-2", 0, + Job.Status.RUNNING); + beans.add(baBean2); + BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(beans, null, null); + + Calendar cal = Calendar.getInstance(); + cal.add(Calendar.MINUTE, -12); //current -12 + Date actualStartForMet = cal.getTime(); + cal.add(Calendar.MINUTE, 2); //current -10 + Date expectedStart = cal.getTime(); + cal.add(Calendar.MINUTE, 1); //current -9 + Date actualStartForMiss = cal.getTime(); + cal.add(Calendar.MINUTE, 3); //current -6 + Date actualEndForMet = cal.getTime(); + cal.add(Calendar.MINUTE, 1); //current -5 + Date expectedEnd = cal.getTime(); + cal.add(Calendar.MINUTE, 2); //current -3 + Date actualEndForMiss = cal.getTime(); + cal.add(Calendar.MINUTE, 8); //current + 5 + Date futureExpectedEnd = cal.getTime(); + + // START_MET, DURATION_MET, END_MET + insertEntriesIntoSLASummaryTable(cjBean1.getId() + "@1", cjBean1.getId(), "testapp-1", + AppType.COORDINATOR_ACTION, EventStatus.END_MET, SLAStatus.MET, expectedStart, + actualStartForMet, 7, 6, expectedEnd, actualEndForMet, actualStartForMet); + + // START_MISS, DURATION_MISS, END_MISS + insertEntriesIntoSLASummaryTable(cjBean1.getId() + "@2", cjBean1.getId(), "testapp-1", + AppType.COORDINATOR_ACTION, EventStatus.END_MISS, SLAStatus.MISS, expectedStart, + actualStartForMiss, 5, 6, expectedEnd, actualEndForMiss, actualStartForMet); + + // // START_MISS, DURATION_MISS (still running, Not Ended, but + // expected Duration/End already passed by now) + insertEntriesIntoSLASummaryTable(cjBean2.getId() + "@1", cjBean2.getId(), "testapp-2", + AppType.COORDINATOR_ACTION, EventStatus.DURATION_MISS, SLAStatus.IN_PROCESS, expectedStart, + actualStartForMiss, 8, 9, futureExpectedEnd, null, actualStartForMet); + + // START_MISS only, (Not Started YET, and Expected Duration/End + // Time not yet passed) + insertEntriesIntoSLASummaryTable(cjBean2.getId() + "@2", cjBean2.getId(), "testapp-2", + AppType.COORDINATOR_ACTION, null, SLAStatus.NOT_STARTED, expectedStart, null, 10, -1, + futureExpectedEnd, null, expectedStart); + + Map<String, String> queryParams = new HashMap<String, String>(); + JSONArray array = null; + + URL url = createURL("", queryParams); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("GET"); + assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + + //test filter bundle ID + queryParams.put(RestConstants.TIME_ZONE_PARAM, "GMT"); + queryParams.put(RestConstants.JOBS_FILTER_PARAM, String.format("bundle=%s",bundleId)); + array = getSLAJSONResponse(queryParams); + assertEquals(4, array.size()); + for(int i=0; i < array.size(); i++) { + JSONObject json = (JSONObject) array.get(i); + String id = (String)json.get(JsonTags.SLA_SUMMARY_ID); + if(id.equals(cjBean1.getId() + "@1")) { + assertEquals(-2L, json.get(JsonTags.SLA_SUMMARY_START_DELAY)); + assertEquals(-1L, json.get(JsonTags.SLA_SUMMARY_DURATION_DELAY)); + assertEquals(-1L, json.get(JsonTags.SLA_SUMMARY_END_DELAY)); + } + } + + //test filter bundle Name + queryParams.clear(); + queryParams.put(RestConstants.TIME_ZONE_PARAM, "GMT"); + queryParams.put(RestConstants.JOBS_FILTER_PARAM, String.format("bundle=%s",bundleName)); + array = getSLAJSONResponse(queryParams); + assertEquals(4, array.size()); + + //test filter bundle ID + EventStatus + queryParams.clear(); + queryParams.put(RestConstants.TIME_ZONE_PARAM, "GMT"); + queryParams.put(RestConstants.JOBS_FILTER_PARAM, String.format("bundle=%s;event_status=END_MISS",bundleId)); + array = getSLAJSONResponse(queryParams); + assertEquals(1, array.size()); + + JSONObject json = (JSONObject) array.get(0); + String parentId = (String) json.get(JsonTags.SLA_SUMMARY_PARENT_ID); + assertTrue(parentId.equals(cjBean1.getId()) || parentId.equals(cjBean2.getId())); + String id = (String) json.get(JsonTags.SLA_SUMMARY_ID); + assertTrue(id.equals(cjBean1.getId() + "@2")); + String es = (String) json.get(JsonTags.SLA_SUMMARY_EVENT_STATUS); + assertTrue(es.contains(EventStatus.END_MISS.toString())); + + // test filter bundle ID + EventStatus + SlaStus + queryParams.clear(); + queryParams.put(RestConstants.JOBS_FILTER_PARAM, String.format("bundle=%s;sla_status=MISS", bundleId)); + array = getSLAJSONResponse(queryParams); + assertEquals(1, array.size()); + + json = (JSONObject) array.get(0); + id = (String) json.get(JsonTags.SLA_SUMMARY_ID); + assertTrue(id.equals(cjBean1.getId() + "@2")); + parentId = (String) json.get(JsonTags.SLA_SUMMARY_PARENT_ID); + assertTrue(parentId.equals(cjBean1.getId())); + assertEquals(1L, json.get(JsonTags.SLA_SUMMARY_START_DELAY)); + assertEquals(1L, json.get(JsonTags.SLA_SUMMARY_DURATION_DELAY)); + assertEquals(2L, json.get(JsonTags.SLA_SUMMARY_END_DELAY)); + + //test filter bundleName + Multiple EventStatus + queryParams.clear(); + queryParams.put(RestConstants.JOBS_FILTER_PARAM, + String.format("bundle=%s;event_status=START_MISS,END_MISS", bundleName)); + array = getSLAJSONResponse(queryParams); + assertEquals(3, array.size()); + + for(int i=0; i < array.size(); i++) { + json = (JSONObject) array.get(i); + id = (String)json.get(JsonTags.SLA_SUMMARY_ID); + assertTrue(id.equals(cjBean1.getId()+"@2") || id.equals(cjBean2.getId()+"@1") || id.equals(cjBean2.getId()+"@2")); + parentId = (String) json.get(JsonTags.SLA_SUMMARY_PARENT_ID); + assertTrue(parentId.equals(cjBean1.getId()) || parentId.equals(cjBean2.getId())); + } + + //test filter bundleName + Multiple EventStatus + Multiple SlaStus + queryParams.clear(); + queryParams.put(RestConstants.JOBS_FILTER_PARAM, + String.format("bundle=%s;event_status=DURATION_MISS;sla_status=IN_PROCESS", bundleName)); + array = getSLAJSONResponse(queryParams); + assertEquals(1, array.size()); + json = (JSONObject) array.get(0); + assertEquals(cjBean2.getId() + "@1", (String) json.get(JsonTags.SLA_SUMMARY_ID)); + assertEquals(cjBean2.getId(), (String) json.get(JsonTags.SLA_SUMMARY_PARENT_ID)); + String eventStatus = (String)json.get(JsonTags.SLA_SUMMARY_EVENT_STATUS); + assertTrue(eventStatus.contains("DURATION_MISS")); + assertTrue(eventStatus.contains("START_MISS")); + assertFalse(eventStatus.contains("END_MISS") || eventStatus.contains("END_MET")); + // actualDuration is null on DB while job is running, populates it in API call + assertEquals(9L, json.get(JsonTags.SLA_SUMMARY_ACTUAL_DURATION)); + assertEquals(1L, json.get(JsonTags.SLA_SUMMARY_DURATION_DELAY)); return null; } }); @@ -147,15 +311,13 @@ public class TestV2SLAServlet extends DagServletTestCase { assertEquals(actualEnd.getTimeInMillis(), json.get(JsonTags.SLA_SUMMARY_ACTUAL_END)); assertEquals(10L, json.get(JsonTags.SLA_SUMMARY_EXPECTED_DURATION)); assertEquals(15L, json.get(JsonTags.SLA_SUMMARY_ACTUAL_DURATION)); - assertEquals(currentTime.getTime(), json.get(JsonTags.SLA_SUMMARY_LAST_MODIFIED)); nominalTime.add(Calendar.HOUR, 1); } } private void insertEntriesIntoSLASummaryTable(int numEntries, String jobIDPrefix, String jobIDSuffix, - String parentId, Date startNominalTime, String appName, AppType appType, Date currentTime) - throws JPAExecutorException { - List<JsonBean> list = new ArrayList<JsonBean>(); + String parentId, Date startNominalTime, String appName, AppType appType, Date currentTime, + EventStatus eventStatus, SLAStatus slaStatus) throws JPAExecutorException { Calendar nominalTime = Calendar.getInstance(); nominalTime.setTime(startNominalTime); for (int i = 1; i <= numEntries; i++) { @@ -165,27 +327,34 @@ public class TestV2SLAServlet extends DagServletTestCase { expectedEnd.add(Calendar.MINUTE, 60); Calendar actualEnd = (Calendar) expectedEnd.clone(); actualEnd.add(Calendar.MINUTE, i); - SLASummaryBean bean = new SLASummaryBean(); - bean.setId(jobIDPrefix + i + jobIDSuffix); - bean.setParentId(parentId); - bean.setAppName(appName); - bean.setAppType(appType); - bean.setJobStatus("RUNNING"); - bean.setEventStatus(EventStatus.END_MISS); - bean.setSLAStatus(SLAStatus.IN_PROCESS); - bean.setNominalTime(nominalTime.getTime()); - bean.setExpectedStart(nominalTime.getTime()); - bean.setActualStart(actualStart.getTime()); - bean.setExpectedEnd(expectedEnd.getTime()); - bean.setActualEnd(actualEnd.getTime()); - bean.setExpectedDuration(10); - bean.setActualDuration(15); - bean.setUser("testuser"); - bean.setLastModifiedTime(currentTime); - list.add(bean); + insertEntriesIntoSLASummaryTable(jobIDPrefix + i + jobIDSuffix, parentId, appName, appType, eventStatus, + slaStatus, nominalTime.getTime(), actualStart.getTime(), ((long) 10), ((long) 15), + expectedEnd.getTime(), actualEnd.getTime(), nominalTime.getTime()); nominalTime.add(Calendar.HOUR, 1); } + } - BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(list, null, null); + private void insertEntriesIntoSLASummaryTable(String jobID, String parentId, String appName, AppType appType, + EventStatus eventStatus, SLAStatus slaStatus, Date expectedStartTime, Date actualStartTime, + long expectedDuration, long actualDuration, Date expectedEndTime, Date actualEndTime, Date nominalTime) + throws JPAExecutorException { + SLASummaryBean bean = new SLASummaryBean(); + bean.setId(jobID); + bean.setParentId(parentId); + bean.setAppName(appName); + bean.setAppType(appType); + bean.setJobStatus("RUNNING"); + bean.setEventStatus(eventStatus); + bean.setSLAStatus(slaStatus); + bean.setNominalTime(nominalTime); + bean.setExpectedStart(expectedStartTime); + bean.setActualStart(actualStartTime); + bean.setExpectedDuration(expectedDuration); + bean.setActualDuration(actualDuration); + bean.setExpectedEnd(expectedEndTime); + bean.setActualEnd(actualEndTime); + bean.setUser("testuser"); + bean.setLastModifiedTime(Calendar.getInstance().getTime()); + SLASummaryQueryExecutor.getInstance().insert(bean); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/1b1ef47a/docs/src/site/twiki/DG_SLAMonitoring.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_SLAMonitoring.twiki b/docs/src/site/twiki/DG_SLAMonitoring.twiki index 106ce8a..7915d84 100644 --- a/docs/src/site/twiki/DG_SLAMonitoring.twiki +++ b/docs/src/site/twiki/DG_SLAMonitoring.twiki @@ -165,6 +165,12 @@ In the REST API, the following filters can be applied while fetching SLA informa * id - id of the workflow job, workflow action or coordinator action * parent_id - Parent id of the workflow job, workflow action or coordinator action * nominal_start and nominal_end - Start and End range for nominal time of the workflow or coordinator. + * bundle - Bundle Job ID or Bundle App Name. Fetches SLA information for actions of all coordinators in that bundle. + * event_status - event stauts such as START_MET/START_MISS/DURATION_MET/DURATION_MISS/END_MET/END_MISS + * sla_status - sla status such as NOT_STARTED/IN_PROCESS/MET/MISS + +multiple event_status and sla_status can be specified with comma separation. When multiple statuses are specified, they are considered as OR. +For example, event_status=START_MET;END_MISS list the coordinator actions where event status is either START_MET OR END_MISS. When timezone query parameter is specified, the expected and actual start/end time returned is formatted. If not specified, the number of milliseconds that have elapsed since January 1, 1970 00:00:00.000 GMT is returned. @@ -187,7 +193,7 @@ GET <oozie-host>:<port>/oozie/v2/sla?timezone=GMT&filter=nominal_start=2013-06-1 msgType : "SLA" appName : "my-sla-app" slaStatus : "IN_PROCESS" - eventStatus : "START_MISS" + jobStatus : "RUNNING" user: "joe" nominalTime: "2013-16-22T05:00Z" expectedStartTime: "2013-16-22T05:10Z" <-- (should start by this time) @@ -218,7 +224,7 @@ GET <oozie-host>:<port>/oozie/v2/sla?timezone=GMT&filter=parent_id=000056-123879 msgType : "SLA" appName : "map-reduce-action" slaStatus : "MISS" - eventStatus : "END_MISS" + jobStatus : "SUCCEEDED" user: "joe" nominalTime: "2013-16-22T05:00Z" expectedStartTime: "2013-16-22T05:10Z" @@ -249,7 +255,7 @@ GET <oozie-host>:<port>/oozie/v2/sla?timezone=GMT&filter=id=000001-1238791320234 msgType : "SLA" appName : "my-coord-app" slaStatus : "MET" - eventStatus : "DURATION_MISS" + jobStatus : "SUCCEEDED" user: "joe" nominalTime: "2013-16-22T05:00Z" expectedStartTime: "2013-16-22T05:10Z" @@ -267,6 +273,59 @@ GET <oozie-host>:<port>/oozie/v2/sla?timezone=GMT&filter=id=000001-1238791320234 Scenario #3 is particularly interesting because it is an overall "MET" because it met its expected End-time, but it is "Duration_Miss" because the actual run (between actual start and actual end) exceeded expected duration. +---+++ Scenario 4: All Coordinator actions in a Bundle +*Request:* +<verbatim> +GET <oozie-host>:<port>/oozie/v2/sla?timezone=GMT&filter=bundle=1234567-150130225116604-oozie-B;event_status=END_MISS +</verbatim> + +*JSON Response* +<verbatim> +{ + id : "000001-1238791320234-oozie-joe-C@1" + parentId : "000001-1238791320234-oozie-joe-C" + appType : "COORDINATOR_ACTION" + msgType : "SLA" + appName : "my-coord-app" + slaStatus : "MET" + eventStatus : "START_MET,DURATION_MISS,END_MISS" + user: "joe" + nominalTime: "2014-01-10T12:00Z" + expectedStartTime: "2014-01-10T12:00Z" + actualStartTime: "2014-01-10T11:59Z" + startDelay: -1 + expectedEndTime: "2014-01-10T13:00Z" + actualEndTime: "2014-01-10T13:05Z" + endDelay: 5 + expectedDuration: 60 + actualDuration: 66 + durationDelay: 6 +} +{ + id : "000001-1238791320234-oozie-joe-C@2" + parentId : "000001-1238791320234-oozie-joe-C" + appType : "COORDINATOR_ACTION" + msgType : "SLA" + appName : "my-coord-app" + slaStatus : "MET" + eventStatus : "START_MISS,DURATION_MET,END_MISS" + user: "joe" + nominalTime: "2014-01-11T12:00Z" + expectedStartTime: "2014-01-11T12:00Z" + actualStartTime: "2014-01-11T12:05Z" + startDelay: 5 + expectedEndTime: "2014-01-11T13:00Z" + actualEndTime: "2014-01-11T13:01Z" + endDelay: 1 + expectedDuration: 60 + actualDuration: 56 + durationDelay: -4 +} +</verbatim> + +Scenario #4 (All Coordinator actions in a Bundle) is to get SLA information of all coordinator actions under bundle job in one call. +startDelay/durationDelay/endDelay values returned indicate how much delay compared to expected time (positive values in case of MISS, and negative values in case of MET). + ---+++ Sample Email Alert <verbatim> Subject: OOZIE - SLA END_MISS (AppName=wf-sla-job, JobID=0000004-130610225200680-oozie-oozi-W) http://git-wip-us.apache.org/repos/asf/oozie/blob/1b1ef47a/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 466fb04..dc753a4 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.2.0 release (trunk - unreleased) +OOZIE-2146 Add option to filter sla information by bundle id or name (ryota) OOZIE-2188 Fix typos in twiki documentation ( jacobtolar via puru) OOZIE-2174 Add missing admin commands to OozieClient and OozieCLI (rkanter) OOZIE-2186 Upgrade Tomcat to 6.0.43 (rkanter)
