OOZIE-3229 [client] [ui] Improved SLA filtering options (asalamon74, andras.piros)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/90a97694 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/90a97694 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/90a97694 Branch: refs/heads/master Commit: 90a97694b76208d6fa50aec34af1bd7e9c9580f8 Parents: f21074a Author: Andras Piros <[email protected]> Authored: Mon Sep 24 10:40:15 2018 +0200 Committer: Andras Piros <[email protected]> Committed: Mon Sep 24 10:40:15 2018 +0200 ---------------------------------------------------------------------- .../org/apache/oozie/client/OozieClient.java | 16 - .../apache/oozie/client/rest/RestConstants.java | 2 + .../java/org/apache/oozie/FilterParser.java | 73 ++ .../sla/SLASummaryGetForFilterJPAExecutor.java | 710 ++++++++++++------- .../org/apache/oozie/servlet/V2SLAServlet.java | 235 +++--- .../java/org/apache/oozie/TestFilterParser.java | 98 +++ ...GetForFilterJPAExecutorFilterCollection.java | 207 ++++++ .../apache/oozie/servlet/TestV2SLAServlet.java | 384 ---------- .../oozie/servlet/TestV2SLAServletBundle.java | 272 +++++++ .../servlet/TestV2SLAServletIntegration.java | 63 ++ .../TestV2SLAServletSLAJSONResponse.java | 307 ++++++++ .../oozie/servlet/V2SLAServletTestCase.java | 152 ++++ docs/src/site/markdown/DG_SLAMonitoring.md | 409 ++++++++++- release-log.txt | 1 + .../main/webapp/console/sla/css/oozie-sla.css | 2 +- .../src/main/webapp/console/sla/js/oozie-sla.js | 52 +- .../src/main/webapp/console/sla/oozie-sla.html | 48 +- 17 files changed, 2182 insertions(+), 849 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/90a97694/client/src/main/java/org/apache/oozie/client/OozieClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java index 949b453..2862d33 100644 --- a/client/src/main/java/org/apache/oozie/client/OozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java @@ -152,22 +152,6 @@ public class OozieClient { public static final String FILTER_APPNAME = "appname"; - public static final String FILTER_SLA_APPNAME = "app_name"; - - public static final String FILTER_SLA_ID = "id"; - - public static final String FILTER_SLA_PARENT_ID = "parent_id"; - - public static final String FILTER_BUNDLE = "bundle"; - - public static final String FILTER_SLA_EVENT_STATUS = "event_status"; - - public static final String FILTER_SLA_STATUS = "sla_status"; - - public static final String FILTER_SLA_NOMINAL_START = "nominal_start"; - - public static final String FILTER_SLA_NOMINAL_END = "nominal_end"; - public static final String FILTER_CREATED_TIME_START = "startcreatedtime"; public static final String FILTER_CREATED_TIME_END = "endcreatedtime"; http://git-wip-us.apache.org/repos/asf/oozie/blob/90a97694/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java index 9873ff3..1353786 100644 --- a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java +++ b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java @@ -51,6 +51,8 @@ public interface RestConstants { String ORDER_PARAM = "order"; + String SORTBY_PARAM = "sortby"; + String ACTION_NAME_PARAM = "action-name"; String JOB_FILTER_PARAM = "filter"; http://git-wip-us.apache.org/repos/asf/oozie/blob/90a97694/core/src/main/java/org/apache/oozie/FilterParser.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/FilterParser.java b/core/src/main/java/org/apache/oozie/FilterParser.java new file mode 100644 index 0000000..cd92a7c --- /dev/null +++ b/core/src/main/java/org/apache/oozie/FilterParser.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie; + +import com.google.common.base.Strings; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.ListMultimap; +import org.apache.oozie.servlet.XServletException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletResponse; +import java.util.StringTokenizer; + +public class FilterParser { + + private static final String PARAMETER_DELIMITER = ";"; + private static final String PARAMETER_EQUALS = "="; + + /** + * Parse filter string to a map with key = filter name and values = filter values + * + * @param filterString the filter string + * @return filter key and values map + * @throws ServletException thrown if failed to parse filter string + */ + public static ListMultimap<String, String> parseFilter(String filterString) throws ServletException { + ListMultimap<String, String> filterFieldMap = LinkedListMultimap.create(); + if (filterString != null) { + StringTokenizer st = new StringTokenizer(filterString, PARAMETER_DELIMITER); + while (st.hasMoreTokens()) { + String token = st.nextToken(); + if (token.contains(PARAMETER_EQUALS)) { + String[] nameValuePair = token.split(PARAMETER_EQUALS); + if (nameValuePair.length != 2) { + filterFormatError(filterString); + } + String key = nameValuePair[0]; + String value = nameValuePair[1]; + if (Strings.isNullOrEmpty(key)) { + filterFormatError(filterString); + } + filterFieldMap.put(key, value); + } + else { + filterFormatError(filterString); + } + } + } + return filterFieldMap; + } + + private static void filterFormatError(String filterString) throws ServletException { + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0401, + String.format("elements must be semicolon-separated name=value pairs but was %s", filterString)); + + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/90a97694/core/src/main/java/org/apache/oozie/executor/jpa/sla/SLASummaryGetForFilterJPAExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/executor/jpa/sla/SLASummaryGetForFilterJPAExecutor.java b/core/src/main/java/org/apache/oozie/executor/jpa/sla/SLASummaryGetForFilterJPAExecutor.java index b54161e..95c30fd 100644 --- a/core/src/main/java/org/apache/oozie/executor/jpa/sla/SLASummaryGetForFilterJPAExecutor.java +++ b/core/src/main/java/org/apache/oozie/executor/jpa/sla/SLASummaryGetForFilterJPAExecutor.java @@ -18,350 +18,526 @@ package org.apache.oozie.executor.jpa.sla; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import javax.persistence.EntityManager; -import javax.persistence.Query; - +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import org.apache.oozie.BundleActionBean; +import org.apache.oozie.BundleJobBean; import org.apache.oozie.ErrorCode; import org.apache.oozie.client.event.SLAEvent; import org.apache.oozie.client.event.SLAEvent.EventStatus; -import org.apache.oozie.client.event.SLAEvent.SLAStatus; import org.apache.oozie.executor.jpa.JPAExecutor; import org.apache.oozie.executor.jpa.JPAExecutorException; +import org.apache.oozie.service.UUIDService; +import org.apache.oozie.servlet.XServletException; import org.apache.oozie.sla.SLASummaryBean; +import org.apache.oozie.util.DateUtils; +import org.apache.oozie.util.Pair; import org.apache.oozie.util.XLog; +import javax.persistence.EntityManager; +import javax.persistence.TypedQuery; +import javax.persistence.criteria.CriteriaBuilder; +import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Order; +import javax.persistence.criteria.Path; +import javax.persistence.criteria.Predicate; +import javax.persistence.criteria.Root; +import javax.persistence.criteria.Subquery; +import javax.persistence.metamodel.Attribute; +import javax.persistence.metamodel.EntityType; +import javax.persistence.metamodel.Metamodel; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletResponse; +import java.sql.Timestamp; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + /** * Load the list of SLASummaryBean (for dashboard) and return the list. */ public class SLASummaryGetForFilterJPAExecutor implements JPAExecutor<List<SLASummaryBean>> { - private static final String selectStr = "SELECT OBJECT(s) FROM SLASummaryBean s WHERE "; - private static final String bundleIdQuery = "SELECT a.coordId FROM BundleActionBean a WHERE a.bundleId=:bundleId"; - private static final String bundleNameQuery = "SELECT a.coordId FROM BundleActionBean a WHERE a.bundleId in " - + "(SELECT b.id from BundleJobBean b WHERE b.appName=:appName)"; - private SLASummaryFilter filter; + private static final String DBFIELD_EXPECTED_START_TS = "expectedStartTS"; + private static final String DBFIELD_ACTUAL_START_TS = "actualStartTS"; + private static final String DBFIELD_EXPECTED_DURATION = "expectedDuration"; + private static final String DBFIELD_ACTUAL_DURATION = "actualDuration"; + private static final String DBFIELD_EXPECTED_END_TS = "expectedEndTS"; + private static final String DBFIELD_ACTUAL_END_TS = "actualEndTS"; + private static final String DBFIELD_EVENT_STATUS = "eventStatus"; + private static final String DBFIELD_CREATED_TIME_TS = "createdTimeTS"; + private static final String DBFIELD_NOMINAL_TIME_TS = "nominalTimeTS"; + private static final String DBFIELD_JOB_STATUS = "jobStatus"; + private static final String DBFIELD_USER = "user"; + private static final String DBFIELD_APP_TYPE = "appType"; + private static final String DBFIELD_APP_NAME = "appName"; + private static final String DBFIELD_SLA_STATUS = "slaStatus"; + private static final String DBFIELD_JOB_ID = "jobId"; + private static final String DBFIELD_PARENT_ID = "parentId"; + private static final String DBFIELD_BUNDLE_ID = "bundleId"; + private static final String DBFIELD_ID = "id"; + private static final String DBFIELD_COORD_ID = "coordId"; + private static final String DEFAULT_SORTBY_COLUMN = DBFIELD_NOMINAL_TIME_TS; + @VisibleForTesting + final FilterCollection filterCollection; private int numMaxResults; - private XLog LOG = XLog.getLog(getClass()); - - - public SLASummaryGetForFilterJPAExecutor(SLASummaryFilter filter, int numMaxResults) { - this.filter = filter; - this.numMaxResults = numMaxResults; - } - - @Override - public String getName() { - return "SLASummaryGetForFilterJPAExecutor"; + private List<String> possibleSortbyColumns; + private String sortbyColumn; + private boolean isDescendingOrder; + private static XLog LOG = XLog.getLog(SLASummaryGetForFilterJPAExecutor.class); + private CriteriaBuilder criteriaBuilder; + private CriteriaQuery<SLASummaryBean> criteriaQuery; + private Root<SLASummaryBean> root; + private List<Predicate> preds; + private static final String multiValueSeparator = ","; + /** + * Matches a bundle_id which has the ${padded_counter}-${startTime}-${systemId}-B format, + * where padded_counter is exactly 7 digits, startTime is exactly 15 digits and + * systemId is at most 10 characters like {@code 1234567-150130225116604-oozie-B} + * + * {@link UUIDService#generateId(UUIDService.ApplicationType)} + */ + private static final Pattern BUNDLE_ID_PATTERN = Pattern.compile("\\d{7}-\\d{15}-.{1,10}-B$"); + + private enum FilterComparator { + LIKE, EQUALS, GREATER_OR_EQUALS, LESSTHAN_OR_EQUALS, IN } - @SuppressWarnings("unchecked") - @Override - public List<SLASummaryBean> execute(EntityManager em) throws JPAExecutorException { - List<SLASummaryBean> ssBean = null; - StringBuilder sb = new StringBuilder(selectStr); - Map<String, Object> queryParams = new LinkedHashMap<String, Object>(); - boolean firstCondition = true; - boolean jobExists = true; - if (filter.getJobId() != null) { - firstCondition = false; - if (filter.getParentId() != null) { - sb.append("(s.jobId = :jobId OR s.parentId = :parentId)"); - queryParams.put("jobId", filter.getJobId()); - queryParams.put("parentId", filter.getParentId()); - } - else { - sb.append("s.jobId = :jobId"); - queryParams.put("jobId", filter.getJobId()); - } + private enum FilterField { + APP_NAME(String.class, DBFIELD_APP_NAME, FilterComparator.LIKE), + APP_TYPE(String.class, DBFIELD_APP_TYPE, FilterComparator.LIKE), + USER_NAME(String.class, DBFIELD_USER, FilterComparator.LIKE), + JOB_STATUS(String.class, DBFIELD_JOB_STATUS, FilterComparator.LIKE), + NOMINAL_START(Timestamp.class, DBFIELD_NOMINAL_TIME_TS, FilterComparator.GREATER_OR_EQUALS), + NOMINAL_END(Timestamp.class, DBFIELD_NOMINAL_TIME_TS, FilterComparator.LESSTHAN_OR_EQUALS), + NOMINAL_AFTER(Timestamp.class, DBFIELD_NOMINAL_TIME_TS, FilterComparator.GREATER_OR_EQUALS), + NOMINAL_BEFORE(Timestamp.class, DBFIELD_NOMINAL_TIME_TS, FilterComparator.LESSTHAN_OR_EQUALS), + CREATED_AFTER(Timestamp.class, DBFIELD_CREATED_TIME_TS, FilterComparator.GREATER_OR_EQUALS), + CREATED_BEFORE(Timestamp.class, DBFIELD_CREATED_TIME_TS, FilterComparator.LESSTHAN_OR_EQUALS), + EXPECTEDSTART_AFTER(Timestamp.class, DBFIELD_EXPECTED_START_TS, FilterComparator.GREATER_OR_EQUALS), + EXPECTEDSTART_BEFORE(Timestamp.class, DBFIELD_EXPECTED_START_TS, FilterComparator.LESSTHAN_OR_EQUALS), + EXPECTEDEND_AFTER(Timestamp.class, DBFIELD_EXPECTED_END_TS, FilterComparator.GREATER_OR_EQUALS), + EXPECTEDEND_BEFORE(Timestamp.class, DBFIELD_EXPECTED_END_TS, FilterComparator.LESSTHAN_OR_EQUALS), + ACTUALSTART_AFTER(Timestamp.class, DBFIELD_ACTUAL_START_TS, FilterComparator.GREATER_OR_EQUALS), + ACTUALSTART_BEFORE(Timestamp.class, DBFIELD_ACTUAL_START_TS, FilterComparator.LESSTHAN_OR_EQUALS), + ACTUALEND_AFTER(Timestamp.class, DBFIELD_ACTUAL_END_TS, FilterComparator.GREATER_OR_EQUALS), + ACTUALEND_BEFORE(Timestamp.class, DBFIELD_ACTUAL_END_TS, FilterComparator.LESSTHAN_OR_EQUALS), + ACTUAL_DURATION_MIN(Integer.class, DBFIELD_ACTUAL_DURATION, FilterComparator.GREATER_OR_EQUALS), + ACTUAL_DURATION_MAX(Integer.class, DBFIELD_ACTUAL_DURATION, FilterComparator.LESSTHAN_OR_EQUALS), + EXPECTED_DURATION_MIN(Integer.class, DBFIELD_EXPECTED_DURATION, FilterComparator.GREATER_OR_EQUALS), + EXPECTED_DURATION_MAX(Integer.class, DBFIELD_EXPECTED_DURATION, FilterComparator.LESSTHAN_OR_EQUALS), + SLA_STATUS(SLAEvent.SLAStatus.class, DBFIELD_SLA_STATUS, FilterComparator.IN), + EVENT_STATUS(SLAEvent.EventStatus.class, null, null), + ID(String.class, DBFIELD_JOB_ID, null), + PARENT_ID(String.class, DBFIELD_PARENT_ID, null), + BUNDLE(String.class, null, null); + + private final Class type; + private final String dbFieldName; + private final FilterComparator filterComparator; + private static final List<Pair<FilterField, FilterField>> intervalFieldPairs = + new ArrayList<Pair<FilterField, FilterField>>() {{ + add(new Pair<>(FilterField.NOMINAL_AFTER, FilterField.NOMINAL_BEFORE)); + add(new Pair<>(FilterField.CREATED_AFTER, FilterField.CREATED_BEFORE)); + add(new Pair<>(FilterField.EXPECTEDSTART_AFTER, FilterField.EXPECTEDSTART_BEFORE)); + add(new Pair<>(FilterField.EXPECTEDEND_AFTER, FilterField.EXPECTEDEND_BEFORE)); + add(new Pair<>(FilterField.ACTUALSTART_AFTER, FilterField.ACTUALSTART_BEFORE)); + add(new Pair<>(FilterField.ACTUALEND_AFTER, FilterField.ACTUALEND_BEFORE)); + add(new Pair<>(FilterField.ACTUAL_DURATION_MIN, FilterField.ACTUAL_DURATION_MAX)); + add(new Pair<>(FilterField.EXPECTED_DURATION_MIN, FilterField.EXPECTED_DURATION_MAX)); + }}; + + private static final List<Pair<FilterField, FilterField>> deprecatedFieldPairs = + new ArrayList<Pair<FilterField, FilterField>>() {{ + add(new Pair<>(FilterField.NOMINAL_START, FilterField.NOMINAL_AFTER)); + add(new Pair<>(FilterField.NOMINAL_END, FilterField.NOMINAL_BEFORE)); + }}; + + FilterField(final Class type, final String dbFieldName, final FilterComparator filterComparator) { + this.type = type; + this.dbFieldName = dbFieldName; + this.filterComparator = filterComparator; } - if (filter.getParentId() != null && filter.getJobId() == null) { - firstCondition = false; - sb.append("s.parentId = :parentId"); - queryParams.put("parentId", filter.getParentId()); - } - if (filter.getBundleId() != null || filter.getBundleName() != null) { - if (firstCondition) { - firstCondition = false; - } - else { - sb.append(" AND "); - } - Query bq; - List<Object> returnList; - try { - if (filter.getBundleId() != null) { - bq = em.createQuery(bundleIdQuery); - bq.setParameter("bundleId", filter.getBundleId()); - } - else { - bq = em.createQuery(bundleNameQuery); - bq.setParameter("appName", filter.getBundleName()); - } - bq.setMaxResults(numMaxResults); - returnList = (List<Object>) bq.getResultList(); + + private static Object convertFromString(final FilterField field, final String valueStr) throws ParseException { + if (String.class.equals(field.type)) { + return valueStr; } - catch (Exception e) { - throw new JPAExecutorException(ErrorCode.E0603, e.getMessage(), e); + else if (Timestamp.class.equals(field.type)) { + Date date = DateUtils.parseDateUTC(valueStr); + return new Timestamp(date.getTime()); } - StringBuilder sub = null; - int ind = 0; - if(returnList.size() == 0) { - jobExists = false; + else if (Integer.class.equals(field.type)) { + return Integer.parseInt(valueStr); } - for (Object obj : returnList) { - String coordId = (String) obj; - if (sub == null) { - sub = new StringBuilder(); - sub.append("s.parentId in (:parentId").append(ind); - } - else { - sub.append(",:parentId").append(ind); + else if (field.type.isEnum()) { + List<String> list = new ArrayList<>(); + String[] statusArr = valueStr.split(multiValueSeparator); + for (String s : statusArr) { + list.add(Enum.valueOf(field.type, s).toString()); } - queryParams.put("parentId" + ind, coordId); - ind++; - } - if(sub != null) { - sub.append(")"); - sb.append(sub.toString()); - } - } - if (filter.getAppName() != null) { - if (firstCondition ){ - firstCondition = false; - } else { - sb.append(" AND "); - } - sb.append("s.appName = :appName"); - queryParams.put("appName", filter.getAppName()); - } - if (filter.getNominalStart() != null) { - if (firstCondition) { - firstCondition = false; - } - else { - sb.append(" AND "); + return list; } - sb.append("s.nominalTimeTS >= :nominalTimeStart"); - queryParams.put("nominalTimeStart", new Timestamp(filter.getNominalStart().getTime())); + return null; } - if (filter.getNominalEnd() != null) { - if (firstCondition) { - firstCondition = false; + private static FilterField findByName(String name) { + if (name==null || !name.equals(name.toLowerCase(Locale.US))) { + return null; } - else { - sb.append(" AND "); + try { + return FilterField.valueOf(name.toUpperCase(Locale.US)); + } catch (IllegalArgumentException e) { + return null; } - sb.append("s.nominalTimeTS <= :nominalTimeEnd"); - queryParams.put("nominalTimeEnd", new Timestamp(filter.getNominalEnd().getTime())); } - if (filter.getEventStatus() != null) { - processEventStatusFilter(filter, queryParams,sb,firstCondition); + private String getColumnName() { + return name().toLowerCase(Locale.US); } + } + + @VisibleForTesting + static class FilterCollection { + private final Map<FilterField, Object> filterValues = new LinkedHashMap<>(); - if (filter.getSLAStatus() != null) { - StringBuilder sub = null; - int ind = 0; - if (firstCondition) { - firstCondition = false; + @VisibleForTesting + void checkAndSetFilterField(String name, String value) throws ServletException, ParseException { + FilterField field = FilterField.findByName(name); + if (field == null) { + throwNewXServletException(ErrorCode.E0401, String.format("Invalid/unsupported names in filter: %s", name)); } else { - sb.append(" AND "); + validateAndSetFilterField(findNonDeprecatedFilterField(field), FilterField.convertFromString(field, value)); } - for (SLAStatus status : filter.getSLAStatus()) { - if (sub == null) { - sub = new StringBuilder(); - sub.append("s.slaStatus in (:slaStatus").append(ind); - } - else { - sub.append(",:slaStatus").append(ind); + } + + @VisibleForTesting + FilterField findNonDeprecatedFilterField(FilterField maybeDeprecatedFilterField) { + for (Pair<FilterField, FilterField> pair : FilterField.deprecatedFieldPairs) { + if (pair.getFirst().equals(maybeDeprecatedFilterField)) { + return pair.getSecond(); } - queryParams.put("slaStatus" + ind, status.toString()); - ind++; - } - if(sub != null) { - sub.append(")"); - sb.append(sub.toString()); } + return maybeDeprecatedFilterField; } - if (jobExists) { - sb.append(" ORDER BY s.nominalTimeTS"); - LOG.debug("Query String: " + sb.toString()); - try { - Query q = em.createQuery(sb.toString()); - for (Map.Entry<String, Object> entry : queryParams.entrySet()) { - q.setParameter(entry.getKey(), entry.getValue()); + private void validateAndSetFilterField(FilterField field, Object value) throws ServletException { + boolean fieldConstraintViolated; + for (Pair<FilterField, FilterField> pair : FilterField.intervalFieldPairs) { + String firstColumnName = pair.getFirst().getColumnName(); + String secondColumnName = pair.getSecond().getColumnName(); + if (pair.getFirst().equals(field)) { + fieldConstraintViolated = checkFieldConstrantViolation((Comparable)value, + (Comparable)getFilterField(secondColumnName)); + } + else if (pair.getSecond().equals(field)) { + fieldConstraintViolated = checkFieldConstrantViolation((Comparable)getFilterField(firstColumnName), + (Comparable)value); + } + else { + fieldConstraintViolated = false; + } + if (fieldConstraintViolated) { + String errorMessage = String.format("should be: field %s <= field %s", firstColumnName, secondColumnName); + throwNewXServletException(ErrorCode.E0302, errorMessage); } - q.setMaxResults(numMaxResults); - ssBean = (List<SLASummaryBean>) q.getResultList(); - } - catch (Exception e) { - throw new JPAExecutorException(ErrorCode.E0603, e.getMessage(), e); } + filterValues.put(field, value); } - return ssBean; - } - private void processEventStatusFilter(SLASummaryFilter filter, Map<String, Object> queryParams, StringBuilder sb, - boolean firstCondition) { - if (firstCondition) { - firstCondition = false; + private void throwNewXServletException(ErrorCode errorCode, String message) throws ServletException { + LOG.error(message); + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, errorCode, message); } - else { - sb.append(" AND "); + + private boolean checkFieldConstrantViolation(Comparable minValue, Comparable maxValue) { + return minValue != null && maxValue != null && minValue.compareTo(maxValue) > 0; } - List<EventStatus> eventStatusList = filter.getEventStatus(); - int ind = 0; - Timestamp currentTime = new Timestamp(new Date().getTime()); - for (EventStatus status : eventStatusList) { - if (ind > 0) { - sb.append(" OR "); - } - if (status.equals(EventStatus.START_MET)) { - sb.append("(s.expectedStartTS IS NOT NULL AND s.actualStartTS IS NOT NULL ").append( - " AND s.expectedStartTS >= s.actualStartTS)"); - } - else if (status.equals(EventStatus.START_MISS)) { - sb.append("((s.expectedStartTS IS NOT NULL AND s.actualStartTS IS NOT NULL ") - .append(" AND s.expectedStartTS <= s.actualStartTS) ") - .append("OR (s.expectedStartTS IS NOT NULL AND s.actualStartTS IS NULL ") - .append(" AND s.expectedStartTS <= :currentTimeStamp))"); - queryParams.put("currentTimeStamp",currentTime); - } - else if (status.equals(EventStatus.DURATION_MET)) { - sb.append("(s.expectedDuration <> -1 AND s.actualDuration <> -1 ").append( - " AND s.expectedDuration >= s.actualDuration) "); - } - else if (status.equals(EventStatus.DURATION_MISS)) { - sb.append("((s.expectedDuration <> -1 AND s.actualDuration <> -1 ") - .append("AND s.expectedDuration < s.actualDuration) ") - .append("OR s.eventStatus = 'DURATION_MISS')"); + @VisibleForTesting + Object getFilterField(String name) { + FilterField field = FilterField.findByName(name); + if (field != null) { + return filterValues.get(field); } - else if (status.equals(EventStatus.END_MET)) { - sb.append("(s.expectedEndTS IS NOT NULL AND s.actualEndTS IS NOT NULL ").append( - " AND s.expectedEndTS <= s.actualEndTS) "); - } - else if (status.equals(EventStatus.END_MISS)) { - sb.append("((s.expectedEndTS IS NOT NULL AND s.actualEndTS IS NOT NULL ") - .append("AND s.expectedEndTS <= s.actualEndTS) ") - .append("OR (s.expectedEndTS IS NOT NULL AND s.actualEndTS IS NULL ") - .append("AND s.expectedEndTS <= :currentTimeStamp))"); - queryParams.put("currentTimeStamp",currentTime); + else { + return null; } - ind++; } } - public static class SLASummaryFilter { - - private String appName; - private String jobId; - private String parentId; - private String bundleId; - private String bundleName; - private List<SLAEvent.EventStatus> eventStatus; - private List<SLAEvent.SLAStatus> slaStatus; - private static String EventStatusSep = ","; - private static String SLAStatusSep = ","; - private Date nominalStart; - private Date nominalEnd; - - public SLASummaryFilter() { - } + public SLASummaryGetForFilterJPAExecutor(int numMaxResults) { + this.filterCollection = new FilterCollection(); + this.numMaxResults = numMaxResults; + } - public String getAppName() { - return appName; - } + @Override + public String getName() { + return SLASummaryGetForFilterJPAExecutor.class.getSimpleName(); + } - public void setAppName(String appName) { - this.appName = appName; - } + @SuppressWarnings("unchecked") + @Override + public List<SLASummaryBean> execute(EntityManager em) throws JPAExecutorException { + initPossibleSortbyColumnList(em); + createCriteriaQuery(em); + TypedQuery<SLASummaryBean> typedQuery = em.createQuery(criteriaQuery); + typedQuery.setMaxResults(numMaxResults); + LOG.debug("Query string: {0}",typedQuery.unwrap(org.apache.openjpa.persistence.QueryImpl.class).getQueryString()); + return typedQuery.getResultList(); + } - public String getJobId() { - return jobId; + private void initPossibleSortbyColumnList(EntityManager em) { + Metamodel metamodel = em.getMetamodel(); + EntityType<SLASummaryBean> slaSummaryBeanEntityType = metamodel.entity(SLASummaryBean.class); + Set<Attribute<SLASummaryBean,?>> slaSummaryBeanAttributes = slaSummaryBeanEntityType.getDeclaredAttributes(); + possibleSortbyColumns = new ArrayList<>(); + for (Attribute<SLASummaryBean,?> attribute : slaSummaryBeanAttributes) { + possibleSortbyColumns.add(attribute.getName()); } + } - public void setJobId(String jobId) { - this.jobId = jobId; - } + private void createCriteriaQuery(final EntityManager em) throws JPAExecutorException { + ensureCriteriaFields(em); + createSelectFrom(); + createWhereCondition(); + createOrderByClause(); + } - public String getParentId() { - return parentId; + private void createOrderByClause() throws JPAExecutorException { + if (Strings.isNullOrEmpty(sortbyColumn)) { + sortbyColumn = DEFAULT_SORTBY_COLUMN; } - - public void setParentId(String parentId) { - this.parentId = parentId; + if (!possibleSortbyColumns.contains(sortbyColumn)) { + String errorMessage = String.format("invalid sortby column: %s", sortbyColumn); + LOG.error(errorMessage); + throw new JPAExecutorException(ErrorCode.E0303, errorMessage); } - public Date getNominalStart() { - return nominalStart; + Path<Object> sortbyColumnPath = root.get(sortbyColumn); + Path<Object> nominalTimeTSPath = root.get(DBFIELD_NOMINAL_TIME_TS); + List<Order> orderList = new ArrayList<>(); + if (isDescendingOrder) { + orderList.add(criteriaBuilder.desc(sortbyColumnPath)); } - - public void setNominalStart(Date nominalStart) { - this.nominalStart = nominalStart; + else { + orderList.add(criteriaBuilder.asc(sortbyColumnPath)); } - - public Date getNominalEnd() { - return nominalEnd; + if (!DBFIELD_NOMINAL_TIME_TS.equals(sortbyColumn)) { + orderList.add(criteriaBuilder.asc(nominalTimeTSPath)); } + criteriaQuery.orderBy(orderList); + } - public void setNominalEnd(Date nominalEnd) { - this.nominalEnd = nominalEnd; - } + private void ensureCriteriaFields(EntityManager em) { + criteriaBuilder = em.getCriteriaBuilder(); + criteriaQuery = criteriaBuilder.createQuery(SLASummaryBean.class); + } - public String getBundleId(){ - return this.bundleId; - } + private void createSelectFrom() { + root = criteriaQuery.from(SLASummaryBean.class); + criteriaQuery.select(root); + } - public void setBundleId(String bundleId) { - this.bundleId = bundleId; + private void createWhereCondition() throws JPAExecutorException { + preds = new ArrayList<>(); + for (Map.Entry<FilterField, Object> entry : filterCollection.filterValues.entrySet()) { + FilterField filterField = entry.getKey(); + Object value = entry.getValue(); + if (filterField.filterComparator != null) { + switch (filterField.filterComparator) { + case LIKE: + preds.add(criteriaBuilder.like(root.<String>get(filterField.dbFieldName), (String)value)); + break; + case EQUALS: + preds.add(criteriaBuilder.equal(root.get(filterField.dbFieldName), value)); + break; + case GREATER_OR_EQUALS: + preds.add(criteriaBuilder.greaterThanOrEqualTo(root.get(filterField.dbFieldName), (Comparable)value)); + break; + case LESSTHAN_OR_EQUALS: + preds.add(criteriaBuilder.lessThanOrEqualTo(root.get(filterField.dbFieldName), (Comparable)value)); + break; + case IN: + preds.add(root.get(filterField.dbFieldName).in((List<String>)value)); + break; + } + } } + createAndAddSpecialCriterias(); + criteriaQuery.where(preds.toArray(new Predicate[0])); + } - public String getBundleName(){ - return this.bundleName; - } + private void createAndAddSpecialCriterias() throws JPAExecutorException { + createAndAddIdAndParentIdCriteria(); + createAndAddBundleFilterCriteria(); + createAndAddEventStatusCriteria(); + } - public void setBundleName(String name){ - this.bundleName = name; - } + private void createAndAddIdAndParentIdCriteria() { + String jobId = (String) getFilterField(DBFIELD_ID); + String parentId = (String) getFilterField( FilterField.PARENT_ID.getColumnName()); - public List<EventStatus> getEventStatus() { - return this.eventStatus; + if (jobId != null && parentId != null) { + preds.add(criteriaBuilder.or( + criteriaBuilder.equal(root.get(FilterField.ID.dbFieldName), jobId), + criteriaBuilder.equal(root.get(FilterField.PARENT_ID.dbFieldName), parentId) + )); + } + else if (jobId != null && parentId == null) { + preds.add(criteriaBuilder.equal(root.get(FilterField.ID.dbFieldName), jobId)); + } + else if (jobId == null && parentId != null) { + preds.add(criteriaBuilder.equal(root.get(FilterField.PARENT_ID.dbFieldName), parentId)); } + } - public void setEventStatus(String str) { - if (this.eventStatus == null) { - this.eventStatus = new ArrayList<EventStatus>(); - } - String[] statusArr = str.split(EventStatusSep); - for (String s : statusArr) { - this.eventStatus.add(SLAEvent.EventStatus.valueOf(s)); - } + private void createAndAddBundleFilterCriteria() { + String bundle = (String) getFilterField(FilterField.BUNDLE.getColumnName()); + + if (bundle == null) { + return; } + Subquery<BundleActionBean> subquery = criteriaQuery.subquery(BundleActionBean.class); + Root<BundleJobBean> subJobBeanRoot = subquery.from(BundleJobBean.class); + Root<BundleActionBean> subActionBeanRoot = subquery.from(BundleActionBean.class); + subquery.select(subActionBeanRoot.get(DBFIELD_COORD_ID)); + Predicate bundleJoinPredicate = criteriaBuilder.equal(subActionBeanRoot.get(DBFIELD_BUNDLE_ID), + subJobBeanRoot.get(DBFIELD_ID)); + if (isBundleId(bundle)) { + subquery.where(bundleJoinPredicate, criteriaBuilder.equal(subActionBeanRoot.get(DBFIELD_BUNDLE_ID), bundle)); + } + else { + subquery.where(bundleJoinPredicate, criteriaBuilder.equal(subJobBeanRoot.get(DBFIELD_APP_NAME), bundle)); + } + preds.add(criteriaBuilder.in(root.get(DBFIELD_PARENT_ID)).value(subquery)); + } - public List<SLAStatus> getSLAStatus() { - return this.slaStatus; + private void createAndAddEventStatusCriteria() throws JPAExecutorException { + String eventStatusFilterFieldName = FilterField.EVENT_STATUS.getColumnName(); + List<String> eventStatusFilterValues = (List<String>)getFilterField(eventStatusFilterFieldName); + if (eventStatusFilterValues != null) { + List<Predicate> eventStatusPreds = new ArrayList<>(); + for (String statusStr : eventStatusFilterValues) { + EventStatus status; + try { + status = SLAEvent.EventStatus.valueOf(statusStr); + } + catch (IllegalArgumentException e) { + throw new JPAExecutorException(ErrorCode.E0303, eventStatusFilterFieldName, statusStr); + } + eventStatusPreds.addAll(EventStatusFilter.createFilterConditionForEventStatus(status, criteriaBuilder, root)); + } + addEventStatusCriteria(eventStatusPreds); } + } - public void setSLAStatus(String str) { - if (this.slaStatus == null) { - this.slaStatus = new ArrayList<SLAStatus>(); + private enum EventStatusFilter { + START_MET_FILTER { + public List<Predicate> createFilterCondition(CriteriaBuilder criteriaBuilder, Root<SLASummaryBean> root) { + return Collections.singletonList(criteriaBuilder.and( + criteriaBuilder.isNotNull(root.get(DBFIELD_EXPECTED_START_TS)), + criteriaBuilder.isNotNull(root.get(DBFIELD_ACTUAL_START_TS)), + criteriaBuilder.ge(root.get(DBFIELD_EXPECTED_START_TS), root.get(DBFIELD_ACTUAL_START_TS)))); + }}, + START_MISS_FILTER { + public List<Predicate> createFilterCondition(CriteriaBuilder criteriaBuilder, Root<SLASummaryBean> root) { + Timestamp currentTime = new Timestamp(new Date().getTime()); + return Arrays.asList(criteriaBuilder.and( + criteriaBuilder.isNotNull(root.get(DBFIELD_EXPECTED_START_TS)), + criteriaBuilder.isNotNull(root.get(DBFIELD_ACTUAL_START_TS)), + criteriaBuilder.lessThanOrEqualTo(root.get(DBFIELD_EXPECTED_START_TS), + root.get(DBFIELD_ACTUAL_START_TS))), + criteriaBuilder.and( + criteriaBuilder.isNotNull(root.get(DBFIELD_EXPECTED_START_TS)), + criteriaBuilder.isNull(root.get(DBFIELD_ACTUAL_START_TS)), + criteriaBuilder.lessThanOrEqualTo(root.get(DBFIELD_EXPECTED_START_TS), currentTime))); + }}, + DURATION_MET_FILTER { + public List<Predicate> createFilterCondition(CriteriaBuilder criteriaBuilder, Root<SLASummaryBean> root) { + return Collections.singletonList(criteriaBuilder.and( + criteriaBuilder.notEqual(root.get(DBFIELD_EXPECTED_DURATION), -1), + criteriaBuilder.notEqual(root.get(DBFIELD_ACTUAL_DURATION), -1), + criteriaBuilder.ge(root.get(DBFIELD_EXPECTED_DURATION), root.get(DBFIELD_ACTUAL_DURATION)))); + }}, + DURATION_MISS_FILTER { + public List<Predicate> createFilterCondition(CriteriaBuilder criteriaBuilder, Root<SLASummaryBean> root) { + return Arrays.asList(criteriaBuilder.and( + criteriaBuilder.notEqual(root.get(DBFIELD_EXPECTED_DURATION), -1), + criteriaBuilder.notEqual(root.get(DBFIELD_ACTUAL_DURATION), -1), + criteriaBuilder.lessThan(root.get(DBFIELD_EXPECTED_DURATION), root.get(DBFIELD_ACTUAL_DURATION))), + criteriaBuilder.equal(root.get(DBFIELD_EVENT_STATUS), EventStatus.DURATION_MISS.name()) + ); + }}, + END_MET_FILTER { + public List<Predicate> createFilterCondition(CriteriaBuilder criteriaBuilder, Root<SLASummaryBean> root) { + return Collections.singletonList(criteriaBuilder.and( + criteriaBuilder.isNotNull(root.get(DBFIELD_EXPECTED_END_TS)), + criteriaBuilder.isNotNull(root.get(DBFIELD_ACTUAL_END_TS)), + criteriaBuilder.greaterThanOrEqualTo(root.get(DBFIELD_EXPECTED_END_TS), + root.get(DBFIELD_ACTUAL_END_TS)))); + }}, + END_MISS_FILTER { + public List<Predicate> createFilterCondition(CriteriaBuilder criteriaBuilder, Root<SLASummaryBean> root) { + Timestamp currentTime = new Timestamp(new Date().getTime()); + return Arrays.asList(criteriaBuilder.and( + criteriaBuilder.isNotNull(root.get(DBFIELD_EXPECTED_END_TS)), + criteriaBuilder.isNotNull(root.get(DBFIELD_ACTUAL_END_TS)), + criteriaBuilder.lessThanOrEqualTo(root.get(DBFIELD_EXPECTED_END_TS), + root.get(DBFIELD_ACTUAL_END_TS))), + criteriaBuilder.and( + criteriaBuilder.isNotNull(root.get(DBFIELD_EXPECTED_END_TS)), + criteriaBuilder.isNull(root.get(DBFIELD_ACTUAL_END_TS)), + criteriaBuilder.lessThanOrEqualTo(root.get(DBFIELD_EXPECTED_END_TS), currentTime) + )); + }}; + + abstract List<Predicate> createFilterCondition(CriteriaBuilder criteriaBuilder, Root<SLASummaryBean> root); + + private static List<Predicate> createFilterConditionForEventStatus(EventStatus eventStatus, + CriteriaBuilder criteriaBuilder, + Root<SLASummaryBean> root) throws JPAExecutorException { + try { + EventStatusFilter eventStatusFilter = EventStatusFilter.valueOf(eventStatus.name() + "_FILTER"); + return eventStatusFilter.createFilterCondition(criteriaBuilder, root); } - String[] statusArr = str.split(SLAStatusSep); - for (String s : statusArr) { - this.slaStatus.add(SLAEvent.SLAStatus.valueOf(s)); + catch (IllegalArgumentException e) { + throw new JPAExecutorException(ErrorCode.E0303, FilterField.EVENT_STATUS.getColumnName(), eventStatus.name()); } } } + private void addEventStatusCriteria(List<Predicate> eventStatusPreds) { + preds.add(criteriaBuilder.or(eventStatusPreds.toArray(new Predicate[0]))); + } + + public void checkAndSetFilterField(String name, String value) throws ServletException, ParseException { + filterCollection.checkAndSetFilterField(name, value); + } + + public void setDescendingOrder(boolean isDescendingOrder) { + this.isDescendingOrder = isDescendingOrder; + } + + public void setSortbyColumn(String sortbyColumn) { + this.sortbyColumn = sortbyColumn; + } + + @VisibleForTesting + Object getFilterField(String name) { + return filterCollection.getFilterField(name); + } + + private boolean isBundleId(String id) { + return BUNDLE_ID_PATTERN.matcher(id).matches(); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/90a97694/core/src/main/java/org/apache/oozie/servlet/V2SLAServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V2SLAServlet.java b/core/src/main/java/org/apache/oozie/servlet/V2SLAServlet.java index 3982d1e..2d0ab65 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V2SLAServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V2SLAServlet.java @@ -18,69 +18,49 @@ package org.apache.oozie.servlet; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Calendar; -import java.util.Collections; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.ListMultimap; import org.apache.oozie.ErrorCode; +import org.apache.oozie.FilterParser; import org.apache.oozie.XException; -import org.apache.oozie.client.OozieClient; -import org.apache.oozie.client.event.SLAEvent.EventStatus; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.command.CommandException; +import org.apache.oozie.executor.jpa.JPAExecutorException; import org.apache.oozie.executor.jpa.SLARegistrationQueryExecutor; import org.apache.oozie.executor.jpa.SLARegistrationQueryExecutor.SLARegQuery; import org.apache.oozie.executor.jpa.sla.SLASummaryGetForFilterJPAExecutor; -import org.apache.oozie.executor.jpa.sla.SLASummaryGetForFilterJPAExecutor.SLASummaryFilter; import org.apache.oozie.service.JPAService; import org.apache.oozie.service.Services; import org.apache.oozie.sla.SLARegistrationBean; import org.apache.oozie.sla.SLASummaryBean; -import org.apache.oozie.util.DateUtils; import org.apache.oozie.util.XLog; import org.json.simple.JSONObject; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + @SuppressWarnings("serial") public class V2SLAServlet extends SLAServlet { private static final String INSTRUMENTATION_NAME = "v2sla"; private static final JsonRestServlet.ResourceInfo RESOURCES_INFO[] = new JsonRestServlet.ResourceInfo[1]; - private static final Set<String> SLA_FILTER_NAMES = new HashSet<String>(); - private Pattern p = Pattern.compile("\\d{7}-\\d{15}-.*-B$"); - - static { - SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_ID); - SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_PARENT_ID); - SLA_FILTER_NAMES.add(OozieClient.FILTER_BUNDLE); - SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_APPNAME); - SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_NOMINAL_START); - SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_NOMINAL_END); - SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_EVENT_STATUS); - SLA_FILTER_NAMES.add(OozieClient.FILTER_SLA_STATUS); - } static { RESOURCES_INFO[0] = new JsonRestServlet.ResourceInfo("", Arrays.asList("GET"), Arrays.asList(new JsonRestServlet.ParameterInfo(RestConstants.JOBS_FILTER_PARAM, String.class, false, - Arrays.asList("GET")))); + Arrays.asList("GET")), new JsonRestServlet.ParameterInfo(RestConstants.ORDER_PARAM, String.class, false, + Arrays.asList("GET")), new JsonRestServlet.ParameterInfo(RestConstants.SORTBY_PARAM, String.class, + false, Arrays.asList("GET")) + )); } public V2SLAServlet() { @@ -90,17 +70,12 @@ public class V2SLAServlet extends SLAServlet { @Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - XLog.getLog(getClass()).debug("Got SLA GET request:" + request.getQueryString()); + XLog.getLog(getClass()).debug("Got SLA GET request: {0}", request.getQueryString()); try { stopCron(); - JSONObject json = getSLASummaryList(request, response); + JSONObject json = getSLASummaryList(request); startCron(); - if (json == null) { - response.setStatus(HttpServletResponse.SC_OK); - } - else { - sendJsonResponse(response, HttpServletResponse.SC_OK, json); - } + sendJsonResponse(response, HttpServletResponse.SC_OK, json); } catch (CommandException ce) { XLog.getLog(getClass()).error("Command exception ", ce); @@ -112,119 +87,103 @@ public class V2SLAServlet extends SLAServlet { } } - private JSONObject getSLASummaryList(HttpServletRequest request, HttpServletResponse response) - throws ServletException, CommandException { - String timeZoneId = request.getParameter(RestConstants.TIME_ZONE_PARAM) == null ? null : request - .getParameter(RestConstants.TIME_ZONE_PARAM); + private JSONObject getSLASummaryList(final HttpServletRequest request) throws ServletException, CommandException { + String timeZoneId = request.getParameter(RestConstants.TIME_ZONE_PARAM); String filterString = request.getParameter(RestConstants.JOBS_FILTER_PARAM); + String orderString = request.getParameter(RestConstants.ORDER_PARAM); + String sortbyString = request.getParameter(RestConstants.SORTBY_PARAM); String maxResults = request.getParameter(RestConstants.LEN_PARAM); int numMaxResults = 1000; // Default + boolean isDescendingOrder = false; // Default if (maxResults != null) { numMaxResults = Integer.parseInt(maxResults); } - if (filterString == null || filterString.equals("")) { + if (Strings.isNullOrEmpty(filterString)) { throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0305, RestConstants.JOBS_FILTER_PARAM); } - try { - Map<String, List<String>> filterList = parseFilter(URLDecoder.decode(filterString, "UTF-8"), SLA_FILTER_NAMES); - SLASummaryFilter filter = new SLASummaryFilter(); - - if (!filterList.containsKey(OozieClient.FILTER_SLA_APPNAME) - && !filterList.containsKey(OozieClient.FILTER_SLA_ID) - && !filterList.containsKey(OozieClient.FILTER_SLA_PARENT_ID) - && !filterList.containsKey(OozieClient.FILTER_BUNDLE) - && !filterList.containsKey(OozieClient.FILTER_SLA_NOMINAL_START) - && !filterList.containsKey(OozieClient.FILTER_SLA_NOMINAL_END)) { - StringBuffer st = new StringBuffer(); - st.append("At least one of the filter parameters - ").append(OozieClient.FILTER_SLA_APPNAME) - .append(",").append(OozieClient.FILTER_SLA_ID).append(",") - .append(OozieClient.FILTER_SLA_PARENT_ID).append(",").append(OozieClient.FILTER_BUNDLE) - .append(",").append(OozieClient.FILTER_SLA_NOMINAL_START).append(" or ") - .append(OozieClient.FILTER_SLA_NOMINAL_END) - .append(" should be specified in the filter query parameter"); - throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0305, st.toString()); - } - - if (filterList.containsKey(OozieClient.FILTER_SLA_ID)) { - filter.setJobId(filterList.get(OozieClient.FILTER_SLA_ID).get(0)); - } - if (filterList.containsKey(OozieClient.FILTER_SLA_PARENT_ID)) { - filter.setParentId(filterList.get(OozieClient.FILTER_SLA_PARENT_ID).get(0)); - } - if (filterList.containsKey(OozieClient.FILTER_BUNDLE)) { - String bundle = filterList.get(OozieClient.FILTER_BUNDLE).get(0); - if (isBundleId(bundle)) { - filter.setBundleId(bundle); - } - else { - filter.setBundleName(bundle); - } - } - if (filterList.containsKey(OozieClient.FILTER_SLA_EVENT_STATUS)) { - filter.setEventStatus(filterList.get(OozieClient.FILTER_SLA_EVENT_STATUS).get(0)); - } - if (filterList.containsKey(OozieClient.FILTER_SLA_STATUS)) { - filter.setSLAStatus(filterList.get(OozieClient.FILTER_SLA_STATUS).get(0)); - } - if (filterList.containsKey(OozieClient.FILTER_SLA_APPNAME)) { - filter.setAppName(filterList.get(OozieClient.FILTER_SLA_APPNAME).get(0)); - } - if (filterList.containsKey(OozieClient.FILTER_SLA_NOMINAL_START)) { - filter.setNominalStart(DateUtils.parseDateUTC(filterList.get(OozieClient.FILTER_SLA_NOMINAL_START).get(0))); - } - if (filterList.containsKey(OozieClient.FILTER_SLA_NOMINAL_END)) { - filter.setNominalEnd(DateUtils.parseDateUTC(filterList.get(OozieClient.FILTER_SLA_NOMINAL_END).get(0))); - } - - JPAService jpaService = Services.get().get(JPAService.class); - List<SLASummaryBean> slaSummaryList = null; - if (jpaService != null) { - slaSummaryList = jpaService.execute(new SLASummaryGetForFilterJPAExecutor(filter, numMaxResults)); - } - else { - XLog.getLog(getClass()).error(ErrorCode.E0610); - } - - List<String> jobIds = new ArrayList<String>(); - if (slaSummaryList != null) { - for (SLASummaryBean summaryBean : slaSummaryList) { - jobIds.add(summaryBean.getId()); - } - } - List<SLARegistrationBean> SLARegistrationList = SLARegistrationQueryExecutor.getInstance().getList( - SLARegQuery.GET_SLA_CONFIGS, jobIds); - - Map<String, Map<String, String>> jobIdSLAConfigMap = new HashMap<String, Map<String, String>>(); - for(SLARegistrationBean registrationBean:SLARegistrationList){ - jobIdSLAConfigMap.put(registrationBean.getId(), registrationBean.getSLAConfigMap()); - } + if (!Strings.isNullOrEmpty(orderString)) { + isDescendingOrder = getOrder(orderString); + } - return SLASummaryBean.toJSONObject(slaSummaryList, jobIdSLAConfigMap, timeZoneId); + try { + ListMultimap<String, String> filterParams = FilterParser.parseFilter(filterString); + return getSLASummaryListByFilterParams(timeZoneId, numMaxResults, filterParams, sortbyString, isDescendingOrder); } catch (XException ex) { throw new CommandException(ex); } - catch (UnsupportedEncodingException e) { - throw new XServletException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, ErrorCode.E0307, - "Unsupported Encoding", e); - } - catch (ParseException e) { + catch (ParseException | IllegalArgumentException e) { throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0303, filterString, e); } + } + + private boolean getOrder(String orderString) throws XServletException { + switch (orderString) { + case "desc": + return true; + case "asc": + return false; + default: + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0303, "order", orderString); + } + } + @VisibleForTesting + JSONObject getSLASummaryListByFilterParams(String timeZoneId, int numMaxResults, ListMultimap<String, String> filterList, + String sortbyColumn, boolean isDescendingOrder) throws + ServletException, ParseException, IllegalArgumentException, JPAExecutorException { + SLASummaryGetForFilterJPAExecutor slaSummaryGetForFilterJPAExecutor = + createSlaSummaryGetForFilterJPAExecutor(numMaxResults, filterList, sortbyColumn, isDescendingOrder); + List<SLASummaryBean> slaSummaryList = filterForSlaSummaryBeans(slaSummaryGetForFilterJPAExecutor); + + List<String> jobIds = new ArrayList<>(); + if (slaSummaryList != null) { + for (SLASummaryBean summaryBean : slaSummaryList) { + jobIds.add(summaryBean.getId()); + } + } + List<SLARegistrationBean> SLARegistrationList = SLARegistrationQueryExecutor.getInstance().getList( + SLARegQuery.GET_SLA_CONFIGS, jobIds); + + Map<String, Map<String, String>> jobIdSLAConfigMap = new HashMap<>(); + for(SLARegistrationBean registrationBean : SLARegistrationList){ + jobIdSLAConfigMap.put(registrationBean.getId(), registrationBean.getSLAConfigMap()); + } + return SLASummaryBean.toJSONObject(slaSummaryList, jobIdSLAConfigMap, timeZoneId); + } + + private List<SLASummaryBean> filterForSlaSummaryBeans(SLASummaryGetForFilterJPAExecutor slaSummaryGetForFilterJPAExecutor) + throws JPAExecutorException, IllegalArgumentException { + JPAService jpaService = Services.get().get(JPAService.class); + List<SLASummaryBean> slaSummaryList = null; + if (jpaService != null) { + slaSummaryList = jpaService.execute(slaSummaryGetForFilterJPAExecutor); + } + else { + XLog.getLog(getClass()).error(ErrorCode.E0610); + } + return slaSummaryList; } - private boolean isBundleId(String id) { - boolean ret = false; - Matcher m = p.matcher(id); - if (m.matches()) { - return true; + private SLASummaryGetForFilterJPAExecutor createSlaSummaryGetForFilterJPAExecutor(int numMaxResults, + ListMultimap<String, String> filterList, + String sortbyColumn, + boolean isDescendingOrder) + throws ServletException, ParseException { + SLASummaryGetForFilterJPAExecutor slaSummaryGetForFilterJPAExecutor = + new SLASummaryGetForFilterJPAExecutor(numMaxResults); + slaSummaryGetForFilterJPAExecutor.setSortbyColumn(sortbyColumn); + slaSummaryGetForFilterJPAExecutor.setDescendingOrder(isDescendingOrder); + + for(String filterName : filterList.keySet()) { + String filterValue = filterList.get(filterName).get(0); + slaSummaryGetForFilterJPAExecutor.checkAndSetFilterField(filterName, filterValue); } - return ret; + return slaSummaryGetForFilterJPAExecutor; } } http://git-wip-us.apache.org/repos/asf/oozie/blob/90a97694/core/src/test/java/org/apache/oozie/TestFilterParser.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/TestFilterParser.java b/core/src/test/java/org/apache/oozie/TestFilterParser.java new file mode 100644 index 0000000..49f47e2 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/TestFilterParser.java @@ -0,0 +1,98 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie; + +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Multimap; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import javax.servlet.ServletException; + +import static org.junit.Assert.assertEquals; + +public class TestFilterParser { + @Rule + public final ExpectedException expectedException = ExpectedException.none(); + + @Test + public void testNullFilter() throws ServletException { + Multimap<String, String> filterMap = FilterParser.parseFilter(null); + assertEquals("Filter map size should be zero for null filter", 0, filterMap.size()); + } + + @Test + public void testEmptyString() throws ServletException { + Multimap<String, String> filterMap = FilterParser.parseFilter(""); + assertEquals("Filter map size should be zero for empty filter", 0, filterMap.size()); + } + + @Test + public void testMissingEquals() throws ServletException { + expectedException.expect(ServletException.class); + FilterParser.parseFilter("keyvalue"); + } + + @Test + public void testMissingKey() throws ServletException { + expectedException.expect(ServletException.class); + FilterParser.parseFilter("=value"); + } + + @Test + public void testTooManyEquals() throws ServletException { + expectedException.expect(ServletException.class); + FilterParser.parseFilter("key=value1=value2"); + } + + @Test + public void testMissingValue() throws ServletException { + expectedException.expect(ServletException.class); + FilterParser.parseFilter("key1="); + } + + @Test + public void testSingleParameter() throws ServletException { + ListMultimap<String, String> filterMap = FilterParser.parseFilter("key1=value1"); + ListMultimap<String, String> expectedMap = LinkedListMultimap.create(); + expectedMap.put("key1", "value1"); + assertEquals("Different filter map", expectedMap, filterMap); + } + + @Test + public void testTwoParameters() throws ServletException { + Multimap<String, String> filterMap = FilterParser.parseFilter("key1=value1;key2=value2"); + ListMultimap<String, String> expectedMap = LinkedListMultimap.create(); + expectedMap.put("key1", "value1"); + expectedMap.put("key2", "value2"); + assertEquals("Different filter map", expectedMap, filterMap); + } + + @Test + public void testRepeatedKeys() throws ServletException { + Multimap<String, String> filterMap = FilterParser.parseFilter("key1=value1;key1=value2"); + ListMultimap<String, String> expectedMap = LinkedListMultimap.create(); + expectedMap.put("key1", "value1"); + expectedMap.put("key1", "value2"); + assertEquals("Different filter map", expectedMap, filterMap); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/90a97694/core/src/test/java/org/apache/oozie/executor/jpa/sla/TestSLASummaryGetForFilterJPAExecutorFilterCollection.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/executor/jpa/sla/TestSLASummaryGetForFilterJPAExecutorFilterCollection.java b/core/src/test/java/org/apache/oozie/executor/jpa/sla/TestSLASummaryGetForFilterJPAExecutorFilterCollection.java new file mode 100644 index 0000000..7f18b8b --- /dev/null +++ b/core/src/test/java/org/apache/oozie/executor/jpa/sla/TestSLASummaryGetForFilterJPAExecutorFilterCollection.java @@ -0,0 +1,207 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.executor.jpa.sla; + +import org.apache.oozie.servlet.XServletException; +import org.apache.oozie.util.DateUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import javax.servlet.ServletException; +import java.sql.Timestamp; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class TestSLASummaryGetForFilterJPAExecutorFilterCollection { + + @Rule + public final ExpectedException expectedException = ExpectedException.none(); + + private SLASummaryGetForFilterJPAExecutor.FilterCollection filterCollection; + + @Before + public void setUp() throws Exception { + filterCollection = new SLASummaryGetForFilterJPAExecutor.FilterCollection(); + } + + @Test + public void testNullParameterName() throws ServletException, ParseException { + expectedException.expect(XServletException.class); + filterCollection.checkAndSetFilterField(null, "value"); + } + + @Test + public void testInvalidParameterName() throws ServletException, ParseException { + expectedException.expect(XServletException.class); + filterCollection.checkAndSetFilterField("app_name_typo", "app_name_value"); + } + + @Test + public void testMixedCaseParameterName() throws ServletException, ParseException { + expectedException.expect(XServletException.class); + filterCollection.checkAndSetFilterField("aPP_name", "app_name_value"); + } + + @Test + public void testInvalidInteger() throws ParseException, ServletException { + expectedException.expect(IllegalArgumentException.class); + filterCollection.checkAndSetFilterField("actual_duration_min", "not a number"); + } + + @Test + public void testStringParameters() throws ServletException, ParseException { + checkSetAndAssertFilterField("app_name", "app_name_value"); + checkSetAndAssertFilterField("app_type", "app_type_value"); + checkSetAndAssertFilterField("user_name", "user_name_value"); + checkSetAndAssertFilterField("job_status", "job_status_value"); + checkSetAndAssertFilterField("id", "id_value"); + checkSetAndAssertFilterField("parent_id", "parent_id_value"); + checkSetAndAssertFilterField("bundle", "bundle_value"); + } + + @Test + public void testIntegerParameters() throws ServletException, ParseException { + checkSetAndAssertFilterField("actual_duration_min", "100", 100); + checkSetAndAssertFilterField("actual_duration_max", "200", 200); + checkSetAndAssertFilterField("expected_duration_min", "300", 300); + checkSetAndAssertFilterField("expected_duration_max", "400", 400); + } + + @Test + public void testTimestampParameters() throws ServletException, ParseException { + String time1 = "2012-06-03T16:00Z"; + Date date1 = DateUtils.parseDateUTC(time1); + Timestamp timestamp1 = new Timestamp(date1.getTime()); + String time2 = "2012-08-03T16:00Z"; + Date date2 = DateUtils.parseDateUTC(time2); + Timestamp timestamp2 = new Timestamp(date2.getTime()); + checkSetAndAssertFilterField("nominal_after", time1, timestamp1); + checkSetAndAssertFilterField("nominal_before", time2, timestamp2); + checkSetAndAssertFilterField("created_after", time1, timestamp1); + checkSetAndAssertFilterField("created_before", time2, timestamp2); + checkSetAndAssertFilterField("expectedstart_after", time1, timestamp1); + checkSetAndAssertFilterField("expectedstart_before", time2, timestamp2); + checkSetAndAssertFilterField("expectedend_after", time1, timestamp1); + checkSetAndAssertFilterField("expectedend_before", time2, timestamp2); + checkSetAndAssertFilterField("actualstart_after", time1, timestamp1); + checkSetAndAssertFilterField("actualstart_before", time2, timestamp2); + checkSetAndAssertFilterField("actualend_after", time1, timestamp1); + checkSetAndAssertFilterField("actualend_before", time2, timestamp2); + } + + @Test + public void testDeprecatedParameter() throws ServletException, ParseException { + String time1 = "2012-06-03T16:00Z"; + Date date1 = DateUtils.parseDateUTC(time1); + Timestamp timestamp1 = new Timestamp(date1.getTime()); + String time2 = "2012-08-03T16:00Z"; + Date date2 = DateUtils.parseDateUTC(time2); + Timestamp timestamp2 = new Timestamp(date2.getTime()); + checkSetAndAssertFilterField("nominal_start", time1, "nominal_after", timestamp1); + checkSetAndAssertFilterField("nominal_end", time2, "nominal_before", timestamp2); + } + + @Test + public void testInvalidSingleSetSLAStatus() throws ParseException, ServletException { + expectedException.expect(IllegalArgumentException.class); + filterCollection.checkAndSetFilterField("sla_status", "IN_PROCESS_TYPO"); + } + + @Test + public void testSingleSetSLAStatus() throws ServletException, ParseException { + checkSetAndAssertFilterField("sla_status", Collections.singletonList("IN_PROCESS"), + Collections.singletonList("IN_PROCESS")); + } + + @Test + public void testMultipleSetSLAStatusSingleSet() throws ServletException, ParseException { + checkSetAndAssertFilterField("sla_status", Collections.singletonList("MET,MISS"), Arrays.asList("MET", "MISS")); + } + + @Test + public void testMultipleSetSLAStatusMultiSet() throws ServletException, ParseException { + checkSetAndAssertFilterField("sla_status", Arrays.asList("MET", "MISS"), Arrays.asList("MISS")); + } + + @Test + public void testSingleSetEventStatus() throws ServletException, ParseException { + checkSetAndAssertFilterField("event_status", Collections.singletonList("START_MET,DURATION_MET"), + Arrays.asList("START_MET", "DURATION_MET")); + } + + @Test + public void testMultipleSetEventStatusSingleSet() throws ServletException, ParseException { + checkSetAndAssertFilterField("event_status", Collections.singletonList("START_MET,DURATION_MISS"), + Arrays.asList("START_MET", "DURATION_MISS")); + } + + @Test + public void testMultipleSetEventStatusMultiSet() throws ServletException, ParseException { + checkSetAndAssertFilterField("event_status", Arrays.asList("START_MET", "DURATION_MISS"), + Collections.singletonList("DURATION_MISS")); + } + + @Test + public void testInvalidIntegerInterval() throws ServletException, ParseException { + filterCollection.checkAndSetFilterField("actual_duration_min", "200"); + expectedException.expect(XServletException.class); + filterCollection.checkAndSetFilterField("actual_duration_max", "100"); + } + + @Test + public void testInvalidTimeInterval() throws ServletException, ParseException { + String time1 = "2018-09-12T16:00Z"; + String time2 = "2018-09-13T16:00Z"; + filterCollection.checkAndSetFilterField("nominal_after", time2); + expectedException.expect(XServletException.class); + filterCollection.checkAndSetFilterField("nominal_before", time1); + } + + private void checkSetAndAssertFilterField(String fieldName, String fieldValue) throws ParseException, ServletException { + checkSetAndAssertFilterField(fieldName, fieldValue, fieldName, fieldValue); + } + + private void checkSetAndAssertFilterField(String fieldName, String fieldValue, Object expectedValue) throws ParseException, + ServletException { + checkSetAndAssertFilterField(fieldName, fieldValue, fieldName, expectedValue); + } + + private void checkSetAndAssertFilterField(String fieldName, String fieldValue, String expectedFieldName, Object expectedValue) + throws ServletException, ParseException { + filterCollection.checkAndSetFilterField(fieldName, fieldValue); + assertEquals("Invalid parameter value", expectedValue, filterCollection.getFilterField(expectedFieldName)); + } + + private void checkSetAndAssertFilterField(String fieldName, List<String> fieldValues, List<String> expectedFieldValues) + throws ServletException, ParseException { + for (String fieldValue : fieldValues) { + filterCollection.checkAndSetFilterField(fieldName, fieldValue); + } + List<String> filterFieldReadAfterSet = (List<String>)filterCollection.getFilterField(fieldName); + String assertMessage = String.format("incorrect %s items", fieldName); + assertEquals(assertMessage, expectedFieldValues, filterFieldReadAfterSet); + } +} \ No newline at end of file
