Repository: oozie Updated Branches: refs/heads/master a015a45ee -> ff9e43fa8
OOZIE-1754 add order(sort) option and exclude filter for coord job Info (ryota) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ff9e43fa Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ff9e43fa Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ff9e43fa Branch: refs/heads/master Commit: ff9e43fa8612f4d099c45518e0f553595e7e0671 Parents: a015a45 Author: egashira <[email protected]> Authored: Thu Apr 10 08:18:58 2014 -0700 Committer: egashira <[email protected]> Committed: Thu Apr 10 08:18:58 2014 -0700 ---------------------------------------------------------------------- .../java/org/apache/oozie/cli/OozieCLI.java | 12 +++- .../org/apache/oozie/client/OozieClient.java | 15 +++-- .../org/apache/oozie/CoordinatorEngine.java | 37 ++++++++--- .../main/java/org/apache/oozie/DagEngine.java | 3 +- .../oozie/command/coord/CoordJobXCommand.java | 14 ++-- .../CoordJobGetActionsSubsetJPAExecutor.java | 45 ++++++++----- .../org/apache/oozie/TestCoordinatorEngine.java | 25 ++++++- .../org/apache/oozie/client/TestOozieCLI.java | 15 ++++- .../TestCoordMaterializeTransitionXCommand.java | 4 +- ...TestCoordJobGetActionsSubsetJPAExecutor.java | 68 ++++++++++++++------ .../servlet/MockCoordinatorEngineService.java | 6 ++ docs/src/site/twiki/DG_CommandLineTool.twiki | 14 ++-- docs/src/site/twiki/WebServicesAPI.twiki | 21 ++++++ release-log.txt | 1 + 14 files changed, 212 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/client/src/main/java/org/apache/oozie/cli/OozieCLI.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java index e1f551d..e3eb3b0 100644 --- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java +++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java @@ -131,6 +131,7 @@ public class OozieCLI { public static final String DATE_OPTION = "date"; public static final String RERUN_REFRESH_OPTION = "refresh"; public static final String RERUN_NOCLEANUP_OPTION = "nocleanup"; + public static final String ORDER_OPTION = "order"; public static final String UPDATE_SHARELIB_OPTION = "sharelibupdate"; @@ -285,7 +286,12 @@ public class OozieCLI { Option offset = new Option(OFFSET_OPTION, true, "job info offset of actions (default '1', requires -info)"); Option len = new Option(LEN_OPTION, true, "number of actions (default TOTAL ACTIONS, requires -info)"); Option filter = new Option(FILTER_OPTION, true, - "status=<S1>[;status=<S2>]* (All Coordinator actions satisfying any one of the status filters will be retreived. Currently, only supported for Coordinator job)"); + "status=<S1>[;status=<S2>]* or status!=<S1>[;status!=<S2>]* " + + "(All Coordinator actions satisfying the status filters will be retreived. " + + "Positive filters '=' concatenated with OR and negative filters '!=' with AND. " + + "Currently, only supported for Coordinator job)"); + Option order = new Option(ORDER_OPTION, true, + "order to show coord actions (default ascending order, 'desc' for descending order, requires -info)"); Option localtime = new Option(LOCAL_TIME_OPTION, false, "use local time (same as passing your time zone to -" + TIME_ZONE_OPTION + "). Overrides -" + TIME_ZONE_OPTION + " option"); Option timezone = new Option(TIME_ZONE_OPTION, true, @@ -338,6 +344,7 @@ public class OozieCLI { jobOptions.addOption(offset); jobOptions.addOption(len); jobOptions.addOption(filter); + jobOptions.addOption(order); jobOptions.addOption(action); jobOptions.addOption(date); jobOptions.addOption(rerun_coord); @@ -969,7 +976,8 @@ public class OozieCLI { s = commandLine.getOptionValue(LEN_OPTION); int len = Integer.parseInt((s != null) ? s : "-1"); String filter = commandLine.getOptionValue(FILTER_OPTION); - printCoordJob(wc.getCoordJobInfo(optionValue, filter, start, len), timeZoneId, + String order = commandLine.getOptionValue(ORDER_OPTION); + printCoordJob(wc.getCoordJobInfo(optionValue, filter, start, len, order), timeZoneId, options.contains(VERBOSE_OPTION)); } else if (optionValue.contains("-C@")) { http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/client/src/main/java/org/apache/oozie/client/OozieClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java index 46c4288..40c9562 100644 --- a/client/src/main/java/org/apache/oozie/client/OozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java @@ -1012,10 +1012,11 @@ public class OozieClient { private class CoordJobInfo extends ClientCallable<CoordinatorJob> { - CoordJobInfo(String jobId, String filter, int start, int len) { + CoordJobInfo(String jobId, String filter, int start, int len, String order) { super("GET", RestConstants.JOB, notEmpty(jobId, "jobId"), prepareParams(RestConstants.JOB_SHOW_PARAM, - RestConstants.JOB_SHOW_INFO, RestConstants.JOB_FILTER_PARAM, filter, RestConstants.OFFSET_PARAM, Integer.toString(start), - RestConstants.LEN_PARAM, Integer.toString(len))); + RestConstants.JOB_SHOW_INFO, RestConstants.JOB_FILTER_PARAM, filter, RestConstants.OFFSET_PARAM, + Integer.toString(start), RestConstants.LEN_PARAM, Integer.toString(len), RestConstants.ORDER_PARAM, + order)); } @Override @@ -1118,7 +1119,7 @@ public class OozieClient { * @throws OozieClientException thrown if the job info could not be retrieved. */ public CoordinatorJob getCoordJobInfo(String jobId) throws OozieClientException { - return new CoordJobInfo(jobId, null, -1, -1).call(); + return new CoordJobInfo(jobId, null, -1, -1, "asc").call(); } /** @@ -1128,11 +1129,13 @@ public class OozieClient { * @param filter filter the status filter * @param start starting index in the list of actions belonging to the job * @param len number of actions to be returned + * @param order order to list coord actions (e.g, desc) * @return the job info. * @throws OozieClientException thrown if the job info could not be retrieved. */ - public CoordinatorJob getCoordJobInfo(String jobId, String filter, int start, int len) throws OozieClientException { - return new CoordJobInfo(jobId, filter, start, len).call(); + public CoordinatorJob getCoordJobInfo(String jobId, String filter, int start, int len, String order) + throws OozieClientException { + return new CoordJobInfo(jobId, filter, start, len, order).call(); } public List<WorkflowJob> getWfsForCoordAction(String coordActionId) throws OozieClientException { http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/core/src/main/java/org/apache/oozie/CoordinatorEngine.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/CoordinatorEngine.java b/core/src/main/java/org/apache/oozie/CoordinatorEngine.java index 3f10024..6a17ce4 100644 --- a/core/src/main/java/org/apache/oozie/CoordinatorEngine.java +++ b/core/src/main/java/org/apache/oozie/CoordinatorEngine.java @@ -67,6 +67,8 @@ public class CoordinatorEngine extends BaseEngine { public final static String COORD_ACTIONS_LOG_MAX_COUNT = "oozie.coord.actions.log.max.count"; private final static int COORD_ACTIONS_LOG_MAX_COUNT_DEFAULT = 50; private int maxNumActionsForLog; + public final static String POSITIVE_FILTER = "positive"; + public final static String NEGATIVE_FILTER = "negative"; /** * Create a system Coordinator engine, with no user and no group. @@ -154,9 +156,9 @@ public class CoordinatorEngine extends BaseEngine { @Override public CoordinatorJobBean getCoordJob(String jobId, String filter, int start, int length, boolean desc) throws BaseEngineException { - List<String> filterList = parseStatusFilter(filter); + Map<String, List<String>> filterMap = parseStatusFilter(filter); try { - return new CoordJobXCommand(jobId, filterList, start, length, desc) + return new CoordJobXCommand(jobId, filterMap, start, length, desc) .call(); } catch (CommandException ex) { @@ -546,18 +548,25 @@ public class CoordinatorEngine extends BaseEngine { } // Parses the filter string (e.g status=RUNNING;status=WAITING) and returns a list of status values - private List<String> parseStatusFilter(String filter) throws CoordinatorEngineException { - List<String> filterList = new ArrayList<String>(); + private Map<String, List<String>> parseStatusFilter(String filter) throws CoordinatorEngineException { + Map<String, List<String>> filterMap = new HashMap<String, List<String>>(); if (filter != null) { //split name;value pairs StringTokenizer st = new StringTokenizer(filter, ";"); while (st.hasMoreTokens()) { String token = st.nextToken(); if (token.contains("=")) { - String[] pair = token.split("="); + boolean negative = false; + String[] pair = null; + if(token.contains("!=")) { + negative = true; + pair = token.split("!="); + }else { + pair = token.split("="); + } if (pair.length != 2) { throw new CoordinatorEngineException(ErrorCode.E0421, token, - "elements must be name=value pairs"); + "elements must be name=value or name!=value pairs"); } if (pair[0].equalsIgnoreCase("status")) { String statusValue = pair[1]; @@ -572,6 +581,18 @@ public class CoordinatorEngine extends BaseEngine { throw new CoordinatorEngineException(ErrorCode.E0421, filter, XLog.format( "invalid status value [{0}]." + " Valid status values are: [{1}]", statusValue, validStatusList)); } + String filterType = negative ? NEGATIVE_FILTER : POSITIVE_FILTER; + String oppositeFilterType = negative ? POSITIVE_FILTER : NEGATIVE_FILTER; + List<String> filterList = filterMap.get(filterType); + if (filterList == null) { + filterList = new ArrayList<String>(); + filterMap.put(filterType, filterList); + } + List<String> oFilterList = filterMap.get(oppositeFilterType); + if (oFilterList != null && oFilterList.contains(statusValue)) { + throw new CoordinatorEngineException(ErrorCode.E0421, filter, XLog.format( + "the status [{0}] specified in both positive and negative filters", statusValue)); + } filterList.add(statusValue); } else { // Check for incorrect filter option @@ -580,11 +601,11 @@ public class CoordinatorEngine extends BaseEngine { } } else { throw new CoordinatorEngineException(ErrorCode.E0421, token, - "elements must be name=value pairs"); + "elements must be name=value or name!=value pairs"); } } } - return filterList; + return filterMap; } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/core/src/main/java/org/apache/oozie/DagEngine.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/DagEngine.java b/core/src/main/java/org/apache/oozie/DagEngine.java index 300d6eb..7d316b8 100644 --- a/core/src/main/java/org/apache/oozie/DagEngine.java +++ b/core/src/main/java/org/apache/oozie/DagEngine.java @@ -495,7 +495,8 @@ public class DagEngine extends BaseEngine { } @Override - public CoordinatorJob getCoordJob(String jobId, String filter, int start, int length, boolean desc) throws BaseEngineException { + public CoordinatorJob getCoordJob(String jobId, String filter, int start, int length, boolean desc) + throws BaseEngineException { throw new BaseEngineException(new XException(ErrorCode.E0301, "cannot get a coordinator job from DagEngine")); } http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/core/src/main/java/org/apache/oozie/command/coord/CoordJobXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordJobXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordJobXCommand.java index 0a030af..c872c48 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordJobXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordJobXCommand.java @@ -20,6 +20,7 @@ package org.apache.oozie.command.coord; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import org.apache.oozie.CoordinatorActionBean; import org.apache.oozie.CoordinatorJobBean; @@ -43,7 +44,7 @@ public class CoordJobXCommand extends CoordinatorXCommand<CoordinatorJobBean> { private int start = 1; private int len = Integer.MAX_VALUE; private boolean desc = false; - private List<String> filterList; + private Map<String, List<String>> filterMap; /** * Constructor for loading a coordinator job information @@ -51,7 +52,7 @@ public class CoordJobXCommand extends CoordinatorXCommand<CoordinatorJobBean> { * @param id coord jobId */ public CoordJobXCommand(String id) { - this(id, Collections.<String>emptyList(), 1, Integer.MAX_VALUE, false); + this(id, null, 1, Integer.MAX_VALUE, false); } /** @@ -60,12 +61,13 @@ public class CoordJobXCommand extends CoordinatorXCommand<CoordinatorJobBean> { * @param id coord jobId * @param start starting index in the list of actions belonging to the job * @param length number of actions to be returned + * @param filetrList */ - public CoordJobXCommand(String id, List<String> filterList, int start, int length, boolean desc) { + public CoordJobXCommand(String id, Map<String, List<String>> filterMap, int start, int length, boolean desc) { super("job.info", "job.info", 1); this.id = ParamChecker.notEmpty(id, "id"); this.getActionInfo = true; - this.filterList = filterList; + this.filterMap = filterMap; this.start = start; this.len = length; this.desc = desc; @@ -130,8 +132,8 @@ public class CoordJobXCommand extends CoordinatorXCommand<CoordinatorJobBean> { coordActions = new ArrayList<CoordinatorActionBean>(); } else { - coordActions = jpaService.execute(new CoordJobGetActionsSubsetJPAExecutor(id, filterList, - start, len, desc)); + coordActions = jpaService.execute(new CoordJobGetActionsSubsetJPAExecutor(id, filterMap, start, + len, desc)); } coordJob.setActions(coordActions); coordJob.setNumActions(numAction); http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/core/src/main/java/org/apache/oozie/executor/jpa/CoordJobGetActionsSubsetJPAExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/executor/jpa/CoordJobGetActionsSubsetJPAExecutor.java b/core/src/main/java/org/apache/oozie/executor/jpa/CoordJobGetActionsSubsetJPAExecutor.java index 21506ab..873f081 100644 --- a/core/src/main/java/org/apache/oozie/executor/jpa/CoordJobGetActionsSubsetJPAExecutor.java +++ b/core/src/main/java/org/apache/oozie/executor/jpa/CoordJobGetActionsSubsetJPAExecutor.java @@ -20,11 +20,13 @@ package org.apache.oozie.executor.jpa; import java.sql.Timestamp; import java.util.ArrayList; import java.util.List; +import java.util.Map; import javax.persistence.EntityManager; import javax.persistence.Query; import org.apache.oozie.CoordinatorActionBean; +import org.apache.oozie.CoordinatorEngine; import org.apache.oozie.ErrorCode; import org.apache.oozie.StringBlob; import org.apache.oozie.client.CoordinatorAction; @@ -41,17 +43,17 @@ public class CoordJobGetActionsSubsetJPAExecutor implements JPAExecutor<List<Coo private int start = 1; private int len = 50; private boolean desc = false; - private List<String> filterList; + private Map<String,List<String>> filterMap; public CoordJobGetActionsSubsetJPAExecutor(String coordJobId) { ParamChecker.notNull(coordJobId, "coordJobId"); this.coordJobId = coordJobId; } - public CoordJobGetActionsSubsetJPAExecutor(String coordJobId, List<String> filterList, int start, int len, boolean desc) { + public CoordJobGetActionsSubsetJPAExecutor(String coordJobId, Map<String, List<String>> filterMap, + int start, int len, boolean desc) { this(coordJobId); - ParamChecker.notNull(filterList, "filterList"); - this.filterList = filterList; + this.filterMap = filterMap; this.start = start; this.len = len; this.desc = desc; @@ -90,13 +92,15 @@ public class CoordJobGetActionsSubsetJPAExecutor implements JPAExecutor<List<Coo } private Query setQueryParameters(Query q, EntityManager em){ - if (!filterList.isEmpty()) { + if (filterMap != null) { // Add the filter clause String query = q.toString(); StringBuilder sbTotal = new StringBuilder(query); int offset = query.lastIndexOf("order"); // Get the 'where' clause for status filters - StringBuilder statusClause = getStatusClause(filterList); + StringBuilder statusClause = new StringBuilder(); + getStatusClause(statusClause, filterMap.get(CoordinatorEngine.POSITIVE_FILTER), true); + getStatusClause(statusClause, filterMap.get(CoordinatorEngine.NEGATIVE_FILTER), false); // Insert 'where' before 'order by' sbTotal.insert(offset, statusClause); q = em.createQuery(sbTotal.toString()); @@ -111,19 +115,28 @@ public class CoordJobGetActionsSubsetJPAExecutor implements JPAExecutor<List<Coo } // Form the where clause to filter by status values - private StringBuilder getStatusClause(List<String> filterList) { - StringBuilder sb = new StringBuilder(); + private StringBuilder getStatusClause(StringBuilder sb, List<String> filterList, boolean positive) { + if (sb == null) { + sb = new StringBuilder(); + } boolean isStatus = false; - for (String statusVal : filterList) { - if (!isStatus) { - sb.append(" and a.statusStr IN (\'" + statusVal + "\'"); - isStatus = true; - } - else { - sb.append(",\'" + statusVal + "\'"); + if (filterList != null && filterList.size() > 0) { + for (String statusVal : filterList) { + if (!isStatus) { + if (positive) { + sb.append(" and a.statusStr IN (\'" + statusVal + "\'"); + } + else { + sb.append(" and a.statusStr NOT IN (\'" + statusVal + "\'"); + } + isStatus = true; + } + else { + sb.append(",\'" + statusVal + "\'"); + } } + sb.append(") "); } - sb.append(") "); return sb; } http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/core/src/test/java/org/apache/oozie/TestCoordinatorEngine.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/TestCoordinatorEngine.java b/core/src/test/java/org/apache/oozie/TestCoordinatorEngine.java index 85ef53b..ea2ca27 100644 --- a/core/src/test/java/org/apache/oozie/TestCoordinatorEngine.java +++ b/core/src/test/java/org/apache/oozie/TestCoordinatorEngine.java @@ -455,13 +455,34 @@ public class TestCoordinatorEngine extends XTestCase { job = ce.getCoordJob(jobId, "", 1, 2, false); assertEquals(job.getActions().size(), 2); + //Check for negative filter + job = ce.getCoordJob(jobId, "status!=RUNNING", 1, 2, false); + assertEquals(job.getActions().size(), 2); + + //Check for multiple negative filter + job = ce.getCoordJob(jobId, "status!=RUNNING;status!=WAITING", 1, 2, false); + assertEquals(job.getActions().size(), 0); + + //Check for combination of positive and negative filter + try { + job = ce.getCoordJob(jobId, "status=WAITING;status!=WAITING", 1, 2, false); + } + catch (CoordinatorEngineException ex) { + assertEquals(ErrorCode.E0421, ex.getErrorCode()); + assertEquals( + "E0421: Invalid job filter [status=WAITING;status!=WAITING], the status [WAITING] " + + "specified in both positive and negative filters", + ex.getMessage()); + } + //Check for missing "=" try { job = ce.getCoordJob(jobId, "statusRUNNING", 1, 2, false); } catch (CoordinatorEngineException ex) { assertEquals(ErrorCode.E0421, ex.getErrorCode()); - assertEquals("E0421: Invalid job filter [statusRUNNING], elements must be name=value pairs", ex.getMessage()); + assertEquals("E0421: Invalid job filter [statusRUNNING], elements must be name=value or name!=value pairs", + ex.getMessage()); } //Check for missing value after "=" @@ -470,7 +491,7 @@ public class TestCoordinatorEngine extends XTestCase { } catch (CoordinatorEngineException ex) { assertEquals(ErrorCode.E0421, ex.getErrorCode()); - assertEquals("E0421: Invalid job filter [status=], elements must be name=value pairs", ex.getMessage()); + assertEquals("E0421: Invalid job filter [status=], elements must be name=value or name!=value pairs", ex.getMessage()); } // Check for invalid status value http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java index e90d28a..76c7a29 100644 --- a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java +++ b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java @@ -846,11 +846,24 @@ public class TestOozieCLI extends DagServletTestCase { MockCoordinatorEngineService.reset(); args = new String[] { "job", "-oozie", oozieUrl, "-info", MockCoordinatorEngineService.JOB_ID + 1 + MockCoordinatorEngineService.JOB_ID_END, - "-len", "10", "-offset", "5" }; + "-len", "10", "-offset", "5", "-order", "desc", "-filter", "status=FAILED"}; assertEquals(0, new OozieCLI().run(args)); assertEquals(RestConstants.JOB_SHOW_INFO, MockCoordinatorEngineService.did); assertEquals(MockCoordinatorEngineService.offset, new Integer(5)); assertEquals(MockCoordinatorEngineService.length, new Integer(10)); + assertEquals(MockCoordinatorEngineService.order, "desc"); + assertEquals(MockCoordinatorEngineService.filter, "status=FAILED"); + + MockCoordinatorEngineService.reset(); + args = new String[] { "job", "-oozie", oozieUrl, "-info", + MockCoordinatorEngineService.JOB_ID + 1 + MockCoordinatorEngineService.JOB_ID_END, + "-len", "10", "-offset", "5", "-order", "desc", "-filter", "status!=FAILED"}; + assertEquals(0, new OozieCLI().run(args)); + assertEquals(RestConstants.JOB_SHOW_INFO, MockCoordinatorEngineService.did); + assertEquals(MockCoordinatorEngineService.offset, new Integer(5)); + assertEquals(MockCoordinatorEngineService.length, new Integer(10)); + assertEquals(MockCoordinatorEngineService.order, "desc"); + assertEquals(MockCoordinatorEngineService.filter, "status!=FAILED"); return null; } http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java index db591e2..6c73585 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordMaterializeTransitionXCommand.java @@ -589,8 +589,8 @@ public class TestCoordMaterializeTransitionXCommand extends XDataTestCase { private void checkCoordActionsNominalTime(String jobId, int number, Date[] nominalTimes) { try { JPAService jpaService = Services.get().get(JPAService.class); - List<CoordinatorActionBean> actions = jpaService.execute(new CoordJobGetActionsSubsetJPAExecutor( - jobId, new ArrayList<String>(), 1, 1000, false)); + List<CoordinatorActionBean> actions = jpaService.execute(new CoordJobGetActionsSubsetJPAExecutor(jobId, + null, 1, 1000, false)); if (actions.size() != number) { fail("Should have " + number + " actions created for job " + jobId + ", but has " + actions.size() + " actions."); http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/core/src/test/java/org/apache/oozie/executor/jpa/TestCoordJobGetActionsSubsetJPAExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/executor/jpa/TestCoordJobGetActionsSubsetJPAExecutor.java b/core/src/test/java/org/apache/oozie/executor/jpa/TestCoordJobGetActionsSubsetJPAExecutor.java index da7d16c..147bdb7 100644 --- a/core/src/test/java/org/apache/oozie/executor/jpa/TestCoordJobGetActionsSubsetJPAExecutor.java +++ b/core/src/test/java/org/apache/oozie/executor/jpa/TestCoordJobGetActionsSubsetJPAExecutor.java @@ -20,10 +20,13 @@ package org.apache.oozie.executor.jpa; import java.util.ArrayList; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.hadoop.fs.Path; import org.apache.oozie.CoordinatorActionBean; +import org.apache.oozie.CoordinatorEngine; import org.apache.oozie.CoordinatorJobBean; import org.apache.oozie.client.CoordinatorAction; import org.apache.oozie.client.CoordinatorJob; @@ -87,8 +90,8 @@ public class TestCoordJobGetActionsSubsetJPAExecutor extends XDataTestCase { throws Exception { JPAService jpaService = Services.get().get(JPAService.class); assertNotNull(jpaService); - CoordJobGetActionsSubsetJPAExecutor actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(jobId, - Collections.<String> emptyList(), start, len, false); + CoordJobGetActionsSubsetJPAExecutor actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(jobId, null, start, + len, false); List<CoordinatorActionBean> actions = jpaService.execute(actionGetCmd); CoordinatorActionBean action = actions.get(0); @@ -146,8 +149,8 @@ public class TestCoordJobGetActionsSubsetJPAExecutor extends XDataTestCase { throws Exception { JPAService jpaService = Services.get().get(JPAService.class); assertNotNull(jpaService); - CoordJobGetActionsSubsetJPAExecutor actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(jobId, - Collections.<String> emptyList(), start, len, order); + CoordJobGetActionsSubsetJPAExecutor actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(jobId, null, start, + len, order); List<CoordinatorActionBean> actions = jpaService.execute(actionGetCmd); return actions; @@ -160,25 +163,50 @@ public class TestCoordJobGetActionsSubsetJPAExecutor extends XDataTestCase { addRecordToCoordActionTable(job.getId(), 1, CoordinatorAction.Status.RUNNING, "coord-action-get.xml", 0); // Add Coordinator action with nominal time: 2009-02-01T23:59Z addRecordToCoordActionTable(job.getId(), 2, CoordinatorAction.Status.WAITING, "coord-action-get.xml", 0); - // Create lists for status filter - List<String> filterList = new ArrayList<String>(); - filterList.add("RUNNING"); - filterList.add("KILLED"); - _testGetActionsSubsetFilter(job.getId(), 1, filterList, 1, 2); - } + // Create lists for status filter to test positive filter + Map<String, List<String>> filterMap = new HashMap<String, List<String>>(); + List<String> positiveFilter = new ArrayList<String>(); + positiveFilter.add("RUNNING"); + positiveFilter.add("KILLED"); + filterMap.put(CoordinatorEngine.POSITIVE_FILTER, positiveFilter); + List<CoordinatorActionBean> actions = _testGetActionsSubsetFilter(job.getId(), 1, filterMap, 1, 2); + assertEquals(actions.size(), 1); + assertEquals(actions.get(0).getActionNumber(), 1); + // Create lists for status filter to test negative filter + filterMap.clear(); + List<String> negativeFilter = new ArrayList<String>(); + negativeFilter.add("WAITING"); + negativeFilter.add("KILLED"); + filterMap.put(CoordinatorEngine.NEGATIVE_FILTER, negativeFilter); + actions = _testGetActionsSubsetFilter(job.getId(), 1, filterMap, 1, 2); + assertEquals(actions.size(), 1); + assertEquals(actions.get(0).getActionNumber(), 1); + + // Test Combination of include/exclude filters - no dup + filterMap.clear(); + filterMap.put(CoordinatorEngine.POSITIVE_FILTER, positiveFilter); + filterMap.put(CoordinatorEngine.NEGATIVE_FILTER, negativeFilter); + actions = _testGetActionsSubsetFilter(job.getId(), 1, filterMap, 1, 2); + assertEquals(actions.size(), 1); + assertEquals(actions.get(0).getActionNumber(), 1); + + // Test Combination of include/exclude filters - dup --> no result + filterMap.clear(); + filterMap.put(CoordinatorEngine.POSITIVE_FILTER, positiveFilter); + filterMap.put(CoordinatorEngine.NEGATIVE_FILTER, positiveFilter); + actions = _testGetActionsSubsetFilter(job.getId(), 1, filterMap, 1, 2); + assertEquals(actions.size(), 0); + } // Check whether actions are retrieved based on the filter values for status - private void _testGetActionsSubsetFilter(String jobId, int actionNum, List<String> filterList, int start, int len) - throws JPAExecutorException { + private List<CoordinatorActionBean> _testGetActionsSubsetFilter(String jobId, int actionNum, + Map<String, List<String>> filterMap, int start, int len) throws JPAExecutorException { JPAService jpaService = Services.get().get(JPAService.class); assertNotNull(jpaService); - CoordJobGetActionsSubsetJPAExecutor actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(jobId, filterList, - start, len, false); + CoordJobGetActionsSubsetJPAExecutor actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(jobId, filterMap, start, + len, false); List<CoordinatorActionBean> actions = jpaService.execute(actionGetCmd); - // As actions are filtered by RUNNING status, only 1 action should be - // returned - assertEquals(actions.size(), 1); - assertEquals(actions.get(0).getActionNumber(), 1); + return actions; } public void testGetActionAllColumns() throws Exception { @@ -200,8 +228,8 @@ public class TestCoordJobGetActionsSubsetJPAExecutor extends XDataTestCase { private void _testGetForInfoAllActions(String jobId, String slaXml, int start, int len) throws Exception { JPAService jpaService = Services.get().get(JPAService.class); assertNotNull(jpaService); - CoordJobGetActionsSubsetJPAExecutor actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(jobId, - Collections.<String> emptyList(), start, len, false); + CoordJobGetActionsSubsetJPAExecutor actionGetCmd = new CoordJobGetActionsSubsetJPAExecutor(jobId, null, start, + len, false); List<CoordinatorActionBean> actions = jpaService.execute(actionGetCmd); CoordinatorActionBean action = actions.get(0); http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java b/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java index f174b06..9892d4b 100644 --- a/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java +++ b/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java @@ -54,6 +54,8 @@ public class MockCoordinatorEngineService extends CoordinatorEngineService { public static String did = null; public static Integer offset = null; public static Integer length = null; + public static String order = null; + public static String filter = null; public static List<CoordinatorJob> coordJobs; public static List<Boolean> started; public static final int INIT_COORD_COUNT = 4; @@ -66,6 +68,8 @@ public class MockCoordinatorEngineService extends CoordinatorEngineService { did = null; offset = null; length = null; + order = null; + filter = null; coordJobs = new ArrayList<CoordinatorJob>(); started = new ArrayList<Boolean>(); for (int i = 0; i < INIT_COORD_COUNT; i++) { @@ -187,6 +191,8 @@ public class MockCoordinatorEngineService extends CoordinatorEngineService { did = RestConstants.JOB_SHOW_INFO; MockCoordinatorEngineService.offset = start; MockCoordinatorEngineService.length = length; + MockCoordinatorEngineService.order = desc ? "desc" : "asc"; + MockCoordinatorEngineService.filter = filter; int idx = validateCoordinatorIdx(jobId); return (CoordinatorJobBean) coordJobs.get(idx); } http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/docs/src/site/twiki/DG_CommandLineTool.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki index 351f0f2..c1bd08a 100644 --- a/docs/src/site/twiki/DG_CommandLineTool.twiki +++ b/docs/src/site/twiki/DG_CommandLineTool.twiki @@ -41,6 +41,12 @@ usage: -info <arg> info of a job -kill <arg> kill a job (coordinator requires -action or -date) -len <arg> number of actions (default TOTAL ACTIONS, requires -info) + -filter <arg> status=<S1>[;status=<S2>]* or status!=<S1>[;status!=<S2>]* + All coordinator actions satisfying the status filters will be retrieved. + Positive filters '=' concatenated with 'OR' and negative filters '!=' with 'AND'. + Currently, only supported for coordinator job. + -order <arg> order to show coordinator actions (default ascending order, 'desc' for descending order, requires -info) + Currently, only supported for coordinator job. -localtime use local time (same as passing your time zone to -timezone). Overrides -timezone option -log <arg> job log @@ -456,11 +462,11 @@ The =localtime= option displays times in local time, if not specified times are The =verbose= option gives more detailed information for all the actions, if checking for workflow job or coordinator job. The =filter= option can be used to filter coordinator actions based on their status. -The filter option syntax is: <code>[status=VALUE][\;status=VALUE]*</code>. +The filter option syntax is: <code>[status=VALUE][\;status=VALUE]* or [status!=VALUE][\;status!=VALUE]*</code>. (Note escape <code>\</code> needed before semicolon to specify multiple names for filter in shell) -Multiple values must be specified as different name value pairs. When multiple filters are specified, -all Coordinator actions that satisfy any one of the filters will be retrieved. -(The query will do an OR among all the filter values for the status) +Multiple values must be specified as different name value pairs. +When multiple positive filters <code>=</code> are specified, all Coordinator actions that satisfy any one of the filters will be retrieved.(The query will do an OR among all the positive filter values for the status) +When multiple negative filters <code>!=</code> are specified, all Coordinator actions that satisfy all of the filters will be retrieved.(The query will do an AND among all the negative filter values for the status) Currently, the filter option can be used only with an =info= option on Coordinator job. An example below shows how the =verbose= option can be used to gather action statistics information for a job: http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/docs/src/site/twiki/WebServicesAPI.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/WebServicesAPI.twiki b/docs/src/site/twiki/WebServicesAPI.twiki index 37c0bc0..351699d 100644 --- a/docs/src/site/twiki/WebServicesAPI.twiki +++ b/docs/src/site/twiki/WebServicesAPI.twiki @@ -1215,6 +1215,27 @@ followed by query params - type=action and scope=<action-number>. One single act GET /oozie/v2/job/0000001-111219170928042-oozie-joe-C?show=allruns&type=action&scope=1 </verbatim> +*Retrieve a subset of actions* + +Query parameters, =offset= and =length= can be specified with a workflow job to retrieve specific actions. Default is offset=0, len=1000 +<verbatim> +GET /oozie/v1/job/0000002-130507145349661-oozie-joe-W?show=info&offset=5&len=10 +</verbatim> +Query parameters, =offset=, =length=, =filter= can be specified with a coordinator job to retrieve specific actions. +Query parameter, =order= with value "desc" can be used to retrieve the latest coordinator actions materialized instead of actions from @1. +Query parameters =filter= can be used to retrieve coodinator actions matching specific status. +Default is offset=0, len=0 for v2/job (i.e., does not return any coordinator actions) and offset=0, len=1000 with v1/job and v0/job. +So if you need actions to be returned with v2 API, specifying =len= parameter is necessary. +Default =order= is "asc". +<verbatim> +GET /oozie/v1/job/0000001-111219170928042-oozie-joe-C?show=info&offset=5&len=10&filter=status%3DKILLED&order=desc +</verbatim> +Note that the filter is URL encoded, its decoded value is <code>status=KILLED</code>. +<verbatim> +GET /oozie/v1/job/0000001-111219170928042-oozie-joe-C?show=info&filter=status%21%3DSUCCEEDED&order=desc +</verbatim> +This retrives coordinator actions except for SUCCEEDED status, which is useful for debugging. + ---++++ Job Application Definition A HTTP GET request retrieves the workflow or a coordinator job definition file. http://git-wip-us.apache.org/repos/asf/oozie/blob/ff9e43fa/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index c241774..00286d9 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.1.0 release (trunk - unreleased) +OOZIE-1754 add order(sort) option and exclude filter for coord job Info (ryota) OOZIE-1761 Improve sharelib purging logic (puru via rohini) OOZIE-1725 add coord EL functions to be used in SLA tag (ryota) OOZIE-1765 JMS Notifications for Workflows not always on the correct topic (rkanter)
