Repository: falcon Updated Branches: refs/heads/master cc80a1754 -> 6e7634e8b
FALCON-1324 Pagination API breaks backward compatibility. Contributed by Ajay Yadava Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/6e7634e8 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/6e7634e8 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/6e7634e8 Branch: refs/heads/master Commit: 6e7634e8b4cff58894fbde5df97579e7ddfaf005 Parents: cc80a17 Author: Ajay Yadava <[email protected]> Authored: Tue Jul 28 14:28:39 2015 +0530 Committer: Ajay Yadava <[email protected]> Committed: Tue Jul 28 14:28:39 2015 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 ++ .../main/java/org/apache/falcon/cli/FalconCLI.java | 8 ++++---- .../java/org/apache/falcon/client/FalconClient.java | 9 ++++----- common/src/main/resources/runtime.properties | 3 ++- .../falcon/resource/AbstractEntityManager.java | 16 ++++++++++++++-- .../falcon/resource/AbstractInstanceManager.java | 3 +-- .../falcon/resource/proxy/InstanceManagerProxy.java | 12 ++++++++---- .../proxy/SchedulableEntityManagerProxy.java | 2 +- src/conf/runtime.properties | 8 +++++++- .../org/apache/falcon/resource/InstanceManager.java | 12 ++++++++---- .../falcon/resource/SchedulableEntityManager.java | 2 +- 11 files changed, 52 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 45d01b0..548325f 100755 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,8 @@ Trunk (Unreleased) FALCON-796 Enable users to triage data processing issues through falcon (Ajay Yadava) IMPROVEMENTS + FALCON-1324 Pagination API breaks backward compatibility(Ajay Yadava). + FALCON-668 FeedReplicator improvement to include more DistCP options(Sowmya Ramesh via Ajay Yadava) FALCON-1320 Adding equals() and hashCode() method in LineageGraphResult.Edge(Pragya Mittal via Ajay Yadava) http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/client/src/main/java/org/apache/falcon/cli/FalconCLI.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/cli/FalconCLI.java b/client/src/main/java/org/apache/falcon/cli/FalconCLI.java index e393f82..148f789 100644 --- a/client/src/main/java/org/apache/falcon/cli/FalconCLI.java +++ b/client/src/main/java/org/apache/falcon/cli/FalconCLI.java @@ -246,8 +246,7 @@ public class FalconCLI { String orderBy = commandLine.getOptionValue(ORDER_BY_OPT); String sortOrder = commandLine.getOptionValue(SORT_ORDER_OPT); Integer offset = parseIntegerInput(commandLine.getOptionValue(OFFSET_OPT), 0, "offset"); - Integer numResults = parseIntegerInput(commandLine.getOptionValue(NUM_RESULTS_OPT), - FalconClient.DEFAULT_NUM_RESULTS, "numResults"); + Integer numResults = parseIntegerInput(commandLine.getOptionValue(NUM_RESULTS_OPT), null, "numResults"); colo = getColo(colo); String instanceAction = "instance"; @@ -347,7 +346,8 @@ public class FalconCLI { OUT.get().println(result); } - private Integer parseIntegerInput(String optionValue, int defaultVal, String optionName) throws FalconCLIException { + private Integer parseIntegerInput(String optionValue, Integer defaultVal, String optionName) + throws FalconCLIException { Integer integer = defaultVal; if (optionValue != null) { try { @@ -418,7 +418,7 @@ public class FalconCLI { String feedInstancePath = commandLine.getOptionValue(PATH_OPT); Integer offset = parseIntegerInput(commandLine.getOptionValue(OFFSET_OPT), 0, "offset"); Integer numResults = parseIntegerInput(commandLine.getOptionValue(NUM_RESULTS_OPT), - FalconClient.DEFAULT_NUM_RESULTS, "numResults"); + null, "numResults"); Integer numInstances = parseIntegerInput(commandLine.getOptionValue(NUM_INSTANCES_OPT), 7, "numInstances"); EntityType entityTypeEnum = null; if (optionsList.contains(LIST_OPT)) { http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/client/src/main/java/org/apache/falcon/client/FalconClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java index 555bdb7..d507371 100644 --- a/client/src/main/java/org/apache/falcon/client/FalconClient.java +++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java @@ -93,7 +93,6 @@ public class FalconClient { private static final String TEMPLATE_SUFFIX = "-template.xml"; private static final String PROPERTIES_SUFFIX = ".properties"; - public static final int DEFAULT_NUM_RESULTS = 10; public static final HostnameVerifier ALL_TRUSTING_HOSTNAME_VERIFIER = new HostnameVerifier() { @Override @@ -441,7 +440,7 @@ public class FalconClient { String filterBy, String orderBy, String sortOrder) throws FalconCLIException { return sendInstanceRequest(Instances.SUMMARY, type, entity, start, end, null, - null, colo, lifeCycles, filterBy, orderBy, sortOrder, 0, DEFAULT_NUM_RESULTS) + null, colo, lifeCycles, filterBy, orderBy, sortOrder, 0, null) .getEntity(InstancesSummaryResult.class); } @@ -450,7 +449,7 @@ public class FalconClient { throws FalconCLIException { return sendInstanceRequest(Instances.LISTING, type, entity, start, end, null, - null, colo, null, "", "", "", 0, DEFAULT_NUM_RESULTS).getEntity(FeedInstanceResult.class); + null, colo, null, "", "", "", 0, null).getEntity(FeedInstanceResult.class); } public InstancesResult killInstances(String type, String entity, String start, @@ -767,7 +766,7 @@ public class FalconClient { String runid, String colo, List<LifeCycle> lifeCycles) throws FalconCLIException { return sendInstanceRequest(instances, type, entity, start, end, props, - runid, colo, lifeCycles, "", "", "", 0, DEFAULT_NUM_RESULTS) + runid, colo, lifeCycles, "", "", "", 0, null) .getEntity(InstancesResult.class); } @@ -776,7 +775,7 @@ public class FalconClient { String runid, String colo, List<LifeCycle> lifeCycles, Boolean isForced) throws FalconCLIException { return sendInstanceRequest(instances, type, entity, start, end, props, - runid, colo, lifeCycles, "", "", "", 0, DEFAULT_NUM_RESULTS, isForced).getEntity(InstancesResult.class); + runid, colo, lifeCycles, "", "", "", 0, null, isForced).getEntity(InstancesResult.class); } http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/common/src/main/resources/runtime.properties ---------------------------------------------------------------------- diff --git a/common/src/main/resources/runtime.properties b/common/src/main/resources/runtime.properties index 3b32463..2159e1a 100644 --- a/common/src/main/resources/runtime.properties +++ b/common/src/main/resources/runtime.properties @@ -23,7 +23,8 @@ *.falcon.replication.workflow.maxmaps=5 *.falcon.replication.workflow.mapbandwidth=100 -*.webservices.default.max.results.per.page=100 +*.webservices.default.max.results.per.page=3000 +*.webservices.default.results.per.page=3000 # Default configs to handle replication for late arriving feeds. *.feed.late.allowed=true http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java index a3801e9..478088d 100644 --- a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java +++ b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java @@ -77,7 +77,7 @@ public abstract class AbstractEntityManager { private static MemoryLocks memoryLocks = MemoryLocks.getInstance(); protected static final int XML_DEBUG_LEN = 10 * 1024; - protected static final String DEFAULT_NUM_RESULTS = "10"; + protected static final Integer DEFAULT_NUM_RESULTS = getDefaultResultsPerPage(); protected static final int MAX_RESULTS = getMaxResultsPerPage(); private AbstractWorkflowEngine workflowEngine; @@ -92,7 +92,7 @@ public abstract class AbstractEntityManager { } private static int getMaxResultsPerPage() { - Integer result = 100; + Integer result = 3000; final String key = "webservices.default.max.results.per.page"; String value = RuntimeProperties.get().getProperty(key, result.toString()); try { @@ -103,6 +103,18 @@ public abstract class AbstractEntityManager { return result; } + private static int getDefaultResultsPerPage() { + Integer result = 3000; + final String key = "webservices.default.results.per.page"; + String value = RuntimeProperties.get().getProperty(key, result.toString()); + try { + result = Integer.valueOf(value); + } catch (NumberFormatException e) { + LOG.warn("Invalid value:{} for key:{} in runtime.properties", value, key); + } + return result; + } + protected void checkColo(String colo) { if (!DeploymentUtil.getCurrentColo().equals(colo)) { throw FalconWebException.newException( http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java index 310e73b..bf42c85 100644 --- a/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java +++ b/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java @@ -76,7 +76,6 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager { private static final long HOUR_IN_MILLIS = 3600000L; protected static final long DAY_IN_MILLIS = 86400000L; private static final long MONTH_IN_MILLIS = 2592000000L; - protected static final String DEFAULT_NUM_RESULTS = "10"; protected EntityType checkType(String type) { if (StringUtils.isEmpty(type)) { @@ -840,7 +839,7 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager { private Pair<Date, Date> getStartAndEndDate(Entity entityObject, String startStr, String endStr) throws FalconException { - return getStartAndEndDate(entityObject, startStr, endStr, Integer.parseInt(DEFAULT_NUM_RESULTS)); + return getStartAndEndDate(entityObject, startStr, endStr, DEFAULT_NUM_RESULTS); } private Pair<Date, Date> getStartAndEndDate(Entity entityObject, String startStr, String endStr, int numResults) http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java index ac3e5db..465a636 100644 --- a/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java +++ b/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java @@ -100,7 +100,8 @@ public class InstanceManagerProxy extends AbstractInstanceManager { @DefaultValue("") @QueryParam("orderBy") final String orderBy, @DefaultValue("") @QueryParam("sortOrder") final String sortOrder, @DefaultValue("0") @QueryParam("offset") final Integer offset, - @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) { + @QueryParam("numResults") final Integer numResults) { + final Integer resultsPerPage = numResults == null ? DEFAULT_NUM_RESULTS : numResults; return new InstanceProxy<InstancesResult>(InstancesResult.class) { @Override protected InstancesResult doExecute(String colo) throws FalconException { @@ -131,7 +132,8 @@ public class InstanceManagerProxy extends AbstractInstanceManager { @DefaultValue("") @QueryParam("orderBy") final String orderBy, @DefaultValue("") @QueryParam("sortOrder") final String sortOrder, @DefaultValue("0") @QueryParam("offset") final Integer offset, - @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) { + @QueryParam("numResults") Integer numResults) { + final Integer resultsPerPage = numResults == null ? DEFAULT_NUM_RESULTS : numResults; return new InstanceProxy<InstancesResult>(InstancesResult.class) { @Override protected InstancesResult doExecute(String colo) throws FalconException { @@ -158,7 +160,8 @@ public class InstanceManagerProxy extends AbstractInstanceManager { @DefaultValue("") @QueryParam("orderBy") final String orderBy, @DefaultValue("") @QueryParam("sortOrder") final String sortOrder, @DefaultValue("0") @QueryParam("offset") final Integer offset, - @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) { + @QueryParam("numResults") final Integer numResults) { + final Integer resultsPerPage = numResults == null ? DEFAULT_NUM_RESULTS : numResults; return new InstanceProxy<InstancesResult>(InstancesResult.class) { @Override protected InstancesResult doExecute(String colo) throws FalconException { @@ -251,7 +254,8 @@ public class InstanceManagerProxy extends AbstractInstanceManager { @DefaultValue("") @QueryParam("orderBy") final String orderBy, @DefaultValue("") @QueryParam("sortOrder") final String sortOrder, @DefaultValue("0") @QueryParam("offset") final Integer offset, - @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) { + @QueryParam("numResults") final Integer numResults) { + final Integer resultsPerPage = numResults == null ? DEFAULT_NUM_RESULTS : numResults; return new InstanceProxy<InstancesResult>(InstancesResult.class) { @Override protected InstancesResult doExecute(String colo) throws FalconException { http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java ---------------------------------------------------------------------- diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java index eb281d3..b9aef71 100644 --- a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java +++ b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java @@ -471,11 +471,11 @@ public class SchedulableEntityManagerProxy extends AbstractSchedulableEntityMana @DefaultValue("") @QueryParam("orderBy") String orderBy, @DefaultValue("asc") @QueryParam("sortOrder") String sortOrder, @DefaultValue("0") @QueryParam("offset") Integer offset, - @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) { if (StringUtils.isNotEmpty(type)) { type = type.substring(1); } + resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage; return super.getEntityList(fields, nameSubsequence, tagKeywords, type, tags, filterBy, orderBy, sortOrder, offset, resultsPerPage); } http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/src/conf/runtime.properties ---------------------------------------------------------------------- diff --git a/src/conf/runtime.properties b/src/conf/runtime.properties index 58dee3d..7a099ee 100644 --- a/src/conf/runtime.properties +++ b/src/conf/runtime.properties @@ -29,7 +29,13 @@ prism.falcon.local.endpoint=https://localhost:15443 # falcon server should have the following properties falcon.current.colo=local -webservices.default.max.results.per.page=100 + + +#default number of results per call, if numResults option is not specified then this value is used. +webservices.default.results.per.page=3000 +# maximum number of results allowed to be returned from server, acts as an upper limit for numResults option +webservices.default.max.results.per.page=3000 + # retry count - to fetch the status from the workflow engine workflow.status.retry.count=30 http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java index 9c5538b..cbc936e 100644 --- a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java +++ b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java @@ -55,7 +55,8 @@ public class InstanceManager extends AbstractInstanceManager { @DefaultValue("") @QueryParam("orderBy") String orderBy, @DefaultValue("") @QueryParam("sortOrder") String sortOrder, @DefaultValue("0") @QueryParam("offset") Integer offset, - @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) { + @QueryParam("numResults") Integer resultsPerPage) { + resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage; return super.getRunningInstances(type, entity, colo, lifeCycles, filterBy, orderBy, sortOrder, offset, resultsPerPage); } @@ -80,7 +81,8 @@ public class InstanceManager extends AbstractInstanceManager { @DefaultValue("") @QueryParam("orderBy") String orderBy, @DefaultValue("") @QueryParam("sortOrder") String sortOrder, @DefaultValue("0") @QueryParam("offset") Integer offset, - @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) { + @QueryParam("numResults") Integer resultsPerPage) { + resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage; return super.getInstances(type, entity, startStr, endStr, colo, lifeCycles, filterBy, orderBy, sortOrder, offset, resultsPerPage); } @@ -101,7 +103,8 @@ public class InstanceManager extends AbstractInstanceManager { @DefaultValue("") @QueryParam("orderBy") String orderBy, @DefaultValue("") @QueryParam("sortOrder") String sortOrder, @DefaultValue("0") @QueryParam("offset") Integer offset, - @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) { + @QueryParam("numResults") Integer resultsPerPage) { + resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage; return super.getStatus(type, entity, startStr, endStr, colo, lifeCycles, filterBy, orderBy, sortOrder, offset, resultsPerPage); } @@ -155,7 +158,8 @@ public class InstanceManager extends AbstractInstanceManager { @DefaultValue("") @QueryParam("orderBy") String orderBy, @DefaultValue("") @QueryParam("sortOrder") String sortOrder, @DefaultValue("0") @QueryParam("offset") Integer offset, - @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) { + @QueryParam("numResults") Integer resultsPerPage) { + resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage; return super.getLogs(type, entity, startStr, endStr, colo, runId, lifeCycles, filterBy, orderBy, sortOrder, offset, resultsPerPage); } http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java ---------------------------------------------------------------------- diff --git a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java index ed6f44e..9a3d6be 100644 --- a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java +++ b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java @@ -76,11 +76,11 @@ public class SchedulableEntityManager extends AbstractSchedulableEntityManager { @DefaultValue("") @QueryParam("orderBy") String orderBy, @DefaultValue("asc") @QueryParam("sortOrder") String sortOrder, @DefaultValue("0") @QueryParam("offset") Integer offset, - @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) { if (StringUtils.isNotEmpty(type)) { type = type.substring(1); } + resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage; return super.getEntityList(fields, nameSubsequence, tagKeywords, type, tags, filterBy, orderBy, sortOrder, offset, resultsPerPage); }
