YARN-6871. Add additional deSelects params in RMWebServices#getAppReport. Contributed by Tanuj Nayak.
(cherry picked from commit 8facf1f976d7e12a846f12baabf54be1b7a49f9d) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b872e5e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b872e5e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b872e5e Branch: refs/heads/resource-types Commit: 4b872e5e9f90626d725ba4dd4965cee4a0d249c7 Parents: a7e34be Author: Sunil G <[email protected]> Authored: Wed Sep 27 14:37:32 2017 +0530 Committer: Subru Krishnan <[email protected]> Committed: Fri Oct 20 17:20:39 2017 -0700 ---------------------------------------------------------------------- .../resourcemanager/webapp/DeSelectFields.java | 11 +- .../resourcemanager/webapp/dao/AppInfo.java | 108 +++++++++++++------ .../webapp/TestRMWebServicesApps.java | 77 ++++++++++++- 3 files changed, 160 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b872e5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java index 258bbfa..c991766 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DeSelectFields.java @@ -91,7 +91,16 @@ public class DeSelectFields { * <code>RESOURCE_REQUESTS</code> is the first * supported type from YARN-6280. */ - RESOURCE_REQUESTS("resourceRequests"); + RESOURCE_REQUESTS("resourceRequests"), + /** + * <code>APP_TIMEOUTS, APP_NODE_LABEL_EXPRESSION, AM_NODE_LABEL_EXPRESSION, + * RESOURCE_INFO</code> are additionally supported parameters added in + * YARN-6871. + */ + TIMEOUTS("timeouts"), + APP_NODE_LABEL_EXPRESSION("appNodeLabelExpression"), + AM_NODE_LABEL_EXPRESSION("amNodeLabelExpression"), + RESOURCE_INFO("resourceInfo"); private final String literals; http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b872e5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 9fb8fb5..db62914 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -121,7 +121,7 @@ public class AppInfo { protected String amNodeLabelExpression; protected ResourcesInfo resourceInfo = null; - protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo(); + private AppTimeoutsInfo timeouts; public AppInfo() { } // JAXB needs this @@ -243,47 +243,87 @@ public class AppInfo { unmanagedApplication = appSubmissionContext.getUnmanagedAM(); appNodeLabelExpression = app.getApplicationSubmissionContext().getNodeLabelExpression(); - amNodeLabelExpression = (unmanagedApplication) ? null - : app.getAMResourceRequests().get(0).getNodeLabelExpression(); + /* + * When the deSelects parameter contains "amNodeLabelExpression", objects + * pertaining to the amNodeLabelExpression are not returned. By default, + * this is not skipped. (YARN-6871) + */ + if(!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) { + amNodeLabelExpression = (unmanagedApplication) ? + null : + app.getAMResourceRequests().get(0).getNodeLabelExpression(); + } + /* + * When the deSelects parameter contains "appNodeLabelExpression", objects + * pertaining to the appNodeLabelExpression are not returned. By default, + * this is not skipped. (YARN-6871) + */ + if (!deSelects.contains(DeSelectType.APP_NODE_LABEL_EXPRESSION)) { + appNodeLabelExpression = + app.getApplicationSubmissionContext().getNodeLabelExpression(); + } + /* + * When the deSelects parameter contains "amNodeLabelExpression", objects + * pertaining to the amNodeLabelExpression are not returned. By default, + * this is not skipped. (YARN-6871) + */ + if (!deSelects.contains(DeSelectType.AM_NODE_LABEL_EXPRESSION)) { + amNodeLabelExpression = (unmanagedApplication) ? + null : + app.getAMResourceRequests().get(0).getNodeLabelExpression(); + } + /* + * When the deSelects parameter contains "resourceInfo", ResourceInfo + * objects are not returned. Default behavior is no skipping. (YARN-6871) + */ // Setting partition based resource usage of application - ResourceScheduler scheduler = rm.getRMContext().getScheduler(); - if (scheduler instanceof CapacityScheduler) { - RMAppAttempt attempt = app.getCurrentAppAttempt(); - if (null != attempt) { - FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler) - .getApplicationAttempt(attempt.getAppAttemptId()); - resourceInfo = null != ficaAppAttempt - ? new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) - : null; + if (!deSelects.contains(DeSelectType.RESOURCE_INFO)) { + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + if (scheduler instanceof CapacityScheduler) { + RMAppAttempt attempt = app.getCurrentAppAttempt(); + if (null != attempt) { + FiCaSchedulerApp ficaAppAttempt = ((CapacityScheduler) scheduler) + .getApplicationAttempt(attempt.getAppAttemptId()); + resourceInfo = null != ficaAppAttempt ? + new ResourcesInfo(ficaAppAttempt.getSchedulingResourceUsage()) : + null; + } } } - Map<ApplicationTimeoutType, Long> applicationTimeouts = - app.getApplicationTimeouts(); - if (applicationTimeouts.isEmpty()) { - // If application is not set timeout, lifetime should be sent as default - // with expiryTime=UNLIMITED and remainingTime=-1 - AppTimeoutInfo timeoutInfo = new AppTimeoutInfo(); - timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME); - timeouts.add(timeoutInfo); - } else { - for (Map.Entry<ApplicationTimeoutType, Long> entry : app - .getApplicationTimeouts().entrySet()) { - AppTimeoutInfo timeout = new AppTimeoutInfo(); - timeout.setTimeoutType(entry.getKey()); - long timeoutInMillis = entry.getValue().longValue(); - timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis)); - if (app.isAppInCompletedStates()) { - timeout.setRemainingTime(0); - } else { - timeout.setRemainingTime(Math - .max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + /* + * When the deSelects parameter contains "appTimeouts", objects pertaining + * to app timeouts are not returned. By default, this is not skipped. + * (YARN-6871) + */ + if (!deSelects.contains(DeSelectType.TIMEOUTS)) { + Map<ApplicationTimeoutType, Long> applicationTimeouts = + app.getApplicationTimeouts(); + if (applicationTimeouts.isEmpty()) { + // If application is not set timeout, lifetime should be sent + // as default with expiryTime=UNLIMITED and remainingTime=-1 + AppTimeoutInfo timeoutInfo = new AppTimeoutInfo(); + timeoutInfo.setTimeoutType(ApplicationTimeoutType.LIFETIME); + timeouts = new AppTimeoutsInfo(); + timeouts.add(timeoutInfo); + } else { + for (Map.Entry<ApplicationTimeoutType, Long> entry : app + .getApplicationTimeouts().entrySet()) { + AppTimeoutInfo timeout = new AppTimeoutInfo(); + timeout.setTimeoutType(entry.getKey()); + long timeoutInMillis = entry.getValue().longValue(); + timeout.setExpiryTime(Times.formatISO8601(timeoutInMillis)); + if (app.isAppInCompletedStates()) { + timeout.setRemainingTime(0); + } else { + timeout.setRemainingTime(Math.max( + (timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + } + timeouts.add(timeout); } - timeouts.add(timeout); } } - } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b872e5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 1cbdec3..b60e581 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1113,8 +1113,83 @@ public class TestRMWebServicesApps extends JerseyTestBase { JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); JSONObject app = array.getJSONObject(0); - assertTrue("resource requests shouldn't exits", + assertTrue("resource requests shouldn't exist", !app.has("resourceRequests")); + + params.clear(); + params.add("deSelects", + DeSelectFields.DeSelectType.AM_NODE_LABEL_EXPRESSION.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("AMNodeLabelExpression shouldn't exist", + !app.has("amNodeLabelExpression")); + + params.clear(); + params.add("deSelects", DeSelectFields.DeSelectType.TIMEOUTS.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("Timeouts shouldn't exist", !app.has("timeouts")); + rm.stop(); + + params.clear(); + params.add("deSelects", + DeSelectFields.DeSelectType.APP_NODE_LABEL_EXPRESSION.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("AppNodeLabelExpression shouldn't exist", + !app.has("appNodeLabelExpression")); + rm.stop(); + + params.clear(); + params + .add("deSelects", DeSelectFields.DeSelectType.RESOURCE_INFO.toString()); + response = + r.path("ws").path("v1").path("cluster").path("apps").queryParams(params) + .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + + json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + apps = json.getJSONObject("apps"); + assertEquals("incorrect number of elements", 1, apps.length()); + array = apps.getJSONArray("app"); + assertEquals("incorrect number of elements", 1, array.length()); + app = array.getJSONObject(0); + assertTrue("Resource info shouldn't exist", !app.has("resourceInfo")); rm.stop(); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
