Author: hitesh
Date: Wed Jul 17 21:43:51 2013
New Revision: 1504290
URL: http://svn.apache.org/r1504290
Log:
Merge r1504288 from trunk to branch-2 for YARN-865. RM webservices can't query
based on application Types. Contributed by Xuan Gong.
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1504290&r1=1504289&r2=1504290&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Wed Jul 17
21:43:51 2013
@@ -486,6 +486,9 @@ Release 2.1.0-beta - 2013-07-02
YARN-922. Change FileSystemRMStateStore to use directories (Jian He via
bikas)
+ YARN-865. RM webservices can't query based on application Types. (Xuan Gong
+ via hitesh)
+
OPTIMIZATIONS
YARN-512. Log aggregation root directory check is more expensive than it
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1504290&r1=1504289&r2=1504290&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
Wed Jul 17 21:43:51 2013
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.re
import java.io.IOException;
import java.util.Collection;
import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import javax.servlet.http.HttpServletRequest;
@@ -231,11 +233,13 @@ public class RMWebServices {
@QueryParam("startedTimeBegin") String startedBegin,
@QueryParam("startedTimeEnd") String startedEnd,
@QueryParam("finishedTimeBegin") String finishBegin,
- @QueryParam("finishedTimeEnd") String finishEnd) {
+ @QueryParam("finishedTimeEnd") String finishEnd,
+ @QueryParam("applicationTypes") Set<String> applicationTypes) {
long num = 0;
boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
+ boolean checkAppTypes = false;
long countNum = 0;
// set values suitable in case both of begin/end not specified
@@ -291,6 +295,27 @@ public class RMWebServices {
"finishTimeEnd must be greater than finishTimeBegin");
}
+ Set<String> appTypes = new HashSet<String>();
+ if (!applicationTypes.isEmpty()) {
+ for (String applicationType : applicationTypes) {
+ if (applicationType != null && !applicationType.trim().isEmpty()) {
+ if (applicationType.indexOf(",") == -1) {
+ appTypes.add(applicationType.trim());
+ } else {
+ String[] types = applicationType.split(",");
+ for (String type : types) {
+ if (!type.trim().isEmpty()) {
+ appTypes.add(type.trim());
+ }
+ }
+ }
+ }
+ }
+ }
+ if (!appTypes.isEmpty()) {
+ checkAppTypes = true;
+ }
+
final ConcurrentMap<ApplicationId, RMApp> apps = rm.getRMContext()
.getRMApps();
AppsInfo allApps = new AppsInfo();
@@ -332,6 +357,10 @@ public class RMWebServices {
continue;
}
}
+ if (checkAppTypes
+ && !appTypes.contains(rmapp.getApplicationType())) {
+ continue;
+ }
if (checkStart
&& (rmapp.getStartTime() < sBegin || rmapp.getStartTime() > sEnd)) {
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1504290&r1=1504289&r2=1504290&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
Wed Jul 17 21:43:51 2013
@@ -638,6 +638,191 @@ public class TestRMWebServicesApps exten
}
@Test
+ public void testAppsQueryAppTypes() throws JSONException, Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ Thread.sleep(1);
+ RMApp app1 = rm.submitApp(1024);
+ amNodeManager.nodeHeartbeat(true);
+ // finish App
+ MockAM am = rm
+ .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId());
+ am.registerAppAttempt();
+ am.unregisterAppAttempt();
+ amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
+ 1, ContainerState.COMPLETE);
+
+ rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ .getShortUserName(), null, false, null, 2, null, "MAPREDUCE");
+ rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ .getShortUserName(), null, false, null, 2, null, "NON-YARN");
+
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("apps").queryParam("applicationTypes", "MAPREDUCE")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ JSONArray array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ assertEquals("MAPREDUCE",
+ array.getJSONObject(0).getString("applicationType"));
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", "YARN")
+ .queryParam("applicationTypes", "MAPREDUCE")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 2, array.length());
+ assertTrue((array.getJSONObject(0).getString("applicationType")
+ .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+ .equals("MAPREDUCE")) ||
+ (array.getJSONObject(1).getString("applicationType").equals("YARN")
+ && array.getJSONObject(0).getString("applicationType")
+ .equals("MAPREDUCE")));
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", "YARN,NON-YARN")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 2, array.length());
+ assertTrue((array.getJSONObject(0).getString("applicationType")
+ .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+ .equals("NON-YARN")) ||
+ (array.getJSONObject(1).getString("applicationType").equals("YARN")
+ && array.getJSONObject(0).getString("applicationType")
+ .equals("NON-YARN")));
+
+ r = resource();
+ response = r.path("ws").path("v1").path("cluster")
+ .path("apps").queryParam("applicationTypes", "")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 3, array.length());
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", "YARN,NON-YARN")
+ .queryParam("applicationTypes", "MAPREDUCE")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 3, array.length());
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", "YARN")
+ .queryParam("applicationTypes", "")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ assertEquals("YARN",
+ array.getJSONObject(0).getString("applicationType"));
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", ",,, ,, YARN ,, ,")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ assertEquals("YARN",
+ array.getJSONObject(0).getString("applicationType"));
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", ",,, ,, ,, ,")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 3, array.length());
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", "YARN, ,NON-YARN, ,,")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 2, array.length());
+ assertTrue((array.getJSONObject(0).getString("applicationType")
+ .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+ .equals("NON-YARN")) ||
+ (array.getJSONObject(1).getString("applicationType").equals("YARN")
+ && array.getJSONObject(0).getString("applicationType")
+ .equals("NON-YARN")));
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", " YARN, , ,,,")
+ .queryParam("applicationTypes", "MAPREDUCE , ,, ,")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 2, array.length());
+ assertTrue((array.getJSONObject(0).getString("applicationType")
+ .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+ .equals("MAPREDUCE")) ||
+ (array.getJSONObject(1).getString("applicationType").equals("YARN")
+ && array.getJSONObject(0).getString("applicationType")
+ .equals("MAPREDUCE")));
+
+ rm.stop();
+ }
+
+ @Test
public void testSingleApp() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1504290&r1=1504289&r2=1504290&view=diff
==============================================================================
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Wed Jul 17 21:43:51 2013
@@ -1119,6 +1119,7 @@ ResourceManager REST API's.
* startedTimeEnd - applications with start time ending with this time,
specified in ms since epoch
* finishedTimeBegin - applications with finish time beginning with this
time, specified in ms since epoch
* finishedTimeEnd - applications with finish time ending with this time,
specified in ms since epoch
+ * applicationTypes - applications matching the given application types,
specified as a comma-separated list.
------
** Elements of the <apps> (Applications) object
