YARN-2546. Made REST API for application creation/submission use numeric and boolean types instead of the string of them. Contributed by Varun Vasudev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72b0881c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72b0881c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72b0881c Branch: refs/heads/HDFS-6581 Commit: 72b0881ca641fa830c907823f674a5c5e39aa15a Parents: c86674a Author: Zhijie Shen <zjs...@apache.org> Authored: Wed Sep 24 17:57:32 2014 -0700 Committer: Zhijie Shen <zjs...@apache.org> Committed: Wed Sep 24 17:57:32 2014 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../webapp/JAXBContextResolver.java | 77 ++++++++++---------- .../TestRMWebServicesAppsModification.java | 9 ++- .../src/site/apt/ResourceManagerRest.apt.vm | 30 ++++---- 4 files changed, 62 insertions(+), 57 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/72b0881c/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4e71c1b..aaac7b7 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -444,6 +444,9 @@ Release 2.6.0 - UNRELEASED YARN-2596. TestWorkPreservingRMRestart fails with FairScheduler. (kasha) + YARN-2546. Made REST API for application creation/submission use numeric and + boolean types instead of the string of them. (Varun Vasudev via zjshen) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/72b0881c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index c577629..ca7edb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -22,62 +22,61 @@ import com.google.inject.Singleton; import com.sun.jersey.api.json.JSONConfiguration; import com.sun.jersey.api.json.JSONJAXBContext; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; +import java.util.*; import javax.ws.rs.ext.ContextResolver; import javax.ws.rs.ext.Provider; import javax.xml.bind.JAXBContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UsersInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*; import org.apache.hadoop.yarn.webapp.RemoteExceptionData; @Singleton @Provider public class JAXBContextResolver implements ContextResolver<JAXBContext> { - private JAXBContext context; - private final Set<Class> types; - - // you have to specify all the dao classes here - private final Class[] cTypes = { AppInfo.class, AppAttemptInfo.class, - AppAttemptsInfo.class, ClusterInfo.class, - CapacitySchedulerQueueInfo.class, FifoSchedulerInfo.class, - SchedulerTypeInfo.class, NodeInfo.class, UserMetricsInfo.class, - CapacitySchedulerInfo.class, ClusterMetricsInfo.class, - SchedulerInfo.class, AppsInfo.class, NodesInfo.class, - RemoteExceptionData.class, CapacitySchedulerQueueInfoList.class, - ResourceInfo.class, UsersInfo.class, UserInfo.class, - ApplicationStatisticsInfo.class, StatisticsItemInfo.class}; + private final Map<Class, JAXBContext> typesContextMap; public JAXBContextResolver() throws Exception { - this.types = new HashSet<Class>(Arrays.asList(cTypes)); - this.context = new JSONJAXBContext(JSONConfiguration.natural() - .rootUnwrapping(false).build(), cTypes); + + JAXBContext context; + JAXBContext unWrappedRootContext; + + // you have to specify all the dao classes here + final Class[] cTypes = + { AppInfo.class, AppAttemptInfo.class, AppAttemptsInfo.class, + ClusterInfo.class, CapacitySchedulerQueueInfo.class, + FifoSchedulerInfo.class, SchedulerTypeInfo.class, NodeInfo.class, + UserMetricsInfo.class, CapacitySchedulerInfo.class, + ClusterMetricsInfo.class, SchedulerInfo.class, AppsInfo.class, + NodesInfo.class, RemoteExceptionData.class, + CapacitySchedulerQueueInfoList.class, ResourceInfo.class, + UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, + StatisticsItemInfo.class }; + // these dao classes need root unwrapping + final Class[] rootUnwrappedTypes = + { NewApplication.class, ApplicationSubmissionContextInfo.class, + ContainerLaunchContextInfo.class, LocalResourceInfo.class, + DelegationToken.class }; + + this.typesContextMap = new HashMap<Class, JAXBContext>(); + context = + new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(false) + .build(), cTypes); + unWrappedRootContext = + new JSONJAXBContext(JSONConfiguration.natural().rootUnwrapping(true) + .build(), rootUnwrappedTypes); + for (Class type : cTypes) { + typesContextMap.put(type, context); + } + for (Class type : rootUnwrappedTypes) { + typesContextMap.put(type, unWrappedRootContext); + } } @Override public JAXBContext getContext(Class<?> objectType) { - return (types.contains(objectType)) ? context : null; + return typesContextMap.get(objectType); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/72b0881c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index 536aa8d..875c83c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -43,6 +43,8 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -214,6 +216,7 @@ public class TestRMWebServicesAppsModification extends JerseyTest { "org.apache.hadoop.yarn.server.resourcemanager.webapp") .contextListenerClass(GuiceServletConfig.class) .filterClass(com.google.inject.servlet.GuiceFilter.class) + .clientConfig(new DefaultClientConfig(JAXBContextResolver.class)) .contextPath("jersey-guice-filter").servletPath("/").build()); switch (run) { case 0: @@ -550,10 +553,10 @@ public class TestRMWebServicesAppsModification extends JerseyTest { } } - // Simple test - just post to /apps/id and validate the response + // Simple test - just post to /apps/new-application and validate the response @Test public void testGetNewApplication() throws Exception { - // client().addFilter(new LoggingFilter(System.out)); + client().addFilter(new LoggingFilter(System.out)); rm.start(); String mediaTypes[] = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; @@ -653,7 +656,7 @@ public class TestRMWebServicesAppsModification extends JerseyTest { // create a test app and submit it via rest(after getting an app-id) then // get the app details from the rmcontext and check that everything matches - // client().addFilter(new LoggingFilter(System.out)); + client().addFilter(new LoggingFilter(System.out)); String lrKey = "example"; String queueName = "testqueue"; String appName = "test"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/72b0881c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index c819f16..37905de 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -2083,12 +2083,12 @@ _01_000001</amContainerLogs> +---+ { - "application-id":"application_1404198295326_0001", + "application-id":"application_1404198295326_0003", "maximum-resource-capability": - { - "memory":"8192", - "vCores":"32" - } + { + "memory":8192, + "vCores":32 + } } +---+ @@ -2257,8 +2257,8 @@ _01_000001</amContainerLogs> "resource":"hdfs://hdfs-namenode:9000/user/testuser/DistributedShell/demo-app/AppMaster.jar", "type":"FILE", "visibility":"APPLICATION", - "size": "43004", - "timestamp": "1405452071209" + "size": 43004, + "timestamp": 1405452071209 } } ] @@ -2290,15 +2290,15 @@ _01_000001</amContainerLogs> ] } }, - "unmanaged-AM":"false", - "max-app-attempts":"2", + "unmanaged-AM":false, + "max-app-attempts":2, "resource": { - "memory":"1024", - "vCores":"1" + "memory":1024, + "vCores":1 }, "application-type":"YARN", - "keep-containers-across-application-attempts":"false" + "keep-containers-across-application-attempts":false } +---+ @@ -2797,8 +2797,8 @@ Server: Jetty(6.1.26) "renewer":"test-renewer", "owner":"cli...@example.com", "kind":"RM_DELEGATION_TOKEN", - "expiration-time":"1405153616489", - "max-validity":"1405672016489" + "expiration-time":1405153616489, + "max-validity":1405672016489 } +---+ @@ -2869,7 +2869,7 @@ Server: Jetty(6.1.26) +---+ { - "expiration-time":"1404112520402" + "expiration-time":1404112520402 } +---+
